Auto merge of #392 - Zeegomo:master, r=pietroalbini

Report agent crashes to the server

fixed #246
Any error encountered by the agent will be reported to the server that will post it to the relevant GitHub thread. Add new `failed` experiment status and command `retry` to queue again jobs marked as failed

Author: bors

docs/agent-http-api.md

@@ -60,6 +60,7 @@ behave this way:
 * `POST /agent-api/complete-experiment` should be called as soon as the agent
   has nothing left to do with the current experiment; after the method returns
   `next-experiment` will return a new experiment
+* `POST /error` should be called only when the agent has encountered an error
 
 ## Available endpoints
 
@@ -227,3 +228,28 @@ The endpoint replies with `true`.
     "result": true
 }
 ```
+
+### `POST /error`
+
+This endpoint tells the Crater server the agent has encountered an error.
+The endpoint expects the error description to be provided as the request body,
+encoded in JSON:
+
+* `error`: a description of the error
+
+For example, this is a valid request data:
+
+```json
+{
+    "error": "pc is not powered on"
+}
+```
+
+The endpoint replies with `true`.
+
+```json
+{
+    "status": "success",
+    "result": true
+}
+```

src/agent/api.rs

@@ -190,4 +190,15 @@ impl AgentApi {
             Ok(())
         })
     }
+
+    pub fn report_error(&self, error: String) -> Fallible<()> {
+        self.retry(|this| {
+            let _: bool = this
+                .build_request(Method::POST, "error")
+                .json(&json!({ "error": error }))
+                .send()?
+                .to_api_response()?;
+            Ok(())
+        })
+    }
 }

src/agent/mod.rs

@@ -2,6 +2,7 @@ mod api;
 mod results;
 
 use crate::agent::api::AgentApi;
+use crate::agent::results::ResultsUploader;
 use crate::config::Config;
 use crate::experiments::Experiment;
 use crate::prelude::*;
@@ -47,15 +48,34 @@ fn run_heartbeat(url: &str, token: &str) {
     });
 }
 
+fn run_experiment(
+    agent: &Agent,
+    db: &ResultsUploader,
+    threads_count: usize,
+    docker_env: &str,
+) -> Fallible<()> {
+    let ex = agent.experiment()?;
+    crate::runner::run_ex(&ex, db, threads_count, &agent.config, docker_env)?;
+    agent.api.complete_experiment()?;
+    Ok(())
+}
+
 pub fn run(url: &str, token: &str, threads_count: usize, docker_env: &str) -> Fallible<()> {
     let agent = Agent::new(url, token)?;
     let db = results::ResultsUploader::new(&agent.api);
 
     run_heartbeat(url, token);
 
     loop {
-        let ex = agent.experiment()?;
-        crate::runner::run_ex(&ex, &db, threads_count, &agent.config, docker_env)?;
-        agent.api.complete_experiment()?;
+        if let Err(err) = run_experiment(&agent, &db, threads_count, docker_env) {
+            utils::report_failure(&err);
+            if let Err(e) = agent
+                .api
+                .report_error(format!("{}", err.find_root_cause()))
+                .with_context(|_| "error encountered")
+            {
+                utils::report_failure(&e);
+            }
+        }
     }
 }

src/experiments.rs

@@ -12,6 +12,7 @@ string_enum!(pub enum Status {
     Queued => "queued",
     Running => "running",
     NeedsReport => "needs-report",
+    Failed => "failed",
     GeneratingReport => "generating-report",
     ReportFailed => "report-failed",
     Completed => "completed",
@@ -226,7 +227,10 @@ impl Experiment {
             )?;
             self.started_at = Some(now);
         // Check if the old status was "running" and there is no completed date
-        } else if self.status == Status::Running && self.completed_at.is_none() {
+        } else if self.status == Status::Running
+            && self.completed_at.is_none()
+            && status != Status::Failed
+        {
             db.execute(
                 "UPDATE experiments SET completed_at = ?1 WHERE name = ?2;",
                 &[&now, &self.name.as_str()],

src/server/routes/agent.rs

@@ -8,6 +8,7 @@ use crate::server::{Data, HttpError};
 use failure::Compat;
 use http::{Response, StatusCode};
 use hyper::Body;
+use std::collections::HashMap;
 use std::sync::Arc;
 use warp::{self, Filter, Rejection};
 
@@ -53,6 +54,14 @@ pub fn routes(
         .and(auth_filter(data.clone(), TokenType::Agent))
         .map(endpoint_heartbeat);
 
+    let error = warp::post2()
+        .and(warp::path("error"))
+        .and(warp::path::end())
+        .and(warp::body::json())
+        .and(data_filter.clone())
+        .and(auth_filter(data.clone(), TokenType::Agent))
+        .map(endpoint_error);
+
     warp::any()
         .and(
             config
@@ -63,6 +72,8 @@ pub fn routes(
                 .or(record_progress)
                 .unify()
                 .or(heartbeat)
+                .unify()
+                .or(error)
                 .unify(),
         )
         .map(handle_results)
@@ -146,6 +157,39 @@ fn endpoint_heartbeat(data: Arc<Data>, auth: AuthDetails) -> Fallible<Response<B
     Ok(ApiResponse::Success { result: true }.into_response()?)
 }
 
+fn endpoint_error(
+    error: HashMap<String, String>,
+    data: Arc<Data>,
+    auth: AuthDetails,
+) -> Fallible<Response<Body>> {
+    let mut ex = Experiment::run_by(&data.db, &Assignee::Agent(auth.name.clone()))?
+        .ok_or_else(|| err_msg("no experiment run by this agent"))?;
+
+    ex.set_status(&data.db, Status::Failed)?;
+
+    if let Some(ref github_issue) = ex.github_issue {
+        Message::new()
+            .line(
+                "rotating_light",
+                format!(
+                    "Experiment **`{}`** has encountered an error: {}",
+                    ex.name,
+                    error.get("error").unwrap_or(&String::from("no error")),
+                ),
+            )
+            .line(
+                "hammer_and_wrench",
+                "If the error is fixed use the `retry` command.",
+            )
+            .note(
+                "sos",
+                "Can someone from the infra team check in on this? @rust-lang/infra",
+            )
+            .send(&github_issue.api_url, &data)?;
+    }
+    Ok(ApiResponse::Success { result: true }.into_response()?)
+}
+
 fn handle_results(resp: Fallible<Response<Body>>) -> Response<Body> {
     match resp {
         Ok(resp) => resp,

src/server/routes/ui/experiments.rs

@@ -25,6 +25,7 @@ impl ExperimentData {
             Status::Queued => ("", "Queued", true),
             Status::Running => ("orange", "Running", true),
             Status::NeedsReport => ("orange", "Needs report", false),
+            Status::Failed => ("red", "Failed", false),
             Status::GeneratingReport => ("orange", "Generating report", false),
             Status::ReportFailed => ("red", "Report failed", false),
             Status::Completed => ("green", "Completed", false),
@@ -62,6 +63,7 @@ pub fn endpoint_queue(data: Arc<Data>) -> Fallible<Response<Body>> {
     let mut queued = Vec::new();
     let mut running = Vec::new();
     let mut needs_report = Vec::new();
+    let mut failed = Vec::new();
     let mut generating_report = Vec::new();
     let mut report_failed = Vec::new();
 
@@ -77,6 +79,7 @@ pub fn endpoint_queue(data: Arc<Data>) -> Fallible<Response<Body>> {
             Status::Queued => queued.push(ex),
             Status::Running => running.push(ex),
             Status::NeedsReport => needs_report.push(ex),
+            Status::Failed => failed.push(ex),
             Status::GeneratingReport => generating_report.push(ex),
             Status::ReportFailed => report_failed.push(ex),
             Status::Completed => unreachable!(),
@@ -87,6 +90,7 @@ pub fn endpoint_queue(data: Arc<Data>) -> Fallible<Response<Body>> {
     experiments.append(&mut report_failed);
     experiments.append(&mut generating_report);
     experiments.append(&mut needs_report);
+    experiments.append(&mut failed);
     experiments.append(&mut running);
     experiments.append(&mut queued);
 

src/server/routes/webhooks/args.rs

@@ -122,6 +122,10 @@ generate_parser!(pub enum Command {
         name: Option<String> = "name",
     })
 
+    "retry" => Retry(RetryArgs {
+        name: Option<String> = "name",
+    })
+
     "reload-acl" => ReloadACL(ReloadACLArgs {})
 
     _ => Edit(EditArgs {

src/server/routes/webhooks/commands.rs

@@ -4,7 +4,9 @@ use crate::experiments::{CapLints, CrateSelect, Experiment, GitHubIssue, Mode, S
 use crate::prelude::*;
 use crate::server::github::Issue;
 use crate::server::messages::{Label, Message};
-use crate::server::routes::webhooks::args::{AbortArgs, EditArgs, RetryReportArgs, RunArgs};
+use crate::server::routes::webhooks::args::{
+    AbortArgs, EditArgs, RetryArgs, RetryReportArgs, RunArgs,
+};
 use crate::server::Data;
 
 pub fn ping(data: &Data, issue: &Issue) -> Fallible<()> {
@@ -108,6 +110,31 @@ pub fn retry_report(data: &Data, issue: &Issue, args: RetryReportArgs) -> Fallib
     }
 }
 
+pub fn retry(data: &Data, issue: &Issue, args: RetryArgs) -> Fallible<()> {
+    let name = get_name(&data.db, issue, args.name)?;
+
+    if let Some(mut experiment) = Experiment::get(&data.db, &name)? {
+        if experiment.status != Status::Failed {
+            bail!("Experiment **`{}`** didn't fail!", name);
+        }
+
+        experiment.set_status(&data.db, Status::Queued)?;
+        data.reports_worker.wake();
+
+        Message::new()
+            .line(
+                "hammer_and_wrench",
+                format!("Experiment **`{}`** queued again.", name),
+            )
+            .set_label(Label::ExperimentQueued)
+            .send(&issue.url, data)?;
+
+        Ok(())
+    } else {
+        bail!("an experiment named **`{}`** doesn't exist!", name);
+    }
+}
+
 pub fn abort(data: &Data, issue: &Issue, args: AbortArgs) -> Fallible<()> {
     let name = get_name(&data.db, issue, args.name)?;
 

src/server/routes/webhooks/mod.rs

@@ -111,6 +111,10 @@ fn process_command(
                 commands::retry_report(data, issue, args)?;
             }
 
+            Command::Retry(args) => {
+                commands::retry(data, issue, args)?;
+            }
+
             Command::Abort(args) => {
                 commands::abort(data, issue, args)?;
             }