Auto merge of #666 - Mark-Simulacrum:improve-backoff, r=Mark-Simulacrum

Improve backoff & metrics

See individual commits for details.

Author: bors

src/agent/api.rs

@@ -1,3 +1,5 @@
+use std::time::Duration;
+
 use crate::agent::Capabilities;
 use crate::crates::Crate;
 use crate::experiments::Experiment;
@@ -62,8 +64,6 @@ impl ResponseExt for ::reqwest::blocking::Response {
     }
 }
 
-const RETRY_AFTER: u64 = 5;
-
 pub struct AgentApi {
     url: String,
     token: String,
@@ -88,6 +88,7 @@ impl AgentApi {
     }
 
     fn retry<T, F: Fn(&Self) -> Fallible<T>>(&self, f: F) -> Fallible<T> {
+        let mut retry_interval = 16u64;
         loop {
             match f(self) {
                 Ok(res) => return Ok(res),
@@ -104,10 +105,19 @@ impl AgentApi {
                     };
 
                     if retry {
-                        warn!("connection to the server failed. retrying in a few seconds...");
-                        ::std::thread::sleep(::std::time::Duration::from_millis(
-                            rand::thread_rng().gen_range(0..(RETRY_AFTER * 1000)),
-                        ));
+                        let sleep_for = Duration::from_millis(
+                            rand::thread_rng().gen_range(500..(retry_interval * 1000)),
+                        );
+                        warn!(
+                            "connection to the server failed. retrying in {:?}...",
+                            sleep_for
+                        );
+                        ::std::thread::sleep(sleep_for);
+                        retry_interval *= 2;
+                        if retry_interval >= 8 * 60 {
+                            retry_interval = 8 * 60;
+                        }
+
                         continue;
                     }
 
@@ -141,7 +151,7 @@ impl AgentApi {
             // healthy.
             crate::agent::set_healthy();
 
-            ::std::thread::sleep(::std::time::Duration::from_secs(120));
+            ::std::thread::sleep(Duration::from_secs(120));
         })
     }
 

src/server/routes/agent.rs

@@ -195,12 +195,6 @@ impl RecordProgressThread {
                 let start = std::time::Instant::now();
 
                 if let Some(ex) = Experiment::get(&db, &result.experiment_name).unwrap() {
-                    metrics.record_completed_jobs(
-                        &worker_name,
-                        &ex.name,
-                        result.data.results.len() as i64,
-                    );
-
                     let db = DatabaseDB::new(&db);
                     if let Err(e) = db.store(&ex, &result.data, EncodingType::Plain) {
                         // Failing to record a result is basically fine -- this
@@ -209,6 +203,12 @@ impl RecordProgressThread {
                         crate::utils::report_failure(&e);
                     }
 
+                    metrics.record_completed_jobs(
+                        &worker_name,
+                        &ex.name,
+                        result.data.results.len() as i64,
+                    );
+
                     if let Err(e) = db.clear_stale_records() {
                         // Not a hard failure. We can continue even if we failed
                         // to clear records from already completed runs...