Auto merge of #536 - pietroalbini:cleanup-caches, r=Mark-Simulacrum

Purge caches before starting an experiment when out of disk space

This PR changes the agent to purge all caches before starting an experiment if there is less than half disk space free on the host machine. This will make running Crater on "low" disk space hosts way less painful.

In theory this only runs the cleanup code when an experiment finishes, but in practice it will run it every 1000 built crates, since distributed runs works internally by splitting an experiment in 1000 crates chunks.

r? @Mark-Simulacrum

Author: bors

Cargo.lock

@@ -63,7 +63,7 @@ hmac = "0.7"
 sha-1 = "0.8"
 rust_team_data = { git = "https://github.com/rust-lang/team" }
 systemstat = "0.1.4"
-rustwide = { version = "0.8.0", features = ["unstable"] }
+rustwide = { version = "0.10.0", features = ["unstable", "unstable-toolchain-ci"] }
 percent-encoding = "2.1.0"
 remove_dir_all = "0.5.2"
 ctrlc = "3.1.3"

Cargo.toml

@@ -9,6 +9,7 @@ use crate::db::{Database, QueryUtils};
 use crate::experiments::Experiment;
 use crate::prelude::*;
 use crate::utils;
+use crate::utils::disk_usage::DiskUsage;
 use failure::Error;
 use rustwide::Workspace;
 use std::collections::BTreeSet;
@@ -17,6 +18,9 @@ use std::ops;
 use std::thread;
 use std::time::Duration;
 
+// Purge all the caches if the disk is more than 50% full.
+const PURGE_CACHES_THRESHOLD: f32 = 0.5;
+
 #[derive(Default, Serialize, Deserialize)]
 pub struct Capabilities {
     #[serde(default)]
@@ -117,6 +121,18 @@ fn run_experiment(
     }
     *past_experiment = Some(ex.name.clone());
 
+    match DiskUsage::fetch() {
+        Ok(usage) => {
+            if usage.is_threshold_reached(PURGE_CACHES_THRESHOLD) {
+                warn!("purging all caches");
+                workspace.purge_all_caches().map_err(|err| (None, err))?;
+            }
+        }
+        Err(err) => {
+            warn!("failed to check the disk usage: {}", err);
+        }
+    }
+
     crate::runner::run_ex(&ex, workspace, &crates, db, threads_count, &agent.config)
         .map_err(|err| (Some(ex), err))?;
     Ok(())

src/agent/mod.rs

@@ -167,7 +167,7 @@ fn run_cargo<DB: WriteResults>(
             } else if !error_codes.is_empty() {
                 Err(e.context(FailureReason::CompilerError(error_codes)).into())
             } else {
-                Err(e)
+                Err(e.into())
             }
         }
     }

src/runner/test.rs

@@ -7,15 +7,13 @@ use crate::runner::{OverrideResult, RunnerState};
 use crate::utils;
 use rustwide::{BuildDirectory, Workspace};
 use std::collections::HashMap;
-use std::path::Path;
 use std::sync::{
     atomic::{AtomicBool, Ordering},
     mpsc::{self, RecvTimeoutError},
     Arc, Mutex,
 };
 use std::thread;
 use std::time::Duration;
-use systemstat::{Filesystem, Platform, System};
 
 pub(super) struct Worker<'a, DB: WriteResults + Sync> {
     name: String,
@@ -185,54 +183,21 @@ impl<'a, DB: WriteResults + Sync> DiskSpaceWatcher<'a, DB> {
     }
 
     fn check(&self) -> Fallible<()> {
-        let fs = match self.current_mount() {
-            Ok(fs) => fs,
-            Err(e) => {
+        let usage = match crate::utils::disk_usage::DiskUsage::fetch() {
+            Ok(usage) => usage,
+            Err(err) => {
                 // TODO: `current_mount` fails sometimes on Windows with ERROR_DEVICE_NOT_READY.
-                warn!("Failed to check space remaining: {}", e);
+                warn!("Failed to check space remaining: {}", err);
                 return Ok(());
             }
         };
 
-        let usage = (fs.total.as_usize() - fs.free.as_usize()) as f32 / fs.total.as_usize() as f32;
-        if usage < self.threshold {
-            info!(
-                "{} disk usage at {}%",
-                fs.fs_mounted_on,
-                (usage * 100.0) as u8
-            );
-        } else {
-            warn!(
-                "{} disk usage at {}%, which is over the threshold of {}%",
-                fs.fs_mounted_on,
-                (usage * 100.0) as u8,
-                (self.threshold * 100.0) as u8,
-            );
-
+        if usage.is_threshold_reached(self.threshold) {
+            warn!("running the scheduled thread cleanup");
             for worker in self.workers {
                 worker.schedule_target_dir_cleanup();
             }
-            warn!("scheduled cleanup");
         }
         Ok(())
     }
-
-    fn current_mount(&self) -> Fallible<Filesystem> {
-        let current_dir = crate::utils::path::normalize_path(&crate::dirs::WORK_DIR);
-        let system = System::new();
-
-        let mut found = None;
-        let mut found_pos = std::usize::MAX;
-        for mount in system.mounts()?.into_iter() {
-            let path = Path::new(&mount.fs_mounted_on);
-            for (i, ancestor) in current_dir.ancestors().enumerate() {
-                if ancestor == path && i < found_pos {
-                    found_pos = i;
-                    found = Some(mount);
-                    break;
-                }
-            }
-        }
-        found.ok_or_else(|| failure::err_msg("failed to find the current mount"))
-    }
 }

src/runner/worker.rs

@@ -0,0 +1,53 @@
+use crate::prelude::*;
+use std::path::Path;
+use systemstat::{Filesystem, Platform, System};
+
+pub(crate) struct DiskUsage {
+    mount_point: String,
+    usage: f32,
+}
+
+impl DiskUsage {
+    pub(crate) fn fetch() -> Fallible<Self> {
+        let fs = current_mount()?;
+        Ok(Self {
+            mount_point: fs.fs_mounted_on.clone(),
+            usage: (fs.total.as_usize() - fs.free.as_usize()) as f32 / fs.total.as_usize() as f32,
+        })
+    }
+
+    pub(crate) fn is_threshold_reached(&self, threshold: f32) -> bool {
+        let usage = (self.usage * 100.0) as u8;
+        if self.usage < threshold {
+            info!("{} disk usage at {}%", self.mount_point, usage);
+            false
+        } else {
+            warn!(
+                "{} disk usage at {}%, which is over the threshold of {}%",
+                self.mount_point,
+                usage,
+                (threshold * 100.0) as u8
+            );
+            true
+        }
+    }
+}
+
+fn current_mount() -> Fallible<Filesystem> {
+    let current_dir = crate::utils::path::normalize_path(&crate::dirs::WORK_DIR);
+    let system = System::new();
+
+    let mut found = None;
+    let mut found_pos = std::usize::MAX;
+    for mount in system.mounts()?.into_iter() {
+        let path = Path::new(&mount.fs_mounted_on);
+        for (i, ancestor) in current_dir.ancestors().enumerate() {
+            if ancestor == path && i < found_pos {
+                found_pos = i;
+                found = Some(mount);
+                break;
+            }
+        }
+    }
+    found.ok_or_else(|| failure::err_msg("failed to find the current mount"))
+}

src/utils/disk_usage.rs

@@ -9,6 +9,7 @@ pub(crate) mod hex;
 pub(crate) mod http;
 #[macro_use]
 mod macros;
+pub(crate) mod disk_usage;
 pub(crate) mod path;
 pub(crate) mod serialize;
 pub mod size;