Cleanup temporary directories at startup

Author: jyn514

Cargo.toml

@@ -82,6 +82,8 @@ memmap2 = "0.5.0"
 # iron dependencies
 iron = "0.6"
 router = "0.6"
+
+# NOTE: if you change this, also double-check that the comment in `queue_builder::remove_tempdirs` is still accurate.
 tempfile = "3.1.0"
 
 # Templating

src/docbuilder/rustwide_builder.rs

@@ -8,7 +8,7 @@ use crate::error::Result;
 use crate::index::api::ReleaseData;
 use crate::repositories::RepositoryStatsUpdater;
 use crate::storage::{rustdoc_archive_path, source_archive_path};
-use crate::utils::{copy_dir_all, parse_rustc_version, CargoMetadata};
+use crate::utils::{copy_dir_all, parse_rustc_version, queue_builder, CargoMetadata};
 use crate::{db::blacklist::is_blacklisted, utils::MetadataPackage};
 use crate::{Config, Context, Index, Metrics, Storage};
 use anyhow::{anyhow, bail, Error};
@@ -314,7 +314,9 @@ impl RustwideBuilder {
         };
         krate.fetch(&self.workspace).map_err(FailureError::compat)?;
 
-        let local_storage = tempfile::Builder::new().prefix("docsrs-docs").tempdir()?;
+        let local_storage = tempfile::Builder::new()
+            .prefix(queue_builder::TEMPDIR_PREFIX)
+            .tempdir()?;
 
         let successful = build_dir
             .build(&self.toolchain, &krate, self.prepare_sandbox(&limits))

src/utils/mod.rs

@@ -18,7 +18,7 @@ mod copy;
 pub(crate) mod daemon;
 mod html;
 mod queue;
-mod queue_builder;
+pub(crate) mod queue_builder;
 mod rustc_version;
 pub(crate) mod sized_buffer;
 

src/utils/queue_builder.rs

@@ -3,8 +3,10 @@ use anyhow::Error;
 use log::{debug, error, info, warn};
 use std::panic::{catch_unwind, AssertUnwindSafe};
 use std::sync::Arc;
-use std::thread;
 use std::time::Duration;
+use std::{fs, io, thread};
+
+pub(crate) const TEMPDIR_PREFIX: &str = "docsrs-docs";
 
 // TODO: change to `fn() -> Result<!, Error>` when never _finally_ stabilizes
 pub fn queue_builder(
@@ -26,6 +28,10 @@ pub fn queue_builder(
     let mut status = BuilderState::Fresh;
 
     loop {
+        if let Err(e) = remove_tempdirs() {
+            report_error(&anyhow::anyhow!(e).context("failed to remove temporary directories"));
+        }
+
         if !matches!(status, BuilderState::QueueInProgress) {
             thread::sleep(Duration::from_secs(60));
         }
@@ -70,3 +76,21 @@ pub fn queue_builder(
         }
     }
 }
+
+/// Sometimes, when the server hits a hard crash or a build thread panics,
+/// rustwide_builder won't actually remove the temporary directories it creates.
+/// Remove them now to avoid running out of disk space.
+fn remove_tempdirs() -> Result<(), io::Error> {
+    // NOTE: hardcodes that `tempfile::tempdir()` uses `std::env::temp_dir`.
+    for entry in std::fs::read_dir(std::env::temp_dir())? {
+        let entry = entry?;
+        if !entry.path().starts_with(TEMPDIR_PREFIX) {
+            continue;
+        }
+        if entry.metadata()?.is_dir() {
+            fs::remove_dir_all(entry.path())?;
+        }
+    }
+
+    Ok(())
+}