support init rustwide_builder periodically

refactor reinitialize workspace

Author: linyihai

src/bin/cratesfyi.rs

@@ -181,7 +181,7 @@ impl CommandLine {
                 let build_queue = ctx.build_queue()?;
                 let config = ctx.config()?;
                 let rustwide_builder = RustwideBuilder::init(&ctx)?;
-                queue_builder(rustwide_builder, build_queue, config)?;
+                queue_builder(&ctx, rustwide_builder, build_queue, config)?;
             }
             Self::StartWebServer { socket_addr } => {
                 // Blocks indefinitely

src/build_queue.rs

@@ -4,8 +4,9 @@ use crate::docbuilder::PackageKind;
 use crate::error::Result;
 use crate::storage::Storage;
 use crate::utils::{get_config, get_crate_priority, report_error, retry, set_config, ConfigName};
+use crate::Context;
 use crate::{Config, Index, InstanceMetrics, RustwideBuilder};
-use anyhow::Context;
+use anyhow::Context as _;
 use fn_error_context::context;
 
 use tracing::{debug, error, info};
@@ -431,7 +432,11 @@ impl BuildQueue {
     /// Builds the top package from the queue. Returns whether there was a package in the queue.
     ///
     /// Note that this will return `Ok(true)` even if the package failed to build.
-    pub(crate) fn build_next_queue_package(&self, builder: &mut RustwideBuilder) -> Result<bool> {
+    pub(crate) fn build_next_queue_package(
+        &self,
+        context: &dyn Context,
+        builder: &mut RustwideBuilder,
+    ) -> Result<bool> {
         let mut processed = false;
         self.process_next_crate(|krate| {
             processed = true;
@@ -442,6 +447,19 @@ impl BuildQueue {
                 .map(|r| PackageKind::Registry(r.as_str()))
                 .unwrap_or(PackageKind::CratesIo);
 
+            if let Err(err) = retry(
+                || {
+                    builder
+                        .reinitialize_workspace_if_interval_passed(context)
+                        .context("Reinitialize workspace failed, locking queue")
+                },
+                3,
+            ) {
+                report_error(&err);
+                self.lock()?;
+                return Err(err);
+            }
+
             match retry(
                 || {
                     builder

src/config.rs

@@ -93,6 +93,7 @@ pub struct Config {
     pub cloudfront_distribution_id_web: Option<String>,
     /// same for the `static.docs.rs` distribution
     pub cloudfront_distribution_id_static: Option<String>,
+    pub(crate) build_workspace_reinitialization_interval: Duration,
 
     // Build params
     pub(crate) build_attempts: u16,
@@ -208,6 +209,10 @@ impl Config {
             build_default_memory_limit: maybe_env("DOCSRS_BUILD_DEFAULT_MEMORY_LIMIT")?,
             include_default_targets: env("DOCSRS_INCLUDE_DEFAULT_TARGETS", true)?,
             disable_memory_limit: env("DOCSRS_DISABLE_MEMORY_LIMIT", false)?,
+            build_workspace_reinitialization_interval: Duration::from_secs(env(
+                "DOCSRS_BUILD_WORKSPACE_REINITIALIZATION_INTERVAL",
+                86400,
+            )?),
         })
     }
 }

src/docbuilder/rustwide_builder.rs

@@ -27,6 +27,7 @@ use std::collections::{HashMap, HashSet};
 use std::fs;
 use std::path::Path;
 use std::sync::Arc;
+use std::time::Instant;
 use tracing::{debug, info, warn};
 
 const USER_AGENT: &str = "docs.rs builder (https://github.com/rust-lang/docs.rs)";
@@ -49,6 +50,30 @@ fn get_configured_toolchain(conn: &mut Client) -> Result<Toolchain> {
     }
 }
 
+fn build_workspace(context: &dyn Context) -> Result<Workspace> {
+    let config = context.config()?;
+
+    let mut builder = WorkspaceBuilder::new(&config.rustwide_workspace, USER_AGENT)
+        .running_inside_docker(config.inside_docker);
+    if let Some(custom_image) = &config.docker_image {
+        let image = match SandboxImage::local(custom_image) {
+            Ok(i) => i,
+            Err(CommandError::SandboxImageMissing(_)) => SandboxImage::remote(custom_image)?,
+            Err(err) => return Err(err.into()),
+        };
+        builder = builder.sandbox_image(image);
+    }
+    if cfg!(test) {
+        builder = builder.fast_init(true);
+    }
+
+    let workspace = builder.init().map_err(FailureError::compat)?;
+    workspace
+        .purge_all_build_dirs()
+        .map_err(FailureError::compat)?;
+    Ok(workspace)
+}
+
 pub enum PackageKind<'a> {
     Local(&'a Path),
     CratesIo,
@@ -65,35 +90,16 @@ pub struct RustwideBuilder {
     index: Arc<Index>,
     rustc_version: String,
     repository_stats_updater: Arc<RepositoryStatsUpdater>,
+    workspace_initialize_time: Instant,
 }
 
 impl RustwideBuilder {
     pub fn init(context: &dyn Context) -> Result<Self> {
         let config = context.config()?;
-
-        let mut builder = WorkspaceBuilder::new(&config.rustwide_workspace, USER_AGENT)
-            .running_inside_docker(config.inside_docker);
-        if let Some(custom_image) = &config.docker_image {
-            let image = match SandboxImage::local(custom_image) {
-                Ok(i) => i,
-                Err(CommandError::SandboxImageMissing(_)) => SandboxImage::remote(custom_image)?,
-                Err(err) => return Err(err.into()),
-            };
-            builder = builder.sandbox_image(image);
-        }
-        if cfg!(test) {
-            builder = builder.fast_init(true);
-        }
-
-        let workspace = builder.init().map_err(FailureError::compat)?;
-        workspace
-            .purge_all_build_dirs()
-            .map_err(FailureError::compat)?;
-
         let pool = context.pool()?;
 
         Ok(RustwideBuilder {
-            workspace,
+            workspace: build_workspace(context)?,
             toolchain: get_configured_toolchain(&mut *pool.get()?)?,
             config,
             db: pool,
@@ -102,9 +108,24 @@ impl RustwideBuilder {
             index: context.index()?,
             rustc_version: String::new(),
             repository_stats_updater: context.repository_stats_updater()?,
+            workspace_initialize_time: Instant::now(),
         })
     }
 
+    pub fn reinitialize_workspace_if_interval_passed(
+        &mut self,
+        context: &dyn Context,
+    ) -> Result<()> {
+        let interval = context.config()?.build_workspace_reinitialization_interval;
+        if self.workspace_initialize_time.elapsed() >= interval {
+            info!("start reinitialize workspace again");
+            self.workspace = build_workspace(context)?;
+            self.workspace_initialize_time = Instant::now();
+        }
+
+        Ok(())
+    }
+
     fn prepare_sandbox(&self, limits: &Limits) -> SandboxBuilder {
         SandboxBuilder::new()
             .cpu_limit(self.config.build_cpu_limit.map(|limit| limit as f32))
@@ -1265,4 +1286,33 @@ mod tests {
             Ok(())
         })
     }
+
+    #[test]
+    #[ignore]
+    fn test_workspace_reinitialize_at_once() {
+        wrapper(|env| {
+            let mut builder = RustwideBuilder::init(env)?;
+            builder.reinitialize_workspace_if_interval_passed(env)?;
+            assert!(builder.build_local_package(Path::new("tests/crates/build-std"))?);
+            Ok(())
+        })
+    }
+
+    #[test]
+    #[ignore]
+    fn test_workspace_reinitialize_after_interval() {
+        use std::thread::sleep;
+        use std::time::Duration;
+        wrapper(|env: &TestEnvironment| {
+            env.override_config(|cfg: &mut Config| {
+                cfg.build_workspace_reinitialization_interval = Duration::from_secs(1)
+            });
+            let mut builder = RustwideBuilder::init(env)?;
+            assert!(builder.build_local_package(Path::new("tests/crates/build-std"))?);
+            sleep(Duration::from_secs(1));
+            builder.reinitialize_workspace_if_interval_passed(env)?;
+            assert!(builder.build_local_package(Path::new("tests/crates/build-std"))?);
+            Ok(())
+        })
+    }
 }

src/utils/daemon.rs

@@ -138,8 +138,9 @@ pub fn start_daemon<C: Context + Send + Sync + 'static>(
     let rustwide_builder = RustwideBuilder::init(&*context)?;
     thread::Builder::new()
         .name("build queue reader".to_string())
-        .spawn(move || {
-            queue_builder(rustwide_builder, build_queue, config).unwrap();
+        .spawn({
+            let context = context.clone();
+            move || queue_builder(&*context, rustwide_builder, build_queue, config).unwrap()
         })
         .unwrap();
 

src/utils/queue_builder.rs

@@ -1,12 +1,14 @@
+use crate::Context;
 use crate::{docbuilder::RustwideBuilder, utils::report_error, BuildQueue, Config};
-use anyhow::{Context, Error};
+use anyhow::{Context as _, Error};
 use std::panic::{catch_unwind, AssertUnwindSafe};
 use std::sync::Arc;
 use std::time::Duration;
 use std::{fs, io, path::Path, thread};
 use tracing::{debug, error, warn};
 
 pub fn queue_builder(
+    context: &dyn Context,
     mut builder: RustwideBuilder,
     build_queue: Arc<BuildQueue>,
     config: Arc<Config>,
@@ -37,7 +39,7 @@ pub fn queue_builder(
         // If a panic occurs while building a crate, lock the queue until an admin has a chance to look at it.
         debug!("Checking build queue");
         let res = catch_unwind(AssertUnwindSafe(|| {
-            match build_queue.build_next_queue_package(&mut builder) {
+            match build_queue.build_next_queue_package(context, &mut builder) {
                 Ok(true) => {}
                 Ok(false) => {
                     debug!("Queue is empty, going back to sleep");