retry crates.io requests when requesting crate-details

Author: syphar

src/build_queue.rs

@@ -3,17 +3,15 @@ use crate::db::{delete_crate, Pool};
 use crate::docbuilder::PackageKind;
 use crate::error::Result;
 use crate::storage::Storage;
-use crate::utils::{get_config, get_crate_priority, report_error, set_config, ConfigName};
+use crate::utils::{get_config, get_crate_priority, report_error, retry, set_config, ConfigName};
 use crate::{Config, Index, Metrics, RustwideBuilder};
 use anyhow::Context;
 use fn_error_context::context;
 
-use tracing::{debug, error, info, warn};
+use tracing::{debug, error, info};
 
 use std::collections::HashMap;
 use std::sync::Arc;
-use std::thread;
-use std::time::Duration;
 
 #[derive(Debug, Clone, Eq, PartialEq, serde::Serialize)]
 pub(crate) struct QueuedCrate {
@@ -262,27 +260,6 @@ impl BuildQueue {
     }
 }
 
-fn retry<T>(mut f: impl FnMut() -> Result<T>, max_attempts: u32) -> Result<T> {
-    for attempt in 1.. {
-        match f() {
-            Ok(result) => return Ok(result),
-            Err(err) => {
-                if attempt > max_attempts {
-                    return Err(err);
-                } else {
-                    let sleep_for = 2u32.pow(attempt);
-                    warn!(
-                        "got error on attempt {}, will try again after {}s:\n{:?}",
-                        attempt, sleep_for, err
-                    );
-                    thread::sleep(Duration::from_secs(sleep_for as u64));
-                }
-            }
-        }
-    }
-    unreachable!()
-}
-
 /// Index methods.
 impl BuildQueue {
     /// Updates registry index repository and adds new crates into build queue.

src/index/api.rs

@@ -1,12 +1,11 @@
+use crate::{error::Result, utils::retry};
 use anyhow::{anyhow, Context};
 use chrono::{DateTime, Utc};
 use reqwest::header::{HeaderValue, ACCEPT, USER_AGENT};
 use semver::Version;
 use serde::Deserialize;
 use url::Url;
 
-use crate::error::Result;
-
 const APP_USER_AGENT: &str = concat!(
     env!("CARGO_PKG_NAME"),
     " ",
@@ -119,7 +118,11 @@ impl Api {
             downloads: i32,
         }
 
-        let response: Response = self.client.get(url).send()?.error_for_status()?.json()?;
+        let response: Response = retry(
+            || Ok(self.client.get(url.clone()).send()?.error_for_status()?),
+            3,
+        )?
+        .json()?;
 
         let version = Version::parse(version)?;
         let version = response

src/utils/mod.rs

@@ -27,6 +27,10 @@ use serde::Serialize;
 use tracing::error;
 pub(crate) mod sized_buffer;
 
+use std::thread;
+use std::time::Duration;
+use tracing::warn;
+
 pub(crate) const APP_USER_AGENT: &str = concat!(
     env!("CARGO_PKG_NAME"),
     " ",
@@ -112,6 +116,27 @@ where
         .context("failed to join thread")?
 }
 
+pub(crate) fn retry<T>(mut f: impl FnMut() -> Result<T>, max_attempts: u32) -> Result<T> {
+    for attempt in 1.. {
+        match f() {
+            Ok(result) => return Ok(result),
+            Err(err) => {
+                if attempt > max_attempts {
+                    return Err(err);
+                } else {
+                    let sleep_for = 2u32.pow(attempt);
+                    warn!(
+                        "got error on attempt {}, will try again after {}s:\n{:?}",
+                        attempt, sleep_for, err
+                    );
+                    thread::sleep(Duration::from_secs(sleep_for as u64));
+                }
+            }
+        }
+    }
+    unreachable!()
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;