Use crates-index instead of rewriting it

This is possible now that it supports bare checkouts. This also takes the
liberty of iterating the crates in parallel, which should give a nice speedup.

Author: jyn514

Cargo.lock

@@ -16,7 +16,7 @@ exclude = [
 ]
 
 [features]
-consistency_check = ["crates-index"]
+consistency_check = ["crates-index", "rayon"]
 
 [dependencies]
 sentry = "0.24.1"
@@ -27,6 +27,7 @@ log = "0.4"
 regex = "1"
 structopt = "0.3"
 crates-index = { version = "0.18.5", optional = true }
+rayon = { version = "1", optional = true }
 crates-index-diff = "8.0.0"
 reqwest = { version = "0.11", features = ["blocking", "json"] } # TODO: Remove blocking when async is ready
 semver = { version = "1.0.4", features = ["serde"] }

Cargo.toml

@@ -1,46 +0,0 @@
-use anyhow::Context;
-use crates_index::Crate;
-pub(crate) struct Crates {
-    repo: git2::Repository,
-}
-
-impl Crates {
-    pub(super) fn new(repo: git2::Repository) -> Self {
-        Self { repo }
-    }
-
-    pub(crate) fn walk(&self, mut f: impl FnMut(Crate)) -> Result<(), anyhow::Error> {
-        log::debug!("Walking crates in index");
-        let tree = self
-            .repo
-            .find_commit(self.repo.refname_to_id("refs/remotes/origin/master")?)?
-            .tree()?;
-
-        let mut result = Ok(());
-
-        tree.walk(git2::TreeWalkMode::PreOrder, |_, entry| {
-            result = (|| {
-                if let Some(blob) = entry.to_object(&self.repo)?.as_blob() {
-                    if let Ok(krate) = Crate::from_slice(blob.content()) {
-                        f(krate);
-                    } else {
-                        log::warn!("not a crate '{}'", entry.name().unwrap());
-                    }
-                }
-                Result::<(), anyhow::Error>::Ok(())
-            })()
-            .with_context(|| {
-                format!(
-                    "loading crate details from '{}'",
-                    entry.name().unwrap_or("<unknown>")
-                )
-            });
-            match result {
-                Ok(_) => git2::TreeWalkResult::Ok,
-                Err(_) => git2::TreeWalkResult::Abort,
-            }
-        })?;
-
-        result
-    }
-}

src/index/crates.rs

@@ -8,8 +8,6 @@ use crate::error::Result;
 use crate::utils::report_error;
 
 pub(crate) mod api;
-#[cfg(feature = "consistency_check")]
-mod crates;
 
 pub struct Index {
     path: PathBuf,
@@ -89,15 +87,18 @@ impl Index {
     }
 
     #[cfg(feature = "consistency_check")]
-    pub(crate) fn crates(&self) -> Result<crates::Crates> {
+    pub(crate) fn crates(&self) -> Result<crates_index::Index> {
         // First ensure the index is up to date, peeking will pull the latest changes without
         // affecting anything else.
         log::debug!("Updating index");
         self.diff()?.peek_changes()?;
-        // It'd be nice to use `crates_index` directly for interacting with the index, but it
-        // doesn't support bare repositories. So we use its `Crate` type but walk the index
-        // ourselves.
-        Ok(crates::Crates::new(git2::Repository::open(&self.path)?))
+        log::debug!("Opening with `crates_index`");
+        // crates_index requires the repo url to match the existing origin or it tries to reinitialize the repo
+        let repo_url = self
+            .repository_url
+            .as_deref()
+            .unwrap_or("https://github.com/rust-lang/crates.io-index");
+        crates_index::Index::with_path(&self.path, repo_url).map_err(Into::into)
     }
 
     pub fn api(&self) -> &Api {

src/index/mod.rs

@@ -1,21 +1,22 @@
 use super::data::{Crate, CrateName, Data, Release, Version};
 use crate::Index;
+use rayon::iter::ParallelIterator;
 
 pub(crate) fn load(index: &Index) -> Result<Data, anyhow::Error> {
-    let mut data = Data::default();
-
-    index.crates()?.walk(|krate| {
-        data.crates.insert(
-            CrateName(krate.name().into()),
-            Crate {
-                releases: krate
+    let crates = index
+        .crates()?
+        .crates_parallel()
+        .map(|krate| {
+            krate.map(|krate| {
+                let releases = krate
                     .versions()
                     .iter()
                     .map(|version| (Version(version.version().into()), Release::default()))
-                    .collect(),
-            },
-        );
-    })?;
+                    .collect();
+                (CrateName(krate.name().into()), Crate { releases })
+            })
+        })
+        .collect::<Result<_, _>>()?;
 
-    Ok(data)
+    Ok(Data { crates })
 }