Auto merge of #4396 - Turbo87:index-crate, r=Turbo87

Extract `cargo-registry-index` package

This PR extracts our code to deal with the package index into a dedicated `cargo-registry-index` crate. This should make it faster to recompile just this part of the code and run the corresponding tests. It also ensures a somewhat clean boundary between the different code areas.

Author: bors

Cargo.lock

@@ -31,6 +31,7 @@ rustdoc-args = [
 [dependencies]
 anyhow = "=1.0.53"
 base64 = "=0.13.0"
+cargo-registry-index = { path = "cargo-registry-index" }
 cargo-registry-markdown = { path = "cargo-registry-markdown" }
 cargo-registry-s3 = { path = "cargo-registry-s3" }
 chrono = { version = "=0.4.19", features = ["serde"] }
@@ -56,7 +57,6 @@ dotenv = "=0.15.0"
 flate2 = "=1.0.22"
 futures-channel = { version = "=0.3.19", default-features = false }
 futures-util = "=0.3.19"
-git2 = "=0.13.25"
 hex = "=0.4.3"
 http = "=0.2.6"
 hyper = { version = "=0.14.16", features = ["client", "http1"] }
@@ -89,6 +89,7 @@ tracing-subscriber = { version = "=0.3.6", features = ["env-filter"] }
 url = "=2.2.2"
 
 [dev-dependencies]
+cargo-registry-index = { path = "cargo-registry-index", features = ["testing"] }
 claim = "=0.5.0"
 conduit-test = "=0.10.0"
 hyper-tls = "=0.5.0"

Cargo.toml

@@ -0,0 +1,24 @@
+[package]
+name = "cargo-registry-index"
+version = "0.0.0"
+license = "MIT OR Apache-2.0"
+repository = "https://github.com/rust-lang/crates.io"
+description = "crates.io package index utilities"
+edition = "2018"
+resolver = "2"
+
+[lib]
+path = "lib.rs"
+
+[features]
+testing = ["serde_json"]
+
+[dependencies]
+anyhow = "=1.0.53"
+base64 = "=0.13.0"
+dotenv = "=0.15.0"
+git2 = "=0.13.25"
+serde = { version = "=1.0.135", features = ["derive"] }
+tempfile = "=3.3.0"
+url = "=2.2.2"
+serde_json = { version = "=1.0.78", optional = true }

cargo-registry-index/Cargo.toml

@@ -1,3 +1,9 @@
+#[macro_use]
+extern crate serde;
+
+#[cfg(feature = "testing")]
+pub mod testing;
+
 use anyhow::{anyhow, Context};
 use std::collections::HashMap;
 use std::io::Write;
@@ -7,8 +13,6 @@ use std::process::Command;
 use tempfile::TempDir;
 use url::Url;
 
-use crate::models::DependencyKind;
-
 static DEFAULT_GIT_SSH_USERNAME: &str = "git";
 
 #[derive(Clone)]
@@ -148,6 +152,14 @@ pub struct Dependency {
     pub package: Option<String>,
 }
 
+#[derive(Copy, Clone, Serialize, Deserialize, Debug)]
+#[serde(rename_all = "lowercase")]
+pub enum DependencyKind {
+    Normal,
+    Build,
+    Dev,
+}
+
 pub struct RepositoryConfig {
     pub index_location: Url,
     pub credentials: Credentials,

src/git.rs renamed to cargo-registry-index/lib.rs

@@ -25,13 +25,10 @@ impl UpstreamIndex {
     }
 
     /// Obtain a list of crates from the index HEAD
-    pub fn crates_from_index_head(
-        &self,
-        crate_name: &str,
-    ) -> anyhow::Result<Vec<cargo_registry::git::Crate>> {
+    pub fn crates_from_index_head(&self, crate_name: &str) -> anyhow::Result<Vec<crate::Crate>> {
         let repo = &self.repository;
 
-        let path = cargo_registry::git::Repository::relative_index_file(crate_name);
+        let path = crate::Repository::relative_index_file(crate_name);
 
         let head = repo.head()?;
         let tree = head.peel_to_tree()?;

src/tests/git.rs renamed to cargo-registry-index/testing.rs

@@ -5,8 +5,8 @@ use std::sync::{Arc, Mutex, MutexGuard, PoisonError};
 use swirl::PerformError;
 
 use crate::db::{DieselPool, DieselPooledConn, PoolError};
-use crate::git::Repository;
 use crate::uploaders::Uploader;
+use cargo_registry_index::Repository;
 
 impl<'a> swirl::db::BorrowedConnection<'a> for DieselPool {
     type Connection = DieselPooledConn<'a>;

src/background_jobs.rs

@@ -13,8 +13,8 @@
 #![warn(clippy::all, rust_2018_idioms)]
 
 use cargo_registry::config;
-use cargo_registry::git::{Repository, RepositoryConfig};
 use cargo_registry::{background_jobs::*, db};
+use cargo_registry_index::{Repository, RepositoryConfig};
 use diesel::r2d2;
 use reqwest::blocking::Client;
 use std::sync::{Arc, Mutex};

src/bin/background-worker.rs

@@ -10,7 +10,6 @@ use std::sync::Arc;
 use swirl::Job;
 
 use crate::controllers::cargo_prelude::*;
-use crate::git;
 use crate::models::{
     insert_version_owner_action, Badge, Category, Crate, DependencyKind, Keyword, NewCrate,
     NewVersion, Rights, VersionAction,
@@ -230,7 +229,7 @@ pub fn publish(req: &mut dyn RequestExt) -> EndpointResult {
         };
 
         // Register this crate in our local git repo.
-        let git_crate = git::Crate {
+        let git_crate = cargo_registry_index::Crate {
             name: name.0,
             vers: vers.to_string(),
             cksum: hex_cksum,
@@ -313,7 +312,7 @@ pub fn add_dependencies(
     conn: &PgConnection,
     deps: &[EncodableCrateDependency],
     target_version_id: i32,
-) -> AppResult<Vec<git::Dependency>> {
+) -> AppResult<Vec<cargo_registry_index::Dependency>> {
     use self::dependencies::dsl::*;
     use diesel::insert_into;
 
@@ -348,14 +347,14 @@ pub fn add_dependencies(
             };
 
             Ok((
-                git::Dependency {
+                cargo_registry_index::Dependency {
                     name,
                     req: dep.version_req.to_string(),
                     features: dep.features.iter().map(|s| s.0.to_string()).collect(),
                     optional: dep.optional,
                     default_features: dep.default_features,
                     target: dep.target.clone(),
-                    kind: dep.kind.or(Some(DependencyKind::Normal)),
+                    kind: dep.kind.or(Some(DependencyKind::Normal)).map(|dk| dk.into()),
                     package,
                 },
                 (

src/controllers/krate/publish.rs

@@ -40,7 +40,6 @@ pub mod config;
 pub mod db;
 mod downloads_counter;
 pub mod email;
-pub mod git;
 pub mod github;
 pub mod metrics;
 pub mod middleware;

src/lib.rs

@@ -4,6 +4,7 @@ use diesel::sql_types::Integer;
 
 use crate::models::{Crate, Version};
 use crate::schema::*;
+use cargo_registry_index::DependencyKind as IndexDependencyKind;
 
 #[derive(Identifiable, Associations, Debug, Queryable, QueryableByName)]
 #[belongs_to(Version)]
@@ -42,6 +43,16 @@ pub enum DependencyKind {
     // if you add a kind here, be sure to update `from_row` below.
 }
 
+impl From<DependencyKind> for IndexDependencyKind {
+    fn from(dk: DependencyKind) -> Self {
+        match dk {
+            DependencyKind::Normal => IndexDependencyKind::Normal,
+            DependencyKind::Build => IndexDependencyKind::Build,
+            DependencyKind::Dev => IndexDependencyKind::Dev,
+        }
+    }
+}
+
 impl FromSql<Integer, Pg> for DependencyKind {
     fn from_sql(bytes: Option<&[u8]>) -> deserialize::Result<Self> {
         match <i32 as FromSql<Integer, Pg>>::from_sql(bytes)? {