Always use the last_seen_reference from the database

Author: Nemo157

src/build_queue.rs

@@ -268,9 +268,25 @@ impl BuildQueue {
     pub fn get_new_crates(&self, index: &Index) -> Result<usize> {
         let mut conn = self.db.get()?;
         let diff = index.diff()?;
-        let (mut changes, oid) = diff.peek_changes_ordered()?;
+
+        let Some(last_seen_reference) = self.last_seen_reference()? else {
+            // we should always have a last seen reference in the database, other than when
+            // initialising a new deployment (e.g. dev/test/staging), in those cases we don't want
+            // to rebuild the entire world, so we get crates-index-diff to fetch the current state
+            // then use the current head as the base and will only start building new crates from
+            // now on
+            let (_, oid) = diff.peek_changes_ordered()?;
+            warn!("no last_seen_reference in database, setting to current head {oid}");
+            self.set_last_seen_reference(oid)?;
+            return Ok(0);
+        };
+        diff.set_last_seen_reference(last_seen_reference)?;
+
+        let (mut changes, new_reference) = diff.peek_changes_ordered()?;
         let mut crates_added = 0;
 
+        debug!("queueing changes from {last_seen_reference} to {new_reference}");
+
         // I believe this will fix ordering of queue if we get more than one crate from changes
         changes.reverse();
 
@@ -341,14 +357,10 @@ impl BuildQueue {
             }
         }
 
-        // additionally set the reference in the database
+        // set the reference in the database
         // so this survives recreating the registry watcher
         // server.
-        self.set_last_seen_reference(oid)?;
-
-        // store the last seen reference as git reference in
-        // the local crates.io index repo.
-        diff.set_last_seen_reference(oid)?;
+        self.set_last_seen_reference(new_reference)?;
 
         Ok(crates_added)
     }

src/utils/daemon.rs

@@ -12,7 +12,7 @@ use anyhow::{anyhow, Context as _, Error};
 use std::sync::Arc;
 use std::thread;
 use std::time::{Duration, Instant};
-use tracing::{debug, error, info};
+use tracing::{debug, info};
 
 /// Run the registry watcher
 /// NOTE: this should only be run once, otherwise crates would be added
@@ -24,22 +24,6 @@ pub fn watch_registry(
 ) -> Result<(), Error> {
     let mut last_gc = Instant::now();
 
-    // On startup we fetch the last seen index reference from
-    // the database and set it in the local index repository.
-    match build_queue.last_seen_reference() {
-        Ok(Some(oid)) => {
-            index.diff()?.set_last_seen_reference(oid)?;
-        }
-        Ok(None) => {}
-        Err(err) => {
-            error!(
-                "queue locked because of invalid last_seen_index_reference in database: {:?}",
-                err
-            );
-            build_queue.lock()?;
-        }
-    }
-
     loop {
         if build_queue.is_locked()? {
             debug!("Queue is locked, skipping checking new crates");