remove cloudfront support & invalidation queueing

Author: syphar

Cargo.lock

@@ -73,7 +73,6 @@ futures-util = "0.3.5"
 async-stream = "0.3.5"
 aws-config = "1.0.0"
 aws-sdk-s3 = "1.3.0"
-aws-sdk-cloudfront = "1.3.0"
 aws-smithy-types-convert = { version = "0.60.0", features = ["convert-chrono"] }
 http = "1.0.0"
 uuid = { version = "1.1.2", features = ["v4"]}

Cargo.toml

@@ -0,0 +1,28 @@
+-- content copy/pasted from `.._initial.up.sql`
+
+CREATE TABLE cdn_invalidation_queue (
+    id bigint NOT NULL,
+    crate character varying(255) NOT NULL,
+    cdn_distribution_id character varying(255) NOT NULL,
+    path_pattern text NOT NULL,
+    queued timestamp with time zone DEFAULT CURRENT_TIMESTAMP NOT NULL,
+    created_in_cdn timestamp with time zone,
+    cdn_reference character varying(255)
+);
+
+
+CREATE SEQUENCE cdn_invalidation_queue_id_seq
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+
+
+ALTER SEQUENCE cdn_invalidation_queue_id_seq OWNED BY cdn_invalidation_queue.id;
+ALTER TABLE ONLY cdn_invalidation_queue ALTER COLUMN id SET DEFAULT nextval('cdn_invalidation_queue_id_seq'::regclass);
+
+CREATE INDEX cdn_invalidation_queue_cdn_reference_idx ON cdn_invalidation_queue USING btree (cdn_reference);
+CREATE INDEX cdn_invalidation_queue_crate_idx ON cdn_invalidation_queue USING btree (crate);
+CREATE INDEX cdn_invalidation_queue_created_in_cdn_idx ON cdn_invalidation_queue USING btree (created_in_cdn);

migrations/20251202040858_remove-cdn-invalidation-queue.down.sql

@@ -0,0 +1 @@
+DROP TABLE cdn_invalidation_queue;

migrations/20251202040858_remove-cdn-invalidation-queue.up.sql

@@ -144,8 +144,6 @@ enum CommandLine {
             value_enum
         )]
         repository_stats_updater: Toggle,
-        #[arg(long = "cdn-invalidator", default_value = "enabled", value_enum)]
-        cdn_invalidator: Toggle,
         #[arg(long = "queue-rebuilds", default_value = "enabled", value_enum)]
         queue_rebuilds: Toggle,
     },
@@ -182,15 +180,11 @@ impl CommandLine {
             Self::Build { subcommand } => subcommand.handle_args(ctx)?,
             Self::StartRegistryWatcher {
                 repository_stats_updater,
-                cdn_invalidator,
                 queue_rebuilds,
             } => {
                 if repository_stats_updater == Toggle::Enabled {
                     docs_rs::utils::daemon::start_background_repository_stats_updater(&ctx)?;
                 }
-                if cdn_invalidator == Toggle::Enabled {
-                    docs_rs::utils::daemon::start_background_cdn_invalidator(&ctx)?;
-                }
                 if queue_rebuilds == Toggle::Enabled {
                     docs_rs::utils::daemon::start_background_queue_rebuild(&ctx)?;
                 }

src/bin/cratesfyi.rs

@@ -303,7 +303,7 @@ impl AsyncBuildQueue {
 
 /// Index methods.
 impl AsyncBuildQueue {
-    async fn queue_crate_invalidation(&self, conn: &mut sqlx::PgConnection, krate: &str) {
+    async fn queue_crate_invalidation(&self, krate: &str) {
         let krate = match krate
             .parse::<KrateName>()
             .with_context(|| format!("can't parse crate name '{}'", krate))
@@ -316,7 +316,7 @@ impl AsyncBuildQueue {
         };
 
         if let Err(err) =
-            cdn::queue_crate_invalidation(conn, &self.config, &self.cdn_metrics, &krate).await
+            cdn::queue_crate_invalidation(&self.config, &self.cdn_metrics, &krate).await
         {
             report_error(&err);
         }
@@ -360,7 +360,7 @@ impl AsyncBuildQueue {
                     Err(err) => report_error(&err),
                 }
 
-                self.queue_crate_invalidation(&mut conn, krate).await;
+                self.queue_crate_invalidation(krate).await;
                 self.remove_crate_from_queue(krate).await?;
                 continue;
             }
@@ -392,8 +392,7 @@ impl AsyncBuildQueue {
                     Err(err) => report_error(&err),
                 }
 
-                self.queue_crate_invalidation(&mut conn, &release.name)
-                    .await;
+                self.queue_crate_invalidation(&release.name).await;
                 self.remove_version_from_queue(&release.name, &version)
                     .await?;
                 continue;
@@ -449,8 +448,7 @@ impl AsyncBuildQueue {
                     report_error(&err);
                 }
 
-                self.queue_crate_invalidation(&mut conn, &release.name)
-                    .await;
+                self.queue_crate_invalidation(&release.name).await;
             }
         }
 
@@ -639,10 +637,8 @@ impl BuildQueue {
 
         self.inner.builder_metrics.total_builds.add(1, &[]);
 
-        self.runtime.block_on(
-            self.inner
-                .queue_crate_invalidation(&mut transaction, &to_process.name),
-        );
+        self.runtime
+            .block_on(self.inner.queue_crate_invalidation(&to_process.name));
 
         let mut increase_attempt_count = || -> Result<()> {
             let attempt: i32 = self.runtime.block_on(
@@ -1405,72 +1401,66 @@ mod tests {
             9
         );
 
-        // no invalidations were run since we don't have a distribution id configured
-        assert!(
-            env.runtime()
-                .block_on(async {
-                    cdn::cloudfront::queued_or_active_crate_invalidations(
-                        &mut *env.async_db().async_conn().await,
-                    )
-                    .await
-                })?
-                .is_empty()
-        );
-
         Ok(())
     }
 
     #[test]
-    fn test_invalidate_cdn_after_build_and_error() -> Result<()> {
+    fn test_invalidate_cdn_after_error() -> Result<()> {
+        let mut fastly_api = mockito::Server::new();
+
         let env = TestEnvironment::with_config_and_runtime(
             TestEnvironment::base_config()
-                .cloudfront_distribution_id_web(Some("distribution_id_web".into()))
-                .cloudfront_distribution_id_static(Some("distribution_id_static".into()))
+                .fastly_api_host(fastly_api.url().parse().unwrap())
+                .fastly_api_token(Some("test-token".into()))
+                .fastly_service_sid(Some("test-sid-1".into()))
                 .build()?,
         )?;
 
         let queue = env.build_queue();
 
-        queue.add_crate("will_succeed", &V1, -1, None)?;
-        queue.add_crate("will_fail", &V1, 0, None)?;
+        let m = fastly_api
+            .mock("POST", "/service/test-sid-1/purge")
+            .with_status(200)
+            .create();
 
-        let fetch_invalidations = || {
-            env.runtime()
-                .block_on(async {
-                    let mut conn = env.async_db().async_conn().await;
-                    cdn::cloudfront::queued_or_active_crate_invalidations(&mut conn).await
-                })
-                .unwrap()
-        };
-
-        assert!(fetch_invalidations().is_empty());
+        queue.add_crate("will_fail", &V1, 0, None)?;
 
         queue.process_next_crate(|krate| {
-            assert_eq!("will_succeed", krate.name);
-            Ok(BuildPackageSummary::default())
+            assert_eq!("will_fail", krate.name);
+            anyhow::bail!("simulate a failure");
         })?;
 
-        let queued_invalidations = fetch_invalidations();
-        assert_eq!(queued_invalidations.len(), 3);
-        assert!(
-            queued_invalidations
-                .iter()
-                .all(|i| i.krate == "will_succeed")
-        );
+        m.expect(1).assert();
+
+        Ok(())
+    }
+    #[test]
+    fn test_invalidate_cdn_after_build() -> Result<()> {
+        let mut fastly_api = mockito::Server::new();
+
+        let env = TestEnvironment::with_config_and_runtime(
+            TestEnvironment::base_config()
+                .fastly_api_host(fastly_api.url().parse().unwrap())
+                .fastly_api_token(Some("test-token".into()))
+                .fastly_service_sid(Some("test-sid-1".into()))
+                .build()?,
+        )?;
+
+        let queue = env.build_queue();
+
+        let m = fastly_api
+            .mock("POST", "/service/test-sid-1/purge")
+            .with_status(200)
+            .create();
+
+        queue.add_crate("will_succeed", &V1, -1, None)?;
 
         queue.process_next_crate(|krate| {
-            assert_eq!("will_fail", krate.name);
-            anyhow::bail!("simulate a failure");
+            assert_eq!("will_succeed", krate.name);
+            Ok(BuildPackageSummary::default())
         })?;
 
-        let queued_invalidations = fetch_invalidations();
-        assert_eq!(queued_invalidations.len(), 6);
-        assert!(
-            queued_invalidations
-                .iter()
-                .skip(3)
-                .all(|i| i.krate == "will_fail")
-        );
+        m.expect(1).assert();
 
         Ok(())
     }