temporary command to recompress broken index files in storage

Author: syphar

.sqlx/query-29e5ac519965abe821ae15c21014d9943a38c16ca49ba014d8227df1b605bad9.json

@@ -59,7 +59,7 @@ bzip2 = "0.6.0"
 getrandom = "0.3.1"
 itertools = { version = "0.14.0" }
 hex = "0.4.3"
-derive_more = { version = "2.0.0", features = ["display", "deref", "from", "into"] }
+derive_more = { version = "2.0.0", features = ["display", "deref", "from", "into", "from_str"] }
 sysinfo = { version = "0.37.2", default-features = false, features = ["system"] }
 derive_builder = "0.20.2"
 

Cargo.toml

@@ -11,8 +11,11 @@ sqlx-prepare ADDITIONAL_ARGS="":
 sqlx-check:
   just sqlx-prepare "--check"
 
-lint: 
-  cargo clippy --all-features --all-targets --workspace --locked -- -D warnings
+lint *args: 
+  cargo clippy --all-features --all-targets --workspace --locked {{ args }} -- -D warnings
+
+lint-fix:
+  just lint --fix --allow-dirty --allow-staged
 
 lint-js *args:
   deno run -A npm:eslint@9 static templates gui-tests eslint.config.js {{ args }}

Justfile

@@ -2,7 +2,7 @@ use anyhow::{Context as _, Result, anyhow};
 use clap::{Parser, Subcommand, ValueEnum};
 use docs_rs::{
     Config, Context, PackageKind, RustwideBuilder,
-    db::{self, CrateId, Overrides, add_path_into_database, types::version::Version},
+    db::{self, CrateId, Overrides, ReleaseId, add_path_into_database, types::version::Version},
     start_background_metrics_webserver, start_web_server,
     utils::{
         ConfigName, get_config, get_crate_pattern_and_priority, list_crate_priorities,
@@ -531,6 +531,16 @@ enum DatabaseSubcommand {
     /// Backfill GitHub/GitLab stats for crates.
     BackfillRepositoryStats,
 
+    /// Recompress broken archive index files in storage.
+    RecompressArchiveIndexes {
+        #[arg(long)]
+        min_release_id: Option<ReleaseId>,
+        #[arg(long)]
+        max_release_id: Option<ReleaseId>,
+        #[arg(long)]
+        concurrency: Option<u8>,
+    },
+
     /// Updates info for a crate from the registry's API
     UpdateCrateRegistryFields {
         #[arg(name = "CRATE")]
@@ -619,6 +629,28 @@ impl DatabaseSubcommand {
                 db::update_crate_data_in_database(&mut conn, &name, &registry_data).await
             })?,
 
+            Self::RecompressArchiveIndexes {
+                min_release_id,
+                max_release_id,
+                concurrency,
+            } => ctx.runtime.block_on(async move {
+                let mut conn = ctx.pool.get_async().await?;
+
+                let (checked, recompressed) = ctx
+                    .async_storage
+                    .recompress_index_files_in_bucket(
+                        &mut conn,
+                        min_release_id,
+                        max_release_id,
+                        concurrency.map(Into::into),
+                    )
+                    .await?;
+
+                println!("{} index files checked", checked);
+                println!("{} index files recompressed", recompressed);
+                Ok::<_, anyhow::Error>(())
+            })?,
+
             Self::AddDirectory { directory } => {
                 ctx.runtime
                     .block_on(add_path_into_database(

src/bin/cratesfyi.rs

@@ -1,4 +1,4 @@
-use derive_more::Display;
+use derive_more::{Display, FromStr};
 use serde::{Deserialize, Serialize};
 
 pub mod dependencies;
@@ -8,7 +8,7 @@ pub mod version;
 #[sqlx(transparent)]
 pub struct CrateId(pub i32);
 
-#[derive(Debug, Clone, Copy, Display, PartialEq, Eq, Hash, Serialize, sqlx::Type)]
+#[derive(Debug, Clone, Copy, Display, PartialEq, Eq, Hash, FromStr, Serialize, sqlx::Type)]
 #[sqlx(transparent)]
 pub struct ReleaseId(pub i32);
 

src/db/types/mod.rs

@@ -1,5 +1,7 @@
-use crate::error::Result;
-use crate::storage::{FileRange, compression::CompressionAlgorithm};
+use crate::{
+    error::Result,
+    storage::{FileRange, compression::CompressionAlgorithm},
+};
 use anyhow::{Context as _, bail};
 use itertools::Itertools as _;
 use sqlx::{Acquire as _, QueryBuilder, Row as _, Sqlite};