refactor: refactor zonky extractor to use generic tar_xz_extractor

Author: brianheineman

examples/zonky/src/main.rs

@@ -9,7 +9,7 @@ use postgresql_embedded::{PostgreSQL, Result, Settings};
 async fn main() -> Result<()> {
     let settings = Settings {
         releases_url: zonky::URL.to_string(),
-        version: VersionReq::parse("=16.2.0")?,
+        version: VersionReq::parse("=16.3.0")?,
         ..Default::default()
     };
     let mut postgresql = PostgreSQL::new(settings);

postgresql_archive/Cargo.toml

@@ -36,7 +36,7 @@ thiserror = { workspace = true }
 tokio = { workspace = true, features = ["full"], optional = true }
 tracing = { workspace = true, features = ["log"] }
 url = { workspace = true }
-xz2 = { workspace = true, optional = true }
+xz2 = { workspace = true }
 zip = { workspace = true }
 
 [dev-dependencies]
@@ -71,7 +71,6 @@ theseus = [
     "sha2",
 ]
 zonky = [
-    "dep:xz2",
     "maven",
 ]
 

postgresql_archive/src/configuration/zonky/extractor.rs

@@ -1,16 +1,13 @@
-use crate::extractor::ExtractDirectories;
+use crate::extractor::{tar_xz_extract, ExtractDirectories};
 use crate::Error::Unexpected;
 use crate::Result;
-use human_bytes::human_bytes;
-use num_format::{Locale, ToFormattedString};
-use std::fs::{create_dir_all, remove_dir_all, remove_file, rename, File};
-use std::io::{copy, BufReader, Cursor};
+use regex::Regex;
+use std::fs::{create_dir_all, remove_dir_all, remove_file, rename};
+use std::io::Cursor;
 use std::path::{Path, PathBuf};
 use std::thread::sleep;
 use std::time::Duration;
-use tar::Archive;
 use tracing::{debug, instrument, warn};
-use xz2::bufread::XzDecoder;
 use zip::ZipArchive;
 
 /// Extracts the compressed tar `bytes` to the [out_dir](Path).
@@ -22,7 +19,6 @@ use zip::ZipArchive;
 #[instrument(skip(bytes))]
 pub fn extract(bytes: &Vec<u8>, extract_directories: ExtractDirectories) -> Result<Vec<PathBuf>> {
     let out_dir = extract_directories.get_path(".")?;
-    let mut files = Vec::new();
     let parent_dir = if let Some(parent) = out_dir.parent() {
         parent
     } else {
@@ -41,7 +37,7 @@ pub fn extract(bytes: &Vec<u8>, extract_directories: ExtractDirectories) -> Resu
             out_dir.to_string_lossy()
         );
         remove_file(&lock_file)?;
-        return Ok(files);
+        return Ok(Vec::new());
     }
 
     let extract_dir = tempfile::tempdir_in(parent_dir)?.into_path();
@@ -66,51 +62,9 @@ pub fn extract(bytes: &Vec<u8>, extract_directories: ExtractDirectories) -> Resu
         return Err(Unexpected("Failed to find archive file".to_string()));
     }
 
-    let input = BufReader::new(Cursor::new(archive_bytes));
-    let decoder = XzDecoder::new(input);
-    let mut archive = Archive::new(decoder);
-    let mut extracted_bytes = 0;
-
-    for archive_entry in archive.entries()? {
-        let mut entry = archive_entry?;
-        let entry_header = entry.header();
-        let entry_type = entry_header.entry_type();
-        let entry_size = entry_header.size()?;
-        #[cfg(unix)]
-        let file_mode = entry_header.mode()?;
-
-        let entry_header_path = entry_header.path()?.to_path_buf();
-        let mut entry_name = extract_dir.clone();
-        entry_name.push(entry_header_path);
-
-        if let Some(parent) = entry_name.parent() {
-            if !parent.exists() {
-                create_dir_all(parent)?;
-            }
-        }
-
-        if entry_type.is_dir() || entry_name.is_dir() {
-            create_dir_all(&entry_name)?;
-        } else if entry_type.is_file() {
-            let mut output_file = File::create(&entry_name)?;
-            copy(&mut entry, &mut output_file)?;
-            extracted_bytes += entry_size;
-
-            #[cfg(unix)]
-            {
-                use std::os::unix::fs::PermissionsExt;
-                output_file.set_permissions(std::fs::Permissions::from_mode(file_mode))?;
-            }
-            files.push(entry_name);
-        } else if entry_type.is_symlink() {
-            #[cfg(unix)]
-            if let Some(symlink_target) = entry.link_name()? {
-                let symlink_path = entry_name.clone();
-                std::os::unix::fs::symlink(symlink_target.as_ref(), symlink_path)?;
-                files.push(entry_name);
-            }
-        }
-    }
+    let mut archive_extract_directories = ExtractDirectories::default();
+    archive_extract_directories.add_mapping(Regex::new(".*")?, extract_dir.clone());
+    let files = tar_xz_extract(&archive_bytes, archive_extract_directories)?;
 
     if out_dir.exists() {
         debug!(
@@ -133,13 +87,6 @@ pub fn extract(bytes: &Vec<u8>, extract_directories: ExtractDirectories) -> Resu
         remove_file(lock_file)?;
     }
 
-    let number_of_files = files.len();
-    debug!(
-        "Extracting {} files totalling {}",
-        number_of_files.to_formatted_string(&Locale::en),
-        human_bytes(extracted_bytes as f64)
-    );
-
     Ok(files)
 }
 

postgresql_archive/src/extractor/mod.rs

@@ -1,8 +1,10 @@
 mod model;
 pub mod registry;
 mod tar_gz_extractor;
+mod tar_xz_extractor;
 mod zip_extractor;
 
 pub use model::ExtractDirectories;
 pub use tar_gz_extractor::extract as tar_gz_extract;
+pub use tar_xz_extractor::extract as tar_xz_extract;
 pub use zip_extractor::extract as zip_extract;

postgresql_archive/src/extractor/tar_xz_extractor.rs

@@ -0,0 +1,81 @@
+use crate::extractor::ExtractDirectories;
+use crate::Error::Unexpected;
+use crate::Result;
+use human_bytes::human_bytes;
+use num_format::{Locale, ToFormattedString};
+use std::fs::{create_dir_all, File};
+use std::io::{copy, BufReader, Cursor};
+use std::path::PathBuf;
+use tar::Archive;
+use tracing::{debug, instrument, warn};
+use xz2::bufread::XzDecoder;
+
+/// Extracts the compressed tar `bytes` to paths defined in `extract_directories`.
+///
+/// # Errors
+/// Returns an error if the extraction fails.
+#[allow(clippy::cast_precision_loss)]
+#[instrument(skip(bytes))]
+pub fn extract(bytes: &Vec<u8>, extract_directories: ExtractDirectories) -> Result<Vec<PathBuf>> {
+    let mut files = Vec::new();
+    let input = BufReader::new(Cursor::new(bytes));
+    let decoder = XzDecoder::new(input);
+    let mut archive = Archive::new(decoder);
+    let mut extracted_bytes = 0;
+
+    for archive_entry in archive.entries()? {
+        let mut entry = archive_entry?;
+        let entry_header = entry.header();
+        let entry_type = entry_header.entry_type();
+        let entry_size = entry_header.size()?;
+        #[cfg(unix)]
+        let file_mode = entry_header.mode()?;
+
+        let entry_header_path = entry_header.path()?.to_path_buf();
+        let prefix = match entry_header_path.components().next() {
+            Some(component) => component.as_os_str().to_str().unwrap_or_default(),
+            None => {
+                return Err(Unexpected(
+                    "Failed to get file header path prefix".to_string(),
+                ));
+            }
+        };
+        let stripped_entry_header_path = entry_header_path.strip_prefix(prefix)?.to_path_buf();
+        let Ok(extract_dir) = extract_directories.get_path(prefix) else {
+            continue;
+        };
+        let mut entry_name = extract_dir.clone();
+        entry_name.push(stripped_entry_header_path);
+
+        if entry_type.is_dir() || entry_name.is_dir() {
+            create_dir_all(&entry_name)?;
+        } else if entry_type.is_file() {
+            let mut output_file = File::create(&entry_name)?;
+            copy(&mut entry, &mut output_file)?;
+            extracted_bytes += entry_size;
+
+            #[cfg(unix)]
+            {
+                use std::os::unix::fs::PermissionsExt;
+                output_file.set_permissions(std::fs::Permissions::from_mode(file_mode))?;
+            }
+            files.push(entry_name);
+        } else if entry_type.is_symlink() {
+            #[cfg(unix)]
+            if let Some(symlink_target) = entry.link_name()? {
+                let symlink_path = entry_name.clone();
+                std::os::unix::fs::symlink(symlink_target.as_ref(), symlink_path)?;
+                files.push(entry_name);
+            }
+        }
+    }
+
+    let number_of_files = files.len();
+    debug!(
+        "Extracted {} files totalling {}",
+        number_of_files.to_formatted_string(&Locale::en),
+        human_bytes(extracted_bytes as f64)
+    );
+
+    Ok(files)
+}