From 75583fbefbb7291cfd48b23820435fbb46f00137 Mon Sep 17 00:00:00 2001 From: Damien Lachaume <135982616+dlachaume@users.noreply.github.com> Date: Tue, 20 May 2025 11:04:41 +0200 Subject: [PATCH 01/20] feat(client-cli): scaffold new command to convert ledger state snapshots --- mithril-client-cli/src/commands/mod.rs | 1 + mithril-client-cli/src/commands/tools/mod.rs | 45 +++++++++++++++++++ .../src/commands/tools/snapshot_converter.rs | 13 ++++++ mithril-client-cli/src/main.rs | 7 ++- 4 files changed, 65 insertions(+), 1 deletion(-) create mode 100644 mithril-client-cli/src/commands/tools/mod.rs create mode 100644 mithril-client-cli/src/commands/tools/snapshot_converter.rs diff --git a/mithril-client-cli/src/commands/mod.rs b/mithril-client-cli/src/commands/mod.rs index 1f7d7229f19..60d782c366f 100644 --- a/mithril-client-cli/src/commands/mod.rs +++ b/mithril-client-cli/src/commands/mod.rs @@ -9,6 +9,7 @@ pub mod cardano_stake_distribution; pub mod cardano_transaction; mod deprecation; pub mod mithril_stake_distribution; +pub mod tools; pub use deprecation::{DeprecatedCommand, Deprecation}; diff --git a/mithril-client-cli/src/commands/tools/mod.rs b/mithril-client-cli/src/commands/tools/mod.rs new file mode 100644 index 00000000000..2881a1811bb --- /dev/null +++ b/mithril-client-cli/src/commands/tools/mod.rs @@ -0,0 +1,45 @@ +//! Tools commands +//! +//! Provides utility subcommands such as converting restored InMemory UTxO-HD ledger snapshot +//! to different flavors (Legacy, LMDB). + +mod snapshot_converter; + +use mithril_client::MithrilResult; +pub use snapshot_converter::*; + +use clap::Subcommand; + +/// Tools commands +#[derive(Subcommand, Debug, Clone)] +pub enum ToolsCommands { + /// UTxO-HD related commands + #[clap(subcommand, name = "utxo-hd")] + UTxOHD(UTxOHDCommands), +} + +impl ToolsCommands { + /// Execute Tools command + pub async fn execute(&self) -> MithrilResult<()> { + match self { + Self::UTxOHD(cmd) => cmd.execute().await, + } + } +} + +/// UTxO-HD related commands +#[derive(Subcommand, Debug, Clone)] +pub enum UTxOHDCommands { + /// Convert a restored `InMemory` ledger snapshot to another flavor. + #[clap(arg_required_else_help = false)] + SnapshotConverter(SnapshotConverterCommand), +} + +impl UTxOHDCommands { + /// Execute UTxO-HD command + pub async fn execute(&self) -> MithrilResult<()> { + match self { + Self::SnapshotConverter(cmd) => cmd.execute().await, + } + } +} diff --git a/mithril-client-cli/src/commands/tools/snapshot_converter.rs b/mithril-client-cli/src/commands/tools/snapshot_converter.rs new file mode 100644 index 00000000000..7c11f2ab023 --- /dev/null +++ b/mithril-client-cli/src/commands/tools/snapshot_converter.rs @@ -0,0 +1,13 @@ +use clap::Parser; + +use mithril_client::MithrilResult; + +#[derive(Parser, Debug, Clone)] +pub struct SnapshotConverterCommand {} + +impl SnapshotConverterCommand { + /// Main command execution + pub async fn execute(&self) -> MithrilResult<()> { + todo!() + } +} diff --git a/mithril-client-cli/src/main.rs b/mithril-client-cli/src/main.rs index df94a659955..02e0820254a 100644 --- a/mithril-client-cli/src/main.rs +++ b/mithril-client-cli/src/main.rs @@ -17,7 +17,8 @@ use mithril_client_cli::commands::{ cardano_db::CardanoDbCommands, cardano_db_v2::CardanoDbV2Commands, cardano_stake_distribution::CardanoStakeDistributionCommands, cardano_transaction::CardanoTransactionCommands, - mithril_stake_distribution::MithrilStakeDistributionCommands, DeprecatedCommand, Deprecation, + mithril_stake_distribution::MithrilStakeDistributionCommands, tools::ToolsCommands, + DeprecatedCommand, Deprecation, }; use mithril_client_cli::{ClapError, CommandContext}; @@ -209,6 +210,9 @@ enum ArtifactCommands { #[clap(alias("doc"), hide(true))] GenerateDoc(GenerateDocCommands), + + #[clap(subcommand)] + Tools(ToolsCommands), } impl ArtifactCommands { @@ -231,6 +235,7 @@ impl ArtifactCommands { Self::GenerateDoc(cmd) => cmd .execute(&mut Args::command()) .map_err(|message| anyhow!(message)), + Self::Tools(cmd) => cmd.execute().await, } } From 1d6fe27457cb8828084cc78147d39e6dabfced33 Mon Sep 17 00:00:00 2001 From: Damien Lachaume <135982616+dlachaume@users.noreply.github.com> Date: Tue, 3 Jun 2025 10:27:54 +0200 Subject: [PATCH 02/20] feat(client-cli): use `unstable` flag for tools command execution --- mithril-client-cli/src/main.rs | 39 +++++++++++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/mithril-client-cli/src/main.rs b/mithril-client-cli/src/main.rs index 02e0820254a..57628c64faf 100644 --- a/mithril-client-cli/src/main.rs +++ b/mithril-client-cli/src/main.rs @@ -235,7 +235,16 @@ impl ArtifactCommands { Self::GenerateDoc(cmd) => cmd .execute(&mut Args::command()) .map_err(|message| anyhow!(message)), - Self::Tools(cmd) => cmd.execute().await, + Self::Tools(cmd) => { + if !context.is_unstable_enabled() { + Err(anyhow!(Self::unstable_flag_missing_message( + "tools", + "utxo-hd snapshot-converter" + ))) + } else { + cmd.execute().await + } + } } } @@ -279,4 +288,32 @@ mod tests { .to_string() .contains("subcommand is only accepted using the --unstable flag.")); } + + #[tokio::test] + async fn fail_if_tools_command_is_used_without_unstable_flag() { + let args = Args::try_parse_from([ + "mithril-client", + "tools", + "utxo-hd", + "snapshot-converter", + "--db-directory", + "whatever", + "--cardano-network", + "preview", + "--cardano-node-version", + "1.2.3", + "--utxo-hd-flavor", + "Legacy", + ]) + .unwrap(); + + let error = args + .execute(Logger::root(slog::Discard, slog::o!())) + .await + .expect_err("Should fail if unstable flag missing"); + + assert!(error + .to_string() + .contains("subcommand is only accepted using the --unstable flag.")); + } } From 4b92a706e7dfb4bfe85aa74972de64ba8f74e015 Mon Sep 17 00:00:00 2001 From: Damien Lachaume <135982616+dlachaume@users.noreply.github.com> Date: Tue, 20 May 2025 12:13:45 +0200 Subject: [PATCH 03/20] feat(client-cli): implement download of Cardano node distribution to retrieve the `snapshot-converter` binary --- Cargo.lock | 2 + mithril-client-cli/Cargo.toml | 8 + mithril-client-cli/src/commands/tools/mod.rs | 2 +- .../src/commands/tools/snapshot_converter.rs | 238 +++++++++++++++++- mithril-client-cli/src/main.rs | 2 - .../github_release.rs | 123 +++++++++ .../github_release_retriever/interface.rs | 27 ++ .../src/utils/github_release_retriever/mod.rs | 7 + .../utils/github_release_retriever/reqwest.rs | 93 +++++++ .../src/utils/http_downloader/mod.rs | 23 ++ .../reqwest_http_downloader.rs | 53 ++++ mithril-client-cli/src/utils/mod.rs | 4 + 12 files changed, 576 insertions(+), 6 deletions(-) create mode 100644 mithril-client-cli/src/utils/github_release_retriever/github_release.rs create mode 100644 mithril-client-cli/src/utils/github_release_retriever/interface.rs create mode 100644 mithril-client-cli/src/utils/github_release_retriever/mod.rs create mode 100644 mithril-client-cli/src/utils/github_release_retriever/reqwest.rs create mode 100644 mithril-client-cli/src/utils/http_downloader/mod.rs create mode 100644 mithril-client-cli/src/utils/http_downloader/reqwest_http_downloader.rs diff --git a/Cargo.lock b/Cargo.lock index 0430d447d7e..726ac6566de 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3878,6 +3878,8 @@ dependencies = [ "mithril-client", "mithril-common", "mithril-doc", + "mockall", + "reqwest", "serde", "serde_json", "slog", diff --git a/mithril-client-cli/Cargo.toml b/mithril-client-cli/Cargo.toml index 4a1749d509d..67a963822d9 100644 --- a/mithril-client-cli/Cargo.toml +++ b/mithril-client-cli/Cargo.toml @@ -38,6 +38,13 @@ indicatif = { version = "0.17.11", features = ["tokio"] } mithril-cli-helper = { path = "../internal/mithril-cli-helper" } mithril-client = { path = "../mithril-client", features = ["fs", "unstable"] } mithril-doc = { path = "../internal/mithril-doc" } +reqwest = { workspace = true, features = [ + "default", + "gzip", + "zstd", + "deflate", + "brotli" +] } serde = { workspace = true } serde_json = { workspace = true } slog = { workspace = true, features = [ @@ -52,3 +59,4 @@ tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } [dev-dependencies] mithril-common = { path = "../mithril-common", features = ["test_tools"] } +mockall = { workspace = true } diff --git a/mithril-client-cli/src/commands/tools/mod.rs b/mithril-client-cli/src/commands/tools/mod.rs index 2881a1811bb..ace75838d72 100644 --- a/mithril-client-cli/src/commands/tools/mod.rs +++ b/mithril-client-cli/src/commands/tools/mod.rs @@ -5,10 +5,10 @@ mod snapshot_converter; -use mithril_client::MithrilResult; pub use snapshot_converter::*; use clap::Subcommand; +use mithril_client::MithrilResult; /// Tools commands #[derive(Subcommand, Debug, Clone)] diff --git a/mithril-client-cli/src/commands/tools/snapshot_converter.rs b/mithril-client-cli/src/commands/tools/snapshot_converter.rs index 7c11f2ab023..abd12d85711 100644 --- a/mithril-client-cli/src/commands/tools/snapshot_converter.rs +++ b/mithril-client-cli/src/commands/tools/snapshot_converter.rs @@ -1,13 +1,245 @@ -use clap::Parser; +use std::{ + env, fmt, + path::{Path, PathBuf}, +}; + +use anyhow::{anyhow, Context}; +use clap::{Parser, ValueEnum}; use mithril_client::MithrilResult; +use crate::utils::{ + GitHubReleaseRetriever, HttpDownloader, ReqwestGitHubApiClient, ReqwestHttpDownloader, +}; + +const GITHUB_ORGANIZATION: &str = "IntersectMBO"; +const GITHUB_REPOSITORY: &str = "cardano-node"; + +const LATEST_DISTRIBUTION_TAG: &str = "latest"; +const PRERELEASE_DISTRIBUTION_TAG: &str = "prerelease"; + +const CARDANO_DISTRIBUTION_TEMP_DIR: &str = "cardano-node-distribution-tmp"; + +#[derive(Debug, Clone, ValueEnum)] +enum UTxOHDFlavor { + #[clap(name = "Legacy")] + Legacy, + #[clap(name = "LMDB")] + Lmdb, +} + +impl fmt::Display for UTxOHDFlavor { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Legacy => write!(f, "Legacy"), + Self::Lmdb => write!(f, "LMDB"), + } + } +} + +/// Clap command to convert a restored `InMemory` Mithril snapshot to another flavor. #[derive(Parser, Debug, Clone)] -pub struct SnapshotConverterCommand {} +pub struct SnapshotConverterCommand { + /// Path to the Cardano node database directory. + #[clap(long)] + db_directory: PathBuf, + + /// Cardano node version of the Mithril signed snapshot. + /// + /// `latest` and `prerelease` are also supported to download the latest or preprelease distribution. + #[clap(long)] + cardano_node_version: String, + + /// UTxO-HD flavor to convert the ledger snapshot to. + #[clap(long)] + utxo_hd_flavor: UTxOHDFlavor, +} impl SnapshotConverterCommand { /// Main command execution pub async fn execute(&self) -> MithrilResult<()> { - todo!() + let distribution_temp_dir = self.db_directory.join(CARDANO_DISTRIBUTION_TEMP_DIR); + std::fs::create_dir(&distribution_temp_dir).with_context(|| { + format!( + "Failed to create directory: {}", + distribution_temp_dir.display() + ) + })?; + + let archive_path = Self::download_cardano_node_distribution( + ReqwestGitHubApiClient::new()?, + ReqwestHttpDownloader::new()?, + &self.cardano_node_version, + &distribution_temp_dir, + ) + .await + .with_context(|| { + "Failed to download 'snapshot-converter' binary from Cardano node distribution" + })?; + + Ok(()) + } + + async fn download_cardano_node_distribution( + github_api_client: impl GitHubReleaseRetriever, + http_downloader: impl HttpDownloader, + tag: &str, + target_dir: &Path, + ) -> MithrilResult { + println!( + "Downloading Cardano node distribution for tag: '{}'...", + tag + ); + let release = match tag { + LATEST_DISTRIBUTION_TAG => github_api_client + .get_latest_release(GITHUB_ORGANIZATION, GITHUB_REPOSITORY) + .await + .with_context(|| "Failed to get latest release")?, + PRERELEASE_DISTRIBUTION_TAG => github_api_client + .get_prerelease(GITHUB_ORGANIZATION, GITHUB_REPOSITORY) + .await + .with_context(|| "Failed to get prerelease")?, + _ => github_api_client + .get_release_by_tag(GITHUB_ORGANIZATION, GITHUB_REPOSITORY, tag) + .await + .with_context(|| format!("Failed to get release by tag: {}", tag))?, + }; + + let asset = release + .get_asset_for_os(env::consts::OS)? + .ok_or_else(|| anyhow!("No asset found for platform: {}", env::consts::OS)) + .with_context(|| { + format!( + "Failed to find asset for current platform: {}", + env::consts::OS + ) + })?; + + let archive_path = http_downloader + .download_file(asset.browser_download_url.parse()?, target_dir, &asset.name) + .await?; + + println!( + "Distribution downloaded successfully. Archive location: {}", + archive_path.display() + ); + + Ok(archive_path) + } +} + +#[cfg(test)] +mod tests { + use mockall::predicate::eq; + use reqwest::Url; + + use mithril_common::temp_dir_create; + + use crate::utils::{GitHubRelease, MockGitHubReleaseRetriever, MockHttpDownloader}; + + use super::*; + + #[tokio::test] + async fn call_get_latest_release_with_latest_tag() { + let temp_dir = temp_dir_create!(); + let release = GitHubRelease::dummy_with_all_supported_assets(); + let asset = release.get_asset_for_os(env::consts::OS).unwrap().unwrap(); + + let cloned_release = release.clone(); + let mut github_api_client = MockGitHubReleaseRetriever::new(); + github_api_client + .expect_get_latest_release() + .with(eq(GITHUB_ORGANIZATION), eq(GITHUB_REPOSITORY)) + .returning(move |_, _| Ok(cloned_release.clone())); + + let mut http_downloader = MockHttpDownloader::new(); + http_downloader + .expect_download_file() + .with( + eq(Url::parse(&asset.browser_download_url).unwrap()), + eq(temp_dir.clone()), + eq(asset.name.clone()), + ) + .returning(|_, _, _| Ok(PathBuf::new())); + + SnapshotConverterCommand::download_cardano_node_distribution( + github_api_client, + http_downloader, + LATEST_DISTRIBUTION_TAG, + &temp_dir, + ) + .await + .unwrap(); + } + + #[tokio::test] + async fn call_get_prerelease_with_prerelease_tag() { + let temp_dir = temp_dir_create!(); + let release = GitHubRelease::dummy_with_all_supported_assets(); + let asset = release.get_asset_for_os(env::consts::OS).unwrap().unwrap(); + + let cloned_release = release.clone(); + let mut github_api_client = MockGitHubReleaseRetriever::new(); + github_api_client + .expect_get_prerelease() + .with(eq(GITHUB_ORGANIZATION), eq(GITHUB_REPOSITORY)) + .returning(move |_, _| Ok(cloned_release.clone())); + + let mut http_downloader = MockHttpDownloader::new(); + http_downloader + .expect_download_file() + .with( + eq(Url::parse(&asset.browser_download_url).unwrap()), + eq(temp_dir.clone()), + eq(asset.name.clone()), + ) + .returning(|_, _, _| Ok(PathBuf::new())); + + SnapshotConverterCommand::download_cardano_node_distribution( + github_api_client, + http_downloader, + PRERELEASE_DISTRIBUTION_TAG, + &temp_dir, + ) + .await + .unwrap(); + } + + #[tokio::test] + async fn call_get_release_by_tag_with_specific_cardano_node_version() { + let cardano_node_version = "10.3.1"; + let temp_dir = temp_dir_create!(); + let release = GitHubRelease::dummy_with_all_supported_assets(); + let asset = release.get_asset_for_os(env::consts::OS).unwrap().unwrap(); + + let cloned_release = release.clone(); + let mut github_api_client = MockGitHubReleaseRetriever::new(); + github_api_client + .expect_get_release_by_tag() + .with( + eq(GITHUB_ORGANIZATION), + eq(GITHUB_REPOSITORY), + eq(cardano_node_version), + ) + .returning(move |_, _, _| Ok(cloned_release.clone())); + + let mut http_downloader = MockHttpDownloader::new(); + http_downloader + .expect_download_file() + .with( + eq(Url::parse(&asset.browser_download_url).unwrap()), + eq(temp_dir.clone()), + eq(asset.name.clone()), + ) + .returning(|_, _, _| Ok(PathBuf::new())); + + SnapshotConverterCommand::download_cardano_node_distribution( + github_api_client, + http_downloader, + cardano_node_version, + &temp_dir, + ) + .await + .unwrap(); } } diff --git a/mithril-client-cli/src/main.rs b/mithril-client-cli/src/main.rs index 57628c64faf..2ac2be52214 100644 --- a/mithril-client-cli/src/main.rs +++ b/mithril-client-cli/src/main.rs @@ -298,8 +298,6 @@ mod tests { "snapshot-converter", "--db-directory", "whatever", - "--cardano-network", - "preview", "--cardano-node-version", "1.2.3", "--utxo-hd-flavor", diff --git a/mithril-client-cli/src/utils/github_release_retriever/github_release.rs b/mithril-client-cli/src/utils/github_release_retriever/github_release.rs new file mode 100644 index 00000000000..937a97bdfc0 --- /dev/null +++ b/mithril-client-cli/src/utils/github_release_retriever/github_release.rs @@ -0,0 +1,123 @@ +use anyhow::anyhow; +use serde::Deserialize; + +use mithril_client::MithrilResult; + +pub const ASSET_PLATFORM_LINUX: &str = "linux"; +pub const ASSET_PLATFORM_MACOS: &str = "macos"; +pub const ASSET_PLATFORM_WINDOWS: &str = "win64"; + +#[derive(Debug, Clone, Deserialize, Eq, PartialEq)] +pub struct GitHubAsset { + pub name: String, + pub browser_download_url: String, +} + +#[derive(Debug, Default, Clone, Deserialize)] +pub struct GitHubRelease { + pub assets: Vec, + pub prerelease: bool, +} + +impl GitHubRelease { + pub fn get_asset_for_os(&self, target_os: &str) -> MithrilResult> { + let os_in_asset_name = match target_os { + "linux" => ASSET_PLATFORM_LINUX, + "macos" => ASSET_PLATFORM_MACOS, + "windows" => ASSET_PLATFORM_WINDOWS, + _ => return Err(anyhow!("Unsupported platform: {}", target_os)), + }; + + let asset = self + .assets + .iter() + .find(|asset| asset.name.contains(os_in_asset_name)); + + Ok(asset) + } + + #[cfg(test)] + pub fn dummy_with_all_supported_assets() -> Self { + GitHubRelease { + assets: vec![ + GitHubAsset { + name: format!("asset-name-{}.tar.gz", ASSET_PLATFORM_LINUX), + browser_download_url: "https://release-assets.com/linux".to_string(), + }, + GitHubAsset { + name: format!("asset-name-{}.tar.gz", ASSET_PLATFORM_MACOS), + browser_download_url: "https://release-assets.com/macos".to_string(), + }, + GitHubAsset { + name: format!("asset-name-{}.zip", ASSET_PLATFORM_WINDOWS), + browser_download_url: "https://release-assets.com/windows".to_string(), + }, + ], + ..GitHubRelease::default() + } + } +} + +#[cfg(test)] +mod tests { + + use super::*; + + fn dummy_asset(os: &str) -> GitHubAsset { + GitHubAsset { + name: format!("asset-name-{}.whatever", os), + browser_download_url: format!("https://release-assets.com/{}", os), + } + } + + #[test] + fn returns_expected_asset_for_each_supported_platform() { + let release = GitHubRelease { + assets: vec![ + dummy_asset(ASSET_PLATFORM_LINUX), + dummy_asset(ASSET_PLATFORM_MACOS), + dummy_asset(ASSET_PLATFORM_WINDOWS), + ], + ..GitHubRelease::default() + }; + + { + let asset = release.get_asset_for_os("linux").unwrap(); + assert_eq!(asset, Some(&dummy_asset(ASSET_PLATFORM_LINUX))); + } + + { + let asset = release.get_asset_for_os("macos").unwrap(); + assert_eq!(asset, Some(&dummy_asset(ASSET_PLATFORM_MACOS))); + } + + { + let asset = release.get_asset_for_os("windows").unwrap(); + assert_eq!(asset, Some(&dummy_asset(ASSET_PLATFORM_WINDOWS))); + } + } + + #[test] + fn returns_none_when_asset_is_missing() { + let release = GitHubRelease { + assets: vec![dummy_asset(ASSET_PLATFORM_LINUX)], + ..GitHubRelease::default() + }; + + let asset = release.get_asset_for_os("macos").unwrap(); + + assert!(asset.is_none()); + } + + #[test] + fn fails_for_unsupported_platform() { + let release = GitHubRelease { + assets: vec![dummy_asset(ASSET_PLATFORM_LINUX)], + ..GitHubRelease::default() + }; + + release + .get_asset_for_os("unsupported") + .expect_err("Should have failed for unsupported platform"); + } +} diff --git a/mithril-client-cli/src/utils/github_release_retriever/interface.rs b/mithril-client-cli/src/utils/github_release_retriever/interface.rs new file mode 100644 index 00000000000..0b1ba7f3df6 --- /dev/null +++ b/mithril-client-cli/src/utils/github_release_retriever/interface.rs @@ -0,0 +1,27 @@ +use async_trait::async_trait; + +use mithril_client::MithrilResult; + +use super::github_release::GitHubRelease; + +/// Trait for interacting with the GitHub API to retrieve Cardano node release. +#[cfg_attr(test, mockall::automock)] +#[async_trait] +pub trait GitHubReleaseRetriever { + /// Retrieves a release by its tag. + async fn get_release_by_tag( + &self, + owner: &str, + repo: &str, + tag: &str, + ) -> MithrilResult; + + /// Retrieves the latest release. + async fn get_latest_release(&self, owner: &str, repo: &str) -> MithrilResult; + + /// Retrieves the prerelease. + async fn get_prerelease(&self, owner: &str, repo: &str) -> MithrilResult; + + /// Retrieves all available releases. + async fn get_all_releases(&self, owner: &str, repo: &str) -> MithrilResult>; +} diff --git a/mithril-client-cli/src/utils/github_release_retriever/mod.rs b/mithril-client-cli/src/utils/github_release_retriever/mod.rs new file mode 100644 index 00000000000..8e8b4155065 --- /dev/null +++ b/mithril-client-cli/src/utils/github_release_retriever/mod.rs @@ -0,0 +1,7 @@ +mod github_release; +mod interface; +mod reqwest; + +pub use github_release::*; +pub use interface::*; +pub use reqwest::*; diff --git a/mithril-client-cli/src/utils/github_release_retriever/reqwest.rs b/mithril-client-cli/src/utils/github_release_retriever/reqwest.rs new file mode 100644 index 00000000000..70f5d9d391e --- /dev/null +++ b/mithril-client-cli/src/utils/github_release_retriever/reqwest.rs @@ -0,0 +1,93 @@ +use anyhow::{anyhow, Context}; +use async_trait::async_trait; +use reqwest::{Client, IntoUrl}; +use serde::de::DeserializeOwned; + +use mithril_client::MithrilResult; + +use super::{GitHubRelease, GitHubReleaseRetriever}; + +pub struct ReqwestGitHubApiClient { + client: Client, +} + +impl ReqwestGitHubApiClient { + pub fn new() -> MithrilResult { + let client = Client::builder() + .user_agent("mithril-client") + .build() + .context("Failed to build Reqwest GitHub API client")?; + + Ok(Self { client }) + } + + async fn download(&self, source_url: U) -> MithrilResult { + let url = source_url + .into_url() + .with_context(|| "Given `source_url` is not a valid Url")?; + let response = self + .client + .get(url.clone()) + .send() + .await + .with_context(|| format!("Failed to send request to GitHub API: {}", url))?; + let body = response.text().await?; + let parsed_body = serde_json::from_str::(&body) + .with_context(|| format!("Failed to parse response from GitHub API: {:?}", body))?; + + Ok(parsed_body) + } +} + +#[async_trait] +impl GitHubReleaseRetriever for ReqwestGitHubApiClient { + async fn get_release_by_tag( + &self, + organization: &str, + repository: &str, + tag: &str, + ) -> MithrilResult { + let url = + format!("https://api.github.com/repos/{organization}/{repository}/releases/tags/{tag}"); + let release = self.download(url).await?; + + Ok(release) + } + + async fn get_latest_release( + &self, + organization: &str, + repository: &str, + ) -> MithrilResult { + let url = + format!("https://api.github.com/repos/{organization}/{repository}/releases/latest"); + let release = self.download(url).await?; + + Ok(release) + } + + async fn get_prerelease( + &self, + organization: &str, + repository: &str, + ) -> MithrilResult { + let releases = self.get_all_releases(organization, repository).await?; + let prerelease = releases + .into_iter() + .find(|release| release.prerelease) + .ok_or_else(|| anyhow!("No prerelease found"))?; + + Ok(prerelease) + } + + async fn get_all_releases( + &self, + organization: &str, + repository: &str, + ) -> MithrilResult> { + let url = format!("https://api.github.com/repos/{organization}/{repository}/releases"); + let releases = self.download(url).await?; + + Ok(releases) + } +} diff --git a/mithril-client-cli/src/utils/http_downloader/mod.rs b/mithril-client-cli/src/utils/http_downloader/mod.rs new file mode 100644 index 00000000000..1341f09d13f --- /dev/null +++ b/mithril-client-cli/src/utils/http_downloader/mod.rs @@ -0,0 +1,23 @@ +mod reqwest_http_downloader; + +pub use reqwest_http_downloader::*; + +use async_trait::async_trait; +use mithril_client::MithrilResult; +use reqwest::Url; +use std::path::{Path, PathBuf}; + +/// Trait for downloading a file over HTTP from a URL, +/// saving it to a target directory with the given filename. +/// +/// Returns the path to the downloaded file. +#[cfg_attr(test, mockall::automock)] +#[async_trait] +pub trait HttpDownloader { + async fn download_file( + &self, + url: Url, + download_dir: &Path, + filename: &str, + ) -> MithrilResult; +} diff --git a/mithril-client-cli/src/utils/http_downloader/reqwest_http_downloader.rs b/mithril-client-cli/src/utils/http_downloader/reqwest_http_downloader.rs new file mode 100644 index 00000000000..4781982dae0 --- /dev/null +++ b/mithril-client-cli/src/utils/http_downloader/reqwest_http_downloader.rs @@ -0,0 +1,53 @@ +use std::{ + fs::File, + io::Write, + path::{Path, PathBuf}, +}; + +use anyhow::Context; +use async_trait::async_trait; +use reqwest::{Client, Url}; + +use mithril_client::MithrilResult; + +use super::HttpDownloader; + +/// [ReqwestHttpDownloader] is an implementation of the [HttpDownloader]. +pub struct ReqwestHttpDownloader { + client: Client, +} + +impl ReqwestHttpDownloader { + /// Creates a new instance of [ReqwestHttpDownloader]. + pub fn new() -> MithrilResult { + let client = Client::builder() + .build() + .with_context(|| "Failed to build Reqwest HTTP client")?; + + Ok(Self { client }) + } +} + +#[async_trait] +impl HttpDownloader for ReqwestHttpDownloader { + async fn download_file( + &self, + url: Url, + download_dir: &Path, + filename: &str, + ) -> MithrilResult { + let response = self + .client + .get(url.clone()) + .send() + .await + .with_context(|| format!("Failed to download file from URL: {}", url))?; + + let bytes = response.bytes().await?; + let download_filepath = download_dir.join(filename); + let mut file = File::create(&download_filepath)?; + file.write_all(&bytes)?; + + Ok(download_filepath) + } +} diff --git a/mithril-client-cli/src/utils/mod.rs b/mithril-client-cli/src/utils/mod.rs index 6d70b6f01f0..114c7e766df 100644 --- a/mithril-client-cli/src/utils/mod.rs +++ b/mithril-client-cli/src/utils/mod.rs @@ -5,6 +5,8 @@ mod cardano_db; mod cardano_db_download_checker; mod expander; mod feedback_receiver; +mod github_release_retriever; +mod http_downloader; mod multi_download_progress_reporter; mod progress_reporter; @@ -12,6 +14,8 @@ pub use cardano_db::*; pub use cardano_db_download_checker::*; pub use expander::*; pub use feedback_receiver::*; +pub use github_release_retriever::*; +pub use http_downloader::*; pub use multi_download_progress_reporter::*; pub use progress_reporter::*; From 4a8905812ade28f6d629f2044ab89eac7181fb5d Mon Sep 17 00:00:00 2001 From: Damien Lachaume <135982616+dlachaume@users.noreply.github.com> Date: Tue, 3 Jun 2025 09:10:32 +0200 Subject: [PATCH 04/20] feat(client-cli): implement unpack of Cardano node distribution --- Cargo.lock | 144 ++++++++++++++++++ mithril-client-cli/Cargo.toml | 3 + .../src/commands/tools/snapshot_converter.rs | 12 +- .../src/utils/archive_unpacker/mod.rs | 18 +++ .../utils/archive_unpacker/tar_gz_unpacker.rs | 102 +++++++++++++ .../src/utils/archive_unpacker/unpacker.rs | 111 ++++++++++++++ .../utils/archive_unpacker/zip_unpacker.rs | 114 ++++++++++++++ mithril-client-cli/src/utils/mod.rs | 2 + 8 files changed, 505 insertions(+), 1 deletion(-) create mode 100644 mithril-client-cli/src/utils/archive_unpacker/mod.rs create mode 100644 mithril-client-cli/src/utils/archive_unpacker/tar_gz_unpacker.rs create mode 100644 mithril-client-cli/src/utils/archive_unpacker/unpacker.rs create mode 100644 mithril-client-cli/src/utils/archive_unpacker/zip_unpacker.rs diff --git a/Cargo.lock b/Cargo.lock index 726ac6566de..ca7a4beeb91 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -173,6 +173,15 @@ version = "1.0.98" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e16d2d3311acee920a9eb8d33b8cbc1787ce4a264e85f964c2404b969bdcd487" +[[package]] +name = "arbitrary" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dde20b3d026af13f561bdd0f15edf01fc734f0dafcedbaf42bba506a9517f223" +dependencies = [ + "derive_arbitrary", +] + [[package]] name = "arc-swap" version = "1.7.1" @@ -839,6 +848,25 @@ version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" +[[package]] +name = "bzip2" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49ecfb22d906f800d4fe833b6282cf4dc1c298f5057ca0b5445e5c209735ca47" +dependencies = [ + "bzip2-sys", +] + +[[package]] +name = "bzip2-sys" +version = "0.1.13+1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "225bff33b2141874fe80d71e07d6eec4f85c5c216453dd96388240f96e1acc14" +dependencies = [ + "cc", + "pkg-config", +] + [[package]] name = "cast" version = "0.3.0" @@ -1155,6 +1183,12 @@ dependencies = [ "tiny-keccak", ] +[[package]] +name = "constant_time_eq" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" + [[package]] name = "convert_case" version = "0.6.0" @@ -1451,6 +1485,12 @@ dependencies = [ "syn 2.0.101", ] +[[package]] +name = "deflate64" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da692b8d1080ea3045efaab14434d40468c3d8657e42abddfffca87b428f4c1b" + [[package]] name = "der" version = "0.7.10" @@ -1486,6 +1526,17 @@ dependencies = [ "serde", ] +[[package]] +name = "derive_arbitrary" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30542c1ad912e0e3d22a1935c290e12e8a29d704a420177a31faad4a601a0800" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.101", +] + [[package]] name = "digest" version = "0.9.0" @@ -1749,6 +1800,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ced92e76e966ca2fd84c8f7aa01a4aea65b0eb6648d72f7c8f3e2764a67fece" dependencies = [ "crc32fast", + "libz-rs-sys", "miniz_oxide", ] @@ -3109,6 +3161,26 @@ version = "0.2.172" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d750af042f7ef4f724306de029d18836c26c1765a54a6a3f094cbd23a7267ffa" +[[package]] +name = "liblzma" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66352d7a8ac12d4877b6e6ea5a9b7650ee094257dc40889955bea5bc5b08c1d0" +dependencies = [ + "liblzma-sys", +] + +[[package]] +name = "liblzma-sys" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5839bad90c3cc2e0b8c4ed8296b80e86040240f81d46b9c0e9bc8dd51ddd3af1" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + [[package]] name = "libm" version = "0.2.15" @@ -3577,6 +3649,15 @@ dependencies = [ "redox_syscall", ] +[[package]] +name = "libz-rs-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6489ca9bd760fe9642d7644e827b0c9add07df89857b0416ee15c1cc1a3b8c5a" +dependencies = [ + "zlib-rs", +] + [[package]] name = "linux-raw-sys" version = "0.9.4" @@ -3870,6 +3951,7 @@ dependencies = [ "clap", "cli-table", "config", + "flate2", "fs2", "futures", "human_bytes", @@ -3886,8 +3968,10 @@ dependencies = [ "slog-async", "slog-bunyan", "slog-term", + "tar", "thiserror 2.0.12", "tokio", + "zip", ] [[package]] @@ -4791,6 +4875,16 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df94ce210e5bc13cb6651479fa48d14f601d9858cfe0467f43ae157023b938d3" +[[package]] +name = "pbkdf2" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" +dependencies = [ + "digest 0.10.7", + "hmac", +] + [[package]] name = "pem" version = "3.0.5" @@ -6133,6 +6227,12 @@ dependencies = [ "rand_core 0.6.4", ] +[[package]] +name = "simd-adler32" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe" + [[package]] name = "similar" version = "2.7.0" @@ -8117,6 +8217,50 @@ dependencies = [ "syn 2.0.101", ] +[[package]] +name = "zip" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "153a6fff49d264c4babdcfa6b4d534747f520e56e8f0f384f3b808c4b64cc1fd" +dependencies = [ + "aes", + "arbitrary", + "bzip2", + "constant_time_eq", + "crc32fast", + "deflate64", + "flate2", + "getrandom 0.3.3", + "hmac", + "indexmap 2.9.0", + "liblzma", + "memchr", + "pbkdf2", + "sha1", + "time", + "zeroize", + "zopfli", + "zstd", +] + +[[package]] +name = "zlib-rs" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "868b928d7949e09af2f6086dfc1e01936064cc7a819253bce650d4e2a2d63ba8" + +[[package]] +name = "zopfli" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edfc5ee405f504cd4984ecc6f14d02d55cfda60fa4b689434ef4102aae150cd7" +dependencies = [ + "bumpalo", + "crc32fast", + "log", + "simd-adler32", +] + [[package]] name = "zstd" version = "0.13.3" diff --git a/mithril-client-cli/Cargo.toml b/mithril-client-cli/Cargo.toml index 67a963822d9..517f9de61b3 100644 --- a/mithril-client-cli/Cargo.toml +++ b/mithril-client-cli/Cargo.toml @@ -31,6 +31,7 @@ chrono = { workspace = true } clap = { workspace = true } cli-table = "0.5.0" config = { workspace = true } +flate2 = "1.1.1" fs2 = "0.4.3" futures = "0.3.31" human_bytes = { version = "0.4.3", features = ["fast"] } @@ -54,8 +55,10 @@ slog = { workspace = true, features = [ slog-async = { workspace = true } slog-bunyan = { workspace = true } slog-term = { workspace = true } +tar = "0.4.44" thiserror = { workspace = true } tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } +zip = "4.0.0" [dev-dependencies] mithril-common = { path = "../mithril-common", features = ["test_tools"] } diff --git a/mithril-client-cli/src/commands/tools/snapshot_converter.rs b/mithril-client-cli/src/commands/tools/snapshot_converter.rs index abd12d85711..2b72672c36e 100644 --- a/mithril-client-cli/src/commands/tools/snapshot_converter.rs +++ b/mithril-client-cli/src/commands/tools/snapshot_converter.rs @@ -9,7 +9,8 @@ use clap::{Parser, ValueEnum}; use mithril_client::MithrilResult; use crate::utils::{ - GitHubReleaseRetriever, HttpDownloader, ReqwestGitHubApiClient, ReqwestHttpDownloader, + ArchiveUnpacker, GitHubReleaseRetriever, HttpDownloader, ReqwestGitHubApiClient, + ReqwestHttpDownloader, }; const GITHUB_ORGANIZATION: &str = "IntersectMBO"; @@ -77,6 +78,15 @@ impl SnapshotConverterCommand { "Failed to download 'snapshot-converter' binary from Cardano node distribution" })?; + ArchiveUnpacker::default() + .unpack(&archive_path, &distribution_temp_dir) + .with_context(|| { + format!( + "Failed to unpack 'snapshot-converter' binary to directory: {}", + distribution_temp_dir.display() + ) + })?; + Ok(()) } diff --git a/mithril-client-cli/src/utils/archive_unpacker/mod.rs b/mithril-client-cli/src/utils/archive_unpacker/mod.rs new file mode 100644 index 00000000000..6c4bf3ff9e0 --- /dev/null +++ b/mithril-client-cli/src/utils/archive_unpacker/mod.rs @@ -0,0 +1,18 @@ +mod tar_gz_unpacker; +mod unpacker; +mod zip_unpacker; + +pub use unpacker::*; + +use mithril_client::MithrilResult; +use std::path::Path; + +/// Trait for supported archive formats (e.g. `.zip`, `.tar.gz`). +#[cfg_attr(test, mockall::automock)] +pub trait ArchiveFormat { + /// Checks whether this format can handle the given archive file. + fn supports(&self, archive_path: &Path) -> bool; + + /// Unpacks the archive into the target directory. + fn unpack(&self, archive_path: &Path, unpack_dir: &Path) -> MithrilResult<()>; +} diff --git a/mithril-client-cli/src/utils/archive_unpacker/tar_gz_unpacker.rs b/mithril-client-cli/src/utils/archive_unpacker/tar_gz_unpacker.rs new file mode 100644 index 00000000000..2e1494030a9 --- /dev/null +++ b/mithril-client-cli/src/utils/archive_unpacker/tar_gz_unpacker.rs @@ -0,0 +1,102 @@ +use std::{fs::File, path::Path}; + +use anyhow::Context; +use flate2::read::GzDecoder; +use tar::Archive; + +use mithril_client::MithrilResult; + +use super::ArchiveFormat; + +#[derive(Debug, Eq, PartialEq)] +pub struct TarGzUnpacker; + +impl ArchiveFormat for TarGzUnpacker { + fn unpack(&self, archive_path: &Path, unpack_dir: &Path) -> MithrilResult<()> { + let archive = File::open(archive_path) + .with_context(|| format!("Could not open archive file '{}'", archive_path.display()))?; + let gzip_decoder = GzDecoder::new(archive); + let mut file_archive = Archive::new(gzip_decoder); + file_archive.unpack(unpack_dir).with_context(|| { + format!( + "Could not unpack '{}' with 'Gzip' to directory '{}'", + archive_path.display(), + unpack_dir.display() + ) + })?; + + Ok(()) + } + + fn supports(&self, path: &Path) -> bool { + path.extension().and_then(|e| e.to_str()) == Some("gz") + } +} + +#[cfg(test)] +mod tests { + use std::fs::{self, File}; + + use flate2::{write::GzEncoder, Compression}; + use tar::{Builder, Header}; + + use mithril_common::{assert_dir_eq, temp_dir_create}; + + use super::*; + + #[test] + fn unpack_tar_archive_extracts_all_files() { + let temp_dir = temp_dir_create!(); + let archive_path = temp_dir.join("archive.tar.gz"); + + { + let tar_gz_file = File::create(&archive_path).unwrap(); + let encoder = GzEncoder::new(tar_gz_file, Compression::default()); + let mut tar_builder = Builder::new(encoder); + + let content = b"root content"; + let mut header = Header::new_gnu(); + header.set_size(content.len() as u64); + header.set_cksum(); + tar_builder + .append_data(&mut header, "root.txt", &content[..]) + .unwrap(); + + let content = b"nested content"; + let mut header = Header::new_gnu(); + header.set_size(content.len() as u64); + header.set_cksum(); + tar_builder + .append_data(&mut header, "nested/dir/nested-file.txt", &content[..]) + .unwrap(); + + tar_builder.finish().unwrap(); + } + + TarGzUnpacker.unpack(&archive_path, &temp_dir).unwrap(); + + assert_dir_eq! { + &temp_dir, + "* nested/ + ** dir/ + *** nested-file.txt + * archive.tar.gz + * root.txt" + }; + + let root_file_content = fs::read_to_string(temp_dir.join("root.txt")).unwrap(); + assert_eq!(root_file_content, "root content"); + + let nested_file_content = + fs::read_to_string(temp_dir.join("nested/dir/nested-file.txt")).unwrap(); + assert_eq!(nested_file_content, "nested content"); + } + + #[test] + fn supported_file_extension() { + assert!(TarGzUnpacker.supports(Path::new("archive.tar.gz"))); + assert!(TarGzUnpacker.supports(Path::new("archive.gz"))); + assert!(!TarGzUnpacker.supports(Path::new("archive.tar"))); + assert!(!TarGzUnpacker.supports(Path::new("archive.whatever"))); + } +} diff --git a/mithril-client-cli/src/utils/archive_unpacker/unpacker.rs b/mithril-client-cli/src/utils/archive_unpacker/unpacker.rs new file mode 100644 index 00000000000..79e7715f71a --- /dev/null +++ b/mithril-client-cli/src/utils/archive_unpacker/unpacker.rs @@ -0,0 +1,111 @@ +use std::path::Path; + +use anyhow::anyhow; + +use mithril_client::MithrilResult; + +use super::{tar_gz_unpacker::TarGzUnpacker, zip_unpacker::ZipUnpacker, ArchiveFormat}; + +pub struct ArchiveUnpacker { + supported_formats: Vec>, +} + +impl Default for ArchiveUnpacker { + fn default() -> Self { + Self { + supported_formats: vec![Box::new(TarGzUnpacker), Box::new(ZipUnpacker)], + } + } +} + +impl ArchiveUnpacker { + fn select_unpacker(&self, archive_path: &Path) -> MithrilResult<&dyn ArchiveFormat> { + self.supported_formats + .iter() + .find(|f| f.supports(archive_path)) + .map(|f| f.as_ref()) + .ok_or_else(|| anyhow!("Unsupported archive format: {}", archive_path.display())) + } + + pub fn unpack(&self, archive_path: &Path, unpack_dir: &Path) -> MithrilResult<()> { + let unpacker = self.select_unpacker(archive_path)?; + unpacker.unpack(archive_path, unpack_dir) + } +} + +#[cfg(test)] +mod tests { + use std::{fs::File, io::Write, path::Path}; + + use flate2::{write::GzEncoder, Compression}; + use tar::{Builder, Header}; + use zip::{write::FileOptions, ZipWriter}; + + use mithril_common::temp_dir_create; + + use super::*; + + #[test] + fn archive_unpacker_unpacks_tar_gz_archive() { + let temp_dir = temp_dir_create!(); + let archive_path = temp_dir.join("archive.tar.gz"); + + { + let tar_gz_file = File::create(&archive_path).unwrap(); + let encoder = GzEncoder::new(tar_gz_file, Compression::default()); + let mut tar_builder = Builder::new(encoder); + + let content = b"whatever content"; + let mut header = Header::new_gnu(); + header.set_size(content.len() as u64); + header.set_cksum(); + tar_builder + .append_data(&mut header, "file.txt", &content[..]) + .unwrap(); + tar_builder.finish().unwrap(); + } + + ArchiveUnpacker::default() + .unpack(&archive_path, &temp_dir) + .unwrap(); + + assert!(temp_dir.join("file.txt").exists()); + } + + #[test] + fn archive_unpacker_unpacks_zip_archive() { + let temp_dir = temp_dir_create!(); + let archive_path = temp_dir.join("archive.zip"); + + { + let zip_file = File::create(&archive_path).unwrap(); + let mut zip_writer = ZipWriter::new(zip_file); + + zip_writer + .start_file("file.txt", FileOptions::<()>::default()) + .unwrap(); + zip_writer.write_all(b"whatever content").unwrap(); + + zip_writer.finish().unwrap(); + } + + ArchiveUnpacker::default() + .unpack(&archive_path, &temp_dir) + .unwrap(); + + assert!(temp_dir.join("file.txt").exists()); + } + + #[test] + fn fails_with_unknown_extension() { + let path = Path::new("whatever.unknown"); + + let archive_unpacker = ArchiveUnpacker::default(); + let result = archive_unpacker.select_unpacker(path); + + assert!( + result.is_err(), + "Should fail with unsupported archive extension." + ); + } +} diff --git a/mithril-client-cli/src/utils/archive_unpacker/zip_unpacker.rs b/mithril-client-cli/src/utils/archive_unpacker/zip_unpacker.rs new file mode 100644 index 00000000000..8ddb54e81f8 --- /dev/null +++ b/mithril-client-cli/src/utils/archive_unpacker/zip_unpacker.rs @@ -0,0 +1,114 @@ +use std::{ + fs::{self, File}, + io, + path::Path, +}; + +use anyhow::Context; +use zip::ZipArchive; + +use mithril_client::MithrilResult; + +use super::ArchiveFormat; + +pub struct ZipUnpacker; + +impl ArchiveFormat for ZipUnpacker { + fn unpack(&self, archive_path: &Path, unpack_dir: &Path) -> MithrilResult<()> { + let file = File::open(archive_path) + .with_context(|| format!("Could not open archive file '{}'", archive_path.display()))?; + let mut archive = ZipArchive::new(file) + .with_context(|| format!("Could not read ZIP archive '{}'", archive_path.display()))?; + + for i in 0..archive.len() { + let mut file = archive.by_index(i)?; + let rel_path = file + .enclosed_name() + .ok_or_else(|| anyhow::anyhow!("File path is unsafe or malformed"))?; + let outpath = unpack_dir.join(rel_path); + + if file.is_dir() { + fs::create_dir_all(&outpath).with_context(|| { + format!("Could not create directory '{}'", outpath.display()) + })?; + } else { + if let Some(parent) = outpath.parent() { + if !parent.exists() { + fs::create_dir_all(parent).with_context(|| { + format!("Could not create directory '{}'", parent.display()) + })?; + } + } + let mut outfile = File::create(&outpath) + .with_context(|| format!("Could not create file '{}'", outpath.display()))?; + io::copy(&mut file, &mut outfile) + .with_context(|| format!("Failed to write file '{}'", outpath.display()))?; + } + } + + Ok(()) + } + + fn supports(&self, path: &Path) -> bool { + path.extension().and_then(|e| e.to_str()) == Some("zip") + } +} + +#[cfg(test)] +mod tests { + use std::io::Write; + + use zip::{write::FileOptions, ZipWriter}; + + use mithril_common::{assert_dir_eq, temp_dir_create}; + + use super::*; + + #[test] + fn unpack_zip_archive_extracts_all_files() { + let temp_dir = temp_dir_create!(); + let archive_path = temp_dir.join("archive.zip"); + + { + let zip_file = fs::File::create(&archive_path).unwrap(); + let mut zip_writer = ZipWriter::new(zip_file); + + zip_writer + .start_file("root.txt", FileOptions::<()>::default()) + .unwrap(); + zip_writer.write_all(b"root content").unwrap(); + + zip_writer + .start_file("nested/dir/nested-file.txt", FileOptions::<()>::default()) + .unwrap(); + zip_writer.write_all(b"nested content").unwrap(); + + zip_writer.finish().unwrap(); + } + + ZipUnpacker.unpack(&archive_path, &temp_dir).unwrap(); + + assert_dir_eq! { + &temp_dir, + "* nested/ + ** dir/ + *** nested-file.txt + * archive.zip + * root.txt" + }; + + let root_file_content = fs::read_to_string(temp_dir.join("root.txt")).unwrap(); + assert_eq!(root_file_content, "root content"); + + let nested_file_content = + fs::read_to_string(temp_dir.join("nested/dir/nested-file.txt")).unwrap(); + assert_eq!(nested_file_content, "nested content"); + } + + #[test] + fn supported_file_extension() { + assert!(ZipUnpacker.supports(Path::new("archive.zip"))); + assert!(ZipUnpacker.supports(Path::new("archive.whatever.zip"))); + assert!(!ZipUnpacker.supports(Path::new("archive.whatever"))); + } +} diff --git a/mithril-client-cli/src/utils/mod.rs b/mithril-client-cli/src/utils/mod.rs index 114c7e766df..949fb9b6442 100644 --- a/mithril-client-cli/src/utils/mod.rs +++ b/mithril-client-cli/src/utils/mod.rs @@ -1,6 +1,7 @@ //! Utilities module //! This module contains tools needed for the commands layer. +mod archive_unpacker; mod cardano_db; mod cardano_db_download_checker; mod expander; @@ -10,6 +11,7 @@ mod http_downloader; mod multi_download_progress_reporter; mod progress_reporter; +pub use archive_unpacker::*; pub use cardano_db::*; pub use cardano_db_download_checker::*; pub use expander::*; From 81b1f7d9bf4b73d1e942ad271caf660716e09782 Mon Sep 17 00:00:00 2001 From: Damien Lachaume <135982616+dlachaume@users.noreply.github.com> Date: Tue, 3 Jun 2025 14:59:40 +0200 Subject: [PATCH 05/20] feat(client-cli): implement directory copy functionality --- mithril-client-cli/src/utils/fs.rs | 109 ++++++++++++++++++++++++++++ mithril-client-cli/src/utils/mod.rs | 2 + 2 files changed, 111 insertions(+) create mode 100644 mithril-client-cli/src/utils/fs.rs diff --git a/mithril-client-cli/src/utils/fs.rs b/mithril-client-cli/src/utils/fs.rs new file mode 100644 index 00000000000..a60a9722b3f --- /dev/null +++ b/mithril-client-cli/src/utils/fs.rs @@ -0,0 +1,109 @@ +use std::{ + fs, + path::{Path, PathBuf}, +}; + +use anyhow::{anyhow, Context}; + +use mithril_client::MithrilResult; + +/// Copies a directory and its contents to a new location. +pub fn copy_dir(source_dir: &Path, target_dir: &Path) -> MithrilResult { + let source_dir_name = source_dir + .file_name() + .ok_or_else(|| anyhow!("Invalid source directory: {}", source_dir.display()))?; + let destination_path = target_dir.join(source_dir_name); + copy_dir_contents(source_dir, &destination_path)?; + + Ok(destination_path) +} + +fn copy_dir_contents(source_dir: &Path, target_dir: &Path) -> MithrilResult<()> { + fs::create_dir_all(target_dir).with_context(|| { + format!( + "Failed to create target directory: {}", + target_dir.display() + ) + })?; + + for entry in fs::read_dir(source_dir)? { + let entry = entry?; + let file_type = entry.file_type()?; + let src_path = entry.path(); + let dst_path = target_dir.join(entry.file_name()); + + if file_type.is_dir() { + copy_dir_contents(&src_path, &dst_path)?; + } else { + fs::copy(&src_path, &dst_path).with_context(|| { + format!( + "Failed to copy file '{}' to '{}'", + src_path.display(), + dst_path.display() + ) + })?; + } + } + + Ok(()) +} + +#[cfg(test)] +mod tests { + use std::fs::File; + + use mithril_common::{assert_dir_eq, temp_dir_create}; + + use super::*; + + #[test] + fn fails_if_source_does_not_exist() { + let temp_dir = temp_dir_create!(); + let dir_not_exist = PathBuf::from("dir_not_exist"); + + copy_dir(&dir_not_exist, &temp_dir) + .expect_err("Expected error when source directory does not exist"); + } + + #[test] + fn returns_copied_directory_path() { + let temp_dir = temp_dir_create!(); + let src = temp_dir.join("dir_to_copy"); + fs::create_dir(&src).unwrap(); + let dst = temp_dir.join("dst"); + + let copied_dir_path = copy_dir(&src, &dst).unwrap(); + + assert_eq!(copied_dir_path, dst.join("dir_to_copy")); + } + + #[test] + fn copies_nested_directories_and_files() { + let temp_dir = temp_dir_create!(); + let src = temp_dir.join("dir_to_copy"); + fs::create_dir(&src).unwrap(); + File::create(src.join("root.txt")).unwrap(); + + let sub_dir1 = src.join("subdir1"); + fs::create_dir(&sub_dir1).unwrap(); + File::create(sub_dir1.join("subdir1.txt")).unwrap(); + + let sub_dir2 = src.join("subdir2"); + fs::create_dir(&sub_dir2).unwrap(); + File::create(sub_dir2.join("subdir2.txt")).unwrap(); + + let dst = temp_dir.join("dst"); + + copy_dir(&src, &dst).unwrap(); + + assert_dir_eq!( + &dst, + "* dir_to_copy/ + ** subdir1/ + *** subdir1.txt + ** subdir2/ + *** subdir2.txt + ** root.txt" + ); + } +} diff --git a/mithril-client-cli/src/utils/mod.rs b/mithril-client-cli/src/utils/mod.rs index 949fb9b6442..631ae02d3ee 100644 --- a/mithril-client-cli/src/utils/mod.rs +++ b/mithril-client-cli/src/utils/mod.rs @@ -6,6 +6,7 @@ mod cardano_db; mod cardano_db_download_checker; mod expander; mod feedback_receiver; +mod fs; mod github_release_retriever; mod http_downloader; mod multi_download_progress_reporter; @@ -16,6 +17,7 @@ pub use cardano_db::*; pub use cardano_db_download_checker::*; pub use expander::*; pub use feedback_receiver::*; +pub use fs::*; pub use github_release_retriever::*; pub use http_downloader::*; pub use multi_download_progress_reporter::*; From 7c45a7ee0763531a1def641228712a83600b9217 Mon Sep 17 00:00:00 2001 From: Damien Lachaume <135982616+dlachaume@users.noreply.github.com> Date: Tue, 3 Jun 2025 09:11:18 +0200 Subject: [PATCH 06/20] feat(client-cli): execute the `snapshot-converter` binary --- .../src/commands/tools/snapshot_converter.rs | 708 +++++++++++++++--- mithril-client-cli/src/main.rs | 2 + 2 files changed, 601 insertions(+), 109 deletions(-) diff --git a/mithril-client-cli/src/commands/tools/snapshot_converter.rs b/mithril-client-cli/src/commands/tools/snapshot_converter.rs index 2b72672c36e..67ef132dbf8 100644 --- a/mithril-client-cli/src/commands/tools/snapshot_converter.rs +++ b/mithril-client-cli/src/commands/tools/snapshot_converter.rs @@ -1,6 +1,8 @@ use std::{ env, fmt, + fs::{create_dir, read_dir}, path::{Path, PathBuf}, + process::Command, }; use anyhow::{anyhow, Context}; @@ -9,7 +11,7 @@ use clap::{Parser, ValueEnum}; use mithril_client::MithrilResult; use crate::utils::{ - ArchiveUnpacker, GitHubReleaseRetriever, HttpDownloader, ReqwestGitHubApiClient, + copy_dir, ArchiveUnpacker, GitHubReleaseRetriever, HttpDownloader, ReqwestGitHubApiClient, ReqwestHttpDownloader, }; @@ -19,7 +21,17 @@ const GITHUB_REPOSITORY: &str = "cardano-node"; const LATEST_DISTRIBUTION_TAG: &str = "latest"; const PRERELEASE_DISTRIBUTION_TAG: &str = "prerelease"; -const CARDANO_DISTRIBUTION_TEMP_DIR: &str = "cardano-node-distribution-tmp"; +const WORK_DIR: &str = "tmp"; +const CARDANO_DISTRIBUTION_DIR: &str = "cardano-node-distribution"; +const SNAPSHOTS_DIR: &str = "snapshots"; + +const SNAPSHOT_CONVERTER_BIN_DIR: &str = "bin"; +const SNAPSHOT_CONVERTER_BIN_NAME_UNIX: &str = "snapshot-converter"; +const SNAPSHOT_CONVERTER_BIN_NAME_WINDOWS: &str = "snapshot-converter.exe"; +const SNAPSHOT_CONVERTER_CONFIG_DIR: &str = "share"; +const SNAPSHOT_CONVERTER_CONFIG_FILE: &str = "config.json"; + +const LEDGER_DIR: &str = "ledger"; #[derive(Debug, Clone, ValueEnum)] enum UTxOHDFlavor { @@ -38,6 +50,23 @@ impl fmt::Display for UTxOHDFlavor { } } +#[derive(Debug, Clone, ValueEnum)] +enum CardanoNetwork { + Preview, + Preprod, + Mainnet, +} + +impl fmt::Display for CardanoNetwork { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Preview => write!(f, "preview"), + Self::Preprod => write!(f, "preprod"), + Self::Mainnet => write!(f, "mainnet"), + } + } +} + /// Clap command to convert a restored `InMemory` Mithril snapshot to another flavor. #[derive(Parser, Debug, Clone)] pub struct SnapshotConverterCommand { @@ -51,6 +80,10 @@ pub struct SnapshotConverterCommand { #[clap(long)] cardano_node_version: String, + /// Cardano network. + #[clap(long)] + cardano_network: CardanoNetwork, + /// UTxO-HD flavor to convert the ledger snapshot to. #[clap(long)] utxo_hd_flavor: UTxOHDFlavor, @@ -59,11 +92,18 @@ pub struct SnapshotConverterCommand { impl SnapshotConverterCommand { /// Main command execution pub async fn execute(&self) -> MithrilResult<()> { - let distribution_temp_dir = self.db_directory.join(CARDANO_DISTRIBUTION_TEMP_DIR); - std::fs::create_dir(&distribution_temp_dir).with_context(|| { + let work_dir = self.db_directory.join(WORK_DIR); + create_dir(&work_dir).with_context(|| { + format!( + "Failed to create snapshot converter work directory: {}", + work_dir.display() + ) + })?; + let distribution_dir = work_dir.join(CARDANO_DISTRIBUTION_DIR); + create_dir(&distribution_dir).with_context(|| { format!( - "Failed to create directory: {}", - distribution_temp_dir.display() + "Failed to create distribution directory: {}", + distribution_dir.display() ) })?; @@ -71,7 +111,7 @@ impl SnapshotConverterCommand { ReqwestGitHubApiClient::new()?, ReqwestHttpDownloader::new()?, &self.cardano_node_version, - &distribution_temp_dir, + &distribution_dir, ) .await .with_context(|| { @@ -79,14 +119,28 @@ impl SnapshotConverterCommand { })?; ArchiveUnpacker::default() - .unpack(&archive_path, &distribution_temp_dir) + .unpack(&archive_path, &distribution_dir) .with_context(|| { format!( "Failed to unpack 'snapshot-converter' binary to directory: {}", - distribution_temp_dir.display() + distribution_dir.display() ) })?; + Self::convert_ledger_state_snapshot( + &work_dir, + &self.db_directory, + &distribution_dir, + &self.cardano_network, + &self.utxo_hd_flavor, + ) + .with_context(|| { + format!( + "Failed to convert ledger snapshot to flavor: {}", + self.utxo_hd_flavor + ) + })?; + Ok(()) } @@ -136,120 +190,556 @@ impl SnapshotConverterCommand { Ok(archive_path) } + + fn convert_ledger_state_snapshot( + work_dir: &Path, + db_dir: &Path, + distribution_dir: &Path, + cardano_network: &CardanoNetwork, + utxo_hd_flavor: &UTxOHDFlavor, + ) -> MithrilResult<()> { + println!( + "Converting ledger state snapshot to '{}' flavor", + utxo_hd_flavor + ); + let snapshots_path = work_dir.join(SNAPSHOTS_DIR); + let copied_snapshot_path = + Self::copy_oldest_ledger_state_snapshot(db_dir, &snapshots_path)?; + let converted_snapshot_path = Self::compute_converted_snapshot_output_path( + &snapshots_path, + &copied_snapshot_path, + utxo_hd_flavor, + )?; + + let converter_bin = + Self::get_snapshot_converter_binary_path(distribution_dir, env::consts::OS)?; + let config_path = + Self::get_snapshot_converter_config_path(distribution_dir, cardano_network); + Self::execute_snapshot_converter( + &converter_bin, + &copied_snapshot_path, + &converted_snapshot_path, + &config_path, + utxo_hd_flavor, + )?; + + Ok(()) + } + + fn execute_snapshot_converter( + bin_path: &Path, + input_path: &Path, + output_path: &Path, + config_path: &Path, + flavor: &UTxOHDFlavor, + ) -> MithrilResult<()> { + Command::new(bin_path) + .arg("Mem") + .arg(input_path) + .arg(flavor.to_string()) + .arg(output_path) + .arg("cardano") + .arg("--config") + .arg(config_path) + .status() + .with_context(|| { + format!( + "Failed to execute snapshot-converter binary at {}", + bin_path.display() + ) + })?; + + Ok(()) + } + + fn get_snapshot_converter_binary_path( + distribution_dir: &Path, + target_os: &str, + ) -> MithrilResult { + let base_path = distribution_dir.join(SNAPSHOT_CONVERTER_BIN_DIR); + + let binary_name = match target_os { + "linux" | "macos" => SNAPSHOT_CONVERTER_BIN_NAME_UNIX, + "windows" => SNAPSHOT_CONVERTER_BIN_NAME_WINDOWS, + _ => return Err(anyhow!("Unsupported platform: {}", target_os)), + }; + + Ok(base_path.join(binary_name)) + } + + fn get_snapshot_converter_config_path( + distribution_dir: &Path, + network: &CardanoNetwork, + ) -> PathBuf { + distribution_dir + .join(SNAPSHOT_CONVERTER_CONFIG_DIR) + .join(network.to_string()) + .join(SNAPSHOT_CONVERTER_CONFIG_FILE) + } + + /// Finds the oldest ledger snapshot (by slot number) in the `ledger/` directory of a Cardano node database. + fn find_oldest_ledger_state_snapshot(db_dir: &Path) -> MithrilResult { + let ledger_dir = db_dir.join(LEDGER_DIR); + + let entries = read_dir(&ledger_dir).with_context(|| { + format!( + "Failed to read ledger state snapshots directory: {}", + ledger_dir.display() + ) + })?; + + let mut min_slot: Option<(u64, PathBuf)> = None; + + for entry in entries { + let entry = entry?; + + let slot = match Self::extract_slot_number(&entry.path()) { + Ok(number) => number, + Err(_) => continue, + }; + + let path = entry.path(); + if path.is_dir() + && (min_slot + .as_ref() + .map(|(min, _)| slot < *min) + .unwrap_or(true)) + { + min_slot = Some((slot, path)); + } + } + + min_slot.map(|(_, path)| path).ok_or_else(|| { + anyhow!( + "No valid ledger state snapshot found in directory: {}", + ledger_dir.display() + ) + }) + } + + fn copy_oldest_ledger_state_snapshot( + db_dir: &Path, + target_dir: &Path, + ) -> MithrilResult { + let snapshot_path = Self::find_oldest_ledger_state_snapshot(db_dir)?; + let copied_snapshot_path = copy_dir(&snapshot_path, target_dir)?; + + Ok(copied_snapshot_path) + } + + fn compute_converted_snapshot_output_path( + snapshots_dir: &Path, + input_snapshot: &Path, + flavor: &UTxOHDFlavor, + ) -> MithrilResult { + let slot_number = Self::extract_slot_number(input_snapshot).with_context(|| { + format!( + "Failed to extract slot number from: {}", + input_snapshot.display() + ) + })?; + + let converted_snapshot_path = snapshots_dir.join(format!( + "{}_{}", + slot_number, + flavor.to_string().to_lowercase() + )); + + Ok(converted_snapshot_path) + } + + fn extract_slot_number(path: &Path) -> MithrilResult { + let file_name = path + .file_name() + .ok_or_else(|| anyhow!("No filename in path: {}", path.display()))?; + + let file_name_str = file_name + .to_str() + .ok_or_else(|| anyhow!("Invalid UTF-8 in path filename: {:?}", file_name))?; + + file_name_str + .parse::() + .with_context(|| format!("Invalid slot number in path filename: {}", file_name_str)) + } } #[cfg(test)] mod tests { - use mockall::predicate::eq; - use reqwest::Url; + use super::*; - use mithril_common::temp_dir_create; + mod download_cardano_node_distribution { + use mockall::predicate::eq; + use reqwest::Url; - use crate::utils::{GitHubRelease, MockGitHubReleaseRetriever, MockHttpDownloader}; + use mithril_common::temp_dir_create; - use super::*; + use crate::utils::{GitHubRelease, MockGitHubReleaseRetriever, MockHttpDownloader}; - #[tokio::test] - async fn call_get_latest_release_with_latest_tag() { - let temp_dir = temp_dir_create!(); - let release = GitHubRelease::dummy_with_all_supported_assets(); - let asset = release.get_asset_for_os(env::consts::OS).unwrap().unwrap(); - - let cloned_release = release.clone(); - let mut github_api_client = MockGitHubReleaseRetriever::new(); - github_api_client - .expect_get_latest_release() - .with(eq(GITHUB_ORGANIZATION), eq(GITHUB_REPOSITORY)) - .returning(move |_, _| Ok(cloned_release.clone())); - - let mut http_downloader = MockHttpDownloader::new(); - http_downloader - .expect_download_file() - .with( - eq(Url::parse(&asset.browser_download_url).unwrap()), - eq(temp_dir.clone()), - eq(asset.name.clone()), + use super::*; + + #[tokio::test] + async fn downloads_latest_release_distribution() { + let temp_dir = temp_dir_create!(); + let release = GitHubRelease::dummy_with_all_supported_assets(); + let asset = release.get_asset_for_os(env::consts::OS).unwrap().unwrap(); + + let cloned_release = release.clone(); + let mut github_api_client = MockGitHubReleaseRetriever::new(); + github_api_client + .expect_get_latest_release() + .with(eq(GITHUB_ORGANIZATION), eq(GITHUB_REPOSITORY)) + .returning(move |_, _| Ok(cloned_release.clone())); + + let mut http_downloader = MockHttpDownloader::new(); + http_downloader + .expect_download_file() + .with( + eq(Url::parse(&asset.browser_download_url).unwrap()), + eq(temp_dir.clone()), + eq(asset.name.clone()), + ) + .returning(|_, _, _| Ok(PathBuf::new())); + + SnapshotConverterCommand::download_cardano_node_distribution( + github_api_client, + http_downloader, + LATEST_DISTRIBUTION_TAG, + &temp_dir, ) - .returning(|_, _, _| Ok(PathBuf::new())); + .await + .unwrap(); + } - SnapshotConverterCommand::download_cardano_node_distribution( - github_api_client, - http_downloader, - LATEST_DISTRIBUTION_TAG, - &temp_dir, - ) - .await - .unwrap(); - } - - #[tokio::test] - async fn call_get_prerelease_with_prerelease_tag() { - let temp_dir = temp_dir_create!(); - let release = GitHubRelease::dummy_with_all_supported_assets(); - let asset = release.get_asset_for_os(env::consts::OS).unwrap().unwrap(); - - let cloned_release = release.clone(); - let mut github_api_client = MockGitHubReleaseRetriever::new(); - github_api_client - .expect_get_prerelease() - .with(eq(GITHUB_ORGANIZATION), eq(GITHUB_REPOSITORY)) - .returning(move |_, _| Ok(cloned_release.clone())); - - let mut http_downloader = MockHttpDownloader::new(); - http_downloader - .expect_download_file() - .with( - eq(Url::parse(&asset.browser_download_url).unwrap()), - eq(temp_dir.clone()), - eq(asset.name.clone()), + #[tokio::test] + async fn downloads_prerelease_distribution() { + let temp_dir = temp_dir_create!(); + let release = GitHubRelease::dummy_with_all_supported_assets(); + let asset = release.get_asset_for_os(env::consts::OS).unwrap().unwrap(); + + let cloned_release = release.clone(); + let mut github_api_client = MockGitHubReleaseRetriever::new(); + github_api_client + .expect_get_prerelease() + .with(eq(GITHUB_ORGANIZATION), eq(GITHUB_REPOSITORY)) + .returning(move |_, _| Ok(cloned_release.clone())); + + let mut http_downloader = MockHttpDownloader::new(); + http_downloader + .expect_download_file() + .with( + eq(Url::parse(&asset.browser_download_url).unwrap()), + eq(temp_dir.clone()), + eq(asset.name.clone()), + ) + .returning(|_, _, _| Ok(PathBuf::new())); + + SnapshotConverterCommand::download_cardano_node_distribution( + github_api_client, + http_downloader, + PRERELEASE_DISTRIBUTION_TAG, + &temp_dir, ) - .returning(|_, _, _| Ok(PathBuf::new())); + .await + .unwrap(); + } - SnapshotConverterCommand::download_cardano_node_distribution( - github_api_client, - http_downloader, - PRERELEASE_DISTRIBUTION_TAG, - &temp_dir, - ) - .await - .unwrap(); - } - - #[tokio::test] - async fn call_get_release_by_tag_with_specific_cardano_node_version() { - let cardano_node_version = "10.3.1"; - let temp_dir = temp_dir_create!(); - let release = GitHubRelease::dummy_with_all_supported_assets(); - let asset = release.get_asset_for_os(env::consts::OS).unwrap().unwrap(); - - let cloned_release = release.clone(); - let mut github_api_client = MockGitHubReleaseRetriever::new(); - github_api_client - .expect_get_release_by_tag() - .with( - eq(GITHUB_ORGANIZATION), - eq(GITHUB_REPOSITORY), - eq(cardano_node_version), + #[tokio::test] + async fn downloads_tagged_release_distribution() { + let cardano_node_version = "10.3.1"; + let temp_dir = temp_dir_create!(); + let release = GitHubRelease::dummy_with_all_supported_assets(); + let asset = release.get_asset_for_os(env::consts::OS).unwrap().unwrap(); + + let cloned_release = release.clone(); + let mut github_api_client = MockGitHubReleaseRetriever::new(); + github_api_client + .expect_get_release_by_tag() + .with( + eq(GITHUB_ORGANIZATION), + eq(GITHUB_REPOSITORY), + eq(cardano_node_version), + ) + .returning(move |_, _, _| Ok(cloned_release.clone())); + + let mut http_downloader = MockHttpDownloader::new(); + http_downloader + .expect_download_file() + .with( + eq(Url::parse(&asset.browser_download_url).unwrap()), + eq(temp_dir.clone()), + eq(asset.name.clone()), + ) + .returning(|_, _, _| Ok(PathBuf::new())); + + SnapshotConverterCommand::download_cardano_node_distribution( + github_api_client, + http_downloader, + cardano_node_version, + &temp_dir, ) - .returning(move |_, _, _| Ok(cloned_release.clone())); - - let mut http_downloader = MockHttpDownloader::new(); - http_downloader - .expect_download_file() - .with( - eq(Url::parse(&asset.browser_download_url).unwrap()), - eq(temp_dir.clone()), - eq(asset.name.clone()), + .await + .unwrap(); + } + } + + mod get_snapshot_converter_binary_path { + use super::*; + + #[test] + fn returns_correct_binary_path_for_linux() { + let distribution_dir = PathBuf::from("/path/to/distribution"); + + let binary_path = SnapshotConverterCommand::get_snapshot_converter_binary_path( + &distribution_dir, + "linux", ) - .returning(|_, _, _| Ok(PathBuf::new())); + .unwrap(); + + assert_eq!( + binary_path, + distribution_dir + .join(SNAPSHOT_CONVERTER_BIN_DIR) + .join(SNAPSHOT_CONVERTER_BIN_NAME_UNIX) + ); + } - SnapshotConverterCommand::download_cardano_node_distribution( - github_api_client, - http_downloader, - cardano_node_version, - &temp_dir, - ) - .await - .unwrap(); + #[test] + fn returns_correct_binary_path_for_macos() { + let distribution_dir = PathBuf::from("/path/to/distribution"); + + let binary_path = SnapshotConverterCommand::get_snapshot_converter_binary_path( + &distribution_dir, + "macos", + ) + .unwrap(); + + assert_eq!( + binary_path, + distribution_dir + .join(SNAPSHOT_CONVERTER_BIN_DIR) + .join(SNAPSHOT_CONVERTER_BIN_NAME_UNIX) + ); + } + + #[test] + fn returns_correct_binary_path_for_windows() { + let distribution_dir = PathBuf::from("/path/to/distribution"); + + let binary_path = SnapshotConverterCommand::get_snapshot_converter_binary_path( + &distribution_dir, + "windows", + ) + .unwrap(); + + assert_eq!( + binary_path, + distribution_dir + .join(SNAPSHOT_CONVERTER_BIN_DIR) + .join(SNAPSHOT_CONVERTER_BIN_NAME_WINDOWS) + ); + } + } + + mod get_snapshot_converter_config_path { + use super::*; + + #[test] + fn returns_config_path_for_mainnet() { + let distribution_dir = PathBuf::from("/path/to/distribution"); + let network = CardanoNetwork::Mainnet; + + let config_path = SnapshotConverterCommand::get_snapshot_converter_config_path( + &distribution_dir, + &network, + ); + + assert_eq!( + config_path, + distribution_dir + .join(SNAPSHOT_CONVERTER_CONFIG_DIR) + .join(network.to_string()) + .join(SNAPSHOT_CONVERTER_CONFIG_FILE) + ); + } + + #[test] + fn returns_config_path_for_preprod() { + let distribution_dir = PathBuf::from("/path/to/distribution"); + let network = CardanoNetwork::Preprod; + + let config_path = SnapshotConverterCommand::get_snapshot_converter_config_path( + &distribution_dir, + &network, + ); + + assert_eq!( + config_path, + distribution_dir + .join(SNAPSHOT_CONVERTER_CONFIG_DIR) + .join(network.to_string()) + .join(SNAPSHOT_CONVERTER_CONFIG_FILE) + ); + } + + #[test] + fn returns_config_path_for_preview() { + let distribution_dir = PathBuf::from("/path/to/distribution"); + let network = CardanoNetwork::Preview; + + let config_path = SnapshotConverterCommand::get_snapshot_converter_config_path( + &distribution_dir, + &network, + ); + + assert_eq!( + config_path, + distribution_dir + .join(SNAPSHOT_CONVERTER_CONFIG_DIR) + .join(network.to_string()) + .join(SNAPSHOT_CONVERTER_CONFIG_FILE) + ); + } + } + + mod extract_slot_number { + use super::*; + + #[test] + fn parses_valid_numeric_path() { + let path = PathBuf::from("/whatever").join("123456"); + + let slot = SnapshotConverterCommand::extract_slot_number(&path).unwrap(); + + assert_eq!(slot, 123456); + } + + #[test] + fn fails_with_non_numeric_filename() { + let path = PathBuf::from("/whatever").join("notanumber"); + + SnapshotConverterCommand::extract_slot_number(&path) + .expect_err("Should fail with non-numeric filename"); + } + + #[test] + fn fails_if_no_filename() { + let path = PathBuf::from("/"); + + SnapshotConverterCommand::extract_slot_number(&path) + .expect_err("Should fail if path has no filename"); + } + } + + mod compute_converted_snapshot_output_path { + use super::*; + + #[test] + fn compute_output_path_from_numeric_file_name() { + let snapshots_dir = PathBuf::from("/snapshots"); + let input_snapshot = PathBuf::from("/whatever").join("123456"); + + { + let snapshot_path = + SnapshotConverterCommand::compute_converted_snapshot_output_path( + &snapshots_dir, + &input_snapshot, + &UTxOHDFlavor::Lmdb, + ) + .unwrap(); + + assert_eq!(snapshot_path, snapshots_dir.join("123456_lmdb")); + } + + { + let snapshot_path = + SnapshotConverterCommand::compute_converted_snapshot_output_path( + &snapshots_dir, + &input_snapshot, + &UTxOHDFlavor::Legacy, + ) + .unwrap(); + + assert_eq!(snapshot_path, snapshots_dir.join("123456_legacy")); + } + } + + #[test] + fn fails_with_invalid_slot_number() { + let snapshots_dir = PathBuf::from("/snapshots"); + let input_snapshot = PathBuf::from("/whatever/notanumber"); + + SnapshotConverterCommand::compute_converted_snapshot_output_path( + &snapshots_dir, + &input_snapshot, + &UTxOHDFlavor::Lmdb, + ) + .expect_err("Should fail with invalid slot number"); + } + } + + mod find_oldest_ledger_state_snapshot { + use std::fs::File; + + use mithril_common::temp_dir_create; + + use super::*; + + #[test] + fn finds_ledger_state_snapshot_with_lowest_slot_number() { + let db_dir = temp_dir_create!(); + let ledger_dir = db_dir.join(LEDGER_DIR); + create_dir(&ledger_dir).unwrap(); + + create_dir(ledger_dir.join("500")).unwrap(); + create_dir(ledger_dir.join("1000")).unwrap(); + create_dir(ledger_dir.join("1500")).unwrap(); + + let found = + SnapshotConverterCommand::find_oldest_ledger_state_snapshot(&db_dir).unwrap(); + + assert_eq!(found, ledger_dir.join("500")); + } + + #[test] + fn returns_snapshot_when_only_one_valid_directory() { + let db_dir = temp_dir_create!(); + let ledger_dir = db_dir.join(LEDGER_DIR); + create_dir(&ledger_dir).unwrap(); + + create_dir(ledger_dir.join("500")).unwrap(); + + let found = + SnapshotConverterCommand::find_oldest_ledger_state_snapshot(&db_dir).unwrap(); + + assert_eq!(found, ledger_dir.join("500")); + } + + #[test] + fn ignores_non_numeric_and_non_directory_entries() { + let temp_dir = temp_dir_create!(); + let ledger_dir = temp_dir.join(LEDGER_DIR); + create_dir(&ledger_dir).unwrap(); + + create_dir(ledger_dir.join("1000")).unwrap(); + File::create(ledger_dir.join("500")).unwrap(); + create_dir(ledger_dir.join("notanumber")).unwrap(); + + let found = + SnapshotConverterCommand::find_oldest_ledger_state_snapshot(&temp_dir).unwrap(); + + assert_eq!(found, ledger_dir.join("1000")); + } + + #[test] + fn returns_error_if_no_valid_snapshot_found() { + let temp_dir = temp_dir_create!(); + let ledger_dir = temp_dir.join(LEDGER_DIR); + create_dir(&ledger_dir).unwrap(); + + File::create(ledger_dir.join("invalid")).unwrap(); + + SnapshotConverterCommand::find_oldest_ledger_state_snapshot(&temp_dir) + .expect_err("Should return error if no valid ledger snapshot directory found"); + } } } diff --git a/mithril-client-cli/src/main.rs b/mithril-client-cli/src/main.rs index 2ac2be52214..57628c64faf 100644 --- a/mithril-client-cli/src/main.rs +++ b/mithril-client-cli/src/main.rs @@ -298,6 +298,8 @@ mod tests { "snapshot-converter", "--db-directory", "whatever", + "--cardano-network", + "preview", "--cardano-node-version", "1.2.3", "--utxo-hd-flavor", From 1b62f3476fde236663ea7badf9232960fdabfb47 Mon Sep 17 00:00:00 2001 From: Damien Lachaume <135982616+dlachaume@users.noreply.github.com> Date: Wed, 4 Jun 2025 15:20:48 +0200 Subject: [PATCH 07/20] feat(client-cli): add `--commit` option to replace ledger state with converted snapshot --- .../src/commands/tools/snapshot_converter.rs | 108 +++++++++++++++++- 1 file changed, 105 insertions(+), 3 deletions(-) diff --git a/mithril-client-cli/src/commands/tools/snapshot_converter.rs b/mithril-client-cli/src/commands/tools/snapshot_converter.rs index 67ef132dbf8..1e0e967232b 100644 --- a/mithril-client-cli/src/commands/tools/snapshot_converter.rs +++ b/mithril-client-cli/src/commands/tools/snapshot_converter.rs @@ -1,6 +1,6 @@ use std::{ env, fmt, - fs::{create_dir, read_dir}, + fs::{create_dir, read_dir, remove_dir_all, rename}, path::{Path, PathBuf}, process::Command, }; @@ -87,6 +87,10 @@ pub struct SnapshotConverterCommand { /// UTxO-HD flavor to convert the ledger snapshot to. #[clap(long)] utxo_hd_flavor: UTxOHDFlavor, + + /// If set, the converted snapshot replaces the current ledger state in the `db_directory`. + #[clap(long)] + commit: bool, } impl SnapshotConverterCommand { @@ -133,6 +137,7 @@ impl SnapshotConverterCommand { &distribution_dir, &self.cardano_network, &self.utxo_hd_flavor, + self.commit, ) .with_context(|| { format!( @@ -197,6 +202,7 @@ impl SnapshotConverterCommand { distribution_dir: &Path, cardano_network: &CardanoNetwork, utxo_hd_flavor: &UTxOHDFlavor, + commit: bool, ) -> MithrilResult<()> { println!( "Converting ledger state snapshot to '{}' flavor", @@ -223,6 +229,14 @@ impl SnapshotConverterCommand { utxo_hd_flavor, )?; + if commit { + Self::commit_converted_snapshot(db_dir, &converted_snapshot_path).with_context( + || "Failed to overwrite the ledger state with the converted snapshot.", + )?; + } else { + println!("Snapshot location: {}", converted_snapshot_path.display()); + } + Ok(()) } @@ -361,18 +375,64 @@ impl SnapshotConverterCommand { .parse::() .with_context(|| format!("Invalid slot number in path filename: {}", file_name_str)) } + + /// Commits the converted snapshot by replacing the current ledger state snapshots in the database directory. + fn commit_converted_snapshot( + db_dir: &Path, + converted_snapshot_path: &Path, + ) -> MithrilResult<()> { + let ledger_dir = db_dir.join(LEDGER_DIR); + println!( + "Upgrading and replacing ledger state in {} with converted snapshot: {}", + ledger_dir.display(), + converted_snapshot_path.display() + ); + + let filename = converted_snapshot_path + .file_name() + .ok_or_else(|| anyhow!("Missing filename in converted snapshot path"))? + .to_string_lossy(); + + let (slot_number, _) = filename + .split_once('_') + .ok_or_else(|| anyhow!("Invalid converted snapshot name format: {}", filename))?; + + remove_dir_all(&ledger_dir).with_context(|| { + format!( + "Failed to remove old ledger state snapshot directory: {}", + ledger_dir.display() + ) + })?; + + create_dir(&ledger_dir).with_context(|| { + format!( + "Failed to recreate ledger state snapshot directory: {}", + ledger_dir.display() + ) + })?; + + let destination = ledger_dir.join(slot_number); + rename(converted_snapshot_path, &destination).with_context(|| { + format!( + "Failed to move converted snapshot to ledger directory: {}", + destination.display() + ) + })?; + + Ok(()) + } } #[cfg(test)] mod tests { + use mithril_common::temp_dir_create; + use super::*; mod download_cardano_node_distribution { use mockall::predicate::eq; use reqwest::Url; - use mithril_common::temp_dir_create; - use crate::utils::{GitHubRelease, MockGitHubReleaseRetriever, MockHttpDownloader}; use super::*; @@ -742,4 +802,46 @@ mod tests { .expect_err("Should return error if no valid ledger snapshot directory found"); } } + + mod commit_converted_snapshot { + use std::fs::File; + + use super::*; + + #[test] + fn moves_converted_snapshot_to_ledger_directory() { + let tmp_dir = temp_dir_create!(); + let ledger_dir = tmp_dir.join(LEDGER_DIR); + create_dir(&ledger_dir).unwrap(); + let previous_snapshot = ledger_dir.join("123"); + File::create(&previous_snapshot).unwrap(); + + let converted_snapshot = tmp_dir.join("456_lmdb"); + File::create(&converted_snapshot).unwrap(); + + assert!(previous_snapshot.exists()); + SnapshotConverterCommand::commit_converted_snapshot(&tmp_dir, &converted_snapshot) + .unwrap(); + + assert!(!previous_snapshot.exists()); + assert!(ledger_dir.join("456").exists()); + } + + #[test] + fn fails_if_converted_snapshot_has_invalid_filename() { + let tmp_dir = temp_dir_create!(); + let ledger_dir = tmp_dir.join(LEDGER_DIR); + create_dir(&ledger_dir).unwrap(); + let previous_snapshot = ledger_dir.join("123"); + File::create(&previous_snapshot).unwrap(); + + let converted_snapshot = tmp_dir.join("456"); + File::create(&converted_snapshot).unwrap(); + + SnapshotConverterCommand::commit_converted_snapshot(&tmp_dir, &converted_snapshot) + .expect_err("Should fail if converted snapshot has invalid filename"); + + assert!(previous_snapshot.exists()); + } + } } From ecc9430eb48233ebfe0f3b0cf8336c71defda999 Mon Sep 17 00:00:00 2001 From: Damien Lachaume <135982616+dlachaume@users.noreply.github.com> Date: Wed, 4 Jun 2025 16:58:15 +0200 Subject: [PATCH 08/20] feat(client-cli): add cleanup logic to delete temporary directories after snapshot conversion or on failure --- .../src/commands/tools/snapshot_converter.rs | 178 ++++++++++++++---- 1 file changed, 144 insertions(+), 34 deletions(-) diff --git a/mithril-client-cli/src/commands/tools/snapshot_converter.rs b/mithril-client-cli/src/commands/tools/snapshot_converter.rs index 1e0e967232b..2f92523d286 100644 --- a/mithril-client-cli/src/commands/tools/snapshot_converter.rs +++ b/mithril-client-cli/src/commands/tools/snapshot_converter.rs @@ -104,49 +104,62 @@ impl SnapshotConverterCommand { ) })?; let distribution_dir = work_dir.join(CARDANO_DISTRIBUTION_DIR); - create_dir(&distribution_dir).with_context(|| { - format!( - "Failed to create distribution directory: {}", - distribution_dir.display() + + let result = { + create_dir(&distribution_dir).with_context(|| { + format!( + "Failed to create distribution directory: {}", + distribution_dir.display() + ) + })?; + + let archive_path = Self::download_cardano_node_distribution( + ReqwestGitHubApiClient::new()?, + ReqwestHttpDownloader::new()?, + &self.cardano_node_version, + &distribution_dir, ) - })?; + .await + .with_context(|| { + "Failed to download 'snapshot-converter' binary from Cardano node distribution" + })?; - let archive_path = Self::download_cardano_node_distribution( - ReqwestGitHubApiClient::new()?, - ReqwestHttpDownloader::new()?, - &self.cardano_node_version, - &distribution_dir, - ) - .await - .with_context(|| { - "Failed to download 'snapshot-converter' binary from Cardano node distribution" - })?; + ArchiveUnpacker::default() + .unpack(&archive_path, &distribution_dir) + .with_context(|| { + format!( + "Failed to unpack 'snapshot-converter' binary to directory: {}", + distribution_dir.display() + ) + })?; - ArchiveUnpacker::default() - .unpack(&archive_path, &distribution_dir) + Self::convert_ledger_state_snapshot( + &work_dir, + &self.db_directory, + &distribution_dir, + &self.cardano_network, + &self.utxo_hd_flavor, + self.commit, + ) .with_context(|| { format!( - "Failed to unpack 'snapshot-converter' binary to directory: {}", - distribution_dir.display() + "Failed to convert ledger snapshot to flavor: {}", + self.utxo_hd_flavor ) })?; - Self::convert_ledger_state_snapshot( - &work_dir, - &self.db_directory, - &distribution_dir, - &self.cardano_network, - &self.utxo_hd_flavor, - self.commit, - ) - .with_context(|| { - format!( - "Failed to convert ledger snapshot to flavor: {}", - self.utxo_hd_flavor - ) - })?; + Ok(()) + }; - Ok(()) + if let Err(e) = Self::cleanup(&work_dir, &distribution_dir, self.commit, result.is_ok()) { + eprintln!( + "Failed to clean up temporary directory {} after execution: {}", + distribution_dir.display(), + e + ); + } + + result } async fn download_cardano_node_distribution( @@ -421,6 +434,29 @@ impl SnapshotConverterCommand { Ok(()) } + + fn cleanup( + work_dir: &Path, + distribution_dir: &Path, + commit: bool, + success: bool, + ) -> MithrilResult<()> { + match (success, commit) { + (true, true) => { + remove_dir_all(distribution_dir)?; + remove_dir_all(work_dir)?; + } + (true, false) => { + remove_dir_all(distribution_dir)?; + } + (false, _) => { + remove_dir_all(distribution_dir)?; + remove_dir_all(work_dir)?; + } + } + + Ok(()) + } } #[cfg(test)] @@ -844,4 +880,78 @@ mod tests { assert!(previous_snapshot.exists()); } } + + mod cleanup { + use super::*; + + #[test] + fn removes_both_dirs_on_success_when_commit_is_true() { + let tmp = temp_dir_create!(); + let work_dir = tmp.join("workdir_dir"); + let distribution_dir = tmp.join("distribution_dir"); + create_dir(&work_dir).unwrap(); + create_dir(&distribution_dir).unwrap(); + + SnapshotConverterCommand::cleanup(&work_dir, &distribution_dir, true, true).unwrap(); + + assert!(!distribution_dir.exists()); + assert!(!work_dir.exists()); + } + + #[test] + fn removes_only_distribution_on_success_when_commit_is_false() { + let tmp = temp_dir_create!(); + let work_dir = tmp.join("workdir_dir"); + let distribution_dir = tmp.join("distribution_dir"); + create_dir(&work_dir).unwrap(); + create_dir(&distribution_dir).unwrap(); + + SnapshotConverterCommand::cleanup(&work_dir, &distribution_dir, false, true).unwrap(); + + assert!(!distribution_dir.exists()); + assert!(work_dir.exists()); + } + + #[test] + fn removes_both_dirs_on_success_when_commit_is_true_and_distribution_is_nested() { + let tmp = temp_dir_create!(); + let work_dir = tmp.join("workdir_dir"); + let distribution_dir = work_dir.join("distribution_dir"); + create_dir(&work_dir).unwrap(); + create_dir(&distribution_dir).unwrap(); + + SnapshotConverterCommand::cleanup(&work_dir, &distribution_dir, true, true).unwrap(); + + assert!(!distribution_dir.exists()); + assert!(!work_dir.exists()); + } + + #[test] + fn removes_only_distribution_on_success_when_commit_is_false_and_distribution_is_nested() { + let tmp = temp_dir_create!(); + let work_dir = tmp.join("workdir_dir"); + let distribution_dir = work_dir.join("distribution_dir"); + create_dir(&work_dir).unwrap(); + create_dir(&distribution_dir).unwrap(); + + SnapshotConverterCommand::cleanup(&work_dir, &distribution_dir, false, true).unwrap(); + + assert!(!distribution_dir.exists()); + assert!(work_dir.exists()); + } + + #[test] + fn removes_both_dirs_on_failure() { + let tmp = temp_dir_create!(); + let work_dir = tmp.join("workdir_dir"); + let distribution_dir = tmp.join("distribution_dir"); + create_dir(&work_dir).unwrap(); + create_dir(&distribution_dir).unwrap(); + + SnapshotConverterCommand::cleanup(&work_dir, &distribution_dir, false, false).unwrap(); + + assert!(!distribution_dir.exists()); + assert!(!work_dir.exists()); + } + } } From a0c94cb8268adc6afc086b357c4713a27c7943cc Mon Sep 17 00:00:00 2001 From: Damien Lachaume <135982616+dlachaume@users.noreply.github.com> Date: Wed, 4 Jun 2025 17:23:39 +0200 Subject: [PATCH 09/20] refactor(client-cli): rephrase error messages and add console output messages remove extra carriage returns --- .../src/commands/tools/snapshot_converter.rs | 29 +++++++------------ 1 file changed, 10 insertions(+), 19 deletions(-) diff --git a/mithril-client-cli/src/commands/tools/snapshot_converter.rs b/mithril-client-cli/src/commands/tools/snapshot_converter.rs index 2f92523d286..47a3a046b3b 100644 --- a/mithril-client-cli/src/commands/tools/snapshot_converter.rs +++ b/mithril-client-cli/src/commands/tools/snapshot_converter.rs @@ -112,7 +112,6 @@ impl SnapshotConverterCommand { distribution_dir.display() ) })?; - let archive_path = Self::download_cardano_node_distribution( ReqwestGitHubApiClient::new()?, ReqwestHttpDownloader::new()?, @@ -120,18 +119,24 @@ impl SnapshotConverterCommand { &distribution_dir, ) .await - .with_context(|| { - "Failed to download 'snapshot-converter' binary from Cardano node distribution" - })?; + .with_context(|| "Failed to download Cardano node distribution")?; + println!( + "Unpacking distribution from archive: {}", + archive_path.display() + ); ArchiveUnpacker::default() .unpack(&archive_path, &distribution_dir) .with_context(|| { format!( - "Failed to unpack 'snapshot-converter' binary to directory: {}", + "Failed to unpack distribution to directory: {}", distribution_dir.display() ) })?; + println!( + "Distribution unpacked successfully to: {}", + distribution_dir.display() + ); Self::convert_ledger_state_snapshot( &work_dir, @@ -186,7 +191,6 @@ impl SnapshotConverterCommand { .await .with_context(|| format!("Failed to get release by tag: {}", tag))?, }; - let asset = release .get_asset_for_os(env::consts::OS)? .ok_or_else(|| anyhow!("No asset found for platform: {}", env::consts::OS)) @@ -196,7 +200,6 @@ impl SnapshotConverterCommand { env::consts::OS ) })?; - let archive_path = http_downloader .download_file(asset.browser_download_url.parse()?, target_dir, &asset.name) .await?; @@ -229,7 +232,6 @@ impl SnapshotConverterCommand { &copied_snapshot_path, utxo_hd_flavor, )?; - let converter_bin = Self::get_snapshot_converter_binary_path(distribution_dir, env::consts::OS)?; let config_path = @@ -284,7 +286,6 @@ impl SnapshotConverterCommand { target_os: &str, ) -> MithrilResult { let base_path = distribution_dir.join(SNAPSHOT_CONVERTER_BIN_DIR); - let binary_name = match target_os { "linux" | "macos" => SNAPSHOT_CONVERTER_BIN_NAME_UNIX, "windows" => SNAPSHOT_CONVERTER_BIN_NAME_WINDOWS, @@ -307,19 +308,16 @@ impl SnapshotConverterCommand { /// Finds the oldest ledger snapshot (by slot number) in the `ledger/` directory of a Cardano node database. fn find_oldest_ledger_state_snapshot(db_dir: &Path) -> MithrilResult { let ledger_dir = db_dir.join(LEDGER_DIR); - let entries = read_dir(&ledger_dir).with_context(|| { format!( "Failed to read ledger state snapshots directory: {}", ledger_dir.display() ) })?; - let mut min_slot: Option<(u64, PathBuf)> = None; for entry in entries { let entry = entry?; - let slot = match Self::extract_slot_number(&entry.path()) { Ok(number) => number, Err(_) => continue, @@ -365,7 +363,6 @@ impl SnapshotConverterCommand { input_snapshot.display() ) })?; - let converted_snapshot_path = snapshots_dir.join(format!( "{}_{}", slot_number, @@ -379,7 +376,6 @@ impl SnapshotConverterCommand { let file_name = path .file_name() .ok_or_else(|| anyhow!("No filename in path: {}", path.display()))?; - let file_name_str = file_name .to_str() .ok_or_else(|| anyhow!("Invalid UTF-8 in path filename: {:?}", file_name))?; @@ -400,30 +396,25 @@ impl SnapshotConverterCommand { ledger_dir.display(), converted_snapshot_path.display() ); - let filename = converted_snapshot_path .file_name() .ok_or_else(|| anyhow!("Missing filename in converted snapshot path"))? .to_string_lossy(); - let (slot_number, _) = filename .split_once('_') .ok_or_else(|| anyhow!("Invalid converted snapshot name format: {}", filename))?; - remove_dir_all(&ledger_dir).with_context(|| { format!( "Failed to remove old ledger state snapshot directory: {}", ledger_dir.display() ) })?; - create_dir(&ledger_dir).with_context(|| { format!( "Failed to recreate ledger state snapshot directory: {}", ledger_dir.display() ) })?; - let destination = ledger_dir.join(slot_number); rename(converted_snapshot_path, &destination).with_context(|| { format!( From efb7743c4c08c3eb52075cae15920b28b408aae0 Mon Sep 17 00:00:00 2001 From: Damien Lachaume <135982616+dlachaume@users.noreply.github.com> Date: Wed, 4 Jun 2025 17:59:14 +0200 Subject: [PATCH 10/20] feat(client-cli): display guidance message for UTxO-HD snapshot conversion after restoration with `--include-ancillary` --- .../src/commands/cardano_db/download.rs | 62 +++++++++++++++--- .../src/commands/cardano_db_v2/download.rs | 64 ++++++++++++++++--- 2 files changed, 110 insertions(+), 16 deletions(-) diff --git a/mithril-client-cli/src/commands/cardano_db/download.rs b/mithril-client-cli/src/commands/cardano_db/download.rs index a593962646b..a6f75e974b3 100644 --- a/mithril-client-cli/src/commands/cardano_db/download.rs +++ b/mithril-client-cli/src/commands/cardano_db/download.rs @@ -185,6 +185,7 @@ impl PreparedCardanoDbDownload { &db_dir, &cardano_db_message, self.is_json_output_enabled(), + self.include_ancillary, )?; Ok(()) @@ -328,6 +329,7 @@ impl PreparedCardanoDbDownload { db_dir: &Path, cardano_db: &Snapshot, json_output: bool, + include_ancillary: bool, ) -> MithrilResult<()> { let canonicalized_filepath = &db_dir.canonicalize().with_context(|| { format!( @@ -336,12 +338,41 @@ impl PreparedCardanoDbDownload { ) })?; + let docker_cmd = format!( + "docker run -v cardano-node-ipc:/ipc -v cardano-node-data:/data --mount type=bind,source=\"{}\",target=/data/db/ -e NETWORK={} ghcr.io/intersectmbo/cardano-node:{}", + canonicalized_filepath.display(), + cardano_db.network, + cardano_db.cardano_node_version + ); + + let snapshot_converter_cmd = |flavor| { + format!( + "mithril-client --unstable tools utxo-hd snapshot-converter --db-directory {} --cardano-node-version {} --utxo-hd-flavor {} --cardano-network {} --commit", + db_dir.display(), + cardano_db.cardano_node_version, + flavor, + cardano_db.network + ) + }; + if json_output { - println!( - r#"{{"timestamp": "{}", "db_directory": "{}"}}"#, - Utc::now().to_rfc3339(), - canonicalized_filepath.display() - ); + let json = if include_ancillary { + serde_json::json!({ + "timestamp": Utc::now().to_rfc3339(), + "db_directory": canonicalized_filepath, + "run_docker_cmd": docker_cmd, + "snapshot_converter_cmd_to_lmdb": snapshot_converter_cmd("LMDB"), + "snapshot_converter_cmd_to_legacy": snapshot_converter_cmd("Legacy") + }) + } else { + serde_json::json!({ + "timestamp": Utc::now().to_rfc3339(), + "db_directory": canonicalized_filepath, + "run_docker_cmd": docker_cmd, + }) + }; + + println!("{}", json); } else { let cardano_node_version = &cardano_db.cardano_node_version; println!( @@ -351,14 +382,29 @@ impl PreparedCardanoDbDownload { If you are using Cardano Docker image, you can restore a Cardano Node with: - docker run -v cardano-node-ipc:/ipc -v cardano-node-data:/data --mount type=bind,source="{}",target=/data/db/ -e NETWORK={} ghcr.io/intersectmbo/cardano-node:{cardano_node_version} + {} "###, cardano_db.digest, db_dir.display(), - canonicalized_filepath.display(), - cardano_db.network, + docker_cmd, ); + + if include_ancillary { + println!( + r###"Upgrade and replace the restored ledger state snapshot to 'LMDB' flavor by running the command: + + {} + + Or to 'Legacy' flavor by running the command: + + {} + + "###, + snapshot_converter_cmd("LMDB"), + snapshot_converter_cmd("Legacy"), + ); + } } Ok(()) diff --git a/mithril-client-cli/src/commands/cardano_db_v2/download.rs b/mithril-client-cli/src/commands/cardano_db_v2/download.rs index b57d3a52552..0b000bd967a 100644 --- a/mithril-client-cli/src/commands/cardano_db_v2/download.rs +++ b/mithril-client-cli/src/commands/cardano_db_v2/download.rs @@ -245,6 +245,9 @@ impl PreparedCardanoDbV2Download { &restoration_options.db_dir, &cardano_db_message, self.is_json_output_enabled(), + restoration_options + .download_unpack_options + .include_ancillary, )?; Ok(()) @@ -488,6 +491,7 @@ impl PreparedCardanoDbV2Download { db_dir: &Path, cardano_db_snapshot: &CardanoDatabaseSnapshot, json_output: bool, + include_ancillary: bool, ) -> MithrilResult<()> { let canonicalized_filepath = &db_dir.canonicalize().with_context(|| { format!( @@ -496,12 +500,41 @@ impl PreparedCardanoDbV2Download { ) })?; + let docker_cmd = format!( + "docker run -v cardano-node-ipc:/ipc -v cardano-node-data:/data --mount type=bind,source=\"{}\",target=/data/db/ -e NETWORK={} ghcr.io/intersectmbo/cardano-node:{}", + canonicalized_filepath.display(), + cardano_db_snapshot.network, + cardano_db_snapshot.cardano_node_version + ); + + let snapshot_converter_cmd = |flavor| { + format!( + "mithril-client --unstable tools utxo-hd snapshot-converter --db-directory {} --cardano-node-version {} --utxo-hd-flavor {} --cardano-network {} --commit", + db_dir.display(), + cardano_db_snapshot.cardano_node_version, + flavor, + cardano_db_snapshot.network + ) + }; + if json_output { - println!( - r#"{{"timestamp": "{}", "db_directory": "{}"}}"#, - Utc::now().to_rfc3339(), - canonicalized_filepath.display() - ); + let json = if include_ancillary { + serde_json::json!({ + "timestamp": Utc::now().to_rfc3339(), + "db_directory": canonicalized_filepath, + "run_docker_cmd": docker_cmd, + "snapshot_converter_cmd_to_lmdb": snapshot_converter_cmd("LMDB"), + "snapshot_converter_cmd_to_legacy": snapshot_converter_cmd("Legacy") + }) + } else { + serde_json::json!({ + "timestamp": Utc::now().to_rfc3339(), + "db_directory": canonicalized_filepath, + "run_docker_cmd": docker_cmd + }) + }; + + println!("{}", json); } else { let cardano_node_version = &cardano_db_snapshot.cardano_node_version; println!( @@ -511,14 +544,29 @@ impl PreparedCardanoDbV2Download { If you are using Cardano Docker image, you can restore a Cardano Node with: - docker run -v cardano-node-ipc:/ipc -v cardano-node-data:/data --mount type=bind,source="{}",target=/data/db/ -e NETWORK={} ghcr.io/intersectmbo/cardano-node:{cardano_node_version} + {} "###, cardano_db_snapshot.hash, db_dir.display(), - canonicalized_filepath.display(), - cardano_db_snapshot.network, + docker_cmd ); + + if include_ancillary { + println!( + r###"Upgrade and replace the restored ledger state snapshot to 'LMDB' flavor by running the command: + + {} + + Or to 'Legacy' flavor by running the command: + + {} + + "###, + snapshot_converter_cmd("LMDB"), + snapshot_converter_cmd("Legacy"), + ); + } } Ok(()) From 25a7bb7667d80eb537983ac015ca4e8005bad10b Mon Sep 17 00:00:00 2001 From: Damien Lachaume <135982616+dlachaume@users.noreply.github.com> Date: Thu, 5 Jun 2025 14:37:59 +0200 Subject: [PATCH 11/20] refactor(client-cli): move `HttpDownloader` trait in a dedicated `interface.rs` sub module --- .../src/utils/http_downloader/interface.rs | 21 ++++++++++++++++++ .../src/utils/http_downloader/mod.rs | 22 ++----------------- 2 files changed, 23 insertions(+), 20 deletions(-) create mode 100644 mithril-client-cli/src/utils/http_downloader/interface.rs diff --git a/mithril-client-cli/src/utils/http_downloader/interface.rs b/mithril-client-cli/src/utils/http_downloader/interface.rs new file mode 100644 index 00000000000..6e6d2e065e1 --- /dev/null +++ b/mithril-client-cli/src/utils/http_downloader/interface.rs @@ -0,0 +1,21 @@ +use std::path::{Path, PathBuf}; + +use async_trait::async_trait; +use reqwest::Url; + +use mithril_client::MithrilResult; + +/// Trait for downloading a file over HTTP from a URL, +/// saving it to a target directory with the given filename. +/// +/// Returns the path to the downloaded file. +#[cfg_attr(test, mockall::automock)] +#[async_trait] +pub trait HttpDownloader { + async fn download_file( + &self, + url: Url, + download_dir: &Path, + filename: &str, + ) -> MithrilResult; +} diff --git a/mithril-client-cli/src/utils/http_downloader/mod.rs b/mithril-client-cli/src/utils/http_downloader/mod.rs index 1341f09d13f..1581ad58d09 100644 --- a/mithril-client-cli/src/utils/http_downloader/mod.rs +++ b/mithril-client-cli/src/utils/http_downloader/mod.rs @@ -1,23 +1,5 @@ +mod interface; mod reqwest_http_downloader; +pub use interface::*; pub use reqwest_http_downloader::*; - -use async_trait::async_trait; -use mithril_client::MithrilResult; -use reqwest::Url; -use std::path::{Path, PathBuf}; - -/// Trait for downloading a file over HTTP from a URL, -/// saving it to a target directory with the given filename. -/// -/// Returns the path to the downloaded file. -#[cfg_attr(test, mockall::automock)] -#[async_trait] -pub trait HttpDownloader { - async fn download_file( - &self, - url: Url, - download_dir: &Path, - filename: &str, - ) -> MithrilResult; -} From 3892a40c9393f5f91f088da02e8e424f603a4ec5 Mon Sep 17 00:00:00 2001 From: Damien Lachaume <135982616+dlachaume@users.noreply.github.com> Date: Thu, 5 Jun 2025 14:42:29 +0200 Subject: [PATCH 12/20] refactor(client-cli): rename `github_release.rs` to `model.rs` --- .../src/utils/github_release_retriever/interface.rs | 2 +- mithril-client-cli/src/utils/github_release_retriever/mod.rs | 4 ++-- .../github_release_retriever/{github_release.rs => model.rs} | 0 3 files changed, 3 insertions(+), 3 deletions(-) rename mithril-client-cli/src/utils/github_release_retriever/{github_release.rs => model.rs} (100%) diff --git a/mithril-client-cli/src/utils/github_release_retriever/interface.rs b/mithril-client-cli/src/utils/github_release_retriever/interface.rs index 0b1ba7f3df6..7c2eec5a09a 100644 --- a/mithril-client-cli/src/utils/github_release_retriever/interface.rs +++ b/mithril-client-cli/src/utils/github_release_retriever/interface.rs @@ -2,7 +2,7 @@ use async_trait::async_trait; use mithril_client::MithrilResult; -use super::github_release::GitHubRelease; +use super::model::GitHubRelease; /// Trait for interacting with the GitHub API to retrieve Cardano node release. #[cfg_attr(test, mockall::automock)] diff --git a/mithril-client-cli/src/utils/github_release_retriever/mod.rs b/mithril-client-cli/src/utils/github_release_retriever/mod.rs index 8e8b4155065..d7676cf377a 100644 --- a/mithril-client-cli/src/utils/github_release_retriever/mod.rs +++ b/mithril-client-cli/src/utils/github_release_retriever/mod.rs @@ -1,7 +1,7 @@ -mod github_release; mod interface; +mod model; mod reqwest; -pub use github_release::*; pub use interface::*; +pub use model::*; pub use reqwest::*; diff --git a/mithril-client-cli/src/utils/github_release_retriever/github_release.rs b/mithril-client-cli/src/utils/github_release_retriever/model.rs similarity index 100% rename from mithril-client-cli/src/utils/github_release_retriever/github_release.rs rename to mithril-client-cli/src/utils/github_release_retriever/model.rs From 28c79efac374a56cefdbae030d12d69a706fcbc0 Mon Sep 17 00:00:00 2001 From: Damien Lachaume <135982616+dlachaume@users.noreply.github.com> Date: Thu, 5 Jun 2025 14:44:57 +0200 Subject: [PATCH 13/20] refactor(client-cli): rename `reqwest_http_downloader.rs` to `reqwest.rs` --- mithril-client-cli/src/utils/http_downloader/mod.rs | 4 ++-- .../{reqwest_http_downloader.rs => reqwest.rs} | 0 2 files changed, 2 insertions(+), 2 deletions(-) rename mithril-client-cli/src/utils/http_downloader/{reqwest_http_downloader.rs => reqwest.rs} (100%) diff --git a/mithril-client-cli/src/utils/http_downloader/mod.rs b/mithril-client-cli/src/utils/http_downloader/mod.rs index 1581ad58d09..f3c8ff4a394 100644 --- a/mithril-client-cli/src/utils/http_downloader/mod.rs +++ b/mithril-client-cli/src/utils/http_downloader/mod.rs @@ -1,5 +1,5 @@ mod interface; -mod reqwest_http_downloader; +mod reqwest; pub use interface::*; -pub use reqwest_http_downloader::*; +pub use reqwest::*; diff --git a/mithril-client-cli/src/utils/http_downloader/reqwest_http_downloader.rs b/mithril-client-cli/src/utils/http_downloader/reqwest.rs similarity index 100% rename from mithril-client-cli/src/utils/http_downloader/reqwest_http_downloader.rs rename to mithril-client-cli/src/utils/http_downloader/reqwest.rs From e574dc162914e5a5c4d3716e75db9570f514368a Mon Sep 17 00:00:00 2001 From: Damien Lachaume <135982616+dlachaume@users.noreply.github.com> Date: Thu, 5 Jun 2025 14:51:06 +0200 Subject: [PATCH 14/20] refactor(client-cli): move `ArchiveFormat` trait in a dedicated `interface.rs` sub module --- .../src/utils/archive_unpacker/interface.rs | 13 +++++++++++++ .../src/utils/archive_unpacker/mod.rs | 15 ++------------- 2 files changed, 15 insertions(+), 13 deletions(-) create mode 100644 mithril-client-cli/src/utils/archive_unpacker/interface.rs diff --git a/mithril-client-cli/src/utils/archive_unpacker/interface.rs b/mithril-client-cli/src/utils/archive_unpacker/interface.rs new file mode 100644 index 00000000000..8afa1f737f0 --- /dev/null +++ b/mithril-client-cli/src/utils/archive_unpacker/interface.rs @@ -0,0 +1,13 @@ +use std::path::Path; + +use mithril_client::MithrilResult; + +/// Trait for supported archive formats (e.g. `.zip`, `.tar.gz`). +#[cfg_attr(test, mockall::automock)] +pub trait ArchiveFormat { + /// Checks whether this format can handle the given archive file. + fn supports(&self, archive_path: &Path) -> bool; + + /// Unpacks the archive into the target directory. + fn unpack(&self, archive_path: &Path, unpack_dir: &Path) -> MithrilResult<()>; +} diff --git a/mithril-client-cli/src/utils/archive_unpacker/mod.rs b/mithril-client-cli/src/utils/archive_unpacker/mod.rs index 6c4bf3ff9e0..b6d895540b8 100644 --- a/mithril-client-cli/src/utils/archive_unpacker/mod.rs +++ b/mithril-client-cli/src/utils/archive_unpacker/mod.rs @@ -1,18 +1,7 @@ +mod interface; mod tar_gz_unpacker; mod unpacker; mod zip_unpacker; +pub use interface::*; pub use unpacker::*; - -use mithril_client::MithrilResult; -use std::path::Path; - -/// Trait for supported archive formats (e.g. `.zip`, `.tar.gz`). -#[cfg_attr(test, mockall::automock)] -pub trait ArchiveFormat { - /// Checks whether this format can handle the given archive file. - fn supports(&self, archive_path: &Path) -> bool; - - /// Unpacks the archive into the target directory. - fn unpack(&self, archive_path: &Path, unpack_dir: &Path) -> MithrilResult<()>; -} From 2bcfaad70eb5bd36d30699c26eb3161e93565d34 Mon Sep 17 00:00:00 2001 From: Damien Lachaume <135982616+dlachaume@users.noreply.github.com> Date: Thu, 5 Jun 2025 15:18:50 +0200 Subject: [PATCH 15/20] refactor(client-cli): only remove contents of the ledger directory, not the directory itself --- .../src/commands/tools/snapshot_converter.rs | 14 +++----- mithril-client-cli/src/utils/fs.rs | 36 +++++++++++++++++++ 2 files changed, 40 insertions(+), 10 deletions(-) diff --git a/mithril-client-cli/src/commands/tools/snapshot_converter.rs b/mithril-client-cli/src/commands/tools/snapshot_converter.rs index 47a3a046b3b..ad3af272ff6 100644 --- a/mithril-client-cli/src/commands/tools/snapshot_converter.rs +++ b/mithril-client-cli/src/commands/tools/snapshot_converter.rs @@ -11,8 +11,8 @@ use clap::{Parser, ValueEnum}; use mithril_client::MithrilResult; use crate::utils::{ - copy_dir, ArchiveUnpacker, GitHubReleaseRetriever, HttpDownloader, ReqwestGitHubApiClient, - ReqwestHttpDownloader, + copy_dir, remove_dir_contents, ArchiveUnpacker, GitHubReleaseRetriever, HttpDownloader, + ReqwestGitHubApiClient, ReqwestHttpDownloader, }; const GITHUB_ORGANIZATION: &str = "IntersectMBO"; @@ -403,15 +403,9 @@ impl SnapshotConverterCommand { let (slot_number, _) = filename .split_once('_') .ok_or_else(|| anyhow!("Invalid converted snapshot name format: {}", filename))?; - remove_dir_all(&ledger_dir).with_context(|| { + remove_dir_contents(&ledger_dir).with_context(|| { format!( - "Failed to remove old ledger state snapshot directory: {}", - ledger_dir.display() - ) - })?; - create_dir(&ledger_dir).with_context(|| { - format!( - "Failed to recreate ledger state snapshot directory: {}", + "Failed to remove contents of ledger directory: {}", ledger_dir.display() ) })?; diff --git a/mithril-client-cli/src/utils/fs.rs b/mithril-client-cli/src/utils/fs.rs index a60a9722b3f..def181bbb02 100644 --- a/mithril-client-cli/src/utils/fs.rs +++ b/mithril-client-cli/src/utils/fs.rs @@ -48,6 +48,26 @@ fn copy_dir_contents(source_dir: &Path, target_dir: &Path) -> MithrilResult<()> Ok(()) } +/// Removes all contents inside the given directory. +pub fn remove_dir_contents(dir: &Path) -> MithrilResult<()> { + if !dir.exists() { + return Ok(()); + } + + for entry in fs::read_dir(dir)? { + let path = entry?.path(); + if path.is_dir() { + fs::remove_dir_all(&path) + .with_context(|| format!("Failed to remove subdirectory: {}", path.display()))?; + } else { + fs::remove_file(&path) + .with_context(|| format!("Failed to remove file: {}", path.display()))?; + } + } + + Ok(()) +} + #[cfg(test)] mod tests { use std::fs::File; @@ -106,4 +126,20 @@ mod tests { ** root.txt" ); } + + #[test] + fn cleans_directory_without_deleting_it() { + let dir = temp_dir_create!().join("dir_to_clean"); + fs::create_dir(&dir).unwrap(); + + File::create(dir.join("file1.txt")).unwrap(); + let sub_dir = dir.join("subdir"); + fs::create_dir(&sub_dir).unwrap(); + File::create(sub_dir.join("file2.txt")).unwrap(); + + remove_dir_contents(&dir).unwrap(); + + assert!(dir.exists()); + assert!(fs::read_dir(&dir).unwrap().next().is_none()); + } } From eb0c8545982c163f5f88a587c781963bae303e2a Mon Sep 17 00:00:00 2001 From: Damien Lachaume <135982616+dlachaume@users.noreply.github.com> Date: Thu, 5 Jun 2025 16:10:25 +0200 Subject: [PATCH 16/20] refactor(client-cli): refactor `find_oldest_ledger_state_snapshot` Extracts and sorts valid ledger snapshots by slot number, then returns the oldest one. --- .../src/commands/tools/snapshot_converter.rs | 90 +++++++++++++------ 1 file changed, 61 insertions(+), 29 deletions(-) diff --git a/mithril-client-cli/src/commands/tools/snapshot_converter.rs b/mithril-client-cli/src/commands/tools/snapshot_converter.rs index ad3af272ff6..d202f79aa32 100644 --- a/mithril-client-cli/src/commands/tools/snapshot_converter.rs +++ b/mithril-client-cli/src/commands/tools/snapshot_converter.rs @@ -305,41 +305,48 @@ impl SnapshotConverterCommand { .join(SNAPSHOT_CONVERTER_CONFIG_FILE) } - /// Finds the oldest ledger snapshot (by slot number) in the `ledger/` directory of a Cardano node database. - fn find_oldest_ledger_state_snapshot(db_dir: &Path) -> MithrilResult { - let ledger_dir = db_dir.join(LEDGER_DIR); - let entries = read_dir(&ledger_dir).with_context(|| { + /// Returns the list of valid ledger snapshot directories sorted in ascending order of slot number. + /// + /// Only directories with numeric names are considered valid snapshots. + fn get_sorted_snapshot_dirs(ledger_dir: &Path) -> MithrilResult> { + let entries = read_dir(ledger_dir).with_context(|| { format!( "Failed to read ledger state snapshots directory: {}", ledger_dir.display() ) })?; - let mut min_slot: Option<(u64, PathBuf)> = None; - - for entry in entries { - let entry = entry?; - let slot = match Self::extract_slot_number(&entry.path()) { - Ok(number) => number, - Err(_) => continue, - }; - - let path = entry.path(); - if path.is_dir() - && (min_slot - .as_ref() - .map(|(min, _)| slot < *min) - .unwrap_or(true)) - { - min_slot = Some((slot, path)); - } - } - min_slot.map(|(_, path)| path).ok_or_else(|| { - anyhow!( - "No valid ledger state snapshot found in directory: {}", - ledger_dir.display() - ) - }) + let mut snapshots = entries + .filter_map(|entry| { + let path = entry.ok()?.path(); + if !path.is_dir() { + return None; + } + SnapshotConverterCommand::extract_slot_number(&path) + .ok() + .map(|slot| (slot, path)) + }) + .collect::>(); + + snapshots.sort_by_key(|(slot, _)| *slot); + + Ok(snapshots) + } + + /// Finds the oldest ledger snapshot (by slot number) in the `ledger/` directory of a Cardano node database. + fn find_oldest_ledger_state_snapshot(db_dir: &Path) -> MithrilResult { + let ledger_dir = db_dir.join(LEDGER_DIR); + let snapshots_by_slot = Self::get_sorted_snapshot_dirs(&ledger_dir)?; + snapshots_by_slot + .into_iter() + .map(|(_, path)| path) + .next() + .ok_or_else(|| { + anyhow!( + "No valid ledger state snapshot found in directory: {}", + ledger_dir.display() + ) + }) } fn copy_oldest_ledger_state_snapshot( @@ -822,6 +829,31 @@ mod tests { SnapshotConverterCommand::find_oldest_ledger_state_snapshot(&temp_dir) .expect_err("Should return error if no valid ledger snapshot directory found"); } + + #[test] + fn get_sorted_snapshot_dirs_returns_sorted_valid_directories() { + let temp_dir = temp_dir_create!(); + let ledger_dir = temp_dir.join(LEDGER_DIR); + create_dir(&ledger_dir).unwrap(); + + create_dir(ledger_dir.join("1500")).unwrap(); + create_dir(ledger_dir.join("1000")).unwrap(); + create_dir(ledger_dir.join("2000")).unwrap(); + File::create(ledger_dir.join("500")).unwrap(); + create_dir(ledger_dir.join("notanumber")).unwrap(); + + let snapshots = + SnapshotConverterCommand::get_sorted_snapshot_dirs(&ledger_dir).unwrap(); + + assert_eq!( + snapshots, + vec![ + (1000, ledger_dir.join("1000")), + (1500, ledger_dir.join("1500")), + (2000, ledger_dir.join("2000")), + ] + ); + } } mod commit_converted_snapshot { From f8ec5ff393f01427ced09624e5b5bd7f5dc8ed30 Mon Sep 17 00:00:00 2001 From: Damien Lachaume <135982616+dlachaume@users.noreply.github.com> Date: Thu, 5 Jun 2025 16:56:16 +0200 Subject: [PATCH 17/20] test(client): add tests for `ReqwestGitHubApiClient` --- Cargo.lock | 1 + mithril-client-cli/Cargo.toml | 1 + .../utils/github_release_retriever/reqwest.rs | 92 +++++++++++++++++++ 3 files changed, 94 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index ca7a4beeb91..f1540d5b322 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3954,6 +3954,7 @@ dependencies = [ "flate2", "fs2", "futures", + "httpmock", "human_bytes", "indicatif", "mithril-cli-helper", diff --git a/mithril-client-cli/Cargo.toml b/mithril-client-cli/Cargo.toml index 517f9de61b3..6dabef899f3 100644 --- a/mithril-client-cli/Cargo.toml +++ b/mithril-client-cli/Cargo.toml @@ -61,5 +61,6 @@ tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } zip = "4.0.0" [dev-dependencies] +httpmock = "0.7.0" mithril-common = { path = "../mithril-common", features = ["test_tools"] } mockall = { workspace = true } diff --git a/mithril-client-cli/src/utils/github_release_retriever/reqwest.rs b/mithril-client-cli/src/utils/github_release_retriever/reqwest.rs index 70f5d9d391e..d75472fc01c 100644 --- a/mithril-client-cli/src/utils/github_release_retriever/reqwest.rs +++ b/mithril-client-cli/src/utils/github_release_retriever/reqwest.rs @@ -31,6 +31,15 @@ impl ReqwestGitHubApiClient { .send() .await .with_context(|| format!("Failed to send request to GitHub API: {}", url))?; + match response.status() { + reqwest::StatusCode::OK => {} + status => { + return Err(anyhow!( + "GitHub API request failed with status code: {}", + status + )); + } + } let body = response.text().await?; let parsed_body = serde_json::from_str::(&body) .with_context(|| format!("Failed to parse response from GitHub API: {:?}", body))?; @@ -91,3 +100,86 @@ impl GitHubReleaseRetriever for ReqwestGitHubApiClient { Ok(releases) } } + +#[cfg(test)] +mod tests { + use httpmock::{Method::GET, MockServer}; + use reqwest::StatusCode; + use serde::Deserialize; + + use super::*; + + #[derive(Debug, Deserialize, PartialEq)] + struct FakeApiResponse { + key: String, + } + + #[tokio::test] + async fn download_succeeds_with_valid_json() { + let server = MockServer::start(); + let _mock = server.mock(|when, then| { + when.method(GET).path("/endpoint"); + then.status(200).body(r#"{ "key": "value" }"#); + }); + let client = ReqwestGitHubApiClient::new().unwrap(); + + let result: FakeApiResponse = client + .download(format!("{}/endpoint", server.base_url())) + .await + .unwrap(); + + assert_eq!( + result, + FakeApiResponse { + key: "value".into() + } + ); + } + + #[tokio::test] + async fn download_fails_on_invalid_json() { + let server = MockServer::start(); + let _mock = server.mock(|when, then| { + when.method(GET).path("/endpoint"); + then.status(200).body("this is not json"); + }); + let client = ReqwestGitHubApiClient::new().unwrap(); + + let result: MithrilResult = client + .download(format!("{}/endpoint", server.base_url())) + .await; + + assert!( + result.is_err(), + "Expected an error with invalid JSON response" + ); + } + + #[tokio::test] + async fn download_fails_on_invalid_url() { + let client = ReqwestGitHubApiClient::new().unwrap(); + + let result: MithrilResult = client.download("not a valid url").await; + + assert!(result.is_err(), "Expected an error for an invalid URL"); + } + + #[tokio::test] + async fn download_fails_when_server_returns_error_and_includes_status_in_error() { + let server = MockServer::start(); + let _mock = server.mock(|when, then| { + when.method(GET).path("/endpoint"); + then.status(StatusCode::INTERNAL_SERVER_ERROR.into()); + }); + let client = ReqwestGitHubApiClient::new().unwrap(); + + let result: MithrilResult = client + .download(format!("{}/endpoint", server.base_url())) + .await; + let error = result.expect_err("Expected an error due to 500 status"); + + assert!(error + .to_string() + .contains(&StatusCode::INTERNAL_SERVER_ERROR.to_string())); + } +} From e007d1726388574d761cf9f0c2d106259378a288 Mon Sep 17 00:00:00 2001 From: Damien Lachaume <135982616+dlachaume@users.noreply.github.com> Date: Thu, 5 Jun 2025 17:35:33 +0200 Subject: [PATCH 18/20] refactor(client-cli): fix and enhance command documentation --- mithril-client-cli/src/commands/tools/mod.rs | 1 + mithril-client-cli/src/commands/tools/snapshot_converter.rs | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/mithril-client-cli/src/commands/tools/mod.rs b/mithril-client-cli/src/commands/tools/mod.rs index ace75838d72..cc2651b52e4 100644 --- a/mithril-client-cli/src/commands/tools/mod.rs +++ b/mithril-client-cli/src/commands/tools/mod.rs @@ -12,6 +12,7 @@ use mithril_client::MithrilResult; /// Tools commands #[derive(Subcommand, Debug, Clone)] +#[command(about = "[unstable] Tools commands")] pub enum ToolsCommands { /// UTxO-HD related commands #[clap(subcommand, name = "utxo-hd")] diff --git a/mithril-client-cli/src/commands/tools/snapshot_converter.rs b/mithril-client-cli/src/commands/tools/snapshot_converter.rs index d202f79aa32..62784e9e6ac 100644 --- a/mithril-client-cli/src/commands/tools/snapshot_converter.rs +++ b/mithril-client-cli/src/commands/tools/snapshot_converter.rs @@ -76,7 +76,7 @@ pub struct SnapshotConverterCommand { /// Cardano node version of the Mithril signed snapshot. /// - /// `latest` and `prerelease` are also supported to download the latest or preprelease distribution. + /// `latest` and `prerelease` are also supported to download the latest or prerelease distribution. #[clap(long)] cardano_node_version: String, From ff0a1277fa7ecb40295a9e7031a87637274c2cb5 Mon Sep 17 00:00:00 2001 From: Damien Lachaume <135982616+dlachaume@users.noreply.github.com> Date: Thu, 5 Jun 2025 14:22:21 +0200 Subject: [PATCH 19/20] chore: update `Cargo.lock` (missing in the previous merge on main) --- Cargo.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index f1540d5b322..27d2b5e23f8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4218,7 +4218,7 @@ dependencies = [ [[package]] name = "mithril-stm" -version = "0.4.1" +version = "0.4.2" dependencies = [ "bincode", "blake2 0.10.6", From 5b39c59efb6c4b19fc4fc917ba698ae6e1f74e02 Mon Sep 17 00:00:00 2001 From: Damien Lachaume <135982616+dlachaume@users.noreply.github.com> Date: Thu, 5 Jun 2025 17:01:09 +0200 Subject: [PATCH 20/20] chore: upgrade crate versions * mithril-client-cli from `0.12.6` to `0.12.7` --- Cargo.lock | 2 +- mithril-client-cli/Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 27d2b5e23f8..76907c70c62 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3943,7 +3943,7 @@ dependencies = [ [[package]] name = "mithril-client-cli" -version = "0.12.6" +version = "0.12.7" dependencies = [ "anyhow", "async-trait", diff --git a/mithril-client-cli/Cargo.toml b/mithril-client-cli/Cargo.toml index 6dabef899f3..18303694869 100644 --- a/mithril-client-cli/Cargo.toml +++ b/mithril-client-cli/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "mithril-client-cli" -version = "0.12.6" +version = "0.12.7" description = "A Mithril Client" authors = { workspace = true } edition = { workspace = true }