Merge pull request #2671 from input-output-hk/ctl/2618-list-invalid-or-missing-immutable-files-in-verify-command-REFACTORING

refactor: listing of invalid or missing immutable files in verify command

Author: turmelclem

Cargo.lock

@@ -327,21 +327,32 @@ async fn main() -> mithril_client::MithrilResult<()> {
         .download_unpack(
             &snapshot,
             &immutable_file_range,
-            target_directory,
+            &target_directory,
             download_unpack_options,
         )
         .await?;
 
+    let verified_digests = client
+        .cardano_database_v2()
+        .download_and_verify_digests(
+            &certificate,
+            &snapshot
+        )
+        .await?;
+
+    let allow_missing_immutables_files = false;
     let merkle_proof = client
         .cardano_database_v2()
-        .compute_merkle_proof(
+        .verify_cardano_database(
             &certificate,
             &snapshot,
             &immutable_file_range,
-            target_directory,
-        )
-        .await?;
-    merkle_proof.verify()?;
+            allow_missing_immutables_files,
+            &target_directory,
+            &verified_digest,
+        ),
+    )
+    .await?;
 
     let message = MessageBuilder::new()
         .compute_cardano_database_message(&certificate, &merkle_proof)

docs/website/root/manual/develop/nodes/mithril-client-library.md

@@ -1,7 +1,7 @@
 [package]
 name = "client-cardano-database-v2"
 description = "Mithril client Cardano database example"
-version = "0.0.13"
+version = "0.0.14"
 authors = ["dev@iohk.io", "mithril-dev@iohk.io"]
 documentation = "https://mithril.network/doc"
 edition = "2021"

examples/client-cardano-database-v2/Cargo.toml

@@ -121,21 +121,27 @@ async fn main() -> MithrilResult<()> {
         println!("Could not send usage statistics to the aggregator: {e:?}");
     }
 
-    println!(
-        "Computing Cardano database snapshot '{}' message...",
-        cardano_database_snapshot.hash
-    );
+    println!("Verifying Cardano database...",);
     let allow_missing_immutables_files = false;
-    let message = wait_spinner(
-        &progress_bar,
-        MessageBuilder::new().compute_cardano_database_message(
+    let merkle_proof = client
+        .cardano_database_v2()
+        .verify_cardano_database(
             &certificate,
             &cardano_database_snapshot,
             &immutable_file_range,
             allow_missing_immutables_files,
             &unpacked_dir,
             &verified_digests,
-        ),
+        )
+        .await?;
+
+    println!(
+        "Computing Cardano database snapshot '{}' message...",
+        cardano_database_snapshot.hash
+    );
+    let message = wait_spinner(
+        &progress_bar,
+        MessageBuilder::new().compute_cardano_database_message(&certificate, &merkle_proof),
     )
     .await?;
 

examples/client-cardano-database-v2/src/main.rs

@@ -1,6 +1,6 @@
 [package]
 name = "mithril-client-cli"
-version = "0.12.27"
+version = "0.12.28"
 description = "A Mithril Client"
 authors = { workspace = true }
 edition = { workspace = true }

mithril-client-cli/Cargo.toml

@@ -65,7 +65,7 @@ impl PreparedCardanoDbV2Download {
         } else {
             ProgressOutputType::Tty
         };
-        let progress_printer = ProgressPrinter::new(progress_output_type, 6);
+        let progress_printer = ProgressPrinter::new(progress_output_type, 7);
         let client = client_builder(context.config_parameters())?
             .add_feedback_receiver(Arc::new(IndicatifFeedbackReceiver::new(
                 progress_output_type,
@@ -147,19 +147,30 @@ impl PreparedCardanoDbV2Download {
             immutable_file_range: restoration_options.immutable_file_range,
             allow_missing: false,
         };
-        let message = shared_steps::compute_cardano_db_snapshot_message(
+
+        let merkle_proof = shared_steps::verify_cardano_database(
             5,
             &progress_printer,
+            &client,
             &certificate,
             &cardano_db_message,
             &options,
             &verified_digests,
         )
+        .await
+        .with_context(|| format!("Can not verify cardano database for hash: '{}'", self.hash))?;
+
+        let message = shared_steps::compute_cardano_db_snapshot_message(
+            6,
+            &progress_printer,
+            &certificate,
+            &merkle_proof,
+        )
         .await?;
 
         shared_steps::verify_message_matches_certificate(
             context.logger(),
-            6,
+            7,
             &progress_printer,
             &certificate,
             &message,

mithril-client-cli/src/commands/cardano_db/download/v2.rs

@@ -6,7 +6,7 @@ use std::path::{Path, PathBuf};
 use mithril_client::{
     CardanoDatabaseSnapshot, Client, MessageBuilder, MithrilCertificate, MithrilResult,
     cardano_database_client::{ImmutableFileRange, VerifiedDigests},
-    common::{ImmutableFileNumber, ProtocolMessage},
+    common::{ImmutableFileNumber, MKProof, ProtocolMessage},
 };
 
 use crate::utils::{CardanoDbUtils, ProgressPrinter};
@@ -69,19 +69,19 @@ pub async fn download_and_verify_digests(
     Ok(verified_digests)
 }
 
-/// Computes the Cardano database snapshot message using the provided certificate and Merkle proof.
-pub async fn compute_cardano_db_snapshot_message(
+pub async fn verify_cardano_database(
     step_number: u16,
     progress_printer: &ProgressPrinter,
+    client: &Client,
     certificate: &MithrilCertificate,
     cardano_database_snapshot: &CardanoDatabaseSnapshot,
     options: &ComputeCardanoDatabaseMessageOptions,
     verified_digest: &VerifiedDigests,
-) -> MithrilResult<ProtocolMessage> {
-    progress_printer.report_step(step_number, "Computing the cardano db snapshot message")?;
-    let message = CardanoDbUtils::wait_spinner(
+) -> MithrilResult<MKProof> {
+    progress_printer.report_step(step_number, "Verifying the cardano database")?;
+    let merkle_proof = CardanoDbUtils::wait_spinner(
         progress_printer,
-        MessageBuilder::new().compute_cardano_database_message(
+        client.cardano_database_v2().verify_cardano_database(
             certificate,
             cardano_database_snapshot,
             &options.immutable_file_range,
@@ -91,6 +91,24 @@ pub async fn compute_cardano_db_snapshot_message(
         ),
     )
     .await
+    .with_context(|| "Can not verify the cardano database")?;
+
+    Ok(merkle_proof)
+}
+
+/// Computes the Cardano database snapshot message using the provided certificate and Merkle proof.
+pub async fn compute_cardano_db_snapshot_message(
+    step_number: u16,
+    progress_printer: &ProgressPrinter,
+    certificate: &MithrilCertificate,
+    merkle_proof: &MKProof,
+) -> MithrilResult<ProtocolMessage> {
+    progress_printer.report_step(step_number, "Computing the cardano db snapshot message")?;
+    let message = CardanoDbUtils::wait_spinner(
+        progress_printer,
+        MessageBuilder::new().compute_cardano_database_message(certificate, merkle_proof),
+    )
+    .await
     .with_context(|| "Can not compute the cardano db snapshot message")?;
 
     Ok(message)

mithril-client-cli/src/commands/cardano_db/shared_steps.rs

@@ -8,8 +8,10 @@ use anyhow::Context;
 use chrono::{DateTime, Utc};
 use clap::Parser;
 use mithril_client::{
-    CardanoDatabaseSnapshot, ComputeCardanoDatabaseMessageError, ImmutableVerificationResult,
-    MithrilResult, cardano_database_client::ImmutableFileRange, common::ImmutableFileNumber,
+    CardanoDatabaseSnapshot, MithrilResult,
+    cardano_database_client::ImmutableFileRange,
+    cardano_database_client::{CardanoDatabaseVerificationError, ImmutableVerificationResult},
+    common::ImmutableFileNumber,
 };
 
 use crate::{
@@ -85,7 +87,7 @@ impl CardanoDbVerifyCommand {
         } else {
             ProgressOutputType::Tty
         };
-        let progress_printer = ProgressPrinter::new(progress_output_type, 4);
+        let progress_printer = ProgressPrinter::new(progress_output_type, 5);
         let client = client_builder(context.config_parameters())?
             .add_feedback_receiver(Arc::new(IndicatifFeedbackReceiver::new(
                 progress_output_type,
@@ -146,19 +148,21 @@ impl CardanoDbVerifyCommand {
             immutable_file_range,
             allow_missing: self.allow_missing,
         };
-        let message = shared_steps::compute_cardano_db_snapshot_message(
+
+        let merkle_proof = shared_steps::verify_cardano_database(
             3,
             &progress_printer,
+            &client,
             &certificate,
             &cardano_db_message,
             &options,
             &verified_digests,
         )
         .await;
 
-        match message {
-            Err(e) => match e.downcast_ref::<ComputeCardanoDatabaseMessageError>() {
-                Some(ComputeCardanoDatabaseMessageError::ImmutableFilesVerification(lists)) => {
+        match merkle_proof {
+            Err(e) => match e.downcast_ref::<CardanoDatabaseVerificationError>() {
+                Some(CardanoDatabaseVerificationError::ImmutableFilesVerification(lists)) => {
                     Self::print_immutables_verification_error(
                         lists,
                         context.is_json_output_enabled(),
@@ -167,10 +171,18 @@ impl CardanoDbVerifyCommand {
                 }
                 _ => Err(e),
             },
-            Ok(message) => {
+            Ok(merkle_proof) => {
+                let message = shared_steps::compute_cardano_db_snapshot_message(
+                    4,
+                    &progress_printer,
+                    &certificate,
+                    &merkle_proof,
+                )
+                .await?;
+
                 shared_steps::verify_message_matches_certificate(
                     &context.logger().clone(),
-                    4,
+                    5,
                     &progress_printer,
                     &certificate,
                     &message,

mithril-client-cli/src/commands/cardano_db/verify.rs

@@ -1,6 +1,6 @@
 [package]
 name = "mithril-client"
-version = "0.12.28"
+version = "0.12.29"
 description = "Mithril client library"
 authors = { workspace = true }
 edition = { workspace = true }

mithril-client/Cargo.toml

@@ -6,14 +6,17 @@ use std::sync::Arc;
 use slog::Logger;
 
 #[cfg(feature = "fs")]
-use mithril_common::messages::{CardanoDatabaseSnapshotMessage, CertificateMessage};
+use mithril_common::{
+    crypto_helper::MKProof,
+    messages::{CardanoDatabaseSnapshotMessage, CertificateMessage},
+};
 
 #[cfg(feature = "fs")]
 use mithril_cardano_node_internal_database::entities::ImmutableFile;
 
 use crate::aggregator_client::AggregatorClient;
 #[cfg(feature = "fs")]
-use crate::cardano_database_client::VerifiedDigests;
+use crate::cardano_database_client::{VerifiedDigests, proving::CardanoDatabaseVerificationError};
 #[cfg(feature = "fs")]
 use crate::feedback::FeedbackSender;
 #[cfg(feature = "fs")]
@@ -111,6 +114,29 @@ impl CardanoDatabaseClient {
             .await
     }
 
+    /// Verify a local cardano database
+    #[cfg(feature = "fs")]
+    pub async fn verify_cardano_database(
+        &self,
+        certificate: &CertificateMessage,
+        cardano_database_snapshot: &CardanoDatabaseSnapshotMessage,
+        immutable_file_range: &ImmutableFileRange,
+        allow_missing: bool,
+        database_dir: &Path,
+        verified_digests: &VerifiedDigests,
+    ) -> Result<MKProof, CardanoDatabaseVerificationError> {
+        self.artifact_prover
+            .verify_cardano_database(
+                certificate,
+                cardano_database_snapshot,
+                immutable_file_range,
+                allow_missing,
+                database_dir,
+                verified_digests,
+            )
+            .await
+    }
+
     /// Checks if immutable directory exists with at least one immutable in it
     #[cfg(feature = "fs")]
     pub fn check_has_immutables(&self, database_dir: &Path) -> MithrilResult<()> {
@@ -157,6 +183,8 @@ pub(crate) mod test_dependency_injector {
         ancillary_verifier: Option<Arc<AncillaryVerifier>>,
         #[cfg(feature = "fs")]
         feedback_receivers: Vec<Arc<dyn FeedbackReceiver>>,
+        #[cfg(feature = "fs")]
+        logger: Logger,
     }
 
     impl CardanoDatabaseClientDependencyInjector {
@@ -175,9 +203,17 @@ pub(crate) mod test_dependency_injector {
                 ancillary_verifier: None,
                 #[cfg(feature = "fs")]
                 feedback_receivers: vec![],
+                #[cfg(feature = "fs")]
+                logger: TestLogger::stdout(),
             }
         }
 
+        #[cfg(feature = "fs")]
+        pub(crate) fn with_logger(self, logger: Logger) -> Self {
+            #[cfg(feature = "fs")]
+            Self { logger, ..self }
+        }
+
         pub(crate) fn with_aggregator_client_mock_config<F>(mut self, config: F) -> Self
         where
             F: FnOnce(&mut MockAggregatorClient),
@@ -230,7 +266,7 @@ pub(crate) mod test_dependency_injector {
                 self.http_file_downloader,
                 self.ancillary_verifier,
                 FeedbackSender::new(&self.feedback_receivers),
-                TestLogger::stdout(),
+                self.logger,
             )
         }