Merge pull request #2637 from input-output-hk/ctl/2618-list-invalid-or-missing-immutable-files-in-verify-command

List invalid or tampered immutable files in verify command

Author: turmelclem

Cargo.lock

@@ -618,6 +618,9 @@ Here is a list of the available parameters:
 | `digest`                   | -                            |          -           | -                          | Digest of the Cardano db snapshot to verify or `latest` for the latest artifact                                       | -             | -       | :heavy_check_mark: |
 | `db_dir`                   | `--db-dir`                   |          -           | -                          | Directory from where the immutable will be verified                                                                   | -             | -       |         -          |
 | `genesis_verification_key` | `--genesis-verification-key` |          -           | `GENESIS_VERIFICATION_KEY` | Genesis verification key to check the certificate chain                                                               | -             | -       |         -          |
+| `start`                    | `--start`                    |          -           | -                          | The first immutable file number to verify                                                                             | -             | -       |         -          |
+| `end`                      | `--end`                      |          -           | -                          | The last immutable file number to verify                                                                              | -             | -       |         -          |
+| `allow_missing`            | `--allow-missing`            |          -           | -                          | If set, the verification will not fail if some immutable files are missing                                            | `false`       | -       |         -          |
 | `run_mode`                 | `--run-mode`                 |          -           | `RUN_MODE`                 | Run Mode                                                                                                              | `dev`         | -       |         -          |
 | `verbose`                  | `--verbose`                  |         `-v`         | -                          | Verbosity level (-v=warning, -vv=info, -vvv=debug, -vvvv=trace)                                                       | `0`           | -       |         -          |
 | `config_directory`         | `--config-directory`         |          -           | -                          | Directory where configuration file is located                                                                         | `./config`    | -       |         -          |

docs/website/root/manual/develop/nodes/mithril-client.md

@@ -1,7 +1,7 @@
 [package]
 name = "client-cardano-database-v2"
 description = "Mithril client Cardano database example"
-version = "0.0.12"
+version = "0.0.13"
 authors = ["dev@iohk.io", "mithril-dev@iohk.io"]
 documentation = "https://mithril.network/doc"
 edition = "2021"

examples/client-cardano-database-v2/Cargo.toml

@@ -17,7 +17,7 @@ use std::time::Duration;
 use tokio::sync::RwLock;
 
 use mithril_client::feedback::{FeedbackReceiver, MithrilEvent, MithrilEventCardanoDatabase};
-use mithril_client::{ClientBuilder, MessageBuilder, MithrilResult};
+use mithril_client::{ClientBuilder, MessageBuilder, MithrilError, MithrilResult};
 
 #[derive(Parser, Debug)]
 #[command(version)]
@@ -82,7 +82,7 @@ async fn main() -> MithrilResult<()> {
         .verify_chain(&cardano_database_snapshot.certificate_hash)
         .await?;
 
-    let immutable_file_range = ImmutableFileRange::From(15000);
+    let immutable_file_range = ImmutableFileRange::From(4000);
     let download_unpack_options = DownloadUnpackOptions {
         allow_override: true,
         include_ancillary: true,
@@ -98,19 +98,11 @@ async fn main() -> MithrilResult<()> {
         )
         .await?;
 
-    println!("Computing Cardano database Merkle proof...",);
-    let merkle_proof = client
+    println!("Downloading and verifying digests file authenticity...");
+    let verified_digests = client
         .cardano_database_v2()
-        .compute_merkle_proof(
-            &certificate,
-            &cardano_database_snapshot,
-            &immutable_file_range,
-            &unpacked_dir,
-        )
+        .download_and_verify_digests(&certificate, &cardano_database_snapshot)
         .await?;
-    merkle_proof
-        .verify()
-        .with_context(|| "Merkle proof verification failed")?;
 
     println!("Sending usage statistics to the aggregator...");
     let full_restoration = immutable_file_range == ImmutableFileRange::Full;
@@ -133,9 +125,17 @@ async fn main() -> MithrilResult<()> {
         "Computing Cardano database snapshot '{}' message...",
         cardano_database_snapshot.hash
     );
+    let allow_missing_immutables_files = false;
     let message = wait_spinner(
         &progress_bar,
-        MessageBuilder::new().compute_cardano_database_message(&certificate, &merkle_proof),
+        MessageBuilder::new().compute_cardano_database_message(
+            &certificate,
+            &cardano_database_snapshot,
+            &immutable_file_range,
+            allow_missing_immutables_files,
+            &unpacked_dir,
+            &verified_digests,
+        ),
     )
     .await?;
 
@@ -261,10 +261,13 @@ fn get_temp_dir() -> MithrilResult<PathBuf> {
     Ok(dir)
 }
 
-async fn wait_spinner<T>(
+pub async fn wait_spinner<T, E>(
     progress_bar: &MultiProgress,
-    future: impl Future<Output = MithrilResult<T>>,
-) -> MithrilResult<T> {
+    future: impl Future<Output = Result<T, E>>,
+) -> MithrilResult<T>
+where
+    MithrilError: From<E>,
+{
     let pb = progress_bar.add(ProgressBar::new_spinner());
     let spinner = async move {
         loop {
@@ -275,6 +278,6 @@ async fn wait_spinner<T>(
 
     tokio::select! {
         _ = spinner => Err(anyhow!("timeout")),
-        res = future => res,
+        res = future => res.map_err(Into::into),
     }
 }

examples/client-cardano-database-v2/src/main.rs

@@ -1,6 +1,6 @@
 [package]
 name = "mithril-client-cli"
-version = "0.12.26"
+version = "0.12.27"
 description = "A Mithril Client"
 authors = { workspace = true }
 edition = { workspace = true }

mithril-client-cli/Cargo.toml

@@ -15,7 +15,10 @@ use mithril_client::{
 use crate::{
     CommandContext,
     commands::{
-        cardano_db::{download::DB_DIRECTORY_NAME, shared_steps},
+        cardano_db::{
+            download::DB_DIRECTORY_NAME,
+            shared_steps::{self, ComputeCardanoDatabaseMessageOptions},
+        },
         client_builder,
     },
     utils::{
@@ -124,22 +127,33 @@ impl PreparedCardanoDbV2Download {
             )
         })?;
 
-        let merkle_proof = shared_steps::compute_verify_merkle_proof(
+        let verified_digests = shared_steps::download_and_verify_digests(
             4,
             &progress_printer,
             &client,
             &certificate,
             &cardano_db_message,
-            &restoration_options.immutable_file_range,
-            &restoration_options.db_dir,
         )
-        .await?;
+        .await
+        .with_context(|| {
+            format!(
+                "Can not download and verify digests file for cardano db snapshot with hash: '{}'",
+                self.hash
+            )
+        })?;
 
+        let options = ComputeCardanoDatabaseMessageOptions {
+            db_dir: restoration_options.db_dir.clone(),
+            immutable_file_range: restoration_options.immutable_file_range,
+            allow_missing: false,
+        };
         let message = shared_steps::compute_cardano_db_snapshot_message(
             5,
             &progress_printer,
             &certificate,
-            &merkle_proof,
+            &cardano_db_message,
+            &options,
+            &verified_digests,
         )
         .await?;
 

mithril-client-cli/src/commands/cardano_db/download/v2.rs

@@ -1,16 +1,22 @@
 use anyhow::{Context, anyhow};
 use chrono::Utc;
 use slog::{Logger, debug, warn};
-use std::path::Path;
+use std::path::{Path, PathBuf};
 
 use mithril_client::{
     CardanoDatabaseSnapshot, Client, MessageBuilder, MithrilCertificate, MithrilResult,
-    cardano_database_client::ImmutableFileRange,
-    common::{ImmutableFileNumber, MKProof, ProtocolMessage},
+    cardano_database_client::{ImmutableFileRange, VerifiedDigests},
+    common::{ImmutableFileNumber, ProtocolMessage},
 };
 
 use crate::utils::{CardanoDbUtils, ProgressPrinter};
 
+pub struct ComputeCardanoDatabaseMessageOptions {
+    pub db_dir: PathBuf,
+    pub immutable_file_range: ImmutableFileRange,
+    pub allow_missing: bool,
+}
+
 pub async fn fetch_certificate_and_verifying_chain(
     step_number: u16,
     progress_printer: &ProgressPrinter,
@@ -47,45 +53,42 @@ pub fn immutable_file_range(
     }
 }
 
-/// Computes and verifies the Merkle proof for the given certificate and database snapshot.
-pub async fn compute_verify_merkle_proof(
+pub async fn download_and_verify_digests(
     step_number: u16,
     progress_printer: &ProgressPrinter,
     client: &Client,
     certificate: &MithrilCertificate,
     cardano_database_snapshot: &CardanoDatabaseSnapshot,
-    immutable_file_range: &ImmutableFileRange,
-    unpacked_dir: &Path,
-) -> MithrilResult<MKProof> {
-    progress_printer.report_step(step_number, "Computing and verifying the Merkle proof…")?;
-    let merkle_proof = client
+) -> MithrilResult<VerifiedDigests> {
+    progress_printer.report_step(step_number, "Downloading and verifying digests…")?;
+    let verified_digests = client
         .cardano_database_v2()
-        .compute_merkle_proof(
-            certificate,
-            cardano_database_snapshot,
-            immutable_file_range,
-            Path::new(&unpacked_dir),
-        )
+        .download_and_verify_digests(certificate, cardano_database_snapshot)
         .await?;
 
-    merkle_proof
-        .verify()
-        .with_context(|| "Merkle proof verification failed")?;
-
-    Ok(merkle_proof)
+    Ok(verified_digests)
 }
 
 /// Computes the Cardano database snapshot message using the provided certificate and Merkle proof.
 pub async fn compute_cardano_db_snapshot_message(
     step_number: u16,
     progress_printer: &ProgressPrinter,
     certificate: &MithrilCertificate,
-    merkle_proof: &MKProof,
+    cardano_database_snapshot: &CardanoDatabaseSnapshot,
+    options: &ComputeCardanoDatabaseMessageOptions,
+    verified_digest: &VerifiedDigests,
 ) -> MithrilResult<ProtocolMessage> {
     progress_printer.report_step(step_number, "Computing the cardano db snapshot message")?;
     let message = CardanoDbUtils::wait_spinner(
         progress_printer,
-        MessageBuilder::new().compute_cardano_database_message(certificate, merkle_proof),
+        MessageBuilder::new().compute_cardano_database_message(
+            certificate,
+            cardano_database_snapshot,
+            &options.immutable_file_range,
+            options.allow_missing,
+            &options.db_dir,
+            verified_digest,
+        ),
     )
     .await
     .with_context(|| "Can not compute the cardano db snapshot message")?;