Merge pull request #2520 from input-output-hk/jpraynaud/2519-fix-missing-immutable-files

jpraynaud · web-flow · commit 0e2ee54f9834 · 2025-05-26T16:56:03.000+02:00
fix: missing `00000` immutable files with Cardano DB v2
diff --git a/.github/workflows/test-client.yml b/.github/workflows/test-client.yml
@@ -239,7 +239,7 @@ jobs:
         if: steps.aggregator_capability_unix.outputs.cardano_database_v2_enabled == 'true' || steps.aggregator_capability_windows.outputs.cardano_database_v2_enabled == 'true'
         shell: bash
         working-directory: ./bin
-        run: ./mithril-client ${{ steps.prepare.outputs.debug_level }} --unstable --origin-tag CI cardano-db-v2 download $CARDANO_DATABASE_V2_SNAPSHOT_HASH ${{ matrix.extra_args }} 2>&1 | tee cdb-download-output.txt
+        run: ./mithril-client ${{ steps.prepare.outputs.debug_level }} --unstable --origin-tag CI cardano-db-v2 download $CARDANO_DATABASE_V2_SNAPSHOT_HASH ${{ matrix.extra_args }} 2>&1 | tee cdb-v2-download-output.txt
 
       - name: Cardano Database V2 Snapshot / verify Cardano node starts successfully
         if: runner.os == 'Linux' && steps.aggregator_capability_unix.outputs.cardano_database_v2_enabled == 'true'
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/mithril-aggregator/Cargo.toml b/mithril-aggregator/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "mithril-aggregator"
-version = "0.7.53"
+version = "0.7.54"
 description = "A Mithril Aggregator server"
 authors = { workspace = true }
 edition = { workspace = true }
diff --git a/mithril-aggregator/src/artifact_builder/cardano_database.rs b/mithril-aggregator/src/artifact_builder/cardano_database.rs
@@ -166,7 +166,7 @@ mod tests {
         let ledger_file_size = 6666;
         let volatile_file_size = 99;
         let cardano_db = DummyCardanoDbBuilder::new("cdb-should_compute_valid_artifact")
-            .with_immutables(&[1, 2, 3])
+            .with_immutables(&[0, 1, 2, 3])
             .append_immutable_trio()
             .set_immutable_trio_file_size(immutable_trio_file_size)
             .with_ledger_files(&["437"])
@@ -227,7 +227,8 @@ mod tests {
         }
 
         let immutable_artifact_builder = {
-            let number_of_immutable_file_loaded = beacon.immutable_file_number;
+            // We upload the immutable files in the [0,beacon.immutable_file_number] range
+            let number_of_immutable_file_loaded = beacon.immutable_file_number + 1;
             let mut immutable_uploader = MockImmutableFilesUploader::new();
             immutable_uploader
                 .expect_batch_upload()
diff --git a/mithril-aggregator/src/artifact_builder/cardano_database_artifacts/immutable.rs b/mithril-aggregator/src/artifact_builder/cardano_database_artifacts/immutable.rs
@@ -193,8 +193,9 @@ impl ImmutableArtifactBuilder {
     ) -> StdResult<(Vec<PathBuf>, CompressionAlgorithm)> {
         let mut archive_paths = vec![];
         let compression_algorithm = self.snapshotter.compression_algorithm();
+        const FIRST_IMMUTABLE_FILE_NUMBER: ImmutableFileNumber = 0;
 
-        for immutable_file_number in 1..=up_to_immutable_file_number {
+        for immutable_file_number in FIRST_IMMUTABLE_FILE_NUMBER..=up_to_immutable_file_number {
             let archive_name_without_extension = format!("{immutable_file_number:05}");
             let archive_name = format!(
                 "{archive_name_without_extension}.{}",
@@ -352,7 +353,7 @@ mod tests {
         let work_dir = get_builder_work_dir("upload_call_archive_creation_and_upload");
         let test_dir = "upload_call_archive_creation_and_upload/cardano_database";
         let cardano_db = DummyCardanoDbBuilder::new(test_dir)
-            .with_immutables(&[1, 2])
+            .with_immutables(&[0, 1, 2])
             .build();
 
         let db_directory = cardano_db.get_dir().to_path_buf();
@@ -368,6 +369,7 @@ mod tests {
 
         let uploader = fake_uploader(
             vec![
+                work_dir.join("00000.tar.gz").to_str().unwrap(),
                 work_dir.join("00001.tar.gz").to_str().unwrap(),
                 work_dir.join("00002.tar.gz").to_str().unwrap(),
             ],
@@ -444,7 +446,7 @@ mod tests {
             let test_dir =
                 "snapshot_immutables_files_up_to_the_given_immutable_file_number/cardano_database";
             let cardano_db = DummyCardanoDbBuilder::new(test_dir)
-                .with_immutables(&[1, 2])
+                .with_immutables(&[0, 1, 2])
                 .build();
 
             let db_directory = cardano_db.get_dir().to_path_buf();
@@ -473,7 +475,11 @@ mod tests {
 
             assert_equivalent(
                 archive_paths,
-                vec![work_dir.join("00001.tar.gz"), work_dir.join("00002.tar.gz")],
+                vec![
+                    work_dir.join("00000.tar.gz"),
+                    work_dir.join("00001.tar.gz"),
+                    work_dir.join("00002.tar.gz"),
+                ],
             )
         }
 
@@ -614,7 +620,7 @@ mod tests {
             let work_dir = get_builder_work_dir("return_all_archives_but_not_rebuild_archives");
             let test_dir = "return_all_archives_but_not_rebuild_archives/cardano_database";
             let cardano_db = DummyCardanoDbBuilder::new(test_dir)
-                .with_immutables(&[1, 2, 3])
+                .with_immutables(&[0, 1, 2, 3])
                 .build();
 
             let db_directory = cardano_db.get_dir().to_path_buf();
@@ -628,6 +634,7 @@ mod tests {
             )
             .unwrap();
 
+            create_fake_file(&work_dir.join("00000.tar.gz"), "00000 content");
             create_fake_file(&work_dir.join("00001.tar.gz"), "00001 content");
             create_fake_file(&work_dir.join("00002.tar.gz"), "00002 content");
 
@@ -647,12 +654,14 @@ mod tests {
             assert_equivalent(
                 archive_paths,
                 vec![
+                    work_dir.join("00000.tar.gz"),
                     work_dir.join("00001.tar.gz"),
                     work_dir.join("00002.tar.gz"),
                     work_dir.join("00003.tar.gz"),
                 ],
             );
             // Check that the existing archives content have not changed
+            assert_file_content!(work_dir.join("00000.tar.gz"), "00000 content");
             assert_file_content!(work_dir.join("00001.tar.gz"), "00001 content");
             assert_file_content!(work_dir.join("00002.tar.gz"), "00002 content");
         }
@@ -666,6 +675,7 @@ mod tests {
                 .expect_compression_algorithm()
                 .returning(|| CompressionAlgorithm::Gzip);
 
+            create_fake_file(&work_dir.join("00000.tar.gz"), "00000 content");
             create_fake_file(&work_dir.join("00001.tar.gz"), "00001 content");
             create_fake_file(&work_dir.join("00002.tar.gz"), "00002 content");
             create_fake_file(&work_dir.join("00003.tar.gz"), "00003 content");
@@ -686,6 +696,7 @@ mod tests {
             assert_equivalent(
                 archive_paths,
                 vec![
+                    work_dir.join("00000.tar.gz"),
                     work_dir.join("00001.tar.gz"),
                     work_dir.join("00002.tar.gz"),
                     work_dir.join("00003.tar.gz"),
diff --git a/mithril-client/Cargo.toml b/mithril-client/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "mithril-client"
-version = "0.12.7"
+version = "0.12.8"
 description = "Mithril client library"
 authors = { workspace = true }
 edition = { workspace = true }
diff --git a/mithril-client/src/cardano_database_client/immutable_file_range.rs b/mithril-client/src/cardano_database_client/immutable_file_range.rs
@@ -26,10 +26,7 @@ impl ImmutableFileRange {
         &self,
         last_immutable_file_number: ImmutableFileNumber,
     ) -> StdResult<RangeInclusive<ImmutableFileNumber>> {
-        // The immutable file numbers start from 1 on all the networks except the 'devnet'
-        // when it is configured with aggressive protocol parameters for fast epochs (used in the e2e tests).
-        // We have taken the choice to consider that the file numbers start from 1 for all the networks.
-        const FIRST_IMMUTABLE_FILE_NUMBER: ImmutableFileNumber = 1;
+        const FIRST_IMMUTABLE_FILE_NUMBER: ImmutableFileNumber = 0;
         let full_range = FIRST_IMMUTABLE_FILE_NUMBER..=last_immutable_file_number;
 
         match self {
@@ -74,7 +71,7 @@ mod tests {
         let result = immutable_file_range
             .to_range_inclusive(last_immutable_file_number)
             .unwrap();
-        assert_eq!(1..=10, result);
+        assert_eq!(0..=10, result);
     }
 
     #[test]
@@ -124,7 +121,7 @@ mod tests {
         let result = immutable_file_range
             .to_range_inclusive(last_immutable_file_number)
             .unwrap();
-        assert_eq!(1..=8, result);
+        assert_eq!(0..=8, result);
 
         let last_immutable_file_number = 7;
         immutable_file_range
