Skip to content

Commit e6135c6

Browse files
AlenarAlenar
authored
Merge pull request #2521 from input-output-hk/djo/2497/adapt_ledger_snapshot_to_utxo-hd
feat: support snapshotting in memory UTxO-HD ledger state snapshots
2 parents 0e2ee54 + 5c0619a commit e6135c6

File tree

16 files changed

+878
-243
lines changed

16 files changed

+878
-243
lines changed

‎Cargo.lock‎

Lines changed: 4 additions & 4 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎mithril-aggregator/Cargo.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
[package]
22
name = "mithril-aggregator"
3-
version = "0.7.54"
3+
version = "0.7.55"
44
description = "A Mithril Aggregator server"
55
authors = { workspace = true }
66
edition = { workspace = true }

‎mithril-aggregator/src/artifact_builder/cardano_database.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -169,7 +169,7 @@ mod tests {
169169
.with_immutables(&[0, 1, 2, 3])
170170
.append_immutable_trio()
171171
.set_immutable_trio_file_size(immutable_trio_file_size)
172-
.with_ledger_files(&["437"])
172+
.with_legacy_ledger_snapshots(&[437])
173173
.set_ledger_file_size(ledger_file_size)
174174
.with_volatile_files(&["blocks-0.dat"])
175175
.set_volatile_file_size(volatile_file_size)

‎mithril-aggregator/src/services/snapshotter/compressed_archive_snapshotter.rs‎

Lines changed: 136 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ use std::path::{Path, PathBuf};
66
use std::sync::Arc;
77

88
use mithril_common::digesters::{
9-
immutable_trio_names, ImmutableFile, LedgerFile, IMMUTABLE_DIR, LEDGER_DIR,
9+
immutable_trio_names, ImmutableFile, LedgerStateSnapshot, IMMUTABLE_DIR, LEDGER_DIR,
1010
};
1111
use mithril_common::entities::{AncillaryFilesManifest, CompressionAlgorithm, ImmutableFileNumber};
1212
use mithril_common::logging::LoggerExtensions;
@@ -258,12 +258,13 @@ impl CompressedArchiveSnapshotter {
258258
.collect();
259259

260260
let db_ledger_dir = self.db_directory.join(LEDGER_DIR);
261-
let ledger_files = LedgerFile::list_all_in_dir(&db_ledger_dir)?;
261+
let ledger_files = LedgerStateSnapshot::list_all_in_dir(&db_ledger_dir)?;
262262
let latest_ledger_files: Vec<PathBuf> = ledger_files
263263
.iter()
264264
.rev()
265265
.take(2)
266-
.map(|ledger_file| PathBuf::from(LEDGER_DIR).join(&ledger_file.filename))
266+
.flat_map(|ledger_state_snapshot| ledger_state_snapshot.get_files_relative_path())
267+
.map(|path| PathBuf::from(LEDGER_DIR).join(path))
267268
.collect();
268269
if latest_ledger_files.is_empty() {
269270
return Err(anyhow!(
@@ -279,6 +280,16 @@ impl CompressedArchiveSnapshotter {
279280
.with_context(|| format!("Can not create folder: `{}`", target_folder.display()))?;
280281

281282
for file in &files_to_snapshot {
283+
// Some files to snapshot are in subfolders (i.e.: in-memory ledger snapshots files)
284+
if let Some(parent_dir) = file.parent() {
285+
let target_parent_dir = target_folder.join(parent_dir);
286+
if !target_parent_dir.exists() {
287+
fs::create_dir_all(&target_parent_dir).with_context(|| {
288+
format!("Can not create folder: `{}`", target_parent_dir.display())
289+
})?;
290+
}
291+
}
292+
282293
let source = self.db_directory.join(file);
283294
let target = target_folder.join(file);
284295
tokio::fs::copy(&source, &target).await.with_context(|| {
@@ -448,7 +459,7 @@ mod tests {
448459
let cardano_db = DummyCardanoDbBuilder::new(current_function!())
449460
.with_immutables(&[1, 2, 3])
450461
.append_immutable_trio()
451-
.with_ledger_files(&["437"])
462+
.with_legacy_ledger_snapshots(&[437])
452463
.with_volatile_files(&["blocks-0.dat"])
453464
.with_non_immutables(&["random_file.txt", "00002.trap"])
454465
.build();
@@ -491,7 +502,7 @@ mod tests {
491502
let test_dir = temp_dir_create!();
492503
let cardano_db = DummyCardanoDbBuilder::new(current_function!())
493504
.with_immutables(&[1, 2, 3])
494-
.with_ledger_files(&["437"])
505+
.with_legacy_ledger_snapshots(&[437])
495506
.with_volatile_files(&["blocks-0.dat"])
496507
.with_non_immutables(&["random_file.txt", "00002.trap"])
497508
.build();
@@ -526,12 +537,12 @@ mod tests {
526537
use super::*;
527538

528539
#[tokio::test]
529-
async fn getting_files_to_include_copy_them_to_a_target_directory_while_keeping_source_dir_structure(
540+
async fn getting_files_to_include_for_legacy_ledger_snapshot_copy_them_to_a_target_directory_while_keeping_source_dir_structure(
530541
) {
531542
let test_dir = temp_dir_create!();
532543
let cardano_db = DummyCardanoDbBuilder::new(current_function!())
533544
.with_immutables(&[1, 2])
534-
.with_ledger_files(&["737"])
545+
.with_legacy_ledger_snapshots(&[737])
535546
.build();
536547
let snapshotter =
537548
snapshotter_for_test(&test_dir, cardano_db.get_dir(), CompressionAlgorithm::Gzip);
@@ -556,6 +567,45 @@ mod tests {
556567
);
557568
}
558569

570+
#[tokio::test]
571+
async fn getting_files_to_include_for_in_memory_ledger_snapshot_copy_them_to_a_target_directory_while_keeping_source_dir_structure(
572+
) {
573+
let test_dir = temp_dir_create!();
574+
let cardano_db = DummyCardanoDbBuilder::new(current_function!())
575+
.with_immutables(&[1, 2])
576+
.with_in_memory_ledger_snapshots(&[737])
577+
.build();
578+
let snapshotter =
579+
snapshotter_for_test(&test_dir, cardano_db.get_dir(), CompressionAlgorithm::Gzip);
580+
let ancillary_snapshot_dir = test_dir.join("ancillary_snapshot");
581+
fs::create_dir(&ancillary_snapshot_dir).unwrap();
582+
583+
snapshotter
584+
.get_files_and_directories_for_ancillary_snapshot(1, &ancillary_snapshot_dir)
585+
.await
586+
.unwrap();
587+
588+
assert_dir_eq!(
589+
&ancillary_snapshot_dir,
590+
format!(
591+
"* {IMMUTABLE_DIR}/
592+
** 00002.chunk
593+
** 00002.primary
594+
** 00002.secondary
595+
* {LEDGER_DIR}/
596+
** 737/
597+
*** {}/
598+
**** {}
599+
*** {}
600+
*** {}",
601+
LedgerStateSnapshot::IN_MEMORY_TABLES,
602+
LedgerStateSnapshot::IN_MEMORY_TVAR,
603+
LedgerStateSnapshot::IN_MEMORY_META,
604+
LedgerStateSnapshot::IN_MEMORY_STATE,
605+
)
606+
);
607+
}
608+
559609
#[tokio::test]
560610
async fn getting_files_to_include_fails_when_no_ledger_file_found() {
561611
let test_dir = temp_dir_create!();
@@ -578,7 +628,7 @@ mod tests {
578628
let test_dir = temp_dir_create!();
579629
let cardano_db = DummyCardanoDbBuilder::new(current_function!())
580630
.with_immutables(&[1, 2])
581-
.with_ledger_files(&["637"])
631+
.with_legacy_ledger_snapshots(&[637])
582632
.build();
583633
let snapshotter = CompressedArchiveSnapshotter {
584634
ancillary_signer: Arc::new(MockAncillarySigner::that_succeeds_with_signature(
@@ -606,7 +656,7 @@ mod tests {
606656
let test_dir = temp_dir_create!();
607657
let cardano_db = DummyCardanoDbBuilder::new(current_function!())
608658
.with_immutables(&[1, 2])
609-
.with_ledger_files(&["637"])
659+
.with_legacy_ledger_snapshots(&[637])
610660
.build();
611661
let snapshotter = CompressedArchiveSnapshotter {
612662
ancillary_signer: Arc::new(MockAncillarySigner::that_fails_with_message("failure")),
@@ -632,7 +682,8 @@ mod tests {
632682
let test_dir = temp_dir_create!();
633683
let cardano_db = DummyCardanoDbBuilder::new(current_function!())
634684
.with_immutables(&[1, 2, 3])
635-
.with_ledger_files(&["437", "537", "637", "737", "9not_included"])
685+
.with_legacy_ledger_snapshots(&[437, 537, 637, 737])
686+
.with_non_ledger_files(&["9not_included"])
636687
.with_volatile_files(&["blocks-0.dat", "blocks-1.dat", "blocks-2.dat"])
637688
.build();
638689
fs::create_dir(cardano_db.get_dir().join("whatever")).unwrap();
@@ -672,7 +723,7 @@ mod tests {
672723
let test_dir = temp_dir_create!();
673724
let cardano_db = DummyCardanoDbBuilder::new(current_function!())
674725
.with_immutables(&[1, 2])
675-
.with_ledger_files(&["737"])
726+
.with_legacy_ledger_snapshots(&[737])
676727
.build();
677728

678729
let snapshotter = CompressedArchiveSnapshotter {
@@ -693,11 +744,12 @@ mod tests {
693744
}
694745

695746
#[tokio::test]
696-
async fn create_archive_generate_sign_and_include_manifest_file() {
747+
async fn create_archive_of_legacy_ledger_snapshot_generate_sign_and_include_manifest_file()
748+
{
697749
let test_dir = temp_dir_create!();
698750
let cardano_db = DummyCardanoDbBuilder::new(current_function!())
699751
.with_immutables(&[1, 2, 3])
700-
.with_ledger_files(&["537", "637", "737"])
752+
.with_legacy_ledger_snapshots(&[537, 637, 737])
701753
.with_non_immutables(&["not_to_include.txt"])
702754
.build();
703755
File::create(cardano_db.get_dir().join("not_to_include_as_well.txt")).unwrap();
@@ -742,6 +794,76 @@ mod tests {
742794
manifest.signature
743795
)
744796
}
797+
798+
#[tokio::test]
799+
async fn create_archive_of_in_memory_ledger_snapshot_generate_sign_and_include_manifest_file(
800+
) {
801+
let test_dir = temp_dir_create!();
802+
let cardano_db = DummyCardanoDbBuilder::new(current_function!())
803+
.with_immutables(&[1, 2, 3])
804+
.with_in_memory_ledger_snapshots(&[537, 637, 737])
805+
.with_non_immutables(&["not_to_include.txt"])
806+
.build();
807+
File::create(cardano_db.get_dir().join("not_to_include_as_well.txt")).unwrap();
808+
809+
let snapshotter = CompressedArchiveSnapshotter {
810+
ancillary_signer: Arc::new(MockAncillarySigner::that_succeeds_with_signature(
811+
fake_keys::signable_manifest_signature()[0],
812+
)),
813+
..snapshotter_for_test(&test_dir, cardano_db.get_dir(), CompressionAlgorithm::Gzip)
814+
};
815+
816+
let archive = snapshotter
817+
.snapshot_ancillary(2, "ancillary")
818+
.await
819+
.unwrap();
820+
let unpacked = archive.unpack_gzip(test_dir);
821+
let manifest_path = unpacked.join(AncillaryFilesManifest::ANCILLARY_MANIFEST_FILE_NAME);
822+
823+
assert!(manifest_path.exists());
824+
825+
let manifest = serde_json::from_reader::<_, AncillaryFilesManifest>(
826+
File::open(&manifest_path).unwrap(),
827+
)
828+
.unwrap();
829+
830+
assert_eq!(
831+
vec![
832+
&PathBuf::from(IMMUTABLE_DIR).join("00003.chunk"),
833+
&PathBuf::from(IMMUTABLE_DIR).join("00003.primary"),
834+
&PathBuf::from(IMMUTABLE_DIR).join("00003.secondary"),
835+
&PathBuf::from(LEDGER_DIR)
836+
.join("637")
837+
.join(LedgerStateSnapshot::IN_MEMORY_META),
838+
&PathBuf::from(LEDGER_DIR)
839+
.join("637")
840+
.join(LedgerStateSnapshot::IN_MEMORY_STATE),
841+
&PathBuf::from(LEDGER_DIR)
842+
.join("637")
843+
.join(LedgerStateSnapshot::IN_MEMORY_TABLES)
844+
.join(LedgerStateSnapshot::IN_MEMORY_TVAR),
845+
&PathBuf::from(LEDGER_DIR)
846+
.join("737")
847+
.join(LedgerStateSnapshot::IN_MEMORY_META),
848+
&PathBuf::from(LEDGER_DIR)
849+
.join("737")
850+
.join(LedgerStateSnapshot::IN_MEMORY_STATE),
851+
&PathBuf::from(LEDGER_DIR)
852+
.join("737")
853+
.join(LedgerStateSnapshot::IN_MEMORY_TABLES)
854+
.join(LedgerStateSnapshot::IN_MEMORY_TVAR),
855+
],
856+
manifest.data.keys().collect::<Vec<_>>()
857+
);
858+
assert_eq!(
859+
Some(
860+
fake_keys::signable_manifest_signature()[0]
861+
.try_into()
862+
.unwrap()
863+
),
864+
manifest.signature
865+
)
866+
}
745867
}
746868

747869
mod compute_immutable_total_and_average_uncompressed_size {
@@ -757,7 +879,7 @@ mod tests {
757879
let cardano_db = DummyCardanoDbBuilder::new(current_function!())
758880
.with_immutables(&[1, 2, 3])
759881
.set_immutable_trio_file_size(immutable_trio_file_size)
760-
.with_ledger_files(&["737"])
882+
.with_legacy_ledger_snapshots(&[737])
761883
.set_ledger_file_size(6666)
762884
.with_volatile_files(&["blocks-0.dat"])
763885
.set_volatile_file_size(99)

‎mithril-aggregator/src/tools/file_archiver/appender.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -390,7 +390,7 @@ mod tests {
390390
let cardano_db = DummyCardanoDbBuilder::new(test_dir)
391391
.with_immutables(&[1, 2, 3])
392392
.set_immutable_trio_file_size(immutable_trio_file_size)
393-
.with_ledger_files(&["437", "537", "637", "737"])
393+
.with_legacy_ledger_snapshots(&[437, 537, 637, 737])
394394
.set_ledger_file_size(ledger_file_size)
395395
.with_volatile_files(&["blocks-0.dat", "blocks-1.dat", "blocks-2.dat"])
396396
.set_volatile_file_size(volatile_file_size)
@@ -480,7 +480,7 @@ mod tests {
480480
let cardano_db = DummyCardanoDbBuilder::new(test_dir)
481481
.with_immutables(&[1, 2])
482482
.set_immutable_trio_file_size(immutable_trio_file_size)
483-
.with_ledger_files(&["437", "537", "637"])
483+
.with_legacy_ledger_snapshots(&[437, 537, 637])
484484
.set_ledger_file_size(ledger_file_size)
485485
.with_volatile_files(&["blocks-0.dat"])
486486
.set_volatile_file_size(volatile_file_size)

‎mithril-client/Cargo.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
[package]
22
name = "mithril-client"
3-
version = "0.12.8"
3+
version = "0.12.9"
44
description = "Mithril client library"
55
authors = { workspace = true }
66
edition = { workspace = true }

‎mithril-client/tests/cardano_db_snapshot_list_get_download_verify.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ async fn cardano_db_snapshot_list_get_download_verify() {
2929
let cardano_db = DummyCardanoDbBuilder::new("cardano_db_snapshot_list_get_download_verify_db")
3030
.with_immutables(&[1, 2, 3, 4])
3131
.append_immutable_trio()
32-
.with_ledger_files(&["437", "537", "637"])
32+
.with_legacy_ledger_snapshots(&[437, 537, 637])
3333
.with_volatile_files(&["blocks-0.dat", "blocks-1.dat"])
3434
.build();
3535
let digester =

‎mithril-client/tests/extensions/snapshot_archives.rs‎

Lines changed: 11 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ use std::path::{Path, PathBuf};
33

44
use mithril_common::crypto_helper::{ManifestSigner, ManifestVerifierSecretKey};
55
use mithril_common::digesters::{
6-
immutable_trio_names, ComputedImmutablesDigests, DummyCardanoDb, IMMUTABLE_DIR,
6+
immutable_trio_names, ComputedImmutablesDigests, DummyCardanoDb, IMMUTABLE_DIR, LEDGER_DIR,
77
};
88
use mithril_common::entities::{AncillaryFilesManifest, CompressionAlgorithm, ImmutableFileNumber};
99
use mithril_common::messages::CardanoDatabaseDigestListItemMessage;
@@ -107,8 +107,8 @@ pub async fn build_ancillary_files_archive(
107107
) {
108108
let db_dir = cardano_db.get_dir();
109109
let ancillary_immutable_number = cardano_db.last_immutable_number().unwrap() + 1;
110-
let last_ledger_file_path = cardano_db
111-
.get_ledger_files()
110+
let last_ledger_state_snapshot = cardano_db
111+
.get_ledger_state_snapshots()
112112
.last()
113113
.expect("Given db should have at least one ledger file");
114114
let archive_name = format!(
@@ -120,15 +120,18 @@ pub async fn build_ancillary_files_archive(
120120
let enc = zstd::Encoder::new(tar_file, 3).unwrap();
121121
let mut tar = tar::Builder::new(enc);
122122

123-
let files_to_include = vec![
124-
PathBuf::from(format!(
125-
"ledger/{}",
126-
last_ledger_file_path.file_name().unwrap().to_string_lossy()
127-
)),
123+
let mut files_to_include = vec![
128124
PathBuf::from(IMMUTABLE_DIR).join(format!("{ancillary_immutable_number:05}.chunk")),
129125
PathBuf::from(IMMUTABLE_DIR).join(format!("{ancillary_immutable_number:05}.primary")),
130126
PathBuf::from(IMMUTABLE_DIR).join(format!("{ancillary_immutable_number:05}.secondary")),
131127
];
128+
files_to_include.extend(
129+
last_ledger_state_snapshot
130+
.get_files_relative_path()
131+
.into_iter()
132+
.map(|p| PathBuf::from(LEDGER_DIR).join(p)),
133+
);
134+
132135
let ancillary_manifest = build_ancillary_manifest(
133136
db_dir,
134137
files_to_include.clone(),

‎mithril-client/tests/snapshot_list_get_show_download_verify.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ async fn snapshot_list_get_show_download_verify() {
2222
let cardano_db = DummyCardanoDbBuilder::new("snapshot_list_get_show_download_verify_db")
2323
.with_immutables(&[1, 2, 3])
2424
.append_immutable_trio()
25-
.with_ledger_files(&["506", "562"])
25+
.with_legacy_ledger_snapshots(&[506, 562])
2626
.build();
2727
let fake_aggregator = FakeAggregator::new();
2828
let test_http_server = fake_aggregator

0 commit comments

Comments
 (0)