feat: support in memory UTxO-HD ledger state snapshots

mithril-aggregator/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "mithril-aggregator"
-version = "0.7.54"
+version = "0.7.55"
 description = "A Mithril Aggregator server"
 authors = { workspace = true }
 edition = { workspace = true }

mithril-aggregator/src/artifact_builder/cardano_database.rs

@@ -169,7 +169,7 @@ mod tests {
             .with_immutables(&[0, 1, 2, 3])
             .append_immutable_trio()
             .set_immutable_trio_file_size(immutable_trio_file_size)
-            .with_ledger_files(&["437"])
+            .with_legacy_ledger_snapshots(&[437])
             .set_ledger_file_size(ledger_file_size)
             .with_volatile_files(&["blocks-0.dat"])
             .set_volatile_file_size(volatile_file_size)

mithril-aggregator/src/services/snapshotter/compressed_archive_snapshotter.rs

@@ -6,7 +6,7 @@ use std::path::{Path, PathBuf};
 use std::sync::Arc;
 
 use mithril_common::digesters::{
-    immutable_trio_names, ImmutableFile, LedgerFile, IMMUTABLE_DIR, LEDGER_DIR,
+    immutable_trio_names, ImmutableFile, LedgerStateSnapshot, IMMUTABLE_DIR, LEDGER_DIR,
 };
 use mithril_common::entities::{AncillaryFilesManifest, CompressionAlgorithm, ImmutableFileNumber};
 use mithril_common::logging::LoggerExtensions;
@@ -258,12 +258,13 @@ impl CompressedArchiveSnapshotter {
             .collect();
 
         let db_ledger_dir = self.db_directory.join(LEDGER_DIR);
-        let ledger_files = LedgerFile::list_all_in_dir(&db_ledger_dir)?;
+        let ledger_files = LedgerStateSnapshot::list_all_in_dir(&db_ledger_dir)?;
         let latest_ledger_files: Vec<PathBuf> = ledger_files
             .iter()
             .rev()
             .take(2)
-            .map(|ledger_file| PathBuf::from(LEDGER_DIR).join(&ledger_file.filename))
+            .flat_map(|ledger_state_snapshot| ledger_state_snapshot.get_files_relative_path())
+            .map(|path| PathBuf::from(LEDGER_DIR).join(path))
             .collect();
         if latest_ledger_files.is_empty() {
             return Err(anyhow!(
@@ -279,6 +280,16 @@ impl CompressedArchiveSnapshotter {
             .with_context(|| format!("Can not create folder: `{}`", target_folder.display()))?;
 
         for file in &files_to_snapshot {
+            // Some files to snapshot are in subfolders (i.e.: in-memory ledger snapshots files)
+            if let Some(parent_dir) = file.parent() {
+                let target_parent_dir = target_folder.join(parent_dir);
+                if !target_parent_dir.exists() {
+                    fs::create_dir_all(&target_parent_dir).with_context(|| {
+                        format!("Can not create folder: `{}`", target_parent_dir.display())
+                    })?;
+                }
+            }
+
             let source = self.db_directory.join(file);
             let target = target_folder.join(file);
             tokio::fs::copy(&source, &target).await.with_context(|| {
@@ -448,7 +459,7 @@ mod tests {
             let cardano_db = DummyCardanoDbBuilder::new(current_function!())
                 .with_immutables(&[1, 2, 3])
                 .append_immutable_trio()
-                .with_ledger_files(&["437"])
+                .with_legacy_ledger_snapshots(&[437])
                 .with_volatile_files(&["blocks-0.dat"])
                 .with_non_immutables(&["random_file.txt", "00002.trap"])
                 .build();
@@ -491,7 +502,7 @@ mod tests {
             let test_dir = temp_dir_create!();
             let cardano_db = DummyCardanoDbBuilder::new(current_function!())
                 .with_immutables(&[1, 2, 3])
-                .with_ledger_files(&["437"])
+                .with_legacy_ledger_snapshots(&[437])
                 .with_volatile_files(&["blocks-0.dat"])
                 .with_non_immutables(&["random_file.txt", "00002.trap"])
                 .build();
@@ -526,12 +537,12 @@ mod tests {
         use super::*;
 
         #[tokio::test]
-        async fn getting_files_to_include_copy_them_to_a_target_directory_while_keeping_source_dir_structure(
+        async fn getting_files_to_include_for_legacy_ledger_snapshot_copy_them_to_a_target_directory_while_keeping_source_dir_structure(
         ) {
             let test_dir = temp_dir_create!();
             let cardano_db = DummyCardanoDbBuilder::new(current_function!())
                 .with_immutables(&[1, 2])
-                .with_ledger_files(&["737"])
+                .with_legacy_ledger_snapshots(&[737])
                 .build();
             let snapshotter =
                 snapshotter_for_test(&test_dir, cardano_db.get_dir(), CompressionAlgorithm::Gzip);
@@ -556,6 +567,45 @@ mod tests {
             );
         }
 
+        #[tokio::test]
+        async fn getting_files_to_include_for_in_memory_ledger_snapshot_copy_them_to_a_target_directory_while_keeping_source_dir_structure(
+        ) {
+            let test_dir = temp_dir_create!();
+            let cardano_db = DummyCardanoDbBuilder::new(current_function!())
+                .with_immutables(&[1, 2])
+                .with_in_memory_ledger_snapshots(&[737])
+                .build();
+            let snapshotter =
+                snapshotter_for_test(&test_dir, cardano_db.get_dir(), CompressionAlgorithm::Gzip);
+            let ancillary_snapshot_dir = test_dir.join("ancillary_snapshot");
+            fs::create_dir(&ancillary_snapshot_dir).unwrap();
+
+            snapshotter
+                .get_files_and_directories_for_ancillary_snapshot(1, &ancillary_snapshot_dir)
+                .await
+                .unwrap();
+
+            assert_dir_eq!(
+                &ancillary_snapshot_dir,
+                format!(
+                    "* {IMMUTABLE_DIR}/
+                     ** 00002.chunk
+                     ** 00002.primary
+                     ** 00002.secondary
+                     * {LEDGER_DIR}/
+                     ** 737/
+                     *** {}/
+                     **** {}
+                     *** {}
+                     *** {}",
+                    LedgerStateSnapshot::IN_MEMORY_TABLES,
+                    LedgerStateSnapshot::IN_MEMORY_TVAR,
+                    LedgerStateSnapshot::IN_MEMORY_META,
+                    LedgerStateSnapshot::IN_MEMORY_STATE,
+                )
+            );
+        }
+
         #[tokio::test]
         async fn getting_files_to_include_fails_when_no_ledger_file_found() {
             let test_dir = temp_dir_create!();
@@ -578,7 +628,7 @@ mod tests {
             let test_dir = temp_dir_create!();
             let cardano_db = DummyCardanoDbBuilder::new(current_function!())
                 .with_immutables(&[1, 2])
-                .with_ledger_files(&["637"])
+                .with_legacy_ledger_snapshots(&[637])
                 .build();
             let snapshotter = CompressedArchiveSnapshotter {
                 ancillary_signer: Arc::new(MockAncillarySigner::that_succeeds_with_signature(
@@ -606,7 +656,7 @@ mod tests {
             let test_dir = temp_dir_create!();
             let cardano_db = DummyCardanoDbBuilder::new(current_function!())
                 .with_immutables(&[1, 2])
-                .with_ledger_files(&["637"])
+                .with_legacy_ledger_snapshots(&[637])
                 .build();
             let snapshotter = CompressedArchiveSnapshotter {
                 ancillary_signer: Arc::new(MockAncillarySigner::that_fails_with_message("failure")),
@@ -632,7 +682,8 @@ mod tests {
             let test_dir = temp_dir_create!();
             let cardano_db = DummyCardanoDbBuilder::new(current_function!())
                 .with_immutables(&[1, 2, 3])
-                .with_ledger_files(&["437", "537", "637", "737", "9not_included"])
+                .with_legacy_ledger_snapshots(&[437, 537, 637, 737])
+                .with_non_ledger_files(&["9not_included"])
                 .with_volatile_files(&["blocks-0.dat", "blocks-1.dat", "blocks-2.dat"])
                 .build();
             fs::create_dir(cardano_db.get_dir().join("whatever")).unwrap();
@@ -672,7 +723,7 @@ mod tests {
             let test_dir = temp_dir_create!();
             let cardano_db = DummyCardanoDbBuilder::new(current_function!())
                 .with_immutables(&[1, 2])
-                .with_ledger_files(&["737"])
+                .with_legacy_ledger_snapshots(&[737])
                 .build();
 
             let snapshotter = CompressedArchiveSnapshotter {
@@ -693,11 +744,12 @@ mod tests {
         }
 
         #[tokio::test]
-        async fn create_archive_generate_sign_and_include_manifest_file() {
+        async fn create_archive_of_legacy_ledger_snapshot_generate_sign_and_include_manifest_file()
+        {
             let test_dir = temp_dir_create!();
             let cardano_db = DummyCardanoDbBuilder::new(current_function!())
                 .with_immutables(&[1, 2, 3])
-                .with_ledger_files(&["537", "637", "737"])
+                .with_legacy_ledger_snapshots(&[537, 637, 737])
                 .with_non_immutables(&["not_to_include.txt"])
                 .build();
             File::create(cardano_db.get_dir().join("not_to_include_as_well.txt")).unwrap();
@@ -742,6 +794,76 @@ mod tests {
                 manifest.signature
             )
         }
+
+        #[tokio::test]
+        async fn create_archive_of_in_memory_ledger_snapshot_generate_sign_and_include_manifest_file(
+        ) {
+            let test_dir = temp_dir_create!();
+            let cardano_db = DummyCardanoDbBuilder::new(current_function!())
+                .with_immutables(&[1, 2, 3])
+                .with_in_memory_ledger_snapshots(&[537, 637, 737])
+                .with_non_immutables(&["not_to_include.txt"])
+                .build();
+            File::create(cardano_db.get_dir().join("not_to_include_as_well.txt")).unwrap();
+
+            let snapshotter = CompressedArchiveSnapshotter {
+                ancillary_signer: Arc::new(MockAncillarySigner::that_succeeds_with_signature(
+                    fake_keys::signable_manifest_signature()[0],
+                )),
+                ..snapshotter_for_test(&test_dir, cardano_db.get_dir(), CompressionAlgorithm::Gzip)
+            };
+
+            let archive = snapshotter
+                .snapshot_ancillary(2, "ancillary")
+                .await
+                .unwrap();
+            let unpacked = archive.unpack_gzip(test_dir);
+            let manifest_path = unpacked.join(AncillaryFilesManifest::ANCILLARY_MANIFEST_FILE_NAME);
+
+            assert!(manifest_path.exists());
+
+            let manifest = serde_json::from_reader::<_, AncillaryFilesManifest>(
+                File::open(&manifest_path).unwrap(),
+            )
+            .unwrap();
+
+            assert_eq!(
+                vec![
+                    &PathBuf::from(IMMUTABLE_DIR).join("00003.chunk"),
+                    &PathBuf::from(IMMUTABLE_DIR).join("00003.primary"),
+                    &PathBuf::from(IMMUTABLE_DIR).join("00003.secondary"),
+                    &PathBuf::from(LEDGER_DIR)
+                        .join("637")
+                        .join(LedgerStateSnapshot::IN_MEMORY_META),
+                    &PathBuf::from(LEDGER_DIR)
+                        .join("637")
+                        .join(LedgerStateSnapshot::IN_MEMORY_STATE),
+                    &PathBuf::from(LEDGER_DIR)
+                        .join("637")
+                        .join(LedgerStateSnapshot::IN_MEMORY_TABLES)
+                        .join(LedgerStateSnapshot::IN_MEMORY_TVAR),
+                    &PathBuf::from(LEDGER_DIR)
+                        .join("737")
+                        .join(LedgerStateSnapshot::IN_MEMORY_META),
+                    &PathBuf::from(LEDGER_DIR)
+                        .join("737")
+                        .join(LedgerStateSnapshot::IN_MEMORY_STATE),
+                    &PathBuf::from(LEDGER_DIR)
+                        .join("737")
+                        .join(LedgerStateSnapshot::IN_MEMORY_TABLES)
+                        .join(LedgerStateSnapshot::IN_MEMORY_TVAR),
+                ],
+                manifest.data.keys().collect::<Vec<_>>()
+            );
+            assert_eq!(
+                Some(
+                    fake_keys::signable_manifest_signature()[0]
+                        .try_into()
+                        .unwrap()
+                ),
+                manifest.signature
+            )
+        }
     }
 
     mod compute_immutable_total_and_average_uncompressed_size {
@@ -757,7 +879,7 @@ mod tests {
             let cardano_db = DummyCardanoDbBuilder::new(current_function!())
                 .with_immutables(&[1, 2, 3])
                 .set_immutable_trio_file_size(immutable_trio_file_size)
-                .with_ledger_files(&["737"])
+                .with_legacy_ledger_snapshots(&[737])
                 .set_ledger_file_size(6666)
                 .with_volatile_files(&["blocks-0.dat"])
                 .set_volatile_file_size(99)

mithril-aggregator/src/tools/file_archiver/appender.rs

@@ -390,7 +390,7 @@ mod tests {
             let cardano_db = DummyCardanoDbBuilder::new(test_dir)
                 .with_immutables(&[1, 2, 3])
                 .set_immutable_trio_file_size(immutable_trio_file_size)
-                .with_ledger_files(&["437", "537", "637", "737"])
+                .with_legacy_ledger_snapshots(&[437, 537, 637, 737])
                 .set_ledger_file_size(ledger_file_size)
                 .with_volatile_files(&["blocks-0.dat", "blocks-1.dat", "blocks-2.dat"])
                 .set_volatile_file_size(volatile_file_size)
@@ -480,7 +480,7 @@ mod tests {
             let cardano_db = DummyCardanoDbBuilder::new(test_dir)
                 .with_immutables(&[1, 2])
                 .set_immutable_trio_file_size(immutable_trio_file_size)
-                .with_ledger_files(&["437", "537", "637"])
+                .with_legacy_ledger_snapshots(&[437, 537, 637])
                 .set_ledger_file_size(ledger_file_size)
                 .with_volatile_files(&["blocks-0.dat"])
                 .set_volatile_file_size(volatile_file_size)

mithril-client/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "mithril-client"
-version = "0.12.8"
+version = "0.12.9"
 description = "Mithril client library"
 authors = { workspace = true }
 edition = { workspace = true }

mithril-client/tests/cardano_db_snapshot_list_get_download_verify.rs

@@ -29,7 +29,7 @@ async fn cardano_db_snapshot_list_get_download_verify() {
     let cardano_db = DummyCardanoDbBuilder::new("cardano_db_snapshot_list_get_download_verify_db")
         .with_immutables(&[1, 2, 3, 4])
         .append_immutable_trio()
-        .with_ledger_files(&["437", "537", "637"])
+        .with_legacy_ledger_snapshots(&[437, 537, 637])
         .with_volatile_files(&["blocks-0.dat", "blocks-1.dat"])
         .build();
     let digester =

mithril-client/tests/extensions/snapshot_archives.rs

@@ -3,7 +3,7 @@ use std::path::{Path, PathBuf};
 
 use mithril_common::crypto_helper::{ManifestSigner, ManifestVerifierSecretKey};
 use mithril_common::digesters::{
-    immutable_trio_names, ComputedImmutablesDigests, DummyCardanoDb, IMMUTABLE_DIR,
+    immutable_trio_names, ComputedImmutablesDigests, DummyCardanoDb, IMMUTABLE_DIR, LEDGER_DIR,
 };
 use mithril_common::entities::{AncillaryFilesManifest, CompressionAlgorithm, ImmutableFileNumber};
 use mithril_common::messages::CardanoDatabaseDigestListItemMessage;
@@ -107,8 +107,8 @@ pub async fn build_ancillary_files_archive(
 ) {
     let db_dir = cardano_db.get_dir();
     let ancillary_immutable_number = cardano_db.last_immutable_number().unwrap() + 1;
-    let last_ledger_file_path = cardano_db
-        .get_ledger_files()
+    let last_ledger_state_snapshot = cardano_db
+        .get_ledger_state_snapshots()
         .last()
         .expect("Given db should have at least one ledger file");
     let archive_name = format!(
@@ -120,15 +120,18 @@ pub async fn build_ancillary_files_archive(
     let enc = zstd::Encoder::new(tar_file, 3).unwrap();
     let mut tar = tar::Builder::new(enc);
 
-    let files_to_include = vec![
-        PathBuf::from(format!(
-            "ledger/{}",
-            last_ledger_file_path.file_name().unwrap().to_string_lossy()
-        )),
+    let mut files_to_include = vec![
         PathBuf::from(IMMUTABLE_DIR).join(format!("{ancillary_immutable_number:05}.chunk")),
         PathBuf::from(IMMUTABLE_DIR).join(format!("{ancillary_immutable_number:05}.primary")),
         PathBuf::from(IMMUTABLE_DIR).join(format!("{ancillary_immutable_number:05}.secondary")),
     ];
+    files_to_include.extend(
+        last_ledger_state_snapshot
+            .get_files_relative_path()
+            .into_iter()
+            .map(|p| PathBuf::from(LEDGER_DIR).join(p)),
+    );
+
     let ancillary_manifest = build_ancillary_manifest(
         db_dir,
         files_to_include.clone(),

mithril-client/tests/snapshot_list_get_show_download_verify.rs

@@ -22,7 +22,7 @@ async fn snapshot_list_get_show_download_verify() {
     let cardano_db = DummyCardanoDbBuilder::new("snapshot_list_get_show_download_verify_db")
         .with_immutables(&[1, 2, 3])
         .append_immutable_trio()
-        .with_ledger_files(&["506", "562"])
+        .with_legacy_ledger_snapshots(&[506, 562])
         .build();
     let fake_aggregator = FakeAggregator::new();
     let test_http_server = fake_aggregator