Merge pull request #2734 from input-output-hk/djo/2704/cdb-by-epoch-route

feat: add `/artifact/cardano-database/epoch/{epoch}` route + support epoch expansion from `latest` in some routes

Author: Alenar

Cargo.lock

@@ -1,6 +1,6 @@
 [package]
 name = "mithril-aggregator"
-version = "0.7.87"
+version = "0.7.88"
 description = "A Mithril Aggregator server"
 authors = { workspace = true }
 edition = { workspace = true }

mithril-aggregator/Cargo.toml

@@ -239,5 +239,17 @@ create table immutable_file_digest (
 drop table pending_certificate;
         "#,
         ),
+        // Migration 36
+        // Add `epoch` virtual column to `signed_entity` table.
+        //
+        // Note: because the epoch in the `beacon` field can be either stored directly as an integer
+        // or as a JSON property, we need to use a coalesce function to get the epoch value.
+        SqlMigration::new(
+            36,
+            r#"
+alter table signed_entity add column epoch as (coalesce(json_extract(beacon, '$.epoch'), beacon));
+create index signed_entity_epoch on signed_entity(epoch);
+        "#,
+        ),
     ]
 }

mithril-aggregator/src/database/migration.rs

@@ -61,14 +61,16 @@ impl GetSignedEntityRecordQuery {
         })
     }
 
-    pub fn cardano_stake_distribution_by_epoch(epoch: Epoch) -> Self {
-        let signed_entity_type_id =
-            SignedEntityTypeDiscriminants::CardanoStakeDistribution.index() as i64;
+    pub fn by_signed_entity_type_and_epoch(
+        signed_entity_type: &SignedEntityTypeDiscriminants,
+        epoch: Epoch,
+    ) -> Self {
+        let signed_entity_type_id = signed_entity_type.index() as i64;
         let epoch = *epoch as i64;
 
         Self {
             condition: WhereCondition::new(
-                "signed_entity_type_id = ?* and beacon = ?*",
+                "signed_entity_type_id = ?* and epoch = ?*",
                 vec![Value::Integer(signed_entity_type_id), Value::Integer(epoch)],
             ),
         }
@@ -93,134 +95,147 @@ impl Query for GetSignedEntityRecordQuery {
 
 #[cfg(test)]
 mod tests {
-    use chrono::DateTime;
-    use mithril_common::{
-        entities::{CardanoDbBeacon, SignedEntityType},
-        test::double::fake_data,
-    };
+    use mithril_common::entities::{BlockNumber, CardanoDbBeacon, SignedEntityType};
     use mithril_persistence::sqlite::ConnectionExtensions;
     use sqlite::ConnectionThreadSafe;
 
     use crate::database::test_helper::{insert_signed_entities, main_db_connection};
 
     use super::*;
 
-    fn create_database_with_cardano_stake_distributions<T: Into<SignedEntityRecord>>(
-        cardano_stake_distributions: Vec<T>,
-    ) -> (ConnectionThreadSafe, Vec<SignedEntityRecord>) {
-        let records = cardano_stake_distributions
-            .into_iter()
-            .map(|cardano_stake_distribution| cardano_stake_distribution.into())
-            .collect::<Vec<_>>();
-
-        let connection = create_database(&records);
-
-        (connection, records)
-    }
-
     fn create_database(records: &[SignedEntityRecord]) -> ConnectionThreadSafe {
         let connection = main_db_connection().unwrap();
         insert_signed_entities(&connection, records.to_vec()).unwrap();
+
         connection
     }
 
     #[test]
-    fn cardano_stake_distribution_by_epoch_returns_records_filtered_by_epoch() {
-        let mut cardano_stake_distributions = fake_data::cardano_stake_distributions(3);
-        cardano_stake_distributions[0].epoch = Epoch(3);
-        cardano_stake_distributions[1].epoch = Epoch(4);
-        cardano_stake_distributions[2].epoch = Epoch(5);
+    fn by_signed_entity_and_epoch_returns_records_filtered_by_epoch() {
+        let records = vec![
+            SignedEntityRecord::fake_with_signed_entity(
+                SignedEntityType::CardanoStakeDistribution(Epoch(3)),
+            ),
+            SignedEntityRecord::fake_with_signed_entity(
+                SignedEntityType::CardanoStakeDistribution(Epoch(4)),
+            ),
+            SignedEntityRecord::fake_with_signed_entity(
+                SignedEntityType::CardanoStakeDistribution(Epoch(5)),
+            ),
+        ];
 
-        let (connection, records) =
-            create_database_with_cardano_stake_distributions(cardano_stake_distributions);
+        let connection = create_database(&records);
 
         let records_retrieved: Vec<SignedEntityRecord> = connection
-            .fetch_collect(
-                GetSignedEntityRecordQuery::cardano_stake_distribution_by_epoch(Epoch(4)),
-            )
+            .fetch_collect(GetSignedEntityRecordQuery::by_signed_entity_type_and_epoch(
+                &SignedEntityTypeDiscriminants::CardanoStakeDistribution,
+                Epoch(4),
+            ))
             .unwrap();
 
         assert_eq!(vec![records[1].clone()], records_retrieved);
     }
 
     #[test]
-    fn cardano_stake_distribution_by_epoch_returns_records_returns_only_cardano_stake_distribution_records()
-     {
-        let cardano_stake_distributions_record: SignedEntityRecord = {
-            let mut cardano_stake_distribution = fake_data::cardano_stake_distribution(Epoch(4));
-            cardano_stake_distribution.hash = "hash-123".to_string();
-            cardano_stake_distribution.into()
-        };
-
-        let snapshots_record = {
-            let mut snapshot = fake_data::snapshots(1)[0].clone();
-            snapshot.beacon.epoch = Epoch(4);
-            SignedEntityRecord::from_snapshot(snapshot, "whatever".to_string(), DateTime::default())
-        };
-
-        let mithril_stake_distribution_record: SignedEntityRecord = {
-            let mithril_stake_distributions = fake_data::mithril_stake_distributions(1);
-            let mut mithril_stake_distribution = mithril_stake_distributions[0].clone();
-            mithril_stake_distribution.epoch = Epoch(4);
-            mithril_stake_distribution.into()
-        };
-
-        let connection = create_database(&[
-            cardano_stake_distributions_record.clone(),
-            snapshots_record,
-            mithril_stake_distribution_record,
-        ]);
+    fn by_signed_entity_and_epoch_returns_records_filtered_by_discriminant() {
+        let records = vec![
+            SignedEntityRecord::fake_with_signed_entity(
+                SignedEntityType::CardanoStakeDistribution(Epoch(3)),
+            ),
+            SignedEntityRecord::fake_with_signed_entity(
+                SignedEntityType::MithrilStakeDistribution(Epoch(3)),
+            ),
+            SignedEntityRecord::fake_with_signed_entity(SignedEntityType::CardanoDatabase(
+                CardanoDbBeacon::new(3, 98),
+            )),
+        ];
 
-        let records_retrieved: Vec<SignedEntityRecord> = connection
-            .fetch_collect(
-                GetSignedEntityRecordQuery::cardano_stake_distribution_by_epoch(Epoch(4)),
-            )
+        let connection = create_database(&records);
+
+        let fetched_msd_records: Vec<SignedEntityRecord> = connection
+            .fetch_collect(GetSignedEntityRecordQuery::by_signed_entity_type_and_epoch(
+                &SignedEntityTypeDiscriminants::MithrilStakeDistribution,
+                Epoch(3),
+            ))
             .unwrap();
+        assert_eq!(vec![records[1].clone()], fetched_msd_records);
 
-        assert_eq!(
-            vec![cardano_stake_distributions_record.clone()],
-            records_retrieved,
-        );
+        let fetched_cdb_records: Vec<SignedEntityRecord> = connection
+            .fetch_collect(GetSignedEntityRecordQuery::by_signed_entity_type_and_epoch(
+                &SignedEntityTypeDiscriminants::CardanoDatabase,
+                Epoch(3),
+            ))
+            .unwrap();
+        assert_eq!(vec![records[2].clone()], fetched_cdb_records);
     }
 
     #[test]
-    fn test_get_signed_entity_records() {
-        let signed_entity_records = SignedEntityRecord::fake_records(5);
+    fn test_get_record_by_id() {
+        let signed_entity_records = vec![
+            SignedEntityRecord::fake_with_signed_entity(
+                SignedEntityType::CardanoStakeDistribution(Epoch(3)),
+            ),
+            SignedEntityRecord::fake_with_signed_entity(SignedEntityType::CardanoTransactions(
+                Epoch(4),
+                BlockNumber(5),
+            )),
+        ];
 
         let connection = main_db_connection().unwrap();
         insert_signed_entities(&connection, signed_entity_records.clone()).unwrap();
 
-        let first_signed_entity_type = signed_entity_records.first().unwrap().to_owned();
-        let signed_entity_records: Vec<SignedEntityRecord> = connection
-            .fetch_collect(GetSignedEntityRecordQuery::by_signed_entity_id(
+        let first_signed_entity_type = signed_entity_records[0].clone();
+        let fetched_record = connection
+            .fetch_first(GetSignedEntityRecordQuery::by_signed_entity_id(
                 &first_signed_entity_type.signed_entity_id,
             ))
             .unwrap();
-        assert_eq!(vec![first_signed_entity_type], signed_entity_records);
+        assert_eq!(Some(first_signed_entity_type), fetched_record);
+    }
 
-        let signed_entity_records: Vec<SignedEntityRecord> = connection
+    #[test]
+    fn test_get_record_by_signed_entity_type() {
+        let signed_entity_records = vec![
+            SignedEntityRecord::fake_with_signed_entity(
+                SignedEntityType::MithrilStakeDistribution(Epoch(2)),
+            ),
+            SignedEntityRecord::fake_with_signed_entity(SignedEntityType::CardanoTransactions(
+                Epoch(4),
+                BlockNumber(5),
+            )),
+            SignedEntityRecord::fake_with_signed_entity(SignedEntityType::CardanoTransactions(
+                Epoch(5),
+                BlockNumber(9),
+            )),
+        ];
+
+        let connection = main_db_connection().unwrap();
+        insert_signed_entities(&connection, signed_entity_records.clone()).unwrap();
+
+        let fetched_tx_records: Vec<SignedEntityRecord> = connection
             .fetch_collect(
                 GetSignedEntityRecordQuery::by_signed_entity_type(
-                    &SignedEntityTypeDiscriminants::CardanoImmutableFilesFull,
+                    &SignedEntityTypeDiscriminants::CardanoTransactions,
                 )
                 .unwrap(),
             )
             .unwrap();
-        let expected_signed_entity_records: Vec<SignedEntityRecord> = signed_entity_records
-            .iter()
-            .filter_map(|se| {
-                (se.signed_entity_type.index()
-                    == SignedEntityType::CardanoImmutableFilesFull(CardanoDbBeacon::default())
-                        .index())
-                .then_some(se.to_owned())
-            })
-            .collect();
-        assert_eq!(expected_signed_entity_records, signed_entity_records);
+        let expected_tx_records: Vec<SignedEntityRecord> =
+            vec![signed_entity_records[2].clone(), signed_entity_records[1].clone()];
+        assert_eq!(expected_tx_records, fetched_tx_records);
+    }
+
+    #[test]
+    fn test_get_all_records() {
+        let signed_entity_records = SignedEntityRecord::fake_records(5);
+
+        let connection = main_db_connection().unwrap();
+        insert_signed_entities(&connection, signed_entity_records.clone()).unwrap();
 
-        let signed_entity_records: Vec<SignedEntityRecord> =
+        let fetched_records: Vec<SignedEntityRecord> =
             connection.fetch_collect(GetSignedEntityRecordQuery::all()).unwrap();
-        let expected_signed_entity_records: Vec<SignedEntityRecord> =
-            signed_entity_records.iter().map(|c| c.to_owned()).collect();
-        assert_eq!(expected_signed_entity_records, signed_entity_records);
+        let expected_signed_entity_records: Vec<_> =
+            signed_entity_records.into_iter().rev().collect();
+        assert_eq!(expected_signed_entity_records, fetched_records);
     }
 }

mithril-aggregator/src/database/query/signed_entity/get_signed_entity.rs

@@ -64,8 +64,68 @@ impl From<MithrilStakeDistribution> for SignedEntityRecord {
     }
 }
 
+#[cfg(test)]
+impl From<CardanoDatabaseSnapshot> for SignedEntityRecord {
+    fn from(value: CardanoDatabaseSnapshot) -> Self {
+        let entity = serde_json::to_string(&value).unwrap();
+
+        SignedEntityRecord {
+            signed_entity_id: value.hash.clone(),
+            signed_entity_type: SignedEntityType::CardanoDatabase(value.beacon),
+            certificate_id: format!("certificate-{}", value.hash),
+            artifact: entity,
+            created_at: DateTime::parse_from_rfc3339("2023-01-19T13:43:05.618857482Z")
+                .unwrap()
+                .with_timezone(&Utc),
+        }
+    }
+}
+
 #[cfg(test)]
 impl SignedEntityRecord {
+    pub(crate) fn fake_with_signed_entity(signed_entity_type: SignedEntityType) -> Self {
+        use mithril_common::test::double::fake_data;
+        fn get_id_and_artifact(artifact: &(impl Artifact + serde::Serialize)) -> (String, String) {
+            (artifact.get_id(), serde_json::to_string(artifact).unwrap())
+        }
+
+        let (id, artifact) = match signed_entity_type.clone() {
+            SignedEntityType::MithrilStakeDistribution(epoch) => {
+                let artifact = fake_data::mithril_stake_distribution(epoch, vec![]);
+                get_id_and_artifact(&artifact)
+            }
+            SignedEntityType::CardanoStakeDistribution(epoch) => {
+                let artifact = fake_data::cardano_stake_distribution(epoch);
+                get_id_and_artifact(&artifact)
+            }
+            SignedEntityType::CardanoImmutableFilesFull(cardano_db_beacon) => {
+                let mut artifact = fake_data::snapshot(cardano_db_beacon.immutable_file_number);
+                artifact.beacon = cardano_db_beacon;
+                get_id_and_artifact(&artifact)
+            }
+            SignedEntityType::CardanoDatabase(cardano_db_beacon) => {
+                let mut artifact =
+                    fake_data::cardano_database_snapshot(cardano_db_beacon.immutable_file_number);
+                artifact.beacon = cardano_db_beacon;
+                get_id_and_artifact(&artifact)
+            }
+            SignedEntityType::CardanoTransactions(_epoch, block_number) => {
+                let artifact = fake_data::cardano_transactions_snapshot(block_number);
+                get_id_and_artifact(&artifact)
+            }
+        };
+
+        SignedEntityRecord {
+            signed_entity_id: id.clone(),
+            signed_entity_type,
+            certificate_id: format!("certificate-{id}"),
+            artifact,
+            created_at: DateTime::parse_from_rfc3339("2023-01-19T13:43:05.618857482Z")
+                .unwrap()
+                .with_timezone(&Utc),
+        }
+    }
+
     pub(crate) fn from_snapshot(
         snapshot: Snapshot,
         certificate_id: String,
@@ -441,8 +501,7 @@ mod tests {
 
     #[test]
     fn test_convert_signed_entity() {
-        let snapshots = fake_data::snapshots(1);
-        let snapshot = snapshots.first().unwrap().to_owned();
+        let snapshot = fake_data::snapshot(1);
         let snapshot_expected = snapshot.clone();
 
         let signed_entity: SignedEntityRecord = SignedEntityRecord::from_snapshot(