Merge pull request #1678 from input-output-hk/djo/common_ctx_block_range_roots_repository

Common Cardano transactions & Block range roots repository

Author: Alenar

Cargo.lock

@@ -1,6 +1,6 @@
 [package]
 name = "mithril-persistence"
-version = "0.1.10"
+version = "0.1.11"
 description = "Common types, interfaces, and utilities to persist data for Mithril nodes."
 authors = { workspace = true }
 edition = { workspace = true }
@@ -16,7 +16,7 @@ anyhow = "1.0.79"
 async-trait = "0.1.77"
 chrono = { version = "0.4.33", features = ["serde"] }
 hex = "0.4.3"
-mithril-common = { path = "../../mithril-common" }
+mithril-common = { path = "../../mithril-common", features = ["fs"] }
 semver = "1.0.21"
 serde = { version = "1.0.196", features = ["derive"] }
 serde_json = "1.0.113"

internal/mithril-persistence/Cargo.toml

@@ -1,6 +1,6 @@
 //! Migration module for cardano transactions store
 //!
-use mithril_persistence::database::SqlMigration;
+use crate::database::SqlMigration;
 
 /// Get all the migrations required by this version of the software.
 /// There shall be one migration per database version. There could be several

mithril-signer/src/database/cardano_transaction_migration.rs renamed to internal/mithril-persistence/src/database/cardano_transaction_migration.rs

@@ -1,17 +1,42 @@
-//! Signed Entity helpers for persistence
+//! Shared hydrator helpers for persistence
 
 use mithril_common::entities::{
     CardanoDbBeacon, Epoch, SignedEntityType, SignedEntityTypeDiscriminants,
 };
 
 use crate::sqlite::HydrationError;
 
-/// Helper struct to hydrate [SignedEntityType].
-pub struct SignedEntityTypeHydrator {}
+/// Helper struct to hydrate common data.
+pub struct Hydrator;
+
+impl Hydrator {
+    /// Read a signed entity beacon column from the database
+    pub fn read_signed_entity_beacon_column<U: sqlite::RowIndex + Clone>(
+        row: &sqlite::Row,
+        column_index: U,
+    ) -> String {
+        // We need to check first that the cell can be read as a string first
+        // (e.g. when beacon json is '{"network": "dev", "epoch": 1, "immutable_file_number": 2}').
+        // If it fails, we fallback on reading the cell as an integer (e.g. when beacon json is '5').
+        // TODO: Maybe there is a better way of doing this.
+        match row.try_read::<&str, _>(column_index.clone()) {
+            Ok(value) => value.to_string(),
+            Err(_) => (row.read::<i64, _>(column_index)).to_string(),
+        }
+    }
+
+    /// Try to convert an i64 field from the database to a u64
+    pub fn try_to_u64(field: &str, value: i64) -> Result<u64, HydrationError> {
+        u64::try_from(value)
+            .map_err(|e|
+                HydrationError::InvalidData(
+                    format!("Integer field {field} (value={value}) is incompatible with u64 representation. Error = {e}")
+                )
+            )
+    }
 
-impl SignedEntityTypeHydrator {
     /// Create a [SignedEntityType] from data coming from the database
-    pub fn hydrate(
+    pub fn hydrate_signed_entity_type(
         signed_entity_type_id: usize,
         beacon_str: &str,
     ) -> Result<SignedEntityType, HydrationError> {

internal/mithril-persistence/src/database/signed_entity_hydrator.rs renamed to internal/mithril-persistence/src/database/hydrator.rs

@@ -1,13 +1,35 @@
 //! database module.
 //! This module contains providers and entities shared between all application types.
 
+pub mod cardano_transaction_migration;
 mod db_version;
-mod signed_entity_hydrator;
+mod hydrator;
+pub(crate) mod provider;
+pub mod record;
+pub mod repository;
 mod version_checker;
 
+pub use db_version::*;
+pub use hydrator::Hydrator;
+pub use version_checker::{DatabaseVersionChecker, SqlMigration};
+
 /// Database version.
 pub type DbVersion = i64;
 
-pub use db_version::*;
-pub use signed_entity_hydrator::SignedEntityTypeHydrator;
-pub use version_checker::{DatabaseVersionChecker, SqlMigration};
+#[cfg(test)]
+pub mod test_helper {
+    use sqlite::ConnectionThreadSafe;
+
+    use mithril_common::StdResult;
+
+    use crate::sqlite::{ConnectionBuilder, ConnectionOptions};
+
+    /// In-memory sqlite database without foreign key support with cardano db migrations applied
+    pub fn cardano_tx_db_connection() -> StdResult<ConnectionThreadSafe> {
+        let connection = ConnectionBuilder::open_memory()
+            .with_options(&[ConnectionOptions::ForceDisableForeignKeys])
+            .with_migrations(crate::database::cardano_transaction_migration::get_migrations())
+            .build()?;
+        Ok(connection)
+    }
+}

internal/mithril-persistence/src/database/mod.rs

@@ -1,10 +1,8 @@
 use mithril_common::entities::BlockNumber;
-use mithril_persistence::sqlite::{
-    Provider, SourceAlias, SqLiteEntity, SqliteConnection, WhereCondition,
-};
 use sqlite::Value;
 
 use crate::database::record::BlockRangeRootRecord;
+use crate::sqlite::{Provider, SourceAlias, SqLiteEntity, SqliteConnection, WhereCondition};
 
 /// Simple queries to retrieve [BlockRangeRootRecord] from the sqlite database.
 pub struct GetBlockRangeRootProvider<'client> {
@@ -22,8 +20,7 @@ impl<'client> GetBlockRangeRootProvider<'client> {
     }
 }
 
-#[cfg(test)]
-impl mithril_persistence::sqlite::GetAllCondition for GetBlockRangeRootProvider<'_> {}
+impl crate::sqlite::GetAllCondition for GetBlockRangeRootProvider<'_> {}
 
 impl<'client> Provider<'client> for GetBlockRangeRootProvider<'client> {
     type Entity = BlockRangeRootRecord;

mithril-signer/src/database/provider/block_range_root/get_block_range_root.rs renamed to internal/mithril-persistence/src/database/provider/block_range_root/get_block_range_root.rs

@@ -1,8 +1,5 @@
-use mithril_persistence::sqlite::{
-    Provider, SourceAlias, SqLiteEntity, SqliteConnection, WhereCondition,
-};
-
 use crate::database::record::IntervalWithoutBlockRangeRootRecord;
+use crate::sqlite::{Provider, SourceAlias, SqLiteEntity, SqliteConnection, WhereCondition};
 
 /// Query that return the interval of block numbers that does not have a block range root.
 pub struct GetIntervalWithoutBlockRangeRootProvider<'client> {

mithril-signer/src/database/provider/block_range_root/get_interval_without_block_range_provider.rs renamed to internal/mithril-persistence/src/database/provider/block_range_root/get_interval_without_block_range_provider.rs

@@ -3,11 +3,9 @@ use std::iter::repeat;
 use sqlite::Value;
 
 use mithril_common::StdResult;
-use mithril_persistence::sqlite::{
-    Provider, SourceAlias, SqLiteEntity, SqliteConnection, WhereCondition,
-};
 
 use crate::database::record::BlockRangeRootRecord;
+use crate::sqlite::{Provider, SourceAlias, SqLiteEntity, SqliteConnection, WhereCondition};
 
 /// Query to insert [BlockRangeRootRecord] in the sqlite database
 pub struct InsertBlockRangeRootProvider<'client> {

mithril-aggregator/src/database/provider/block_range_root/insert_block_range.rs renamed to internal/mithril-persistence/src/database/provider/block_range_root/insert_block_range.rs

@@ -3,13 +3,11 @@ use std::ops::Range;
 use sqlite::Value;
 
 use mithril_common::entities::{BlockNumber, BlockRange, TransactionHash};
-#[cfg(test)]
-use mithril_persistence::sqlite::GetAllCondition;
-use mithril_persistence::sqlite::{
-    Provider, SourceAlias, SqLiteEntity, SqliteConnection, WhereCondition,
-};
 
 use crate::database::record::CardanoTransactionRecord;
+use crate::sqlite::{
+    GetAllCondition, Provider, SourceAlias, SqLiteEntity, SqliteConnection, WhereCondition,
+};
 
 /// Simple queries to retrieve [CardanoTransaction] from the sqlite database.
 pub struct GetCardanoTransactionProvider<'client> {
@@ -90,5 +88,4 @@ impl<'client> Provider<'client> for GetCardanoTransactionProvider<'client> {
     }
 }
 
-#[cfg(test)]
 impl GetAllCondition for GetCardanoTransactionProvider<'_> {}