diff --git a/deny.toml b/deny.toml index bb2a4118f..85be20882 100644 --- a/deny.toml +++ b/deny.toml @@ -39,3 +39,8 @@ name = "ring" [[licenses.clarify.license-files]] hash = 3171872035 path = "LICENSE" + +[sources] +allow-git = [ + "https://github.com/n0-computer/iroh", +] \ No newline at end of file diff --git a/src/api/blobs.rs b/src/api/blobs.rs index 897e0371c..4f8384cca 100644 --- a/src/api/blobs.rs +++ b/src/api/blobs.rs @@ -144,7 +144,6 @@ impl Blobs { /// clears the protections before. /// /// Users should rely only on garbage collection for blob deletion. - #[cfg(feature = "fs-store")] pub(crate) async fn delete_with_opts(&self, options: DeleteOptions) -> RequestResult<()> { trace!("{options:?}"); self.client.rpc(options).await??; @@ -152,7 +151,6 @@ impl Blobs { } /// See [`Self::delete_with_opts`]. - #[cfg(feature = "fs-store")] pub(crate) async fn delete( &self, hashes: impl IntoIterator>, diff --git a/src/api/blobs/reader.rs b/src/api/blobs/reader.rs index 5077c2632..294d916ef 100644 --- a/src/api/blobs/reader.rs +++ b/src/api/blobs/reader.rs @@ -225,10 +225,11 @@ mod tests { protocol::ChunkRangesExt, store::{ fs::{ - tests::{create_n0_bao, test_data, INTERESTING_SIZES}, + tests::{test_data, INTERESTING_SIZES}, FsStore, }, mem::MemStore, + util::tests::create_n0_bao, }, }; diff --git a/src/api/remote.rs b/src/api/remote.rs index dcfbc4fb4..09754d566 100644 --- a/src/api/remote.rs +++ b/src/api/remote.rs @@ -1073,10 +1073,11 @@ mod tests { protocol::{ChunkRangesExt, ChunkRangesSeq, GetRequest}, store::{ fs::{ - tests::{create_n0_bao, test_data, INTERESTING_SIZES}, + tests::{test_data, INTERESTING_SIZES}, FsStore, }, mem::MemStore, + util::tests::create_n0_bao, }, tests::{add_test_hash_seq, add_test_hash_seq_incomplete}, }; diff --git a/src/store/fs.rs b/src/store/fs.rs index 48946abd6..46d391178 100644 --- a/src/store/fs.rs +++ b/src/store/fs.rs @@ -91,7 +91,6 @@ use bytes::Bytes; use delete_set::{BaoFilePart, ProtectHandle}; use entity_manager::{EntityManagerState, SpawnArg}; use entry_state::{DataLocation, OutboardLocation}; -use gc::run_gc; use import::{ImportEntry, ImportSource}; use irpc::{channel::mpsc, RpcMessage}; use meta::list_blobs; @@ -120,6 +119,7 @@ use crate::{ }, util::entity_manager::{self, ActiveEntityState}, }, + gc::run_gc, util::{BaoTreeSender, FixedSize, MemOrFile, ValueOrPoisioned}, IROH_BLOCK_SIZE, }, @@ -141,7 +141,6 @@ use entry_state::EntryState; use import::{import_byte_stream, import_bytes, import_path, ImportEntryMsg}; use options::Options; use tracing::Instrument; -mod gc; use crate::{ api::{ @@ -1498,10 +1497,7 @@ pub mod tests { use core::panic; use std::collections::{HashMap, HashSet}; - use bao_tree::{ - io::{outboard::PreOrderMemOutboard, round_up_to_chunks_groups}, - ChunkRanges, - }; + use bao_tree::{io::round_up_to_chunks_groups, ChunkRanges}; use n0_future::{stream, Stream, StreamExt}; use testresult::TestResult; use walkdir::WalkDir; @@ -1510,7 +1506,7 @@ pub mod tests { use crate::{ api::blobs::Bitfield, store::{ - util::{read_checksummed, SliceInfoExt, Tag}, + util::{read_checksummed, tests::create_n0_bao, SliceInfoExt, Tag}, IROH_BLOCK_SIZE, }, }; @@ -1527,17 +1523,6 @@ pub mod tests { 1024 * 1024 * 8, // data file, outboard file ]; - /// Create n0 flavoured bao. Note that this can be used to request ranges below a chunk group size, - /// which can not be exported via bao because we don't store hashes below the chunk group level. - pub fn create_n0_bao(data: &[u8], ranges: &ChunkRanges) -> anyhow::Result<(Hash, Vec)> { - let outboard = PreOrderMemOutboard::create(data, IROH_BLOCK_SIZE); - let mut encoded = Vec::new(); - let size = data.len() as u64; - encoded.extend_from_slice(&size.to_le_bytes()); - bao_tree::io::sync::encode_ranges_validated(data, &outboard, ranges, &mut encoded)?; - Ok((outboard.root.into(), encoded)) - } - pub fn round_up_request(size: u64, ranges: &ChunkRanges) -> ChunkRanges { let last_chunk = ChunkNum::chunks(size); let data_range = ChunkRanges::from(..last_chunk); diff --git a/src/store/fs/options.rs b/src/store/fs/options.rs index f7dfa82f6..afd723c5b 100644 --- a/src/store/fs/options.rs +++ b/src/store/fs/options.rs @@ -4,8 +4,8 @@ use std::{ time::Duration, }; -pub use super::gc::{GcConfig, ProtectCb, ProtectOutcome}; use super::{meta::raw_outboard_size, temp_name}; +pub use crate::store::gc::{GcConfig, ProtectCb, ProtectOutcome}; use crate::Hash; /// Options for directories used by the file store. diff --git a/src/store/fs/gc.rs b/src/store/gc.rs similarity index 95% rename from src/store/fs/gc.rs rename to src/store/gc.rs index da7836e76..abb9903e4 100644 --- a/src/store/fs/gc.rs +++ b/src/store/gc.rs @@ -240,12 +240,9 @@ pub async fn run_gc(store: Store, config: GcConfig) { #[cfg(test)] mod tests { - use std::{ - io::{self}, - path::Path, - }; + use std::io::{self}; - use bao_tree::{io::EncodeError, ChunkNum}; + use bao_tree::io::EncodeError; use range_collections::RangeSet2; use testresult::TestResult; @@ -253,7 +250,6 @@ mod tests { use crate::{ api::{blobs::AddBytesOptions, ExportBaoError, RequestError, Store}, hashseq::HashSeq, - store::fs::{options::PathOptions, tests::create_n0_bao}, BlobFormat, }; @@ -266,6 +262,7 @@ mod tests { let et = blobs.add_slice("e").temp_tag().await?; let ft = blobs.add_slice("f").temp_tag().await?; let gt = blobs.add_slice("g").temp_tag().await?; + let ht = blobs.add_slice("h").with_named_tag("h").await?; let a = *at.hash(); let b = *bt.hash(); let c = *ct.hash(); @@ -273,6 +270,7 @@ mod tests { let e = *et.hash(); let f = *ft.hash(); let g = *gt.hash(); + let h = ht.hash; store.tags().set("c", *ct.hash_and_format()).await?; let dehs = [d, e].into_iter().collect::(); let hehs = blobs @@ -292,6 +290,7 @@ mod tests { store.tags().set("fg", *fghs.hash_and_format()).await?; drop(fghs); drop(bt); + store.tags().delete("h").await?; let mut live = HashSet::new(); gc_run_once(store, &mut live).await?; // a is protected because we keep the temp tag @@ -313,12 +312,19 @@ mod tests { assert!(store.has(f).await?); assert!(live.contains(&g)); assert!(store.has(g).await?); + // h is not protected because we deleted the tag before gc ran + assert!(!live.contains(&h)); + assert!(!store.has(h).await?); drop(at); drop(hehs); Ok(()) } - async fn gc_file_delete(path: &Path, store: &Store) -> TestResult<()> { + #[cfg(feature = "fs-store")] + async fn gc_file_delete(path: &std::path::Path, store: &Store) -> TestResult<()> { + use bao_tree::ChunkNum; + + use crate::store::{fs::options::PathOptions, util::tests::create_n0_bao}; let mut live = HashSet::new(); let options = PathOptions::new(&path.join("db")); // create a large complete file and check that the data and outboard files are deleted by gc @@ -366,6 +372,7 @@ mod tests { } #[tokio::test] + #[cfg(feature = "fs-store")] async fn gc_smoke_fs() -> TestResult { tracing_subscriber::fmt::try_init().ok(); let testdir = tempfile::tempdir()?; @@ -385,6 +392,7 @@ mod tests { } #[tokio::test] + #[cfg(feature = "fs-store")] async fn gc_check_deletion_fs() -> TestResult { tracing_subscriber::fmt::try_init().ok(); let testdir = tempfile::tempdir()?; diff --git a/src/store/mem.rs b/src/store/mem.rs index e5529e7fa..b5a3a8ce6 100644 --- a/src/store/mem.rs +++ b/src/store/mem.rs @@ -58,6 +58,7 @@ use crate::{ }, protocol::ChunkRangesExt, store::{ + gc::{run_gc, GcConfig}, util::{SizeInfo, SparseMemFile, Tag}, IROH_BLOCK_SIZE, }, @@ -66,7 +67,9 @@ use crate::{ }; #[derive(Debug, Default)] -pub struct Options {} +pub struct Options { + pub gc_config: Option, +} #[derive(Debug, Clone)] #[repr(transparent)] @@ -113,6 +116,10 @@ impl MemStore { } pub fn new() -> Self { + Self::new_with_opts(Options::default()) + } + + pub fn new_with_opts(opts: Options) -> Self { let (sender, receiver) = tokio::sync::mpsc::channel(32); tokio::spawn( Actor { @@ -130,7 +137,13 @@ impl MemStore { } .run(), ); - Self::from_sender(sender.into()) + + let store = Self::from_sender(sender.into()); + if let Some(gc_config) = opts.gc_config { + tokio::spawn(run_gc(store.deref().clone(), gc_config)); + } + + store } } diff --git a/src/store/mod.rs b/src/store/mod.rs index 4fdb30606..9d7290da5 100644 --- a/src/store/mod.rs +++ b/src/store/mod.rs @@ -7,6 +7,7 @@ use bao_tree::BlockSize; #[cfg(feature = "fs-store")] pub mod fs; +mod gc; pub mod mem; pub mod readonly_mem; mod test; diff --git a/src/store/util.rs b/src/store/util.rs index 7bc3a3227..03be152bb 100644 --- a/src/store/util.rs +++ b/src/store/util.rs @@ -404,3 +404,22 @@ impl bao_tree::io::mixed::Sender for BaoTreeSender { self.0.send(item).await } } + +#[cfg(test)] +#[cfg(feature = "fs-store")] +pub mod tests { + use bao_tree::{io::outboard::PreOrderMemOutboard, ChunkRanges}; + + use crate::{hash::Hash, store::IROH_BLOCK_SIZE}; + + /// Create n0 flavoured bao. Note that this can be used to request ranges below a chunk group size, + /// which can not be exported via bao because we don't store hashes below the chunk group level. + pub fn create_n0_bao(data: &[u8], ranges: &ChunkRanges) -> anyhow::Result<(Hash, Vec)> { + let outboard = PreOrderMemOutboard::create(data, IROH_BLOCK_SIZE); + let mut encoded = Vec::new(); + let size = data.len() as u64; + encoded.extend_from_slice(&size.to_le_bytes()); + bao_tree::io::sync::encode_ranges_validated(data, &outboard, ranges, &mut encoded)?; + Ok((outboard.root.into(), encoded)) + } +} diff --git a/src/tests.rs b/src/tests.rs index 09b2e5b33..c280f0004 100644 --- a/src/tests.rs +++ b/src/tests.rs @@ -19,11 +19,11 @@ use crate::{ provider::events::{AbortReason, EventMask, EventSender, ProviderMessage, RequestUpdate}, store::{ fs::{ - tests::{create_n0_bao, test_data, INTERESTING_SIZES}, + tests::{test_data, INTERESTING_SIZES}, FsStore, }, mem::MemStore, - util::observer::Combine, + util::{observer::Combine, tests::create_n0_bao}, }, util::sink::Drain, BlobFormat, Hash, HashAndFormat,