From ee8a6fdc2f943bbd2fd46d53a12da8e09b7355bc Mon Sep 17 00:00:00 2001 From: Ruediger Klaehn Date: Wed, 30 Oct 2024 17:57:36 +0200 Subject: [PATCH 01/20] Remove rpc types and blobs client and use the one from iroh-blobs --- Cargo.lock | 9 +- Cargo.toml | 3 +- iroh-cli/src/commands/blobs.rs | 2 +- iroh/examples/collection-provide.rs | 13 +- iroh/examples/hello-world-provide.rs | 9 +- iroh/src/client.rs | 17 +- iroh/src/client/blobs.rs | 1677 -------------------------- iroh/src/client/docs.rs | 9 +- iroh/src/client/tags.rs | 60 - iroh/src/node.rs | 10 +- iroh/src/node/rpc.rs | 8 +- iroh/src/rpc_protocol.rs | 10 +- iroh/src/rpc_protocol/blobs.rs | 319 ----- iroh/src/rpc_protocol/net.rs | 1 + iroh/src/rpc_protocol/tags.rs | 109 -- iroh/tests/provide.rs | 16 +- 16 files changed, 56 insertions(+), 2216 deletions(-) delete mode 100644 iroh/src/client/blobs.rs delete mode 100644 iroh/src/client/tags.rs delete mode 100644 iroh/src/rpc_protocol/blobs.rs delete mode 100644 iroh/src/rpc_protocol/tags.rs diff --git a/Cargo.lock b/Cargo.lock index c8cdf6109cb..1ce1f066746 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2547,7 +2547,6 @@ dependencies = [ [[package]] name = "iroh-blobs" version = "0.27.0" -source = "git+https://github.com/n0-computer/iroh-blobs?branch=main#378fc0fd8257c0beeb3a655a8378226a630b2d73" dependencies = [ "anyhow", "async-channel", @@ -2557,6 +2556,7 @@ dependencies = [ "derive_more", "futures-buffered", "futures-lite 2.3.0", + "futures-util", "genawaiter", "hashlink", "hex", @@ -2566,26 +2566,33 @@ dependencies = [ "iroh-net", "iroh-quinn", "iroh-router", + "nested_enum_utils", "num_cpus", "oneshot", "parking_lot", "pin-project", + "portable-atomic", "postcard", + "quic-rpc 0.13.0", + "quic-rpc-derive", "rand", "range-collections", "redb 1.5.1", "redb 2.1.1", + "ref-cast", "reflink-copy", "self_cell", "serde", "serde-error", "smallvec", + "strum 0.26.3", "tempfile", "thiserror", "tokio", "tokio-util", "tracing", "tracing-futures", + "walkdir", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index c069d76c1a9..d2be0a7940a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -54,6 +54,7 @@ iroh-metrics = { path = "./iroh-metrics" } iroh-test = { path = "./iroh-test" } iroh-router = { path = "./iroh-router" } -iroh-blobs = { git = "https://github.com/n0-computer/iroh-blobs", branch = "main" } +# iroh-blobs = { git = "https://github.com/n0-computer/iroh-blobs", branch = "main" } iroh-gossip = { git = "https://github.com/n0-computer/iroh-gossip", branch = "main" } iroh-docs = { git = "https://github.com/n0-computer/iroh-docs", branch = "main" } +iroh-blobs = { path = "../iroh-blobs" } \ No newline at end of file diff --git a/iroh-cli/src/commands/blobs.rs b/iroh-cli/src/commands/blobs.rs index 0be5657a87f..7c199df3e34 100644 --- a/iroh-cli/src/commands/blobs.rs +++ b/iroh-cli/src/commands/blobs.rs @@ -370,7 +370,7 @@ impl BlobCommands { BlobFormat::Raw }; let status = iroh.blobs().status(hash).await?; - let ticket = iroh.blobs().share(hash, format, addr_options).await?; + let ticket: BlobTicket = todo!(); let (blob_status, size) = match (status, format) { (BlobStatus::Complete { size }, BlobFormat::Raw) => ("blob", size), diff --git a/iroh/examples/collection-provide.rs b/iroh/examples/collection-provide.rs index 429288a2f2e..06d9f6dcfee 100644 --- a/iroh/examples/collection-provide.rs +++ b/iroh/examples/collection-provide.rs @@ -7,7 +7,7 @@ //! run this example from the project root: //! $ cargo run --example collection-provide use iroh::blobs::{format::collection::Collection, util::SetTagOption, BlobFormat}; -use iroh_base::node_addr::AddrInfoOptions; +use iroh_base::{node_addr::AddrInfoOptions, ticket::BlobTicket}; use tracing_subscriber::{prelude::*, EnvFilter}; // set the RUST_LOG env var to one of {debug,info,warn} to see logging info @@ -44,14 +44,9 @@ async fn main() -> anyhow::Result<()> { // create a ticket // tickets wrap all details needed to get a collection - let ticket = node - .blobs() - .share( - hash, - BlobFormat::HashSeq, - AddrInfoOptions::RelayAndAddresses, - ) - .await?; + let mut addr = node.net().node_addr().await?; + addr.apply_options(AddrInfoOptions::RelayAndAddresses); + let ticket = BlobTicket::new(addr, hash, BlobFormat::HashSeq)?; // print some info about the node println!("serving hash: {}", ticket.hash()); diff --git a/iroh/examples/hello-world-provide.rs b/iroh/examples/hello-world-provide.rs index 15c21801dea..a74bc3bae35 100644 --- a/iroh/examples/hello-world-provide.rs +++ b/iroh/examples/hello-world-provide.rs @@ -3,7 +3,7 @@ //! This is using an in memory database and a random node id. //! run this example from the project root: //! $ cargo run --example hello-world-provide -use iroh_base::node_addr::AddrInfoOptions; +use iroh_base::{node_addr::AddrInfoOptions, ticket::BlobTicket}; use tracing_subscriber::{prelude::*, EnvFilter}; // set the RUST_LOG env var to one of {debug,info,warn} to see logging info @@ -27,10 +27,9 @@ async fn main() -> anyhow::Result<()> { let res = node.blobs().add_bytes("Hello, world!").await?; // create a ticket - let ticket = node - .blobs() - .share(res.hash, res.format, AddrInfoOptions::RelayAndAddresses) - .await?; + let mut addr = node.net().node_addr().await?; + addr.apply_options(AddrInfoOptions::RelayAndAddresses); + let ticket = BlobTicket::new(addr, res.hash, res.format)?; // print some info about the node println!("serving hash: {}", ticket.hash()); diff --git a/iroh/src/client.rs b/iroh/src/client.rs index 54505682a03..514e4ca9646 100644 --- a/iroh/src/client.rs +++ b/iroh/src/client.rs @@ -19,10 +19,10 @@ pub(crate) use self::quic::{connect_raw as quic_connect_raw, RPC_ALPN}; pub use self::{docs::Doc, net::NodeStatus}; pub mod authors; -pub mod blobs; pub mod docs; pub mod net; -pub mod tags; +pub use iroh_blobs::rpc::client::blobs; +pub use iroh_blobs::rpc::client::tags; /// Iroh rpc connection - boxed so that we can have a concrete type. pub(crate) type RpcConnection = BoxedServiceConnection; @@ -60,8 +60,11 @@ impl Iroh { } /// Returns the blobs client. - pub fn blobs(&self) -> &blobs::Client { - blobs::Client::ref_cast(&self.rpc) + pub fn blobs( + &self, + ) -> iroh_blobs::rpc::client::blobs::Client, RpcService> + { + iroh_blobs::rpc::client::blobs::Client::new(self.rpc.clone().map()) } /// Returns the docs client. @@ -75,8 +78,10 @@ impl Iroh { } /// Returns the tags client. - pub fn tags(&self) -> &tags::Client { - tags::Client::ref_cast(&self.rpc) + pub fn tags( + &self, + ) -> iroh_blobs::rpc::client::tags::Client, RpcService> { + iroh_blobs::rpc::client::tags::Client::new(self.rpc.clone().map()) } /// Returns the gossip client. diff --git a/iroh/src/client/blobs.rs b/iroh/src/client/blobs.rs deleted file mode 100644 index 7bc4d889ad3..00000000000 --- a/iroh/src/client/blobs.rs +++ /dev/null @@ -1,1677 +0,0 @@ -//! API for blobs management. -//! -//! The main entry point is the [`Client`]. -//! -//! You obtain a [`Client`] via [`Iroh::blobs()`](crate::client::Iroh::blobs). -//! -//! ## Interacting with the local blob store -//! -//! ### Importing data -//! -//! There are several ways to import data into the local blob store: -//! -//! - [`add_bytes`](Client::add_bytes) -//! imports in memory data. -//! - [`add_stream`](Client::add_stream) -//! imports data from a stream of bytes. -//! - [`add_reader`](Client::add_reader) -//! imports data from an [async reader](tokio::io::AsyncRead). -//! - [`add_from_path`](Client::add_from_path) -//! imports data from a file. -//! -//! The last method imports data from a file on the local filesystem. -//! This is the most efficient way to import large amounts of data. -//! -//! ### Exporting data -//! -//! There are several ways to export data from the local blob store: -//! -//! - [`read_to_bytes`](Client::read_to_bytes) reads data into memory. -//! - [`read`](Client::read) creates a [reader](Reader) to read data from. -//! - [`export`](Client::export) eports data to a file on the local filesystem. -//! -//! ## Interacting with remote nodes -//! -//! - [`download`](Client::download) downloads data from a remote node. -//! - [`share`](Client::share) allows creating a ticket to share data with a -//! remote node. -//! -//! ## Interacting with the blob store itself -//! -//! These are more advanced operations that are usually not needed in normal -//! operation. -//! -//! - [`consistency_check`](Client::consistency_check) checks the internal -//! consistency of the local blob store. -//! - [`validate`](Client::validate) validates the locally stored data against -//! their BLAKE3 hashes. -//! - [`delete_blob`](Client::delete_blob) deletes a blob from the local store. -//! -//! ### Batch operations -//! -//! For complex update operations, there is a [`batch`](Client::batch) API that -//! allows you to add multiple blobs in a single logical batch. -//! -//! Operations in a batch return [temporary tags](crate::blobs::TempTag) that -//! protect the added data from garbage collection as long as the batch is -//! alive. -//! -//! To store the data permanently, a temp tag needs to be upgraded to a -//! permanent tag using [`persist`](crate::client::blobs::Batch::persist) or -//! [`persist_to`](crate::client::blobs::Batch::persist_to). -use std::{ - future::Future, - io, - path::PathBuf, - pin::Pin, - sync::Arc, - task::{Context, Poll}, -}; - -use anyhow::{anyhow, Context as _, Result}; -use bytes::Bytes; -use futures_lite::{Stream, StreamExt}; -use futures_util::SinkExt; -use genawaiter::sync::{Co, Gen}; -use iroh_base::{node_addr::AddrInfoOptions, ticket::BlobTicket}; -pub use iroh_blobs::net_protocol::DownloadMode; -use iroh_blobs::{ - export::ExportProgress as BytesExportProgress, - format::collection::{Collection, SimpleStore}, - get::db::DownloadProgress as BytesDownloadProgress, - net_protocol::BlobDownloadRequest, - store::{BaoBlobSize, ConsistencyCheckProgress, ExportFormat, ExportMode, ValidateProgress}, - util::SetTagOption, - BlobFormat, Hash, Tag, -}; -use iroh_net::NodeAddr; -use portable_atomic::{AtomicU64, Ordering}; -use quic_rpc::client::BoxStreamSync; -use ref_cast::RefCast; -use serde::{Deserialize, Serialize}; -use tokio::io::{AsyncRead, AsyncReadExt, ReadBuf}; -use tokio_util::io::{ReaderStream, StreamReader}; -use tracing::warn; - -mod batch; -pub use batch::{AddDirOpts, AddFileOpts, AddReaderOpts, Batch}; - -use super::{flatten, tags, Iroh, RpcClient}; -use crate::rpc_protocol::{ - blobs::{ - AddPathRequest, AddStreamRequest, AddStreamUpdate, BatchCreateRequest, BatchCreateResponse, - BlobStatusRequest, ConsistencyCheckRequest, CreateCollectionRequest, - CreateCollectionResponse, DeleteRequest, ExportRequest, ListIncompleteRequest, ListRequest, - ReadAtRequest, ReadAtResponse, ValidateRequest, - }, - node::StatusRequest, -}; - -/// Iroh blobs client. -#[derive(Debug, Clone, RefCast)] -#[repr(transparent)] -pub struct Client { - pub(super) rpc: RpcClient, -} - -impl<'a> From<&'a Iroh> for &'a RpcClient { - fn from(client: &'a Iroh) -> &'a RpcClient { - &client.blobs().rpc - } -} - -impl Client { - /// Check if a blob is completely stored on the node. - /// - /// Note that this will return false for blobs that are partially stored on - /// the node. - pub async fn status(&self, hash: Hash) -> Result { - let status = self.rpc.rpc(BlobStatusRequest { hash }).await??; - Ok(status.0) - } - - /// Check if a blob is completely stored on the node. - /// - /// This is just a convenience wrapper around `status` that returns a boolean. - pub async fn has(&self, hash: Hash) -> Result { - match self.status(hash).await { - Ok(BlobStatus::Complete { .. }) => Ok(true), - Ok(_) => Ok(false), - Err(err) => Err(err), - } - } - - /// Create a new batch for adding data. - /// - /// A batch is a context in which temp tags are created and data is added to the node. Temp tags - /// are automatically deleted when the batch is dropped, leading to the data being garbage collected - /// unless a permanent tag is created for it. - pub async fn batch(&self) -> Result { - let (updates, mut stream) = self.rpc.bidi(BatchCreateRequest).await?; - let BatchCreateResponse::Id(batch) = stream.next().await.context("expected scope id")??; - let rpc = self.rpc.clone(); - Ok(Batch::new(batch, rpc, updates, 1024)) - } - - /// Stream the contents of a a single blob. - /// - /// Returns a [`Reader`], which can report the size of the blob before reading it. - pub async fn read(&self, hash: Hash) -> Result { - Reader::from_rpc_read(&self.rpc, hash).await - } - - /// Read offset + len from a single blob. - /// - /// If `len` is `None` it will read the full blob. - pub async fn read_at(&self, hash: Hash, offset: u64, len: ReadAtLen) -> Result { - Reader::from_rpc_read_at(&self.rpc, hash, offset, len).await - } - - /// Read all bytes of single blob. - /// - /// This allocates a buffer for the full blob. Use only if you know that the blob you're - /// reading is small. If not sure, use [`Self::read`] and check the size with - /// [`Reader::size`] before calling [`Reader::read_to_bytes`]. - pub async fn read_to_bytes(&self, hash: Hash) -> Result { - Reader::from_rpc_read(&self.rpc, hash) - .await? - .read_to_bytes() - .await - } - - /// Read all bytes of single blob at `offset` for length `len`. - /// - /// This allocates a buffer for the full length. - pub async fn read_at_to_bytes(&self, hash: Hash, offset: u64, len: ReadAtLen) -> Result { - Reader::from_rpc_read_at(&self.rpc, hash, offset, len) - .await? - .read_to_bytes() - .await - } - - /// Import a blob from a filesystem path. - /// - /// `path` should be an absolute path valid for the file system on which - /// the node runs. - /// If `in_place` is true, Iroh will assume that the data will not change and will share it in - /// place without copying to the Iroh data directory. - pub async fn add_from_path( - &self, - path: PathBuf, - in_place: bool, - tag: SetTagOption, - wrap: WrapOption, - ) -> Result { - let stream = self - .rpc - .server_streaming(AddPathRequest { - path, - in_place, - tag, - wrap, - }) - .await?; - Ok(AddProgress::new(stream)) - } - - /// Create a collection from already existing blobs. - /// - /// For automatically clearing the tags for the passed in blobs you can set - /// `tags_to_delete` to those tags, and they will be deleted once the collection is created. - pub async fn create_collection( - &self, - collection: Collection, - tag: SetTagOption, - tags_to_delete: Vec, - ) -> anyhow::Result<(Hash, Tag)> { - let CreateCollectionResponse { hash, tag } = self - .rpc - .rpc(CreateCollectionRequest { - collection, - tag, - tags_to_delete, - }) - .await??; - Ok((hash, tag)) - } - - /// Write a blob by passing an async reader. - pub async fn add_reader( - &self, - reader: impl AsyncRead + Unpin + Send + 'static, - tag: SetTagOption, - ) -> anyhow::Result { - const CAP: usize = 1024 * 64; // send 64KB per request by default - let input = ReaderStream::with_capacity(reader, CAP); - self.add_stream(input, tag).await - } - - /// Write a blob by passing a stream of bytes. - pub async fn add_stream( - &self, - input: impl Stream> + Send + Unpin + 'static, - tag: SetTagOption, - ) -> anyhow::Result { - let (mut sink, progress) = self.rpc.bidi(AddStreamRequest { tag }).await?; - let mut input = input.map(|chunk| match chunk { - Ok(chunk) => Ok(AddStreamUpdate::Chunk(chunk)), - Err(err) => { - warn!("Abort send, reason: failed to read from source stream: {err:?}"); - Ok(AddStreamUpdate::Abort) - } - }); - tokio::spawn(async move { - // TODO: Is it important to catch this error? It should also result in an error on the - // response stream. If we deem it important, we could one-shot send it into the - // BlobAddProgress and return from there. Not sure. - if let Err(err) = sink.send_all(&mut input).await { - warn!("Failed to send input stream to remote: {err:?}"); - } - }); - - Ok(AddProgress::new(progress)) - } - - /// Write a blob by passing bytes. - pub async fn add_bytes(&self, bytes: impl Into) -> anyhow::Result { - let input = futures_lite::stream::once(Ok(bytes.into())); - self.add_stream(input, SetTagOption::Auto).await?.await - } - - /// Write a blob by passing bytes, setting an explicit tag name. - pub async fn add_bytes_named( - &self, - bytes: impl Into, - name: impl Into, - ) -> anyhow::Result { - let input = futures_lite::stream::once(Ok(bytes.into())); - self.add_stream(input, SetTagOption::Named(name.into())) - .await? - .await - } - - /// Validate hashes on the running node. - /// - /// If `repair` is true, repair the store by removing invalid data. - pub async fn validate( - &self, - repair: bool, - ) -> Result>> { - let stream = self - .rpc - .server_streaming(ValidateRequest { repair }) - .await?; - Ok(stream.map(|res| res.map_err(anyhow::Error::from))) - } - - /// Validate hashes on the running node. - /// - /// If `repair` is true, repair the store by removing invalid data. - pub async fn consistency_check( - &self, - repair: bool, - ) -> Result>> { - let stream = self - .rpc - .server_streaming(ConsistencyCheckRequest { repair }) - .await?; - Ok(stream.map(|r| r.map_err(anyhow::Error::from))) - } - - /// Download a blob from another node and add it to the local database. - pub async fn download(&self, hash: Hash, node: NodeAddr) -> Result { - self.download_with_opts( - hash, - DownloadOptions { - format: BlobFormat::Raw, - nodes: vec![node], - tag: SetTagOption::Auto, - mode: DownloadMode::Queued, - }, - ) - .await - } - - /// Download a hash sequence from another node and add it to the local database. - pub async fn download_hash_seq(&self, hash: Hash, node: NodeAddr) -> Result { - self.download_with_opts( - hash, - DownloadOptions { - format: BlobFormat::HashSeq, - nodes: vec![node], - tag: SetTagOption::Auto, - mode: DownloadMode::Queued, - }, - ) - .await - } - - /// Download a blob, with additional options. - pub async fn download_with_opts( - &self, - hash: Hash, - opts: DownloadOptions, - ) -> Result { - let DownloadOptions { - format, - nodes, - tag, - mode, - } = opts; - let stream = self - .rpc - .server_streaming(BlobDownloadRequest { - hash, - format, - nodes, - tag, - mode, - }) - .await?; - Ok(DownloadProgress::new( - stream.map(|res| res.map_err(anyhow::Error::from)), - )) - } - - /// Export a blob from the internal blob store to a path on the node's filesystem. - /// - /// `destination` should be an writeable, absolute path on the local node's filesystem. - /// - /// If `format` is set to [`ExportFormat::Collection`], and the `hash` refers to a collection, - /// all children of the collection will be exported. See [`ExportFormat`] for details. - /// - /// The `mode` argument defines if the blob should be copied to the target location or moved out of - /// the internal store into the target location. See [`ExportMode`] for details. - pub async fn export( - &self, - hash: Hash, - destination: PathBuf, - format: ExportFormat, - mode: ExportMode, - ) -> Result { - let req = ExportRequest { - hash, - path: destination, - format, - mode, - }; - let stream = self.rpc.server_streaming(req).await?; - Ok(ExportProgress::new( - stream.map(|r| r.map_err(anyhow::Error::from)), - )) - } - - /// List all complete blobs. - pub async fn list(&self) -> Result>> { - let stream = self.rpc.server_streaming(ListRequest).await?; - Ok(flatten(stream)) - } - - /// List all incomplete (partial) blobs. - pub async fn list_incomplete(&self) -> Result>> { - let stream = self.rpc.server_streaming(ListIncompleteRequest).await?; - Ok(flatten(stream)) - } - - /// Read the content of a collection. - pub async fn get_collection(&self, hash: Hash) -> Result { - Collection::load(hash, self).await - } - - /// List all collections. - pub fn list_collections(&self) -> Result>> { - let this = self.clone(); - Ok(Gen::new(|co| async move { - if let Err(cause) = this.list_collections_impl(&co).await { - co.yield_(Err(cause)).await; - } - })) - } - - async fn list_collections_impl(&self, co: &Co>) -> Result<()> { - let tags = self.tags_client(); - let mut tags = tags.list_hash_seq().await?; - while let Some(tag) = tags.next().await { - let tag = tag?; - if let Ok(collection) = self.get_collection(tag.hash).await { - let info = CollectionInfo { - tag: tag.name, - hash: tag.hash, - total_blobs_count: Some(collection.len() as u64 + 1), - total_blobs_size: Some(0), - }; - co.yield_(Ok(info)).await; - } - } - Ok(()) - } - - /// Delete a blob. - /// - /// **Warning**: this operation deletes the blob from the local store even - /// if it is tagged. You should usually not do this manually, but rely on the - /// node to remove data that is not tagged. - pub async fn delete_blob(&self, hash: Hash) -> Result<()> { - self.rpc.rpc(DeleteRequest { hash }).await??; - Ok(()) - } - - /// Share a blob. - pub async fn share( - &self, - hash: Hash, - blob_format: BlobFormat, - addr_options: AddrInfoOptions, - ) -> Result { - let mut addr = self.rpc.rpc(StatusRequest).await??.addr; - addr.apply_options(addr_options); - let ticket = BlobTicket::new(addr, hash, blob_format).expect("correct ticket"); - - Ok(ticket) - } - - fn tags_client(&self) -> tags::Client { - tags::Client { - rpc: self.rpc.clone(), - } - } -} - -impl SimpleStore for Client { - async fn load(&self, hash: Hash) -> anyhow::Result { - self.read_to_bytes(hash).await - } -} - -/// Defines the way to read bytes. -#[derive(Debug, Serialize, Deserialize, Default, Clone, Copy)] -pub enum ReadAtLen { - /// Reads all available bytes. - #[default] - All, - /// Reads exactly this many bytes, erroring out on larger or smaller. - Exact(u64), - /// Reads at most this many bytes. - AtMost(u64), -} - -impl ReadAtLen { - pub(crate) fn as_result_len(&self, size_remaining: u64) -> u64 { - match self { - ReadAtLen::All => size_remaining, - ReadAtLen::Exact(len) => *len, - ReadAtLen::AtMost(len) => std::cmp::min(*len, size_remaining), - } - } -} - -/// Whether to wrap the added data in a collection. -#[derive(Debug, Serialize, Deserialize, Default, Clone)] -pub enum WrapOption { - /// Do not wrap the file or directory. - #[default] - NoWrap, - /// Wrap the file or directory in a collection. - Wrap { - /// Override the filename in the wrapping collection. - name: Option, - }, -} - -/// Status information about a blob. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub enum BlobStatus { - /// The blob is not stored at all. - NotFound, - /// The blob is only stored partially. - Partial { - /// The size of the currently stored partial blob. - size: BaoBlobSize, - }, - /// The blob is stored completely. - Complete { - /// The size of the blob. - size: u64, - }, -} - -/// Outcome of a blob add operation. -#[derive(Debug, Clone)] -pub struct AddOutcome { - /// The hash of the blob - pub hash: Hash, - /// The format the blob - pub format: BlobFormat, - /// The size of the blob - pub size: u64, - /// The tag of the blob - pub tag: Tag, -} - -/// Information about a stored collection. -#[derive(Debug, Serialize, Deserialize)] -pub struct CollectionInfo { - /// Tag of the collection - pub tag: Tag, - - /// Hash of the collection - pub hash: Hash, - /// Number of children in the collection - /// - /// This is an optional field, because the data is not always available. - pub total_blobs_count: Option, - /// Total size of the raw data referred to by all links - /// - /// This is an optional field, because the data is not always available. - pub total_blobs_size: Option, -} - -/// Information about a complete blob. -#[derive(Debug, Serialize, Deserialize)] -pub struct BlobInfo { - /// Location of the blob - pub path: String, - /// The hash of the blob - pub hash: Hash, - /// The size of the blob - pub size: u64, -} - -/// Information about an incomplete blob. -#[derive(Debug, Serialize, Deserialize)] -pub struct IncompleteBlobInfo { - /// The size we got - pub size: u64, - /// The size we expect - pub expected_size: u64, - /// The hash of the blob - pub hash: Hash, -} - -/// Progress stream for blob add operations. -#[derive(derive_more::Debug)] -pub struct AddProgress { - #[debug(skip)] - stream: Pin< - Box> + Send + Unpin + 'static>, - >, - current_total_size: Arc, -} - -impl AddProgress { - fn new( - stream: (impl Stream< - Item = Result, impl Into>, - > + Send - + Unpin - + 'static), - ) -> Self { - let current_total_size = Arc::new(AtomicU64::new(0)); - let total_size = current_total_size.clone(); - let stream = stream.map(move |item| match item { - Ok(item) => { - let item = item.into(); - if let iroh_blobs::provider::AddProgress::Found { size, .. } = &item { - total_size.fetch_add(*size, Ordering::Relaxed); - } - Ok(item) - } - Err(err) => Err(err.into()), - }); - Self { - stream: Box::pin(stream), - current_total_size, - } - } - /// Finish writing the stream, ignoring all intermediate progress events. - /// - /// Returns a [`AddOutcome`] which contains a tag, format, hash and a size. - /// When importing a single blob, this is the hash and size of that blob. - /// When importing a collection, the hash is the hash of the collection and the size - /// is the total size of all imported blobs (but excluding the size of the collection blob - /// itself). - pub async fn finish(self) -> Result { - self.await - } -} - -impl Stream for AddProgress { - type Item = Result; - fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - Pin::new(&mut self.stream).poll_next(cx) - } -} - -impl Future for AddProgress { - type Output = Result; - - fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { - loop { - match Pin::new(&mut self.stream).poll_next(cx) { - Poll::Pending => return Poll::Pending, - Poll::Ready(None) => { - return Poll::Ready(Err(anyhow!("Response stream ended prematurely"))) - } - Poll::Ready(Some(Err(err))) => return Poll::Ready(Err(err)), - Poll::Ready(Some(Ok(msg))) => match msg { - iroh_blobs::provider::AddProgress::AllDone { hash, format, tag } => { - let outcome = AddOutcome { - hash, - format, - tag, - size: self.current_total_size.load(Ordering::Relaxed), - }; - return Poll::Ready(Ok(outcome)); - } - iroh_blobs::provider::AddProgress::Abort(err) => { - return Poll::Ready(Err(err.into())); - } - _ => {} - }, - } - } - } -} - -/// Outcome of a blob download operation. -#[derive(Debug, Clone)] -pub struct DownloadOutcome { - /// The size of the data we already had locally - pub local_size: u64, - /// The size of the data we downloaded from the network - pub downloaded_size: u64, - /// Statistics about the download - pub stats: iroh_blobs::get::Stats, -} - -/// Progress stream for blob download operations. -#[derive(derive_more::Debug)] -pub struct DownloadProgress { - #[debug(skip)] - stream: Pin> + Send + Unpin + 'static>>, - current_local_size: Arc, - current_network_size: Arc, -} - -impl DownloadProgress { - /// Create a [`DownloadProgress`] that can help you easily poll the [`BytesDownloadProgress`] stream from your download until it is finished or errors. - pub fn new( - stream: (impl Stream, impl Into>> - + Send - + Unpin - + 'static), - ) -> Self { - let current_local_size = Arc::new(AtomicU64::new(0)); - let current_network_size = Arc::new(AtomicU64::new(0)); - - let local_size = current_local_size.clone(); - let network_size = current_network_size.clone(); - - let stream = stream.map(move |item| match item { - Ok(item) => { - let item = item.into(); - match &item { - BytesDownloadProgress::FoundLocal { size, .. } => { - local_size.fetch_add(size.value(), Ordering::Relaxed); - } - BytesDownloadProgress::Found { size, .. } => { - network_size.fetch_add(*size, Ordering::Relaxed); - } - _ => {} - } - - Ok(item) - } - Err(err) => Err(err.into()), - }); - Self { - stream: Box::pin(stream), - current_local_size, - current_network_size, - } - } - - /// Finish writing the stream, ignoring all intermediate progress events. - /// - /// Returns a [`DownloadOutcome`] which contains the size of the content we downloaded and the size of the content we already had locally. - /// When importing a single blob, this is the size of that blob. - /// When importing a collection, this is the total size of all imported blobs (but excluding the size of the collection blob itself). - pub async fn finish(self) -> Result { - self.await - } -} - -impl Stream for DownloadProgress { - type Item = Result; - fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - Pin::new(&mut self.stream).poll_next(cx) - } -} - -impl Future for DownloadProgress { - type Output = Result; - - fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { - loop { - match Pin::new(&mut self.stream).poll_next(cx) { - Poll::Pending => return Poll::Pending, - Poll::Ready(None) => { - return Poll::Ready(Err(anyhow!("Response stream ended prematurely"))) - } - Poll::Ready(Some(Err(err))) => return Poll::Ready(Err(err)), - Poll::Ready(Some(Ok(msg))) => match msg { - BytesDownloadProgress::AllDone(stats) => { - let outcome = DownloadOutcome { - local_size: self.current_local_size.load(Ordering::Relaxed), - downloaded_size: self.current_network_size.load(Ordering::Relaxed), - stats, - }; - return Poll::Ready(Ok(outcome)); - } - BytesDownloadProgress::Abort(err) => { - return Poll::Ready(Err(err.into())); - } - _ => {} - }, - } - } - } -} - -/// Outcome of a blob export operation. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct ExportOutcome { - /// The total size of the exported data. - total_size: u64, -} - -/// Progress stream for blob export operations. -#[derive(derive_more::Debug)] -pub struct ExportProgress { - #[debug(skip)] - stream: Pin> + Send + Unpin + 'static>>, - current_total_size: Arc, -} - -impl ExportProgress { - /// Create a [`ExportProgress`] that can help you easily poll the [`BytesExportProgress`] stream from your - /// download until it is finished or errors. - pub fn new( - stream: (impl Stream, impl Into>> - + Send - + Unpin - + 'static), - ) -> Self { - let current_total_size = Arc::new(AtomicU64::new(0)); - let total_size = current_total_size.clone(); - let stream = stream.map(move |item| match item { - Ok(item) => { - let item = item.into(); - if let BytesExportProgress::Found { size, .. } = &item { - let size = size.value(); - total_size.fetch_add(size, Ordering::Relaxed); - } - - Ok(item) - } - Err(err) => Err(err.into()), - }); - Self { - stream: Box::pin(stream), - current_total_size, - } - } - - /// Finish writing the stream, ignoring all intermediate progress events. - /// - /// Returns a [`ExportOutcome`] which contains the size of the content we exported. - pub async fn finish(self) -> Result { - self.await - } -} - -impl Stream for ExportProgress { - type Item = Result; - fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - Pin::new(&mut self.stream).poll_next(cx) - } -} - -impl Future for ExportProgress { - type Output = Result; - - fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { - loop { - match Pin::new(&mut self.stream).poll_next(cx) { - Poll::Pending => return Poll::Pending, - Poll::Ready(None) => { - return Poll::Ready(Err(anyhow!("Response stream ended prematurely"))) - } - Poll::Ready(Some(Err(err))) => return Poll::Ready(Err(err)), - Poll::Ready(Some(Ok(msg))) => match msg { - BytesExportProgress::AllDone => { - let outcome = ExportOutcome { - total_size: self.current_total_size.load(Ordering::Relaxed), - }; - return Poll::Ready(Ok(outcome)); - } - BytesExportProgress::Abort(err) => { - return Poll::Ready(Err(err.into())); - } - _ => {} - }, - } - } - } -} - -/// Data reader for a single blob. -/// -/// Implements [`AsyncRead`]. -#[derive(derive_more::Debug)] -pub struct Reader { - size: u64, - response_size: u64, - is_complete: bool, - #[debug("StreamReader")] - stream: tokio_util::io::StreamReader>, Bytes>, -} - -impl Reader { - fn new( - size: u64, - response_size: u64, - is_complete: bool, - stream: BoxStreamSync<'static, io::Result>, - ) -> Self { - Self { - size, - response_size, - is_complete, - stream: StreamReader::new(stream), - } - } - - pub(crate) async fn from_rpc_read(rpc: &RpcClient, hash: Hash) -> anyhow::Result { - Self::from_rpc_read_at(rpc, hash, 0, ReadAtLen::All).await - } - - async fn from_rpc_read_at( - rpc: &RpcClient, - hash: Hash, - offset: u64, - len: ReadAtLen, - ) -> anyhow::Result { - let stream = rpc - .server_streaming(ReadAtRequest { hash, offset, len }) - .await?; - let mut stream = flatten(stream); - - let (size, is_complete) = match stream.next().await { - Some(Ok(ReadAtResponse::Entry { size, is_complete })) => (size, is_complete), - Some(Err(err)) => return Err(err), - Some(Ok(_)) => return Err(anyhow!("Expected header frame, but got data frame")), - None => return Err(anyhow!("Expected header frame, but RPC stream was dropped")), - }; - - let stream = stream.map(|item| match item { - Ok(ReadAtResponse::Data { chunk }) => Ok(chunk), - Ok(_) => Err(io::Error::new(io::ErrorKind::Other, "Expected data frame")), - Err(err) => Err(io::Error::new(io::ErrorKind::Other, format!("{err}"))), - }); - let len = len.as_result_len(size.value() - offset); - Ok(Self::new(size.value(), len, is_complete, Box::pin(stream))) - } - - /// Total size of this blob. - pub fn size(&self) -> u64 { - self.size - } - - /// Whether this blob has been downloaded completely. - /// - /// Returns false for partial blobs for which some chunks are missing. - pub fn is_complete(&self) -> bool { - self.is_complete - } - - /// Read all bytes of the blob. - pub async fn read_to_bytes(&mut self) -> anyhow::Result { - let mut buf = Vec::with_capacity(self.response_size as usize); - self.read_to_end(&mut buf).await?; - Ok(buf.into()) - } -} - -impl AsyncRead for Reader { - fn poll_read( - mut self: Pin<&mut Self>, - cx: &mut Context<'_>, - buf: &mut ReadBuf<'_>, - ) -> Poll> { - Pin::new(&mut self.stream).poll_read(cx, buf) - } -} - -impl Stream for Reader { - type Item = io::Result; - - fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - Pin::new(&mut self.stream).get_pin_mut().poll_next(cx) - } - - fn size_hint(&self) -> (usize, Option) { - self.stream.get_ref().size_hint() - } -} - -/// Options to configure a download request. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct DownloadOptions { - /// The format of the data to download. - pub format: BlobFormat, - /// Source nodes to download from. - /// - /// If set to more than a single node, they will all be tried. If `mode` is set to - /// [`DownloadMode::Direct`], they will be tried sequentially until a download succeeds. - /// If `mode` is set to [`DownloadMode::Queued`], the nodes may be dialed in parallel, - /// if the concurrency limits permit. - pub nodes: Vec, - /// Optional tag to tag the data with. - pub tag: SetTagOption, - /// Whether to directly start the download or add it to the download queue. - pub mode: DownloadMode, -} - -#[cfg(test)] -mod tests { - use iroh_blobs::hashseq::HashSeq; - use iroh_net::NodeId; - use rand::RngCore; - use testresult::TestResult; - use tokio::{io::AsyncWriteExt, sync::mpsc}; - - use super::*; - - #[tokio::test] - async fn test_blob_create_collection() -> Result<()> { - let _guard = iroh_test::logging::setup(); - - let node = crate::node::Node::memory().spawn().await?; - - // create temp file - let temp_dir = tempfile::tempdir().context("tempdir")?; - - let in_root = temp_dir.path().join("in"); - tokio::fs::create_dir_all(in_root.clone()) - .await - .context("create dir all")?; - - let mut paths = Vec::new(); - for i in 0..5 { - let path = in_root.join(format!("test-{i}")); - let size = 100; - let mut buf = vec![0u8; size]; - rand::thread_rng().fill_bytes(&mut buf); - let mut file = tokio::fs::File::create(path.clone()) - .await - .context("create file")?; - file.write_all(&buf.clone()).await.context("write_all")?; - file.flush().await.context("flush")?; - paths.push(path); - } - - let client = node.client(); - - let mut collection = Collection::default(); - let mut tags = Vec::new(); - // import files - for path in &paths { - let import_outcome = client - .blobs() - .add_from_path( - path.to_path_buf(), - false, - SetTagOption::Auto, - WrapOption::NoWrap, - ) - .await - .context("import file")? - .finish() - .await - .context("import finish")?; - - collection.push( - path.file_name().unwrap().to_str().unwrap().to_string(), - import_outcome.hash, - ); - tags.push(import_outcome.tag); - } - - let (hash, tag) = client - .blobs() - .create_collection(collection, SetTagOption::Auto, tags) - .await?; - - let collections: Vec<_> = client.blobs().list_collections()?.try_collect().await?; - - assert_eq!(collections.len(), 1); - { - let CollectionInfo { - tag, - hash, - total_blobs_count, - .. - } = &collections[0]; - assert_eq!(tag, tag); - assert_eq!(hash, hash); - // 5 blobs + 1 meta - assert_eq!(total_blobs_count, &Some(5 + 1)); - } - - // check that "temp" tags have been deleted - let tags: Vec<_> = client.tags().list().await?.try_collect().await?; - assert_eq!(tags.len(), 1); - assert_eq!(tags[0].hash, hash); - assert_eq!(tags[0].name, tag); - assert_eq!(tags[0].format, BlobFormat::HashSeq); - - Ok(()) - } - - #[tokio::test] - async fn test_blob_read_at() -> Result<()> { - // let _guard = iroh_test::logging::setup(); - - let node = crate::node::Node::memory().spawn().await?; - - // create temp file - let temp_dir = tempfile::tempdir().context("tempdir")?; - - let in_root = temp_dir.path().join("in"); - tokio::fs::create_dir_all(in_root.clone()) - .await - .context("create dir all")?; - - let path = in_root.join("test-blob"); - let size = 1024 * 128; - let buf: Vec = (0..size).map(|i| i as u8).collect(); - let mut file = tokio::fs::File::create(path.clone()) - .await - .context("create file")?; - file.write_all(&buf.clone()).await.context("write_all")?; - file.flush().await.context("flush")?; - - let client = node.client(); - - let import_outcome = client - .blobs() - .add_from_path( - path.to_path_buf(), - false, - SetTagOption::Auto, - WrapOption::NoWrap, - ) - .await - .context("import file")? - .finish() - .await - .context("import finish")?; - - let hash = import_outcome.hash; - - // Read everything - let res = client.blobs().read_to_bytes(hash).await?; - assert_eq!(&res, &buf[..]); - - // Read at smaller than blob_get_chunk_size - let res = client - .blobs() - .read_at_to_bytes(hash, 0, ReadAtLen::Exact(100)) - .await?; - assert_eq!(res.len(), 100); - assert_eq!(&res[..], &buf[0..100]); - - let res = client - .blobs() - .read_at_to_bytes(hash, 20, ReadAtLen::Exact(120)) - .await?; - assert_eq!(res.len(), 120); - assert_eq!(&res[..], &buf[20..140]); - - // Read at equal to blob_get_chunk_size - let res = client - .blobs() - .read_at_to_bytes(hash, 0, ReadAtLen::Exact(1024 * 64)) - .await?; - assert_eq!(res.len(), 1024 * 64); - assert_eq!(&res[..], &buf[0..1024 * 64]); - - let res = client - .blobs() - .read_at_to_bytes(hash, 20, ReadAtLen::Exact(1024 * 64)) - .await?; - assert_eq!(res.len(), 1024 * 64); - assert_eq!(&res[..], &buf[20..(20 + 1024 * 64)]); - - // Read at larger than blob_get_chunk_size - let res = client - .blobs() - .read_at_to_bytes(hash, 0, ReadAtLen::Exact(10 + 1024 * 64)) - .await?; - assert_eq!(res.len(), 10 + 1024 * 64); - assert_eq!(&res[..], &buf[0..(10 + 1024 * 64)]); - - let res = client - .blobs() - .read_at_to_bytes(hash, 20, ReadAtLen::Exact(10 + 1024 * 64)) - .await?; - assert_eq!(res.len(), 10 + 1024 * 64); - assert_eq!(&res[..], &buf[20..(20 + 10 + 1024 * 64)]); - - // full length - let res = client - .blobs() - .read_at_to_bytes(hash, 20, ReadAtLen::All) - .await?; - assert_eq!(res.len(), 1024 * 128 - 20); - assert_eq!(&res[..], &buf[20..]); - - // size should be total - let reader = client - .blobs() - .read_at(hash, 0, ReadAtLen::Exact(20)) - .await?; - assert_eq!(reader.size(), 1024 * 128); - assert_eq!(reader.response_size, 20); - - // last chunk - exact - let res = client - .blobs() - .read_at_to_bytes(hash, 1024 * 127, ReadAtLen::Exact(1024)) - .await?; - assert_eq!(res.len(), 1024); - assert_eq!(res, &buf[1024 * 127..]); - - // last chunk - open - let res = client - .blobs() - .read_at_to_bytes(hash, 1024 * 127, ReadAtLen::All) - .await?; - assert_eq!(res.len(), 1024); - assert_eq!(res, &buf[1024 * 127..]); - - // last chunk - larger - let mut res = client - .blobs() - .read_at(hash, 1024 * 127, ReadAtLen::AtMost(2048)) - .await?; - assert_eq!(res.size, 1024 * 128); - assert_eq!(res.response_size, 1024); - let res = res.read_to_bytes().await?; - assert_eq!(res.len(), 1024); - assert_eq!(res, &buf[1024 * 127..]); - - // out of bounds - too long - let res = client - .blobs() - .read_at(hash, 0, ReadAtLen::Exact(1024 * 128 + 1)) - .await; - let err = res.unwrap_err(); - assert!(err.to_string().contains("out of bound")); - - // out of bounds - offset larger than blob - let res = client - .blobs() - .read_at(hash, 1024 * 128 + 1, ReadAtLen::All) - .await; - let err = res.unwrap_err(); - assert!(err.to_string().contains("out of range")); - - // out of bounds - offset + length too large - let res = client - .blobs() - .read_at(hash, 1024 * 127, ReadAtLen::Exact(1025)) - .await; - let err = res.unwrap_err(); - assert!(err.to_string().contains("out of bound")); - - Ok(()) - } - - #[tokio::test] - async fn test_blob_get_collection() -> Result<()> { - let _guard = iroh_test::logging::setup(); - - let node = crate::node::Node::memory().spawn().await?; - - // create temp file - let temp_dir = tempfile::tempdir().context("tempdir")?; - - let in_root = temp_dir.path().join("in"); - tokio::fs::create_dir_all(in_root.clone()) - .await - .context("create dir all")?; - - let mut paths = Vec::new(); - for i in 0..5 { - let path = in_root.join(format!("test-{i}")); - let size = 100; - let mut buf = vec![0u8; size]; - rand::thread_rng().fill_bytes(&mut buf); - let mut file = tokio::fs::File::create(path.clone()) - .await - .context("create file")?; - file.write_all(&buf.clone()).await.context("write_all")?; - file.flush().await.context("flush")?; - paths.push(path); - } - - let client = node.client(); - - let mut collection = Collection::default(); - let mut tags = Vec::new(); - // import files - for path in &paths { - let import_outcome = client - .blobs() - .add_from_path( - path.to_path_buf(), - false, - SetTagOption::Auto, - WrapOption::NoWrap, - ) - .await - .context("import file")? - .finish() - .await - .context("import finish")?; - - collection.push( - path.file_name().unwrap().to_str().unwrap().to_string(), - import_outcome.hash, - ); - tags.push(import_outcome.tag); - } - - let (hash, _tag) = client - .blobs() - .create_collection(collection, SetTagOption::Auto, tags) - .await?; - - let collection = client.blobs().get_collection(hash).await?; - - // 5 blobs - assert_eq!(collection.len(), 5); - - Ok(()) - } - - #[tokio::test] - async fn test_blob_share() -> Result<()> { - let _guard = iroh_test::logging::setup(); - - let node = crate::node::Node::memory().spawn().await?; - - // create temp file - let temp_dir = tempfile::tempdir().context("tempdir")?; - - let in_root = temp_dir.path().join("in"); - tokio::fs::create_dir_all(in_root.clone()) - .await - .context("create dir all")?; - - let path = in_root.join("test-blob"); - let size = 1024 * 128; - let buf: Vec = (0..size).map(|i| i as u8).collect(); - let mut file = tokio::fs::File::create(path.clone()) - .await - .context("create file")?; - file.write_all(&buf.clone()).await.context("write_all")?; - file.flush().await.context("flush")?; - - let client = node.client(); - - let import_outcome = client - .blobs() - .add_from_path( - path.to_path_buf(), - false, - SetTagOption::Auto, - WrapOption::NoWrap, - ) - .await - .context("import file")? - .finish() - .await - .context("import finish")?; - - let ticket = client - .blobs() - .share(import_outcome.hash, BlobFormat::Raw, Default::default()) - .await?; - assert_eq!(ticket.hash(), import_outcome.hash); - - let status = client.blobs().status(import_outcome.hash).await?; - assert_eq!(status, BlobStatus::Complete { size }); - - Ok(()) - } - - #[derive(Debug, Clone)] - struct BlobEvents { - sender: mpsc::Sender, - } - - impl BlobEvents { - fn new(cap: usize) -> (Self, mpsc::Receiver) { - let (s, r) = mpsc::channel(cap); - (Self { sender: s }, r) - } - } - - impl iroh_blobs::provider::CustomEventSender for BlobEvents { - fn send(&self, event: iroh_blobs::provider::Event) -> futures_lite::future::Boxed<()> { - let sender = self.sender.clone(); - Box::pin(async move { - sender.send(event).await.ok(); - }) - } - - fn try_send(&self, event: iroh_blobs::provider::Event) { - self.sender.try_send(event).ok(); - } - } - - #[tokio::test] - async fn test_blob_provide_events() -> Result<()> { - let _guard = iroh_test::logging::setup(); - - let (node1_events, mut node1_events_r) = BlobEvents::new(16); - let node1 = crate::node::Node::memory() - .blobs_events(node1_events) - .spawn() - .await?; - - let (node2_events, mut node2_events_r) = BlobEvents::new(16); - let node2 = crate::node::Node::memory() - .blobs_events(node2_events) - .spawn() - .await?; - - let import_outcome = node1.blobs().add_bytes(&b"hello world"[..]).await?; - - // Download in node2 - let node1_addr = node1.net().node_addr().await?; - let res = node2 - .blobs() - .download(import_outcome.hash, node1_addr) - .await? - .await?; - dbg!(&res); - assert_eq!(res.local_size, 0); - assert_eq!(res.downloaded_size, 11); - - node1.shutdown().await?; - node2.shutdown().await?; - - let mut ev1 = Vec::new(); - while let Some(ev) = node1_events_r.recv().await { - ev1.push(ev); - } - // assert_eq!(ev1.len(), 3); - assert!(matches!( - ev1[0], - iroh_blobs::provider::Event::ClientConnected { .. } - )); - assert!(matches!( - ev1[1], - iroh_blobs::provider::Event::GetRequestReceived { .. } - )); - assert!(matches!( - ev1[2], - iroh_blobs::provider::Event::TransferProgress { .. } - )); - assert!(matches!( - ev1[3], - iroh_blobs::provider::Event::TransferCompleted { .. } - )); - dbg!(&ev1); - - let mut ev2 = Vec::new(); - while let Some(ev) = node2_events_r.recv().await { - ev2.push(ev); - } - - // Node 2 did not provide anything - assert!(ev2.is_empty()); - Ok(()) - } - /// Download a existing blob from oneself - #[tokio::test] - async fn test_blob_get_self_existing() -> TestResult<()> { - let _guard = iroh_test::logging::setup(); - - let node = crate::node::Node::memory().spawn().await?; - let node_id = node.node_id(); - let client = node.client(); - - let AddOutcome { hash, size, .. } = client.blobs().add_bytes("foo").await?; - - // Direct - let res = client - .blobs() - .download_with_opts( - hash, - DownloadOptions { - format: BlobFormat::Raw, - nodes: vec![node_id.into()], - tag: SetTagOption::Auto, - mode: DownloadMode::Direct, - }, - ) - .await? - .await?; - - assert_eq!(res.local_size, size); - assert_eq!(res.downloaded_size, 0); - - // Queued - let res = client - .blobs() - .download_with_opts( - hash, - DownloadOptions { - format: BlobFormat::Raw, - nodes: vec![node_id.into()], - tag: SetTagOption::Auto, - mode: DownloadMode::Queued, - }, - ) - .await? - .await?; - - assert_eq!(res.local_size, size); - assert_eq!(res.downloaded_size, 0); - - Ok(()) - } - - /// Download a missing blob from oneself - #[tokio::test] - async fn test_blob_get_self_missing() -> TestResult<()> { - let _guard = iroh_test::logging::setup(); - - let node = crate::node::Node::memory().spawn().await?; - let node_id = node.node_id(); - let client = node.client(); - - let hash = Hash::from_bytes([0u8; 32]); - - // Direct - let res = client - .blobs() - .download_with_opts( - hash, - DownloadOptions { - format: BlobFormat::Raw, - nodes: vec![node_id.into()], - tag: SetTagOption::Auto, - mode: DownloadMode::Direct, - }, - ) - .await? - .await; - assert!(res.is_err()); - assert_eq!( - res.err().unwrap().to_string().as_str(), - "No nodes to download from provided" - ); - - // Queued - let res = client - .blobs() - .download_with_opts( - hash, - DownloadOptions { - format: BlobFormat::Raw, - nodes: vec![node_id.into()], - tag: SetTagOption::Auto, - mode: DownloadMode::Queued, - }, - ) - .await? - .await; - assert!(res.is_err()); - assert_eq!( - res.err().unwrap().to_string().as_str(), - "No provider nodes found" - ); - - Ok(()) - } - - /// Download a existing collection. Check that things succeed and no download is performed. - #[tokio::test] - async fn test_blob_get_existing_collection() -> TestResult<()> { - let _guard = iroh_test::logging::setup(); - - let node = crate::node::Node::memory().spawn().await?; - // We use a nonexisting node id because we just want to check that this succeeds without - // hitting the network. - let node_id = NodeId::from_bytes(&[0u8; 32])?; - let client = node.client(); - - let mut collection = Collection::default(); - let mut tags = Vec::new(); - let mut size = 0; - for value in ["iroh", "is", "cool"] { - let import_outcome = client.blobs().add_bytes(value).await.context("add bytes")?; - collection.push(value.to_string(), import_outcome.hash); - tags.push(import_outcome.tag); - size += import_outcome.size; - } - - let (hash, _tag) = client - .blobs() - .create_collection(collection, SetTagOption::Auto, tags) - .await?; - - // load the hashseq and collection header manually to calculate our expected size - let hashseq_bytes = client.blobs().read_to_bytes(hash).await?; - size += hashseq_bytes.len() as u64; - let hashseq = HashSeq::try_from(hashseq_bytes)?; - let collection_header_bytes = client - .blobs() - .read_to_bytes(hashseq.into_iter().next().expect("header to exist")) - .await?; - size += collection_header_bytes.len() as u64; - - // Direct - let res = client - .blobs() - .download_with_opts( - hash, - DownloadOptions { - format: BlobFormat::HashSeq, - nodes: vec![node_id.into()], - tag: SetTagOption::Auto, - mode: DownloadMode::Direct, - }, - ) - .await? - .await - .context("direct (download)")?; - - assert_eq!(res.local_size, size); - assert_eq!(res.downloaded_size, 0); - - // Queued - let res = client - .blobs() - .download_with_opts( - hash, - DownloadOptions { - format: BlobFormat::HashSeq, - nodes: vec![node_id.into()], - tag: SetTagOption::Auto, - mode: DownloadMode::Queued, - }, - ) - .await? - .await - .context("queued")?; - - assert_eq!(res.local_size, size); - assert_eq!(res.downloaded_size, 0); - - Ok(()) - } - - #[tokio::test] - #[cfg_attr(target_os = "windows", ignore = "flaky")] - async fn test_blob_delete_mem() -> Result<()> { - let _guard = iroh_test::logging::setup(); - - let node = crate::node::Node::memory().spawn().await?; - - let res = node.blobs().add_bytes(&b"hello world"[..]).await?; - - let hashes: Vec<_> = node.blobs().list().await?.try_collect().await?; - assert_eq!(hashes.len(), 1); - assert_eq!(hashes[0].hash, res.hash); - - // delete - node.blobs().delete_blob(res.hash).await?; - - let hashes: Vec<_> = node.blobs().list().await?.try_collect().await?; - assert!(hashes.is_empty()); - - Ok(()) - } - - #[tokio::test] - async fn test_blob_delete_fs() -> Result<()> { - let _guard = iroh_test::logging::setup(); - - let dir = tempfile::tempdir()?; - let node = crate::node::Node::persistent(dir.path()) - .await? - .spawn() - .await?; - - let res = node.blobs().add_bytes(&b"hello world"[..]).await?; - - let hashes: Vec<_> = node.blobs().list().await?.try_collect().await?; - assert_eq!(hashes.len(), 1); - assert_eq!(hashes[0].hash, res.hash); - - // delete - node.blobs().delete_blob(res.hash).await?; - - let hashes: Vec<_> = node.blobs().list().await?.try_collect().await?; - assert!(hashes.is_empty()); - - Ok(()) - } -} diff --git a/iroh/src/client/docs.rs b/iroh/src/client/docs.rs index cdceaede8b6..1fa7c9e5f1a 100644 --- a/iroh/src/client/docs.rs +++ b/iroh/src/client/docs.rs @@ -475,14 +475,19 @@ impl Entry { /// /// You can pass either a [`Doc`] or the `Iroh` client by reference as `client`. pub async fn content_reader(&self, client: impl Into<&RpcClient>) -> Result { - blobs::Reader::from_rpc_read(client.into(), self.content_hash()).await + let client: RpcClient = client.into().clone(); + let client: quic_rpc::RpcClient = client.map(); + blobs::Reader::from_rpc_read(&client, self.content_hash()).await } /// Reads all content of an [`Entry`] into a buffer. /// /// You can pass either a [`Doc`] or the `Iroh` client by reference as `client`. pub async fn content_bytes(&self, client: impl Into<&RpcClient>) -> Result { - blobs::Reader::from_rpc_read(client.into(), self.content_hash()) + let client: RpcClient = client.into().clone(); + let client: quic_rpc::RpcClient = client.map(); + + blobs::Reader::from_rpc_read(&client, self.content_hash()) .await? .read_to_bytes() .await diff --git a/iroh/src/client/tags.rs b/iroh/src/client/tags.rs deleted file mode 100644 index 1b0accf03aa..00000000000 --- a/iroh/src/client/tags.rs +++ /dev/null @@ -1,60 +0,0 @@ -//! API for tag management. -//! -//! The purpose of tags is to mark information as important to prevent it -//! from being garbage-collected (if the garbage collector is turned on). -//! Currently this is used for blobs. -//! -//! The main entry point is the [`Client`]. -//! -//! You obtain a [`Client`] via [`Iroh::tags()`](crate::client::Iroh::tags). -//! -//! [`Client::list`] can be used to list all tags. -//! [`Client::list_hash_seq`] can be used to list all tags with a hash_seq format. -//! -//! [`Client::delete`] can be used to delete a tag. -use anyhow::Result; -use futures_lite::{Stream, StreamExt}; -use iroh_blobs::{BlobFormat, Hash, Tag}; -use ref_cast::RefCast; -use serde::{Deserialize, Serialize}; - -use super::RpcClient; -use crate::rpc_protocol::tags::{DeleteRequest, ListRequest}; - -/// Iroh tags client. -#[derive(Debug, Clone, RefCast)] -#[repr(transparent)] -pub struct Client { - pub(super) rpc: RpcClient, -} - -impl Client { - /// Lists all tags. - pub async fn list(&self) -> Result>> { - let stream = self.rpc.server_streaming(ListRequest::all()).await?; - Ok(stream.map(|res| res.map_err(anyhow::Error::from))) - } - - /// Lists all tags with a hash_seq format. - pub async fn list_hash_seq(&self) -> Result>> { - let stream = self.rpc.server_streaming(ListRequest::hash_seq()).await?; - Ok(stream.map(|res| res.map_err(anyhow::Error::from))) - } - - /// Deletes a tag. - pub async fn delete(&self, name: Tag) -> Result<()> { - self.rpc.rpc(DeleteRequest { name }).await??; - Ok(()) - } -} - -/// Information about a tag. -#[derive(Debug, Serialize, Deserialize)] -pub struct TagInfo { - /// Name of the tag - pub name: Tag, - /// Format of the data - pub format: BlobFormat, - /// Hash of the data - pub hash: Hash, -} diff --git a/iroh/src/node.rs b/iroh/src/node.rs index bd2df139442..db07cce3e88 100644 --- a/iroh/src/node.rs +++ b/iroh/src/node.rs @@ -497,7 +497,7 @@ fn node_address_for_storage(info: RemoteInfo) -> Option { mod tests { use anyhow::{bail, Context}; use bytes::Bytes; - use iroh_base::node_addr::AddrInfoOptions; + use iroh_base::{node_addr::AddrInfoOptions, ticket::BlobTicket}; use iroh_blobs::{provider::AddProgress, util::SetTagOption, BlobFormat}; use iroh_net::{key::SecretKey, relay::RelayMode, test_utils::DnsPkarrServer, NodeAddr}; @@ -518,11 +518,9 @@ mod tests { .hash; let _drop_guard = node.cancel_token().drop_guard(); - let ticket = node - .blobs() - .share(hash, BlobFormat::Raw, AddrInfoOptions::RelayAndAddresses) - .await - .unwrap(); + let mut addr = node.net().node_addr().await.unwrap(); + addr.apply_options(AddrInfoOptions::RelayAndAddresses); + let ticket = BlobTicket::new(addr, hash, BlobFormat::Raw).unwrap(); println!("addrs: {:?}", ticket.node_addr().info); assert!(!ticket.node_addr().info.direct_addresses.is_empty()); } diff --git a/iroh/src/node/rpc.rs b/iroh/src/node/rpc.rs index a6dc9f3a2b1..a38993abda4 100644 --- a/iroh/src/node/rpc.rs +++ b/iroh/src/node/rpc.rs @@ -189,7 +189,7 @@ impl Handler { async fn handle_blobs_request( self, msg: blobs::Request, - chan: RpcChannel, + chan: RpcChannel, ) -> Result<(), RpcServerError> { use blobs::Request::*; debug!("handling blob request: {msg}"); @@ -231,7 +231,7 @@ impl Handler { async fn handle_tags_request( self, msg: tags::Request, - chan: RpcChannel, + chan: RpcChannel, ) -> Result<(), RpcServerError> { use tags::Request::*; match msg { @@ -322,8 +322,8 @@ impl Handler { match msg { Net(msg) => self.handle_net_request(msg, chan).await, Node(msg) => self.handle_node_request(msg, chan).await, - Blobs(msg) => self.handle_blobs_request(msg, chan).await, - Tags(msg) => self.handle_tags_request(msg, chan).await, + BlobsAndTags(iroh_blobs::rpc::proto::Request::Blobs(msg)) => self.handle_blobs_request(msg, chan.map()).await, + BlobsAndTags(iroh_blobs::rpc::proto::Request::Tags(msg)) => self.handle_tags_request(msg, chan.map()).await, Authors(msg) => self.handle_authors_request(msg, chan).await, Docs(msg) => self.handle_docs_request(msg, chan).await, Gossip(msg) => self.handle_gossip_request(msg, chan).await, diff --git a/iroh/src/rpc_protocol.rs b/iroh/src/rpc_protocol.rs index d062c6c4440..7c776968c9f 100644 --- a/iroh/src/rpc_protocol.rs +++ b/iroh/src/rpc_protocol.rs @@ -18,11 +18,11 @@ use serde::{Deserialize, Serialize}; pub mod authors; -pub mod blobs; pub mod docs; pub mod net; pub mod node; -pub mod tags; +pub use iroh_blobs::rpc::proto::blobs; +pub use iroh_blobs::rpc::proto::tags; /// The RPC service for the iroh provider process. #[derive(Debug, Clone)] @@ -35,11 +35,10 @@ pub struct RpcService; pub enum Request { Node(node::Request), Net(net::Request), - Blobs(blobs::Request), Docs(docs::Request), - Tags(tags::Request), Authors(authors::Request), Gossip(iroh_gossip::RpcRequest), + BlobsAndTags(iroh_blobs::rpc::proto::Request), } /// The response enum, listing all possible responses. @@ -49,11 +48,10 @@ pub enum Request { pub enum Response { Node(node::Response), Net(net::Response), - Blobs(blobs::Response), - Tags(tags::Response), Docs(docs::Response), Authors(authors::Response), Gossip(iroh_gossip::RpcResponse), + BlobsAndTags(iroh_blobs::rpc::proto::Response), } impl quic_rpc::Service for RpcService { diff --git a/iroh/src/rpc_protocol/blobs.rs b/iroh/src/rpc_protocol/blobs.rs deleted file mode 100644 index b27c83809b9..00000000000 --- a/iroh/src/rpc_protocol/blobs.rs +++ /dev/null @@ -1,319 +0,0 @@ -use std::path::PathBuf; - -use bytes::Bytes; -use iroh_base::hash::Hash; -use iroh_blobs::{ - export::ExportProgress, - format::collection::Collection, - get::db::DownloadProgress, - net_protocol::{BatchId, BlobDownloadRequest}, - provider::{AddProgress, BatchAddPathProgress}, - store::{ - BaoBlobSize, ConsistencyCheckProgress, ExportFormat, ExportMode, ImportMode, - ValidateProgress, - }, - util::SetTagOption, - BlobFormat, HashAndFormat, Tag, -}; -use nested_enum_utils::enum_conversions; -use quic_rpc_derive::rpc_requests; -use serde::{Deserialize, Serialize}; - -use super::RpcService; -use crate::{ - client::blobs::{BlobInfo, BlobStatus, IncompleteBlobInfo, ReadAtLen, WrapOption}, - node::{RpcError, RpcResult}, -}; - -#[allow(missing_docs)] -#[derive(strum::Display, Debug, Serialize, Deserialize)] -#[enum_conversions(super::Request)] -#[rpc_requests(RpcService)] -pub enum Request { - #[server_streaming(response = RpcResult)] - ReadAt(ReadAtRequest), - #[bidi_streaming(update = AddStreamUpdate, response = AddStreamResponse)] - AddStream(AddStreamRequest), - AddStreamUpdate(AddStreamUpdate), - #[server_streaming(response = AddPathResponse)] - AddPath(AddPathRequest), - #[server_streaming(response = DownloadResponse)] - Download(BlobDownloadRequest), - #[server_streaming(response = ExportResponse)] - Export(ExportRequest), - #[server_streaming(response = RpcResult)] - List(ListRequest), - #[server_streaming(response = RpcResult)] - ListIncomplete(ListIncompleteRequest), - #[rpc(response = RpcResult<()>)] - Delete(DeleteRequest), - #[server_streaming(response = ValidateProgress)] - Validate(ValidateRequest), - #[server_streaming(response = ConsistencyCheckProgress)] - Fsck(ConsistencyCheckRequest), - #[rpc(response = RpcResult)] - CreateCollection(CreateCollectionRequest), - #[rpc(response = RpcResult)] - BlobStatus(BlobStatusRequest), - - #[bidi_streaming(update = BatchUpdate, response = BatchCreateResponse)] - BatchCreate(BatchCreateRequest), - BatchUpdate(BatchUpdate), - #[bidi_streaming(update = BatchAddStreamUpdate, response = BatchAddStreamResponse)] - BatchAddStream(BatchAddStreamRequest), - BatchAddStreamUpdate(BatchAddStreamUpdate), - #[server_streaming(response = BatchAddPathResponse)] - BatchAddPath(BatchAddPathRequest), - #[rpc(response = RpcResult<()>)] - BatchCreateTempTag(BatchCreateTempTagRequest), -} - -#[allow(missing_docs)] -#[derive(strum::Display, Debug, Serialize, Deserialize)] -#[enum_conversions(super::Response)] -pub enum Response { - ReadAt(RpcResult), - AddStream(AddStreamResponse), - AddPath(AddPathResponse), - List(RpcResult), - ListIncomplete(RpcResult), - Download(DownloadResponse), - Fsck(ConsistencyCheckProgress), - Export(ExportResponse), - Validate(ValidateProgress), - CreateCollection(RpcResult), - BlobStatus(RpcResult), - BatchCreate(BatchCreateResponse), - BatchAddStream(BatchAddStreamResponse), - BatchAddPath(BatchAddPathResponse), -} - -/// A request to the node to provide the data at the given path -/// -/// Will produce a stream of [`AddProgress`] messages. -#[derive(Debug, Serialize, Deserialize)] -pub struct AddPathRequest { - /// The path to the data to provide. - /// - /// This should be an absolute path valid for the file system on which - /// the node runs. Usually the cli will run on the same machine as the - /// node, so this should be an absolute path on the cli machine. - pub path: PathBuf, - /// True if the provider can assume that the data will not change, so it - /// can be shared in place. - pub in_place: bool, - /// Tag to tag the data with. - pub tag: SetTagOption, - /// Whether to wrap the added data in a collection - pub wrap: WrapOption, -} - -/// Wrapper around [`AddProgress`]. -#[derive(Debug, Serialize, Deserialize, derive_more::Into)] -pub struct AddPathResponse(pub AddProgress); - -/// Progress response for [`BlobDownloadRequest`] -#[derive(Debug, Clone, Serialize, Deserialize, derive_more::From, derive_more::Into)] -pub struct DownloadResponse(pub DownloadProgress); - -/// A request to the node to download and share the data specified by the hash. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ExportRequest { - /// The hash of the blob to export. - pub hash: Hash, - /// The filepath to where the data should be saved - /// - /// This should be an absolute path valid for the file system on which - /// the node runs. - pub path: PathBuf, - /// Set to [`ExportFormat::Collection`] if the `hash` refers to a [`Collection`] and you want - /// to export all children of the collection into individual files. - pub format: ExportFormat, - /// The mode of exporting. - /// - /// The default is [`ExportMode::Copy`]. See [`ExportMode`] for details. - pub mode: ExportMode, -} - -/// Progress response for [`ExportRequest`] -#[derive(Debug, Clone, Serialize, Deserialize, derive_more::From, derive_more::Into)] -pub struct ExportResponse(pub ExportProgress); - -/// A request to the node to validate the integrity of all provided data -#[derive(Debug, Serialize, Deserialize)] -pub struct ConsistencyCheckRequest { - /// repair the store by dropping inconsistent blobs - pub repair: bool, -} - -/// A request to the node to validate the integrity of all provided data -#[derive(Debug, Serialize, Deserialize)] -pub struct ValidateRequest { - /// repair the store by downgrading blobs from complete to partial - pub repair: bool, -} - -/// List all blobs, including collections -#[derive(Debug, Serialize, Deserialize)] -pub struct ListRequest; - -/// List all blobs, including collections -#[derive(Debug, Serialize, Deserialize)] -pub struct ListIncompleteRequest; - -/// Get the bytes for a hash -#[derive(Serialize, Deserialize, Debug)] -pub struct ReadAtRequest { - /// Hash to get bytes for - pub hash: Hash, - /// Offset to start reading at - pub offset: u64, - /// Length of the data to get - pub len: ReadAtLen, -} - -/// Response to [`ReadAtRequest`] -#[derive(Serialize, Deserialize, Debug)] -pub enum ReadAtResponse { - /// The entry header. - Entry { - /// The size of the blob - size: BaoBlobSize, - /// Whether the blob is complete - is_complete: bool, - }, - /// Chunks of entry data. - Data { - /// The data chunk - chunk: Bytes, - }, -} - -/// Write a blob from a byte stream -#[derive(Serialize, Deserialize, Debug)] -pub struct AddStreamRequest { - /// Tag to tag the data with. - pub tag: SetTagOption, -} - -/// Write a blob from a byte stream -#[derive(Serialize, Deserialize, Debug)] -pub enum AddStreamUpdate { - /// A chunk of stream data - Chunk(Bytes), - /// Abort the request due to an error on the client side - Abort, -} - -/// Wrapper around [`AddProgress`]. -#[derive(Debug, Serialize, Deserialize, derive_more::Into)] -pub struct AddStreamResponse(pub AddProgress); - -/// Delete a blob -#[derive(Debug, Serialize, Deserialize)] -pub struct DeleteRequest { - /// Name of the tag - pub hash: Hash, -} - -/// Create a collection. -#[derive(Debug, Serialize, Deserialize)] -pub struct CreateCollectionRequest { - /// The collection - pub collection: Collection, - /// Tag option. - pub tag: SetTagOption, - /// Tags that should be deleted after creation. - pub tags_to_delete: Vec, -} - -/// A response to a create collection request -#[derive(Debug, Serialize, Deserialize)] -pub struct CreateCollectionResponse { - /// The resulting hash. - pub hash: Hash, - /// The resulting tag. - pub tag: Tag, -} - -#[derive(Debug, Serialize, Deserialize)] -pub struct BlobStatusRequest { - /// The hash of the blob - pub hash: Hash, -} - -/// The response to a status request -#[derive(Debug, Serialize, Deserialize, derive_more::From, derive_more::Into)] -pub struct BlobStatusResponse(pub BlobStatus); - -/// Request to create a new scope for temp tags -#[derive(Debug, Serialize, Deserialize)] -pub struct BatchCreateRequest; - -/// Update to a temp tag scope -#[derive(Debug, Serialize, Deserialize)] -pub enum BatchUpdate { - /// Drop of a remote temp tag - Drop(HashAndFormat), - /// Message to check that the connection is still alive - Ping, -} - -/// Response to a temp tag scope request -#[derive(Debug, Serialize, Deserialize)] -pub enum BatchCreateResponse { - /// We got the id of the scope - Id(BatchId), -} - -/// Create a temp tag with a given hash and format -#[derive(Debug, Serialize, Deserialize)] -pub struct BatchCreateTempTagRequest { - /// Content to protect - pub content: HashAndFormat, - /// Batch to create the temp tag in - pub batch: BatchId, -} - -/// Write a blob from a byte stream -#[derive(Serialize, Deserialize, Debug)] -pub struct BatchAddStreamRequest { - /// What format to use for the blob - pub format: BlobFormat, - /// Batch to create the temp tag in - pub batch: BatchId, -} - -/// Write a blob from a byte stream -#[derive(Serialize, Deserialize, Debug)] -pub enum BatchAddStreamUpdate { - /// A chunk of stream data - Chunk(Bytes), - /// Abort the request due to an error on the client side - Abort, -} - -/// Wrapper around [`AddProgress`]. -#[derive(Debug, Serialize, Deserialize)] -pub enum BatchAddStreamResponse { - Abort(RpcError), - OutboardProgress { offset: u64 }, - Result { hash: Hash }, -} - -/// Write a blob from a byte stream -#[derive(Serialize, Deserialize, Debug)] -pub struct BatchAddPathRequest { - /// The path to the data to provide. - pub path: PathBuf, - /// Add the data in place - pub import_mode: ImportMode, - /// What format to use for the blob - pub format: BlobFormat, - /// Batch to create the temp tag in - pub batch: BatchId, -} - -/// Response to a batch add path request -#[derive(Serialize, Deserialize, Debug)] -pub struct BatchAddPathResponse(pub BatchAddPathProgress); diff --git a/iroh/src/rpc_protocol/net.rs b/iroh/src/rpc_protocol/net.rs index 35e0768ede7..de04f6d655a 100644 --- a/iroh/src/rpc_protocol/net.rs +++ b/iroh/src/rpc_protocol/net.rs @@ -37,6 +37,7 @@ pub enum Response { RemoteInfosIter(RpcResult), RemoteInfo(RpcResult), Watch(WatchResponse), + Unit(RpcResult<()>), } /// List network path information about all the remote nodes known by this node. diff --git a/iroh/src/rpc_protocol/tags.rs b/iroh/src/rpc_protocol/tags.rs deleted file mode 100644 index 34e4edd9ae0..00000000000 --- a/iroh/src/rpc_protocol/tags.rs +++ /dev/null @@ -1,109 +0,0 @@ -use iroh_blobs::{net_protocol::BatchId, HashAndFormat, Tag}; -use nested_enum_utils::enum_conversions; -use quic_rpc_derive::rpc_requests; -use serde::{Deserialize, Serialize}; - -use super::RpcService; -use crate::{client::tags::TagInfo, node::RpcResult}; - -#[allow(missing_docs)] -#[derive(strum::Display, Debug, Serialize, Deserialize)] -#[enum_conversions(super::Request)] -#[rpc_requests(RpcService)] -pub enum Request { - #[rpc(response = RpcResult)] - Create(CreateRequest), - #[rpc(response = RpcResult<()>)] - Set(SetRequest), - #[rpc(response = RpcResult<()>)] - DeleteTag(DeleteRequest), - #[server_streaming(response = TagInfo)] - ListTags(ListRequest), -} - -#[allow(missing_docs)] -#[derive(strum::Display, Debug, Serialize, Deserialize)] -#[enum_conversions(super::Response)] -pub enum Response { - Create(RpcResult), - ListTags(TagInfo), - DeleteTag(RpcResult<()>), -} - -/// Determine how to sync the db after a modification operation -#[derive(Debug, Serialize, Deserialize, Default)] -pub enum SyncMode { - /// Fully sync the db - #[default] - Full, - /// Do not sync the db - None, -} - -/// Create a tag -#[derive(Debug, Serialize, Deserialize)] -pub struct CreateRequest { - /// Value of the tag - pub value: HashAndFormat, - /// Batch to use, none for global - pub batch: Option, - /// Sync mode - pub sync: SyncMode, -} - -/// Set or delete a tag -#[derive(Debug, Serialize, Deserialize)] -pub struct SetRequest { - /// Name of the tag - pub name: Tag, - /// Value of the tag, None to delete - pub value: Option, - /// Batch to use, none for global - pub batch: Option, - /// Sync mode - pub sync: SyncMode, -} - -/// List all collections -/// -/// Lists all collections that have been explicitly added to the database. -#[derive(Debug, Serialize, Deserialize)] -pub struct ListRequest { - /// List raw tags - pub raw: bool, - /// List hash seq tags - pub hash_seq: bool, -} - -impl ListRequest { - /// List all tags - pub fn all() -> Self { - Self { - raw: true, - hash_seq: true, - } - } - - /// List raw tags - pub fn raw() -> Self { - Self { - raw: true, - hash_seq: false, - } - } - - /// List hash seq tags - pub fn hash_seq() -> Self { - Self { - raw: false, - hash_seq: true, - } - } -} - -/// Delete a tag -#[derive(Debug, Serialize, Deserialize)] -pub struct DeleteRequest { - /// Name of the tag - pub name: Tag, -} diff --git a/iroh/tests/provide.rs b/iroh/tests/provide.rs index 7d380dc4bb2..5d9214b62dd 100644 --- a/iroh/tests/provide.rs +++ b/iroh/tests/provide.rs @@ -10,7 +10,7 @@ use bao_tree::{blake3, ChunkNum, ChunkRanges}; use bytes::Bytes; use futures_lite::FutureExt; use iroh::node::{Builder, DocsStorage}; -use iroh_base::node_addr::AddrInfoOptions; +use iroh_base::{node_addr::AddrInfoOptions, ticket::BlobTicket}; use iroh_blobs::{ format::collection::Collection, get::{ @@ -386,15 +386,11 @@ async fn test_run_ticket() { let node = test_node(db).spawn().await.unwrap(); let _drop_guard = node.cancel_token().drop_guard(); - let ticket = node - .blobs() - .share( - hash, - BlobFormat::HashSeq, - AddrInfoOptions::RelayAndAddresses, - ) - .await - .unwrap(); + let mut addr = node.net().node_addr().await.unwrap(); + addr.apply_options(AddrInfoOptions::RelayAndAddresses); + let ticket = BlobTicket::new(addr, hash, iroh_blobs::BlobFormat::HashSeq) + .expect("ticket creation failed"); + tokio::time::timeout(Duration::from_secs(10), async move { let request = GetRequest::all(hash); run_collection_get_request(SecretKey::generate(), ticket.node_addr().clone(), request).await From 06b55c7f057777e33525dc8da9b57b1aff9f2655 Mon Sep 17 00:00:00 2001 From: Ruediger Klaehn Date: Wed, 30 Oct 2024 19:15:52 +0200 Subject: [PATCH 02/20] remove some warnings --- iroh-cli/src/commands/blobs.rs | 4 +- iroh/src/client.rs | 3 - iroh/src/node/rpc.rs | 254 +++++++++++++++++---------------- iroh/tests/provide.rs | 4 +- 4 files changed, 138 insertions(+), 127 deletions(-) diff --git a/iroh-cli/src/commands/blobs.rs b/iroh-cli/src/commands/blobs.rs index 7c199df3e34..d40db80dd54 100644 --- a/iroh-cli/src/commands/blobs.rs +++ b/iroh-cli/src/commands/blobs.rs @@ -370,7 +370,9 @@ impl BlobCommands { BlobFormat::Raw }; let status = iroh.blobs().status(hash).await?; - let ticket: BlobTicket = todo!(); + let mut addr: NodeAddr = iroh.net().node_addr().await?; + addr.apply_options(addr_options); + let ticket = BlobTicket::new(addr, hash, format)?; let (blob_status, size) = match (status, format) { (BlobStatus::Complete { size }, BlobFormat::Raw) => ("blob", size), diff --git a/iroh/src/client.rs b/iroh/src/client.rs index 514e4ca9646..9214d1c341a 100644 --- a/iroh/src/client.rs +++ b/iroh/src/client.rs @@ -24,9 +24,6 @@ pub mod net; pub use iroh_blobs::rpc::client::blobs; pub use iroh_blobs::rpc::client::tags; -/// Iroh rpc connection - boxed so that we can have a concrete type. -pub(crate) type RpcConnection = BoxedServiceConnection; - // Keep this type exposed, otherwise every occurrence of `RpcClient` in the API // will show up as `RpcClient>` in the docs. /// Iroh rpc client - boxed so that we can have a concrete type. diff --git a/iroh/src/node/rpc.rs b/iroh/src/node/rpc.rs index a38993abda4..963b170f403 100644 --- a/iroh/src/node/rpc.rs +++ b/iroh/src/node/rpc.rs @@ -189,7 +189,11 @@ impl Handler { async fn handle_blobs_request( self, msg: blobs::Request, - chan: RpcChannel, + chan: RpcChannel< + iroh_blobs::rpc::proto::RpcService, + IrohServerEndpoint, + crate::rpc_protocol::RpcService, + >, ) -> Result<(), RpcServerError> { use blobs::Request::*; debug!("handling blob request: {msg}"); @@ -231,7 +235,11 @@ impl Handler { async fn handle_tags_request( self, msg: tags::Request, - chan: RpcChannel, + chan: RpcChannel< + iroh_blobs::rpc::proto::RpcService, + IrohServerEndpoint, + crate::rpc_protocol::RpcService, + >, ) -> Result<(), RpcServerError> { use tags::Request::*; match msg { @@ -322,8 +330,12 @@ impl Handler { match msg { Net(msg) => self.handle_net_request(msg, chan).await, Node(msg) => self.handle_node_request(msg, chan).await, - BlobsAndTags(iroh_blobs::rpc::proto::Request::Blobs(msg)) => self.handle_blobs_request(msg, chan.map()).await, - BlobsAndTags(iroh_blobs::rpc::proto::Request::Tags(msg)) => self.handle_tags_request(msg, chan.map()).await, + BlobsAndTags(iroh_blobs::rpc::proto::Request::Blobs(msg)) => { + self.handle_blobs_request(msg, chan.map()).await + } + BlobsAndTags(iroh_blobs::rpc::proto::Request::Tags(msg)) => { + self.handle_tags_request(msg, chan.map()).await + } Authors(msg) => self.handle_authors_request(msg, chan).await, Docs(msg) => self.handle_docs_request(msg, chan).await, Gossip(msg) => self.handle_gossip_request(msg, chan).await, @@ -640,6 +652,123 @@ impl Handler { Ok(()) } + #[allow(clippy::unused_async)] + async fn node_stats(self, _req: StatsRequest) -> RpcResult { + #[cfg(feature = "metrics")] + let res = Ok(StatsResponse { + stats: crate::metrics::get_metrics().map_err(|e| RpcError::new(&*e))?, + }); + + #[cfg(not(feature = "metrics"))] + let res = Err(RpcError::new(&*anyhow::anyhow!("metrics are disabled"))); + + res + } + + async fn node_status(self, _: StatusRequest) -> RpcResult { + Ok(NodeStatus { + addr: self + .inner + .endpoint + .node_addr() + .await + .map_err(|e| RpcError::new(&*e))?, + listen_addrs: self + .inner + .local_endpoint_addresses() + .await + .unwrap_or_default(), + version: env!("CARGO_PKG_VERSION").to_string(), + rpc_addr: self.inner.rpc_addr, + }) + } + + #[allow(clippy::unused_async)] + async fn node_id(self, _: IdRequest) -> RpcResult { + Ok(self.inner.endpoint.secret_key().public()) + } + + async fn node_addr(self, _: AddrRequest) -> RpcResult { + let addr = self + .inner + .endpoint + .node_addr() + .await + .map_err(|e| RpcError::new(&*e))?; + Ok(addr) + } + + #[allow(clippy::unused_async)] + async fn node_relay(self, _: RelayRequest) -> RpcResult> { + Ok(self.inner.endpoint.home_relay()) + } + + #[allow(clippy::unused_async)] + async fn node_shutdown(self, request: ShutdownRequest) { + if request.force { + info!("hard shutdown requested"); + std::process::exit(0); + } else { + // trigger a graceful shutdown + info!("graceful shutdown requested"); + self.inner.cancel_token.cancel(); + } + } + + fn node_watch(self, _: NodeWatchRequest) -> impl Stream { + futures_lite::stream::unfold((), |()| async move { + tokio::time::sleep(HEALTH_POLL_WAIT).await; + Some(( + WatchResponse { + version: env!("CARGO_PKG_VERSION").to_string(), + }, + (), + )) + }) + } + + async fn tags_set(self, msg: tags::SetRequest) -> RpcResult<()> { + let blobs = self.blobs(); + blobs + .store() + .set_tag(msg.name, msg.value) + .await + .map_err(|e| RpcError::new(&e))?; + if let SyncMode::Full = msg.sync { + blobs.store().sync().await.map_err(|e| RpcError::new(&e))?; + } + if let Some(batch) = msg.batch { + if let Some(content) = msg.value.as_ref() { + blobs + .batches() + .await + .remove_one(batch, content) + .map_err(|e| RpcError::new(&*e))?; + } + } + Ok(()) + } + + async fn tags_create(self, msg: tags::CreateRequest) -> RpcResult { + let blobs = self.blobs(); + let tag = blobs + .store() + .create_tag(msg.value) + .await + .map_err(|e| RpcError::new(&e))?; + if let SyncMode::Full = msg.sync { + blobs.store().sync().await.map_err(|e| RpcError::new(&e))?; + } + if let Some(batch) = msg.batch { + blobs + .batches() + .await + .remove_one(batch, &msg.value) + .map_err(|e| RpcError::new(&*e))?; + } + Ok(tag) + } + fn blob_download(self, msg: BlobDownloadRequest) -> impl Stream { let (sender, receiver) = async_channel::bounded(1024); let endpoint = self.inner.endpoint.clone(); @@ -808,123 +937,6 @@ impl Handler { Ok(()) } - #[allow(clippy::unused_async)] - async fn node_stats(self, _req: StatsRequest) -> RpcResult { - #[cfg(feature = "metrics")] - let res = Ok(StatsResponse { - stats: crate::metrics::get_metrics().map_err(|e| RpcError::new(&*e))?, - }); - - #[cfg(not(feature = "metrics"))] - let res = Err(RpcError::new(&*anyhow::anyhow!("metrics are disabled"))); - - res - } - - async fn node_status(self, _: StatusRequest) -> RpcResult { - Ok(NodeStatus { - addr: self - .inner - .endpoint - .node_addr() - .await - .map_err(|e| RpcError::new(&*e))?, - listen_addrs: self - .inner - .local_endpoint_addresses() - .await - .unwrap_or_default(), - version: env!("CARGO_PKG_VERSION").to_string(), - rpc_addr: self.inner.rpc_addr, - }) - } - - #[allow(clippy::unused_async)] - async fn node_id(self, _: IdRequest) -> RpcResult { - Ok(self.inner.endpoint.secret_key().public()) - } - - async fn node_addr(self, _: AddrRequest) -> RpcResult { - let addr = self - .inner - .endpoint - .node_addr() - .await - .map_err(|e| RpcError::new(&*e))?; - Ok(addr) - } - - #[allow(clippy::unused_async)] - async fn node_relay(self, _: RelayRequest) -> RpcResult> { - Ok(self.inner.endpoint.home_relay()) - } - - #[allow(clippy::unused_async)] - async fn node_shutdown(self, request: ShutdownRequest) { - if request.force { - info!("hard shutdown requested"); - std::process::exit(0); - } else { - // trigger a graceful shutdown - info!("graceful shutdown requested"); - self.inner.cancel_token.cancel(); - } - } - - async fn tags_set(self, msg: tags::SetRequest) -> RpcResult<()> { - let blobs = self.blobs(); - blobs - .store() - .set_tag(msg.name, msg.value) - .await - .map_err(|e| RpcError::new(&e))?; - if let SyncMode::Full = msg.sync { - blobs.store().sync().await.map_err(|e| RpcError::new(&e))?; - } - if let Some(batch) = msg.batch { - if let Some(content) = msg.value.as_ref() { - blobs - .batches() - .await - .remove_one(batch, content) - .map_err(|e| RpcError::new(&*e))?; - } - } - Ok(()) - } - - async fn tags_create(self, msg: tags::CreateRequest) -> RpcResult { - let blobs = self.blobs(); - let tag = blobs - .store() - .create_tag(msg.value) - .await - .map_err(|e| RpcError::new(&e))?; - if let SyncMode::Full = msg.sync { - blobs.store().sync().await.map_err(|e| RpcError::new(&e))?; - } - if let Some(batch) = msg.batch { - blobs - .batches() - .await - .remove_one(batch, &msg.value) - .map_err(|e| RpcError::new(&*e))?; - } - Ok(tag) - } - - fn node_watch(self, _: NodeWatchRequest) -> impl Stream { - futures_lite::stream::unfold((), |()| async move { - tokio::time::sleep(HEALTH_POLL_WAIT).await; - Some(( - WatchResponse { - version: env!("CARGO_PKG_VERSION").to_string(), - }, - (), - )) - }) - } - async fn batch_create_temp_tag(self, msg: BatchCreateTempTagRequest) -> RpcResult<()> { let blobs = self.blobs(); let tag = blobs.store().temp_tag(msg.content); diff --git a/iroh/tests/provide.rs b/iroh/tests/provide.rs index 5d9214b62dd..87ae66f3842 100644 --- a/iroh/tests/provide.rs +++ b/iroh/tests/provide.rs @@ -19,7 +19,7 @@ use iroh_blobs::{ }, protocol::{GetRequest, RangeSpecSeq}, store::{MapMut, Store}, - BlobFormat, Hash, + Hash, }; use iroh_net::{defaults::staging::default_relay_map, key::SecretKey, NodeAddr, NodeId}; use rand::RngCore; @@ -390,7 +390,7 @@ async fn test_run_ticket() { addr.apply_options(AddrInfoOptions::RelayAndAddresses); let ticket = BlobTicket::new(addr, hash, iroh_blobs::BlobFormat::HashSeq) .expect("ticket creation failed"); - + tokio::time::timeout(Duration::from_secs(10), async move { let request = GetRequest::all(hash); run_collection_get_request(SecretKey::generate(), ticket.node_addr().clone(), request).await From 5ec42686c3684775adbc97fbae22f7e85824d011 Mon Sep 17 00:00:00 2001 From: Ruediger Klaehn Date: Wed, 30 Oct 2024 22:14:37 +0200 Subject: [PATCH 03/20] WIP --- iroh/src/node/rpc.rs | 358 +++++++++++++++++++++---------------------- 1 file changed, 179 insertions(+), 179 deletions(-) diff --git a/iroh/src/node/rpc.rs b/iroh/src/node/rpc.rs index 963b170f403..d06be0f57cf 100644 --- a/iroh/src/node/rpc.rs +++ b/iroh/src/node/rpc.rs @@ -342,185 +342,6 @@ impl Handler { } } - fn local_pool_handle(&self) -> LocalPoolHandle { - self.inner.local_pool_handle.clone() - } - - async fn blob_status(self, msg: BlobStatusRequest) -> RpcResult { - let blobs = self.blobs(); - let entry = blobs - .store() - .get(&msg.hash) - .await - .map_err(|e| RpcError::new(&e))?; - Ok(BlobStatusResponse(match entry { - Some(entry) => { - if entry.is_complete() { - BlobStatus::Complete { - size: entry.size().value(), - } - } else { - BlobStatus::Partial { size: entry.size() } - } - } - None => BlobStatus::NotFound, - })) - } - - async fn blob_list_impl(self, co: &Co>) -> io::Result<()> { - use bao_tree::io::fsm::Outboard; - - let blobs = self.blobs(); - let db = blobs.store(); - for blob in db.blobs().await? { - let blob = blob?; - let Some(entry) = db.get(&blob).await? else { - continue; - }; - let hash = entry.hash(); - let size = entry.outboard().await?.tree().size(); - let path = "".to_owned(); - co.yield_(Ok(BlobInfo { hash, size, path })).await; - } - Ok(()) - } - - async fn blob_list_incomplete_impl( - self, - co: &Co>, - ) -> io::Result<()> { - let blobs = self.blobs(); - let db = blobs.store(); - for hash in db.partial_blobs().await? { - let hash = hash?; - let Ok(Some(entry)) = db.get_mut(&hash).await else { - continue; - }; - if entry.is_complete() { - continue; - } - let size = 0; - let expected_size = entry.size().value(); - co.yield_(Ok(IncompleteBlobInfo { - hash, - size, - expected_size, - })) - .await; - } - Ok(()) - } - - fn blob_list( - self, - _msg: ListRequest, - ) -> impl Stream> + Send + 'static { - Gen::new(|co| async move { - if let Err(e) = self.blob_list_impl(&co).await { - co.yield_(Err(RpcError::new(&e))).await; - } - }) - } - - fn blob_list_incomplete( - self, - _msg: ListIncompleteRequest, - ) -> impl Stream> + Send + 'static { - Gen::new(move |co| async move { - if let Err(e) = self.blob_list_incomplete_impl(&co).await { - co.yield_(Err(RpcError::new(&e))).await; - } - }) - } - - async fn blob_delete_tag(self, msg: TagDeleteRequest) -> RpcResult<()> { - self.blobs_store() - .set_tag(msg.name, None) - .await - .map_err(|e| RpcError::new(&e))?; - Ok(()) - } - - async fn blob_delete_blob(self, msg: DeleteRequest) -> RpcResult<()> { - self.blobs_store() - .delete(vec![msg.hash]) - .await - .map_err(|e| RpcError::new(&e))?; - Ok(()) - } - - fn blob_list_tags(self, msg: ListTagsRequest) -> impl Stream + Send + 'static { - tracing::info!("blob_list_tags"); - let blobs = self.blobs(); - Gen::new(|co| async move { - let tags = blobs.store().tags().await.unwrap(); - #[allow(clippy::manual_flatten)] - for item in tags { - if let Ok((name, HashAndFormat { hash, format })) = item { - if (format.is_raw() && msg.raw) || (format.is_hash_seq() && msg.hash_seq) { - co.yield_(TagInfo { name, hash, format }).await; - } - } - } - }) - } - - /// Invoke validate on the database and stream out the result - fn blob_validate( - self, - msg: ValidateRequest, - ) -> impl Stream + Send + 'static { - let (tx, rx) = async_channel::bounded(1); - let tx2 = tx.clone(); - let blobs = self.blobs(); - tokio::task::spawn(async move { - if let Err(e) = blobs - .store() - .validate(msg.repair, AsyncChannelProgressSender::new(tx).boxed()) - .await - { - tx2.send(ValidateProgress::Abort(RpcError::new(&e))) - .await - .ok(); - } - }); - rx - } - - /// Invoke validate on the database and stream out the result - fn blob_consistency_check( - self, - msg: ConsistencyCheckRequest, - ) -> impl Stream + Send + 'static { - let (tx, rx) = async_channel::bounded(1); - let tx2 = tx.clone(); - let blobs = self.blobs(); - tokio::task::spawn(async move { - if let Err(e) = blobs - .store() - .consistency_check(msg.repair, AsyncChannelProgressSender::new(tx).boxed()) - .await - { - tx2.send(ConsistencyCheckProgress::Abort(RpcError::new(&e))) - .await - .ok(); - } - }); - rx - } - - fn blob_add_from_path(self, msg: AddPathRequest) -> impl Stream { - // provide a little buffer so that we don't slow down the sender - let (tx, rx) = async_channel::bounded(32); - let tx2 = tx.clone(); - self.local_pool_handle().spawn_detached(|| async move { - if let Err(e) = self.blob_add_from_path0(msg, tx).await { - tx2.send(AddProgress::Abort(RpcError::new(&*e))).await.ok(); - } - }); - rx.map(AddPathResponse) - } - fn doc_import_file(self, msg: ImportFileRequest) -> impl Stream { // provide a little buffer so that we don't slow down the sender let (tx, rx) = async_channel::bounded(32); @@ -727,6 +548,185 @@ impl Handler { }) } + fn local_pool_handle(&self) -> LocalPoolHandle { + self.inner.local_pool_handle.clone() + } + + async fn blob_status(self, msg: BlobStatusRequest) -> RpcResult { + let blobs = self.blobs(); + let entry = blobs + .store() + .get(&msg.hash) + .await + .map_err(|e| RpcError::new(&e))?; + Ok(BlobStatusResponse(match entry { + Some(entry) => { + if entry.is_complete() { + BlobStatus::Complete { + size: entry.size().value(), + } + } else { + BlobStatus::Partial { size: entry.size() } + } + } + None => BlobStatus::NotFound, + })) + } + + async fn blob_list_impl(self, co: &Co>) -> io::Result<()> { + use bao_tree::io::fsm::Outboard; + + let blobs = self.blobs(); + let db = blobs.store(); + for blob in db.blobs().await? { + let blob = blob?; + let Some(entry) = db.get(&blob).await? else { + continue; + }; + let hash = entry.hash(); + let size = entry.outboard().await?.tree().size(); + let path = "".to_owned(); + co.yield_(Ok(BlobInfo { hash, size, path })).await; + } + Ok(()) + } + + async fn blob_list_incomplete_impl( + self, + co: &Co>, + ) -> io::Result<()> { + let blobs = self.blobs(); + let db = blobs.store(); + for hash in db.partial_blobs().await? { + let hash = hash?; + let Ok(Some(entry)) = db.get_mut(&hash).await else { + continue; + }; + if entry.is_complete() { + continue; + } + let size = 0; + let expected_size = entry.size().value(); + co.yield_(Ok(IncompleteBlobInfo { + hash, + size, + expected_size, + })) + .await; + } + Ok(()) + } + + fn blob_list( + self, + _msg: ListRequest, + ) -> impl Stream> + Send + 'static { + Gen::new(|co| async move { + if let Err(e) = self.blob_list_impl(&co).await { + co.yield_(Err(RpcError::new(&e))).await; + } + }) + } + + fn blob_list_incomplete( + self, + _msg: ListIncompleteRequest, + ) -> impl Stream> + Send + 'static { + Gen::new(move |co| async move { + if let Err(e) = self.blob_list_incomplete_impl(&co).await { + co.yield_(Err(RpcError::new(&e))).await; + } + }) + } + + async fn blob_delete_tag(self, msg: TagDeleteRequest) -> RpcResult<()> { + self.blobs_store() + .set_tag(msg.name, None) + .await + .map_err(|e| RpcError::new(&e))?; + Ok(()) + } + + async fn blob_delete_blob(self, msg: DeleteRequest) -> RpcResult<()> { + self.blobs_store() + .delete(vec![msg.hash]) + .await + .map_err(|e| RpcError::new(&e))?; + Ok(()) + } + + fn blob_list_tags(self, msg: ListTagsRequest) -> impl Stream + Send + 'static { + tracing::info!("blob_list_tags"); + let blobs = self.blobs(); + Gen::new(|co| async move { + let tags = blobs.store().tags().await.unwrap(); + #[allow(clippy::manual_flatten)] + for item in tags { + if let Ok((name, HashAndFormat { hash, format })) = item { + if (format.is_raw() && msg.raw) || (format.is_hash_seq() && msg.hash_seq) { + co.yield_(TagInfo { name, hash, format }).await; + } + } + } + }) + } + + /// Invoke validate on the database and stream out the result + fn blob_validate( + self, + msg: ValidateRequest, + ) -> impl Stream + Send + 'static { + let (tx, rx) = async_channel::bounded(1); + let tx2 = tx.clone(); + let blobs = self.blobs(); + tokio::task::spawn(async move { + if let Err(e) = blobs + .store() + .validate(msg.repair, AsyncChannelProgressSender::new(tx).boxed()) + .await + { + tx2.send(ValidateProgress::Abort(RpcError::new(&e))) + .await + .ok(); + } + }); + rx + } + + /// Invoke validate on the database and stream out the result + fn blob_consistency_check( + self, + msg: ConsistencyCheckRequest, + ) -> impl Stream + Send + 'static { + let (tx, rx) = async_channel::bounded(1); + let tx2 = tx.clone(); + let blobs = self.blobs(); + tokio::task::spawn(async move { + if let Err(e) = blobs + .store() + .consistency_check(msg.repair, AsyncChannelProgressSender::new(tx).boxed()) + .await + { + tx2.send(ConsistencyCheckProgress::Abort(RpcError::new(&e))) + .await + .ok(); + } + }); + rx + } + + fn blob_add_from_path(self, msg: AddPathRequest) -> impl Stream { + // provide a little buffer so that we don't slow down the sender + let (tx, rx) = async_channel::bounded(32); + let tx2 = tx.clone(); + self.local_pool_handle().spawn_detached(|| async move { + if let Err(e) = self.blob_add_from_path0(msg, tx).await { + tx2.send(AddProgress::Abort(RpcError::new(&*e))).await.ok(); + } + }); + rx.map(AddPathResponse) + } + async fn tags_set(self, msg: tags::SetRequest) -> RpcResult<()> { let blobs = self.blobs(); blobs From ec7ca5915c99e6d5be7d4368d5b3f8aa0efea967 Mon Sep 17 00:00:00 2001 From: Ruediger Klaehn Date: Thu, 31 Oct 2024 12:10:32 +0200 Subject: [PATCH 04/20] Remove the entire server side of the rpc handling --- iroh/src/node/builder.rs | 1 + iroh/src/node/rpc.rs | 894 ++++----------------------------------- iroh/src/rpc_protocol.rs | 2 - 3 files changed, 74 insertions(+), 823 deletions(-) diff --git a/iroh/src/node/builder.rs b/iroh/src/node/builder.rs index beca58c211d..c1c505f136b 100644 --- a/iroh/src/node/builder.rs +++ b/iroh/src/node/builder.rs @@ -847,6 +847,7 @@ impl ProtocolBuilder { self.local_pool_handle().clone(), blob_events, downloader, + self.endpoint().clone(), ); self = self.accept(iroh_blobs::protocol::ALPN.to_vec(), Arc::new(blobs_proto)); diff --git a/iroh/src/node/rpc.rs b/iroh/src/node/rpc.rs index d06be0f57cf..bd10752fd4d 100644 --- a/iroh/src/node/rpc.rs +++ b/iroh/src/node/rpc.rs @@ -1,35 +1,23 @@ use std::{ fmt::Debug, - io, sync::{Arc, Mutex}, time::Duration, }; use anyhow::{anyhow, Result}; -use futures_buffered::BufferedStreamExt; use futures_lite::{Stream, StreamExt}; -use futures_util::FutureExt; -use genawaiter::sync::{Co, Gen}; use iroh_blobs::{ export::ExportProgress, - format::collection::Collection, - get::db::DownloadProgress, - net_protocol::{BlobDownloadRequest, Blobs as BlobsProtocol}, - provider::{AddProgress, BatchAddPathProgress}, - store::{ - ConsistencyCheckProgress, ExportFormat, ImportProgress, MapEntry, Store as BaoStore, - ValidateProgress, - }, + net_protocol::Blobs as BlobsProtocol, + store::{ExportFormat, ImportProgress, Store as BaoStore}, util::{ local_pool::LocalPoolHandle, progress::{AsyncChannelProgressSender, ProgressSender}, - SetTagOption, }, - BlobFormat, HashAndFormat, Tag, + BlobFormat, HashAndFormat, }; use iroh_docs::{engine::Engine, net::DOCS_ALPN}; use iroh_gossip::net::{Gossip, GOSSIP_ALPN}; -use iroh_io::AsyncSliceReader; use iroh_net::{relay::RelayUrl, NodeAddr, NodeId}; use iroh_router::Router; use quic_rpc::server::{RpcChannel, RpcServerError}; @@ -39,23 +27,10 @@ use tracing::{debug, info, warn}; use super::IrohServerEndpoint; use crate::{ - client::{ - blobs::{BlobInfo, BlobStatus, IncompleteBlobInfo, WrapOption}, - tags::TagInfo, - NodeStatus, - }, + client::NodeStatus, node::NodeInner, rpc_protocol::{ authors, - blobs::{ - self, AddPathRequest, AddPathResponse, AddStreamRequest, AddStreamResponse, - AddStreamUpdate, BatchAddPathRequest, BatchAddPathResponse, BatchAddStreamRequest, - BatchAddStreamResponse, BatchAddStreamUpdate, BatchCreateRequest, BatchCreateResponse, - BatchCreateTempTagRequest, BatchUpdate, BlobStatusRequest, BlobStatusResponse, - ConsistencyCheckRequest, CreateCollectionRequest, CreateCollectionResponse, - DeleteRequest, DownloadResponse, ExportRequest, ExportResponse, ListIncompleteRequest, - ListRequest, ReadAtRequest, ReadAtResponse, ValidateRequest, - }, docs::{ ExportFileRequest, ExportFileResponse, ImportFileRequest, ImportFileResponse, Request as DocsRequest, SetHashRequest, @@ -66,7 +41,6 @@ use crate::{ WatchResponse, }, node::{self, ShutdownRequest, StatsRequest, StatsResponse, StatusRequest}, - tags::{self, DeleteRequest as TagDeleteRequest, ListRequest as ListTagsRequest, SyncMode}, Request, RpcService, }, }; @@ -74,11 +48,6 @@ use crate::{ mod docs; const HEALTH_POLL_WAIT: Duration = Duration::from_secs(1); -/// Chunk size for getting blobs over RPC -const RPC_BLOB_GET_CHUNK_SIZE: usize = 1024 * 64; -/// Channel cap for getting blobs over RPC -const RPC_BLOB_GET_CHANNEL_CAP: usize = 2; - pub(crate) type RpcError = serde_error::Error; pub(crate) type RpcResult = Result; @@ -186,69 +155,81 @@ impl Handler { } } - async fn handle_blobs_request( - self, - msg: blobs::Request, - chan: RpcChannel< - iroh_blobs::rpc::proto::RpcService, - IrohServerEndpoint, - crate::rpc_protocol::RpcService, - >, - ) -> Result<(), RpcServerError> { - use blobs::Request::*; - debug!("handling blob request: {msg}"); - match msg { - List(msg) => chan.server_streaming(msg, self, Self::blob_list).await, - ListIncomplete(msg) => { - chan.server_streaming(msg, self, Self::blob_list_incomplete) - .await - } - CreateCollection(msg) => chan.rpc(msg, self, Self::create_collection).await, - Delete(msg) => chan.rpc(msg, self, Self::blob_delete_blob).await, - AddPath(msg) => { - chan.server_streaming(msg, self, Self::blob_add_from_path) - .await - } - Download(msg) => chan.server_streaming(msg, self, Self::blob_download).await, - Export(msg) => chan.server_streaming(msg, self, Self::blob_export).await, - Validate(msg) => chan.server_streaming(msg, self, Self::blob_validate).await, - Fsck(msg) => { - chan.server_streaming(msg, self, Self::blob_consistency_check) - .await - } - ReadAt(msg) => chan.server_streaming(msg, self, Self::blob_read_at).await, - AddStream(msg) => chan.bidi_streaming(msg, self, Self::blob_add_stream).await, - AddStreamUpdate(_msg) => Err(RpcServerError::UnexpectedUpdateMessage), - BlobStatus(msg) => chan.rpc(msg, self, Self::blob_status).await, - BatchCreate(msg) => chan.bidi_streaming(msg, self, Self::batch_create).await, - BatchUpdate(_) => Err(RpcServerError::UnexpectedStartMessage), - BatchAddStream(msg) => chan.bidi_streaming(msg, self, Self::batch_add_stream).await, - BatchAddStreamUpdate(_) => Err(RpcServerError::UnexpectedStartMessage), - BatchAddPath(msg) => { - chan.server_streaming(msg, self, Self::batch_add_from_path) - .await - } - BatchCreateTempTag(msg) => chan.rpc(msg, self, Self::batch_create_temp_tag).await, - } - } - - async fn handle_tags_request( + async fn handle_blobs_and_tags_request( self, - msg: tags::Request, + msg: iroh_blobs::rpc::proto::Request, chan: RpcChannel< iroh_blobs::rpc::proto::RpcService, IrohServerEndpoint, crate::rpc_protocol::RpcService, >, ) -> Result<(), RpcServerError> { - use tags::Request::*; - match msg { - ListTags(msg) => chan.server_streaming(msg, self, Self::blob_list_tags).await, - DeleteTag(msg) => chan.rpc(msg, self, Self::blob_delete_tag).await, - Create(msg) => chan.rpc(msg, self, Self::tags_create).await, - Set(msg) => chan.rpc(msg, self, Self::tags_set).await, - } - } + self.blobs().handle_rpc_request(msg, chan).await + } + + // async fn handle_blobs_request( + // self, + // msg: blobs::Request, + // chan: RpcChannel< + // iroh_blobs::rpc::proto::RpcService, + // IrohServerEndpoint, + // crate::rpc_protocol::RpcService, + // >, + // ) -> Result<(), RpcServerError> { + // use blobs::Request::*; + // debug!("handling blob request: {msg}"); + // match msg { + // List(msg) => chan.server_streaming(msg, self, Self::blob_list).await, + // ListIncomplete(msg) => { + // chan.server_streaming(msg, self, Self::blob_list_incomplete) + // .await + // } + // CreateCollection(msg) => chan.rpc(msg, self, Self::create_collection).await, + // Delete(msg) => chan.rpc(msg, self, Self::blob_delete_blob).await, + // AddPath(msg) => { + // chan.server_streaming(msg, self, Self::blob_add_from_path) + // .await + // } + // Download(msg) => chan.server_streaming(msg, self, Self::blob_download).await, + // Export(msg) => chan.server_streaming(msg, self, Self::blob_export).await, + // Validate(msg) => chan.server_streaming(msg, self, Self::blob_validate).await, + // Fsck(msg) => { + // chan.server_streaming(msg, self, Self::blob_consistency_check) + // .await + // } + // ReadAt(msg) => chan.server_streaming(msg, self, Self::blob_read_at).await, + // AddStream(msg) => chan.bidi_streaming(msg, self, Self::blob_add_stream).await, + // AddStreamUpdate(_msg) => Err(RpcServerError::UnexpectedUpdateMessage), + // BlobStatus(msg) => chan.rpc(msg, self, Self::blob_status).await, + // BatchCreate(msg) => chan.bidi_streaming(msg, self, Self::batch_create).await, + // BatchUpdate(_) => Err(RpcServerError::UnexpectedStartMessage), + // BatchAddStream(msg) => chan.bidi_streaming(msg, self, Self::batch_add_stream).await, + // BatchAddStreamUpdate(_) => Err(RpcServerError::UnexpectedStartMessage), + // BatchAddPath(msg) => { + // chan.server_streaming(msg, self, Self::batch_add_from_path) + // .await + // } + // BatchCreateTempTag(msg) => chan.rpc(msg, self, Self::batch_create_temp_tag).await, + // } + // } + + // async fn handle_tags_request( + // self, + // msg: tags::Request, + // chan: RpcChannel< + // iroh_blobs::rpc::proto::RpcService, + // IrohServerEndpoint, + // crate::rpc_protocol::RpcService, + // >, + // ) -> Result<(), RpcServerError> { + // use tags::Request::*; + // match msg { + // ListTags(msg) => chan.server_streaming(msg, self, Self::blob_list_tags).await, + // DeleteTag(msg) => chan.rpc(msg, self, Self::blob_delete_tag).await, + // Create(msg) => chan.rpc(msg, self, Self::tags_create).await, + // Set(msg) => chan.rpc(msg, self, Self::tags_set).await, + // } + // } async fn handle_gossip_request( self, @@ -330,12 +311,7 @@ impl Handler { match msg { Net(msg) => self.handle_net_request(msg, chan).await, Node(msg) => self.handle_node_request(msg, chan).await, - BlobsAndTags(iroh_blobs::rpc::proto::Request::Blobs(msg)) => { - self.handle_blobs_request(msg, chan.map()).await - } - BlobsAndTags(iroh_blobs::rpc::proto::Request::Tags(msg)) => { - self.handle_tags_request(msg, chan.map()).await - } + BlobsAndTags(msg) => self.handle_blobs_and_tags_request(msg, chan.map()).await, Authors(msg) => self.handle_authors_request(msg, chan).await, Docs(msg) => self.handle_docs_request(msg, chan).await, Gossip(msg) => self.handle_gossip_request(msg, chan).await, @@ -552,685 +528,6 @@ impl Handler { self.inner.local_pool_handle.clone() } - async fn blob_status(self, msg: BlobStatusRequest) -> RpcResult { - let blobs = self.blobs(); - let entry = blobs - .store() - .get(&msg.hash) - .await - .map_err(|e| RpcError::new(&e))?; - Ok(BlobStatusResponse(match entry { - Some(entry) => { - if entry.is_complete() { - BlobStatus::Complete { - size: entry.size().value(), - } - } else { - BlobStatus::Partial { size: entry.size() } - } - } - None => BlobStatus::NotFound, - })) - } - - async fn blob_list_impl(self, co: &Co>) -> io::Result<()> { - use bao_tree::io::fsm::Outboard; - - let blobs = self.blobs(); - let db = blobs.store(); - for blob in db.blobs().await? { - let blob = blob?; - let Some(entry) = db.get(&blob).await? else { - continue; - }; - let hash = entry.hash(); - let size = entry.outboard().await?.tree().size(); - let path = "".to_owned(); - co.yield_(Ok(BlobInfo { hash, size, path })).await; - } - Ok(()) - } - - async fn blob_list_incomplete_impl( - self, - co: &Co>, - ) -> io::Result<()> { - let blobs = self.blobs(); - let db = blobs.store(); - for hash in db.partial_blobs().await? { - let hash = hash?; - let Ok(Some(entry)) = db.get_mut(&hash).await else { - continue; - }; - if entry.is_complete() { - continue; - } - let size = 0; - let expected_size = entry.size().value(); - co.yield_(Ok(IncompleteBlobInfo { - hash, - size, - expected_size, - })) - .await; - } - Ok(()) - } - - fn blob_list( - self, - _msg: ListRequest, - ) -> impl Stream> + Send + 'static { - Gen::new(|co| async move { - if let Err(e) = self.blob_list_impl(&co).await { - co.yield_(Err(RpcError::new(&e))).await; - } - }) - } - - fn blob_list_incomplete( - self, - _msg: ListIncompleteRequest, - ) -> impl Stream> + Send + 'static { - Gen::new(move |co| async move { - if let Err(e) = self.blob_list_incomplete_impl(&co).await { - co.yield_(Err(RpcError::new(&e))).await; - } - }) - } - - async fn blob_delete_tag(self, msg: TagDeleteRequest) -> RpcResult<()> { - self.blobs_store() - .set_tag(msg.name, None) - .await - .map_err(|e| RpcError::new(&e))?; - Ok(()) - } - - async fn blob_delete_blob(self, msg: DeleteRequest) -> RpcResult<()> { - self.blobs_store() - .delete(vec![msg.hash]) - .await - .map_err(|e| RpcError::new(&e))?; - Ok(()) - } - - fn blob_list_tags(self, msg: ListTagsRequest) -> impl Stream + Send + 'static { - tracing::info!("blob_list_tags"); - let blobs = self.blobs(); - Gen::new(|co| async move { - let tags = blobs.store().tags().await.unwrap(); - #[allow(clippy::manual_flatten)] - for item in tags { - if let Ok((name, HashAndFormat { hash, format })) = item { - if (format.is_raw() && msg.raw) || (format.is_hash_seq() && msg.hash_seq) { - co.yield_(TagInfo { name, hash, format }).await; - } - } - } - }) - } - - /// Invoke validate on the database and stream out the result - fn blob_validate( - self, - msg: ValidateRequest, - ) -> impl Stream + Send + 'static { - let (tx, rx) = async_channel::bounded(1); - let tx2 = tx.clone(); - let blobs = self.blobs(); - tokio::task::spawn(async move { - if let Err(e) = blobs - .store() - .validate(msg.repair, AsyncChannelProgressSender::new(tx).boxed()) - .await - { - tx2.send(ValidateProgress::Abort(RpcError::new(&e))) - .await - .ok(); - } - }); - rx - } - - /// Invoke validate on the database and stream out the result - fn blob_consistency_check( - self, - msg: ConsistencyCheckRequest, - ) -> impl Stream + Send + 'static { - let (tx, rx) = async_channel::bounded(1); - let tx2 = tx.clone(); - let blobs = self.blobs(); - tokio::task::spawn(async move { - if let Err(e) = blobs - .store() - .consistency_check(msg.repair, AsyncChannelProgressSender::new(tx).boxed()) - .await - { - tx2.send(ConsistencyCheckProgress::Abort(RpcError::new(&e))) - .await - .ok(); - } - }); - rx - } - - fn blob_add_from_path(self, msg: AddPathRequest) -> impl Stream { - // provide a little buffer so that we don't slow down the sender - let (tx, rx) = async_channel::bounded(32); - let tx2 = tx.clone(); - self.local_pool_handle().spawn_detached(|| async move { - if let Err(e) = self.blob_add_from_path0(msg, tx).await { - tx2.send(AddProgress::Abort(RpcError::new(&*e))).await.ok(); - } - }); - rx.map(AddPathResponse) - } - - async fn tags_set(self, msg: tags::SetRequest) -> RpcResult<()> { - let blobs = self.blobs(); - blobs - .store() - .set_tag(msg.name, msg.value) - .await - .map_err(|e| RpcError::new(&e))?; - if let SyncMode::Full = msg.sync { - blobs.store().sync().await.map_err(|e| RpcError::new(&e))?; - } - if let Some(batch) = msg.batch { - if let Some(content) = msg.value.as_ref() { - blobs - .batches() - .await - .remove_one(batch, content) - .map_err(|e| RpcError::new(&*e))?; - } - } - Ok(()) - } - - async fn tags_create(self, msg: tags::CreateRequest) -> RpcResult { - let blobs = self.blobs(); - let tag = blobs - .store() - .create_tag(msg.value) - .await - .map_err(|e| RpcError::new(&e))?; - if let SyncMode::Full = msg.sync { - blobs.store().sync().await.map_err(|e| RpcError::new(&e))?; - } - if let Some(batch) = msg.batch { - blobs - .batches() - .await - .remove_one(batch, &msg.value) - .map_err(|e| RpcError::new(&*e))?; - } - Ok(tag) - } - - fn blob_download(self, msg: BlobDownloadRequest) -> impl Stream { - let (sender, receiver) = async_channel::bounded(1024); - let endpoint = self.inner.endpoint.clone(); - let progress = AsyncChannelProgressSender::new(sender); - - let blobs_protocol = self - .router - .get_protocol::>(iroh_blobs::protocol::ALPN) - .expect("missing blobs"); - - self.local_pool_handle().spawn_detached(move || async move { - if let Err(err) = blobs_protocol - .download(endpoint, msg, progress.clone()) - .await - { - progress - .send(DownloadProgress::Abort(RpcError::new(&*err))) - .await - .ok(); - } - }); - - receiver.map(DownloadResponse) - } - - fn blob_export(self, msg: ExportRequest) -> impl Stream { - let (tx, rx) = async_channel::bounded(1024); - let progress = AsyncChannelProgressSender::new(tx); - self.local_pool_handle().spawn_detached(move || async move { - let res = iroh_blobs::export::export( - self.blobs().store(), - msg.hash, - msg.path, - msg.format, - msg.mode, - progress.clone(), - ) - .await; - match res { - Ok(()) => progress.send(ExportProgress::AllDone).await.ok(), - Err(err) => progress - .send(ExportProgress::Abort(RpcError::new(&*err))) - .await - .ok(), - }; - }); - rx.map(ExportResponse) - } - - async fn blob_add_from_path0( - self, - msg: AddPathRequest, - progress: async_channel::Sender, - ) -> anyhow::Result<()> { - use std::collections::BTreeMap; - - use iroh_blobs::store::ImportMode; - - let blobs = self.blobs(); - let progress = AsyncChannelProgressSender::new(progress); - let names = Arc::new(Mutex::new(BTreeMap::new())); - // convert import progress to provide progress - let import_progress = progress.clone().with_filter_map(move |x| match x { - ImportProgress::Found { id, name } => { - names.lock().unwrap().insert(id, name); - None - } - ImportProgress::Size { id, size } => { - let name = names.lock().unwrap().remove(&id)?; - Some(AddProgress::Found { id, name, size }) - } - ImportProgress::OutboardProgress { id, offset } => { - Some(AddProgress::Progress { id, offset }) - } - ImportProgress::OutboardDone { hash, id } => Some(AddProgress::Done { hash, id }), - _ => None, - }); - let AddPathRequest { - wrap, - path: root, - in_place, - tag, - } = msg; - // Check that the path is absolute and exists. - anyhow::ensure!(root.is_absolute(), "path must be absolute"); - anyhow::ensure!( - root.exists(), - "trying to add missing path: {}", - root.display() - ); - - let import_mode = match in_place { - true => ImportMode::TryReference, - false => ImportMode::Copy, - }; - - let create_collection = match wrap { - WrapOption::Wrap { .. } => true, - WrapOption::NoWrap => root.is_dir(), - }; - - let temp_tag = if create_collection { - // import all files below root recursively - let data_sources = crate::util::fs::scan_path(root, wrap)?; - let blobs = self.blobs(); - - const IO_PARALLELISM: usize = 4; - let result: Vec<_> = futures_lite::stream::iter(data_sources) - .map(|source| { - let import_progress = import_progress.clone(); - let blobs = blobs.clone(); - async move { - let name = source.name().to_string(); - let (tag, size) = blobs - .store() - .import_file( - source.path().to_owned(), - import_mode, - BlobFormat::Raw, - import_progress, - ) - .await?; - let hash = *tag.hash(); - io::Result::Ok((name, hash, size, tag)) - } - }) - .buffered_ordered(IO_PARALLELISM) - .try_collect() - .await?; - - // create a collection - let (collection, _child_tags): (Collection, Vec<_>) = result - .into_iter() - .map(|(name, hash, _, tag)| ((name, hash), tag)) - .unzip(); - - collection.store(blobs.store()).await? - } else { - // import a single file - let (tag, _size) = blobs - .store() - .import_file(root, import_mode, BlobFormat::Raw, import_progress) - .await?; - tag - }; - - let hash_and_format = temp_tag.inner(); - let HashAndFormat { hash, format } = *hash_and_format; - let tag = match tag { - SetTagOption::Named(tag) => { - blobs - .store() - .set_tag(tag.clone(), Some(*hash_and_format)) - .await?; - tag - } - SetTagOption::Auto => blobs.store().create_tag(*hash_and_format).await?, - }; - progress - .send(AddProgress::AllDone { - hash, - format, - tag: tag.clone(), - }) - .await?; - Ok(()) - } - - async fn batch_create_temp_tag(self, msg: BatchCreateTempTagRequest) -> RpcResult<()> { - let blobs = self.blobs(); - let tag = blobs.store().temp_tag(msg.content); - blobs.batches().await.store(msg.batch, tag); - Ok(()) - } - - fn batch_add_stream( - self, - msg: BatchAddStreamRequest, - stream: impl Stream + Send + Unpin + 'static, - ) -> impl Stream { - let (tx, rx) = async_channel::bounded(32); - let this = self.clone(); - - self.local_pool_handle().spawn_detached(|| async move { - if let Err(err) = this.batch_add_stream0(msg, stream, tx.clone()).await { - tx.send(BatchAddStreamResponse::Abort(RpcError::new(&*err))) - .await - .ok(); - } - }); - rx - } - - fn batch_add_from_path( - self, - msg: BatchAddPathRequest, - ) -> impl Stream { - // provide a little buffer so that we don't slow down the sender - let (tx, rx) = async_channel::bounded(32); - let tx2 = tx.clone(); - self.local_pool_handle().spawn_detached(|| async move { - if let Err(e) = self.batch_add_from_path0(msg, tx).await { - tx2.send(BatchAddPathProgress::Abort(RpcError::new(&*e))) - .await - .ok(); - } - }); - rx.map(BatchAddPathResponse) - } - - async fn batch_add_stream0( - self, - msg: BatchAddStreamRequest, - stream: impl Stream + Send + Unpin + 'static, - progress: async_channel::Sender, - ) -> anyhow::Result<()> { - let blobs = self.blobs(); - let progress = AsyncChannelProgressSender::new(progress); - - let stream = stream.map(|item| match item { - BatchAddStreamUpdate::Chunk(chunk) => Ok(chunk), - BatchAddStreamUpdate::Abort => { - Err(io::Error::new(io::ErrorKind::Interrupted, "Remote abort")) - } - }); - - let import_progress = progress.clone().with_filter_map(move |x| match x { - ImportProgress::OutboardProgress { offset, .. } => { - Some(BatchAddStreamResponse::OutboardProgress { offset }) - } - _ => None, - }); - let (temp_tag, _len) = blobs - .store() - .import_stream(stream, msg.format, import_progress) - .await?; - let hash = temp_tag.inner().hash; - blobs.batches().await.store(msg.batch, temp_tag); - progress - .send(BatchAddStreamResponse::Result { hash }) - .await?; - Ok(()) - } - - async fn batch_add_from_path0( - self, - msg: BatchAddPathRequest, - progress: async_channel::Sender, - ) -> anyhow::Result<()> { - let progress = AsyncChannelProgressSender::new(progress); - // convert import progress to provide progress - let import_progress = progress.clone().with_filter_map(move |x| match x { - ImportProgress::Size { size, .. } => Some(BatchAddPathProgress::Found { size }), - ImportProgress::OutboardProgress { offset, .. } => { - Some(BatchAddPathProgress::Progress { offset }) - } - ImportProgress::OutboardDone { hash, .. } => Some(BatchAddPathProgress::Done { hash }), - _ => None, - }); - let BatchAddPathRequest { - path: root, - import_mode, - format, - batch, - } = msg; - // Check that the path is absolute and exists. - anyhow::ensure!(root.is_absolute(), "path must be absolute"); - anyhow::ensure!( - root.exists(), - "trying to add missing path: {}", - root.display() - ); - let blobs = self.blobs(); - let (tag, _) = blobs - .store() - .import_file(root, import_mode, format, import_progress) - .await?; - let hash = *tag.hash(); - blobs.batches().await.store(batch, tag); - - progress.send(BatchAddPathProgress::Done { hash }).await?; - Ok(()) - } - - fn blob_add_stream( - self, - msg: AddStreamRequest, - stream: impl Stream + Send + Unpin + 'static, - ) -> impl Stream { - let (tx, rx) = async_channel::bounded(32); - let this = self.clone(); - - self.local_pool_handle().spawn_detached(|| async move { - if let Err(err) = this.blob_add_stream0(msg, stream, tx.clone()).await { - tx.send(AddProgress::Abort(RpcError::new(&*err))).await.ok(); - } - }); - - rx.map(AddStreamResponse) - } - - async fn blob_add_stream0( - self, - msg: AddStreamRequest, - stream: impl Stream + Send + Unpin + 'static, - progress: async_channel::Sender, - ) -> anyhow::Result<()> { - let progress = AsyncChannelProgressSender::new(progress); - - let stream = stream.map(|item| match item { - AddStreamUpdate::Chunk(chunk) => Ok(chunk), - AddStreamUpdate::Abort => { - Err(io::Error::new(io::ErrorKind::Interrupted, "Remote abort")) - } - }); - - let name_cache = Arc::new(Mutex::new(None)); - let import_progress = progress.clone().with_filter_map(move |x| match x { - ImportProgress::Found { id: _, name } => { - let _ = name_cache.lock().unwrap().insert(name); - None - } - ImportProgress::Size { id, size } => { - let name = name_cache.lock().unwrap().take()?; - Some(AddProgress::Found { id, name, size }) - } - ImportProgress::OutboardProgress { id, offset } => { - Some(AddProgress::Progress { id, offset }) - } - ImportProgress::OutboardDone { hash, id } => Some(AddProgress::Done { hash, id }), - _ => None, - }); - let blobs = self.blobs(); - let (temp_tag, _len) = blobs - .store() - .import_stream(stream, BlobFormat::Raw, import_progress) - .await?; - let hash_and_format = *temp_tag.inner(); - let HashAndFormat { hash, format } = hash_and_format; - let tag = match msg.tag { - SetTagOption::Named(tag) => { - blobs - .store() - .set_tag(tag.clone(), Some(hash_and_format)) - .await?; - tag - } - SetTagOption::Auto => blobs.store().create_tag(hash_and_format).await?, - }; - progress - .send(AddProgress::AllDone { hash, tag, format }) - .await?; - Ok(()) - } - - fn blob_read_at( - self, - req: ReadAtRequest, - ) -> impl Stream> + Send + 'static { - let (tx, rx) = async_channel::bounded(RPC_BLOB_GET_CHANNEL_CAP); - let db = self.blobs_store(); - self.local_pool_handle().spawn_detached(move || async move { - if let Err(err) = read_loop(req, db, tx.clone(), RPC_BLOB_GET_CHUNK_SIZE).await { - tx.send(RpcResult::Err(RpcError::new(&*err))).await.ok(); - } - }); - - async fn read_loop( - req: ReadAtRequest, - db: D, - tx: async_channel::Sender>, - max_chunk_size: usize, - ) -> anyhow::Result<()> { - let entry = db.get(&req.hash).await?; - let entry = entry.ok_or_else(|| anyhow!("Blob not found"))?; - let size = entry.size(); - - anyhow::ensure!( - req.offset <= size.value(), - "requested offset is out of range: {} > {:?}", - req.offset, - size - ); - - let len: usize = req - .len - .as_result_len(size.value() - req.offset) - .try_into()?; - - anyhow::ensure!( - req.offset + len as u64 <= size.value(), - "requested range is out of bounds: offset: {}, len: {} > {:?}", - req.offset, - len, - size - ); - - tx.send(Ok(ReadAtResponse::Entry { - size, - is_complete: entry.is_complete(), - })) - .await?; - let mut reader = entry.data_reader().await?; - - let (num_chunks, chunk_size) = if len <= max_chunk_size { - (1, len) - } else { - let num_chunks = len / max_chunk_size + (len % max_chunk_size != 0) as usize; - (num_chunks, max_chunk_size) - }; - - let mut read = 0u64; - for i in 0..num_chunks { - let chunk_size = if i == num_chunks - 1 { - // last chunk might be smaller - len - read as usize - } else { - chunk_size - }; - let chunk = reader.read_at(req.offset + read, chunk_size).await?; - let chunk_len = chunk.len(); - if !chunk.is_empty() { - tx.send(Ok(ReadAtResponse::Data { chunk })).await?; - } - if chunk_len < chunk_size { - break; - } else { - read += chunk_len as u64; - } - } - Ok(()) - } - - rx - } - - fn batch_create( - self, - _: BatchCreateRequest, - mut updates: impl Stream + Send + Unpin + 'static, - ) -> impl Stream { - let blobs = self.blobs(); - async move { - let batch = blobs.batches().await.create(); - tokio::spawn(async move { - while let Some(item) = updates.next().await { - match item { - BatchUpdate::Drop(content) => { - // this can not fail, since we keep the batch alive. - // therefore it is safe to ignore the result. - let _ = blobs.batches().await.remove_one(batch, &content); - } - BatchUpdate::Ping => {} - } - } - blobs.batches().await.remove(batch); - }); - BatchCreateResponse::Id(batch) - } - .into_stream() - } - fn remote_infos_iter( self, _: RemoteInfosIterRequest, @@ -1265,51 +562,6 @@ impl Handler { .map_err(|e| RpcError::new(&*e))?; Ok(()) } - - async fn create_collection( - self, - req: CreateCollectionRequest, - ) -> RpcResult { - let CreateCollectionRequest { - collection, - tag, - tags_to_delete, - } = req; - - let blobs = self.blobs(); - - let temp_tag = collection - .store(blobs.store()) - .await - .map_err(|e| RpcError::new(&*e))?; - let hash_and_format = temp_tag.inner(); - let HashAndFormat { hash, .. } = *hash_and_format; - let tag = match tag { - SetTagOption::Named(tag) => { - blobs - .store() - .set_tag(tag.clone(), Some(*hash_and_format)) - .await - .map_err(|e| RpcError::new(&e))?; - tag - } - SetTagOption::Auto => blobs - .store() - .create_tag(*hash_and_format) - .await - .map_err(|e| RpcError::new(&e))?, - }; - - for tag in tags_to_delete { - blobs - .store() - .set_tag(tag, None) - .await - .map_err(|e| RpcError::new(&e))?; - } - - Ok(CreateCollectionResponse { hash, tag }) - } } fn docs_disabled() -> RpcError { diff --git a/iroh/src/rpc_protocol.rs b/iroh/src/rpc_protocol.rs index 7c776968c9f..88b2b520a74 100644 --- a/iroh/src/rpc_protocol.rs +++ b/iroh/src/rpc_protocol.rs @@ -21,8 +21,6 @@ pub mod authors; pub mod docs; pub mod net; pub mod node; -pub use iroh_blobs::rpc::proto::blobs; -pub use iroh_blobs::rpc::proto::tags; /// The RPC service for the iroh provider process. #[derive(Debug, Clone)] From 5c19048bd78edaeeba388633fe6ed39803d0a061 Mon Sep 17 00:00:00 2001 From: Ruediger Klaehn Date: Thu, 31 Oct 2024 12:25:08 +0200 Subject: [PATCH 05/20] Use git dependency for iroh-blobs --- Cargo.lock | 1 + Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 1ce1f066746..bf4a6987461 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2547,6 +2547,7 @@ dependencies = [ [[package]] name = "iroh-blobs" version = "0.27.0" +source = "git+https://github.com/n0-computer/iroh-blobs?branch=add-rpc#b3d0c58e80f15dbe1bb148793efe3fd164d91054" dependencies = [ "anyhow", "async-channel", diff --git a/Cargo.toml b/Cargo.toml index d2be0a7940a..3df379e0ad6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -57,4 +57,4 @@ iroh-router = { path = "./iroh-router" } # iroh-blobs = { git = "https://github.com/n0-computer/iroh-blobs", branch = "main" } iroh-gossip = { git = "https://github.com/n0-computer/iroh-gossip", branch = "main" } iroh-docs = { git = "https://github.com/n0-computer/iroh-docs", branch = "main" } -iroh-blobs = { path = "../iroh-blobs" } \ No newline at end of file +iroh-blobs = { git = "https://github.com/n0-computer/iroh-blobs", branch = "add-rpc" } \ No newline at end of file From 03b617876862b8c1c2cef99d81a5332a00c6d17d Mon Sep 17 00:00:00 2001 From: Ruediger Klaehn Date: Thu, 31 Oct 2024 12:40:27 +0200 Subject: [PATCH 06/20] fmt --- iroh/src/client.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/iroh/src/client.rs b/iroh/src/client.rs index 9214d1c341a..63322241d9e 100644 --- a/iroh/src/client.rs +++ b/iroh/src/client.rs @@ -21,8 +21,7 @@ pub use self::{docs::Doc, net::NodeStatus}; pub mod authors; pub mod docs; pub mod net; -pub use iroh_blobs::rpc::client::blobs; -pub use iroh_blobs::rpc::client::tags; +pub use iroh_blobs::rpc::client::{blobs, tags}; // Keep this type exposed, otherwise every occurrence of `RpcClient` in the API // will show up as `RpcClient>` in the docs. From c3bf803e315eaa487ec265335c0f83ff3962bb80 Mon Sep 17 00:00:00 2001 From: Ruediger Klaehn Date: Thu, 31 Oct 2024 13:43:23 +0200 Subject: [PATCH 07/20] Try different exports --- iroh-cli/src/commands/blobs.rs | 12 +- iroh-cli/src/commands/docs.rs | 3 +- iroh/examples/custom-protocol.rs | 2 +- iroh/src/client.rs | 3 +- iroh/src/client/blobs.rs | 8 + iroh/src/client/blobs/batch.rs | 463 ------------------------------- iroh/src/client/docs.rs | 11 +- iroh/src/node.rs | 8 +- iroh/src/util/fs.rs | 3 +- iroh/tests/batch.rs | 8 +- 10 files changed, 35 insertions(+), 486 deletions(-) create mode 100644 iroh/src/client/blobs.rs delete mode 100644 iroh/src/client/blobs/batch.rs diff --git a/iroh-cli/src/commands/blobs.rs b/iroh-cli/src/commands/blobs.rs index d40db80dd54..ac3b5f6f1cb 100644 --- a/iroh-cli/src/commands/blobs.rs +++ b/iroh-cli/src/commands/blobs.rs @@ -19,20 +19,18 @@ use iroh::{ base::{node_addr::AddrInfoOptions, ticket::BlobTicket}, blobs::{ get::{db::DownloadProgress, progress::BlobProgress, Stats}, + net_protocol::DownloadMode, provider::AddProgress, + rpc::client::blobs::{ + BlobInfo, BlobStatus, CollectionInfo, DownloadOptions, IncompleteBlobInfo, WrapOption, + }, store::{ ConsistencyCheckProgress, ExportFormat, ExportMode, ReportLevel, ValidateProgress, }, util::SetTagOption, BlobFormat, Hash, HashAndFormat, Tag, }, - client::{ - blobs::{ - BlobInfo, BlobStatus, CollectionInfo, DownloadMode, DownloadOptions, - IncompleteBlobInfo, WrapOption, - }, - Iroh, - }, + client::Iroh, net::{key::PublicKey, relay::RelayUrl, NodeAddr}, }; use tokio::io::AsyncWriteExt; diff --git a/iroh-cli/src/commands/docs.rs b/iroh-cli/src/commands/docs.rs index 4846db12089..e5d8e4c1de1 100644 --- a/iroh-cli/src/commands/docs.rs +++ b/iroh-cli/src/commands/docs.rs @@ -17,9 +17,8 @@ use futures_lite::{Stream, StreamExt}; use indicatif::{HumanBytes, HumanDuration, MultiProgress, ProgressBar, ProgressStyle}; use iroh::{ base::{base32::fmt_short, node_addr::AddrInfoOptions}, - blobs::{provider::AddProgress, util::SetTagOption, Hash, Tag}, + blobs::{provider::AddProgress, rpc::client::blobs::WrapOption, util::SetTagOption, Hash, Tag}, client::{ - blobs::WrapOption, docs::{Doc, Entry, LiveEvent, Origin, ShareMode}, Iroh, }, diff --git a/iroh/examples/custom-protocol.rs b/iroh/examples/custom-protocol.rs index c6aeae5a5ff..b6bdd56d387 100644 --- a/iroh/examples/custom-protocol.rs +++ b/iroh/examples/custom-protocol.rs @@ -118,7 +118,7 @@ async fn main() -> Result<()> { // Print out our query results. for hash in hashes { - read_and_print(node.blobs(), hash).await?; + read_and_print(&node.blobs(), hash).await?; } } } diff --git a/iroh/src/client.rs b/iroh/src/client.rs index 63322241d9e..af637de318a 100644 --- a/iroh/src/client.rs +++ b/iroh/src/client.rs @@ -19,9 +19,10 @@ pub(crate) use self::quic::{connect_raw as quic_connect_raw, RPC_ALPN}; pub use self::{docs::Doc, net::NodeStatus}; pub mod authors; +pub mod blobs; pub mod docs; pub mod net; -pub use iroh_blobs::rpc::client::{blobs, tags}; +// pub use iroh_blobs::rpc::client::{blobs, tags}; // Keep this type exposed, otherwise every occurrence of `RpcClient` in the API // will show up as `RpcClient>` in the docs. diff --git a/iroh/src/client/blobs.rs b/iroh/src/client/blobs.rs new file mode 100644 index 00000000000..94a2c71608c --- /dev/null +++ b/iroh/src/client/blobs.rs @@ -0,0 +1,8 @@ +//! Reexport of iroh-blobs rpc client + +use quic_rpc::client::BoxedServiceConnection; + +use super::RpcService; +/// Reexport of iroh-blobs rpc client +pub type Client = + iroh_blobs::rpc::client::blobs::Client, RpcService>; diff --git a/iroh/src/client/blobs/batch.rs b/iroh/src/client/blobs/batch.rs deleted file mode 100644 index 9dad9797ba6..00000000000 --- a/iroh/src/client/blobs/batch.rs +++ /dev/null @@ -1,463 +0,0 @@ -use std::{ - io, - path::PathBuf, - sync::{Arc, Mutex}, -}; - -use anyhow::{anyhow, Context, Result}; -use bytes::Bytes; -use futures_buffered::BufferedStreamExt; -use futures_lite::StreamExt; -use futures_util::{sink::Buffer, FutureExt, SinkExt, Stream}; -use iroh_blobs::{ - format::collection::Collection, - net_protocol::BatchId, - provider::BatchAddPathProgress, - store::ImportMode, - util::{SetTagOption, TagDrop}, - BlobFormat, HashAndFormat, Tag, TempTag, -}; -use quic_rpc::client::UpdateSink; -use tokio::io::AsyncRead; -use tokio_util::io::ReaderStream; -use tracing::{debug, warn}; - -use super::WrapOption; -use crate::{ - client::{RpcClient, RpcConnection, RpcService}, - rpc_protocol::{ - blobs::{ - BatchAddPathRequest, BatchAddStreamRequest, BatchAddStreamResponse, - BatchAddStreamUpdate, BatchCreateTempTagRequest, BatchUpdate, - }, - tags::{self, SyncMode}, - }, -}; - -/// A scope in which blobs can be added. -#[derive(derive_more::Debug)] -struct BatchInner { - /// The id of the scope. - batch: BatchId, - /// The rpc client. - rpc: RpcClient, - /// The stream to send drop - #[debug(skip)] - updates: Mutex, BatchUpdate>>, -} - -/// A batch for write operations. -/// -/// This serves mostly as a scope for temporary tags. -/// -/// It is not a transaction, so things in a batch are not atomic. Also, there is -/// no isolation between batches. -#[derive(derive_more::Debug)] -pub struct Batch(Arc); - -impl TagDrop for BatchInner { - fn on_drop(&self, content: &HashAndFormat) { - let mut updates = self.updates.lock().unwrap(); - // make a spirited attempt to notify the server that we are dropping the content - // - // this will occasionally fail, but that's acceptable. The temp tags for the batch - // will be cleaned up as soon as the entire batch is dropped. - // - // E.g. a typical scenario is that you create a large array of temp tags, and then - // store them in a hash sequence and then drop the array. You will get many drops - // at the same time, and might get a send failure here. - // - // But that just means that the server will clean up the temp tags when the batch is - // dropped. - updates.feed(BatchUpdate::Drop(*content)).now_or_never(); - updates.flush().now_or_never(); - } -} - -/// Options for adding a file as a blob -#[derive(Debug, Clone, Copy, Default)] -pub struct AddFileOpts { - /// The import mode - pub import_mode: ImportMode, - /// The format of the blob - pub format: BlobFormat, -} - -/// Options for adding a directory as a collection -#[derive(Debug, Clone)] -pub struct AddDirOpts { - /// The import mode - pub import_mode: ImportMode, - /// Whether to preserve the directory name - pub wrap: WrapOption, - /// Io parallelism - pub io_parallelism: usize, -} - -impl Default for AddDirOpts { - fn default() -> Self { - Self { - import_mode: ImportMode::TryReference, - wrap: WrapOption::NoWrap, - io_parallelism: 4, - } - } -} - -/// Options for adding a directory as a collection -#[derive(Debug, Clone)] -pub struct AddReaderOpts { - /// The format of the blob - pub format: BlobFormat, - /// Size of the chunks to send - pub chunk_size: usize, -} - -impl Default for AddReaderOpts { - fn default() -> Self { - Self { - format: BlobFormat::Raw, - chunk_size: 1024 * 64, - } - } -} - -impl Batch { - pub(super) fn new( - batch: BatchId, - rpc: RpcClient, - updates: UpdateSink, - buffer_size: usize, - ) -> Self { - let updates = updates.buffer(buffer_size); - Self(Arc::new(BatchInner { - batch, - rpc, - updates: updates.into(), - })) - } - - /// Write a blob by passing bytes. - pub async fn add_bytes(&self, bytes: impl Into) -> Result { - self.add_bytes_with_opts(bytes, Default::default()).await - } - - /// Import a blob from a filesystem path, using the default options. - /// - /// For more control, use [`Self::add_file_with_opts`]. - pub async fn add_file(&self, path: PathBuf) -> Result<(TempTag, u64)> { - self.add_file_with_opts(path, AddFileOpts::default()).await - } - - /// Add a directory as a hashseq in iroh collection format - pub async fn add_dir(&self, root: PathBuf) -> Result { - self.add_dir_with_opts(root, Default::default()).await - } - - /// Write a blob by passing an async reader. - /// - /// This will consume the stream in 64KB chunks, and use a format of [BlobFormat::Raw]. - /// - /// For more options, see [`Self::add_reader_with_opts`]. - pub async fn add_reader( - &self, - reader: impl AsyncRead + Unpin + Send + 'static, - ) -> anyhow::Result { - self.add_reader_with_opts(reader, Default::default()).await - } - - /// Write a blob by passing a stream of bytes. - pub async fn add_stream( - &self, - input: impl Stream> + Send + Unpin + 'static, - ) -> Result { - self.add_stream_with_opts(input, Default::default()).await - } - - /// Creates a temp tag to protect some content (blob or hashseq) from being deleted. - /// - /// This is a lower-level API. The other functions in [`Batch`] already create [`TempTag`]s automatically. - /// - /// [`TempTag`]s allow you to protect some data from deletion while a download is ongoing, - /// even if you don't want to protect it permanently. - pub async fn temp_tag(&self, content: HashAndFormat) -> Result { - // Notify the server that we want one temp tag for the given content - self.0 - .rpc - .rpc(BatchCreateTempTagRequest { - batch: self.0.batch, - content, - }) - .await??; - // Only after success of the above call, we can create the corresponding local temp tag - Ok(self.local_temp_tag(content, None)) - } - - /// Write a blob by passing an async reader. - /// - /// This consumes the stream in chunks using `opts.chunk_size`. A good default is 64KB. - pub async fn add_reader_with_opts( - &self, - reader: impl AsyncRead + Unpin + Send + 'static, - opts: AddReaderOpts, - ) -> anyhow::Result { - let AddReaderOpts { format, chunk_size } = opts; - let input = ReaderStream::with_capacity(reader, chunk_size); - self.add_stream_with_opts(input, format).await - } - - /// Write a blob by passing bytes. - pub async fn add_bytes_with_opts( - &self, - bytes: impl Into, - format: BlobFormat, - ) -> Result { - let input = futures_lite::stream::once(Ok(bytes.into())); - self.add_stream_with_opts(input, format).await - } - - /// Import a blob from a filesystem path. - /// - /// `path` should be an absolute path valid for the file system on which - /// the node runs, which refers to a file. - /// - /// If you use [`ImportMode::TryReference`], Iroh will assume that the data will not - /// change and will share it in place without copying to the Iroh data directory - /// if appropriate. However, for tiny files, Iroh will copy the data. - /// - /// If you use [`ImportMode::Copy`], Iroh will always copy the data. - /// - /// Will return a temp tag for the added blob, as well as the size of the file. - pub async fn add_file_with_opts( - &self, - path: PathBuf, - opts: AddFileOpts, - ) -> Result<(TempTag, u64)> { - let AddFileOpts { - import_mode, - format, - } = opts; - anyhow::ensure!( - path.is_absolute(), - "Path must be absolute, but got: {:?}", - path - ); - anyhow::ensure!(path.is_file(), "Path does not refer to a file: {:?}", path); - let mut stream = self - .0 - .rpc - .server_streaming(BatchAddPathRequest { - path, - import_mode, - format, - batch: self.0.batch, - }) - .await?; - let mut res_hash = None; - let mut res_size = None; - while let Some(item) = stream.next().await { - match item?.0 { - BatchAddPathProgress::Abort(cause) => { - Err(cause)?; - } - BatchAddPathProgress::Done { hash } => { - res_hash = Some(hash); - } - BatchAddPathProgress::Found { size } => { - res_size = Some(size); - } - _ => {} - } - } - let hash = res_hash.context("Missing hash")?; - let size = res_size.context("Missing size")?; - Ok(( - self.local_temp_tag(HashAndFormat { hash, format }, Some(size)), - size, - )) - } - - /// Add a directory as a hashseq in iroh collection format - /// - /// This can also be used to add a single file as a collection, if - /// wrap is set to [WrapOption::Wrap]. - /// - /// However, if you want to add a single file as a raw blob, use add_file instead. - pub async fn add_dir_with_opts(&self, root: PathBuf, opts: AddDirOpts) -> Result { - let AddDirOpts { - import_mode, - wrap, - io_parallelism, - } = opts; - anyhow::ensure!(root.is_absolute(), "Path must be absolute"); - - // let (send, recv) = flume::bounded(32); - // let import_progress = FlumeProgressSender::new(send); - - // import all files below root recursively - let data_sources = crate::util::fs::scan_path(root, wrap)?; - let opts = AddFileOpts { - import_mode, - format: BlobFormat::Raw, - }; - let result: Vec<_> = futures_lite::stream::iter(data_sources) - .map(|source| { - // let import_progress = import_progress.clone(); - async move { - let name = source.name().to_string(); - let (tag, size) = self - .add_file_with_opts(source.path().to_owned(), opts) - .await?; - let hash = *tag.hash(); - anyhow::Ok((name, hash, size, tag)) - } - }) - .buffered_ordered(io_parallelism) - .try_collect() - .await?; - - // create a collection - let (collection, child_tags): (Collection, Vec<_>) = result - .into_iter() - .map(|(name, hash, _, tag)| ((name, hash), tag)) - .unzip(); - - let tag = self.add_collection(collection).await?; - drop(child_tags); - Ok(tag) - } - - /// Write a blob by passing a stream of bytes. - /// - /// For convenient interop with common sources of data, this function takes a stream of `io::Result`. - /// If you have raw bytes, you need to wrap them in `io::Result::Ok`. - pub async fn add_stream_with_opts( - &self, - mut input: impl Stream> + Send + Unpin + 'static, - format: BlobFormat, - ) -> Result { - let (mut sink, mut stream) = self - .0 - .rpc - .bidi(BatchAddStreamRequest { - batch: self.0.batch, - format, - }) - .await?; - let mut size = 0u64; - while let Some(item) = input.next().await { - match item { - Ok(chunk) => { - size += chunk.len() as u64; - sink.send(BatchAddStreamUpdate::Chunk(chunk)) - .await - .map_err(|err| anyhow!("Failed to send input stream to remote: {err:?}"))?; - } - Err(err) => { - warn!("Abort send, reason: failed to read from source stream: {err:?}"); - sink.send(BatchAddStreamUpdate::Abort) - .await - .map_err(|err| anyhow!("Failed to send input stream to remote: {err:?}"))?; - break; - } - } - } - // this is needed for the remote to notice that the stream is closed - drop(sink); - let mut res = None; - while let Some(item) = stream.next().await { - match item? { - BatchAddStreamResponse::Abort(cause) => { - Err(cause)?; - } - BatchAddStreamResponse::Result { hash } => { - res = Some(hash); - } - _ => {} - } - } - let hash = res.context("Missing answer")?; - Ok(self.local_temp_tag(HashAndFormat { hash, format }, Some(size))) - } - - /// Add a collection. - /// - /// This is a convenience function that converts the collection into two blobs - /// (the metadata and the hash sequence) and adds them, returning a temp tag for - /// the hash sequence. - /// - /// Note that this does not guarantee that the data that the collection refers to - /// actually exists. It will just create 2 blobs, the metadata and the hash sequence - /// itself. - pub async fn add_collection(&self, collection: Collection) -> Result { - self.add_blob_seq(collection.to_blobs()).await - } - - /// Add a sequence of blobs, where the last is a hash sequence. - /// - /// It is a common pattern in iroh to have a hash sequence with one or more - /// blobs of metadata, and the remaining blobs being the actual data. E.g. - /// a collection is a hash sequence where the first child is the metadata. - pub async fn add_blob_seq(&self, iter: impl Iterator) -> Result { - let mut blobs = iter.peekable(); - // put the tags somewhere - let mut tags = vec![]; - loop { - let blob = blobs.next().context("Failed to get next blob")?; - if blobs.peek().is_none() { - return self.add_bytes_with_opts(blob, BlobFormat::HashSeq).await; - } else { - tags.push(self.add_bytes(blob).await?); - } - } - } - - /// Upgrades a temp tag to a persistent tag. - pub async fn persist(&self, tt: TempTag) -> Result { - let tag = self - .0 - .rpc - .rpc(tags::CreateRequest { - value: tt.hash_and_format(), - batch: Some(self.0.batch), - sync: SyncMode::Full, - }) - .await??; - Ok(tag) - } - - /// Upgrades a temp tag to a persistent tag with a specific name. - pub async fn persist_to(&self, tt: TempTag, tag: Tag) -> Result<()> { - self.0 - .rpc - .rpc(tags::SetRequest { - name: tag, - value: Some(tt.hash_and_format()), - batch: Some(self.0.batch), - sync: SyncMode::Full, - }) - .await??; - Ok(()) - } - - /// Upgrades a temp tag to a persistent tag with either a specific name or - /// an automatically generated name. - pub async fn persist_with_opts(&self, tt: TempTag, opts: SetTagOption) -> Result { - match opts { - SetTagOption::Auto => self.persist(tt).await, - SetTagOption::Named(tag) => { - self.persist_to(tt, tag.clone()).await?; - Ok(tag) - } - } - } - - /// Creates a temp tag for the given hash and format, without notifying the server. - /// - /// Caution: only do this for data for which you know the server side has created a temp tag. - fn local_temp_tag(&self, inner: HashAndFormat, _size: Option) -> TempTag { - let on_drop: Arc = self.0.clone(); - let on_drop = Some(Arc::downgrade(&on_drop)); - TempTag::new(inner, on_drop) - } -} diff --git a/iroh/src/client/docs.rs b/iroh/src/client/docs.rs index 1fa7c9e5f1a..caff14c23b8 100644 --- a/iroh/src/client/docs.rs +++ b/iroh/src/client/docs.rs @@ -31,7 +31,7 @@ use quic_rpc::message::RpcMsg; use ref_cast::RefCast; use serde::{Deserialize, Serialize}; -use super::{blobs, flatten, RpcClient}; +use super::{flatten, RpcClient}; use crate::rpc_protocol::{ docs::{ CloseRequest, CreateRequest, DelRequest, DelResponse, DocListRequest, DocSubscribeRequest, @@ -474,10 +474,13 @@ impl Entry { /// Reads the content of an [`Entry`] as a streaming [`blobs::Reader`]. /// /// You can pass either a [`Doc`] or the `Iroh` client by reference as `client`. - pub async fn content_reader(&self, client: impl Into<&RpcClient>) -> Result { + pub async fn content_reader( + &self, + client: impl Into<&RpcClient>, + ) -> Result { let client: RpcClient = client.into().clone(); let client: quic_rpc::RpcClient = client.map(); - blobs::Reader::from_rpc_read(&client, self.content_hash()).await + iroh_blobs::rpc::client::blobs::Reader::from_rpc_read(&client, self.content_hash()).await } /// Reads all content of an [`Entry`] into a buffer. @@ -487,7 +490,7 @@ impl Entry { let client: RpcClient = client.into().clone(); let client: quic_rpc::RpcClient = client.map(); - blobs::Reader::from_rpc_read(&client, self.content_hash()) + iroh_blobs::rpc::client::blobs::Reader::from_rpc_read(&client, self.content_hash()) .await? .read_to_bytes() .await diff --git a/iroh/src/node.rs b/iroh/src/node.rs index db07cce3e88..7a5a2a717f3 100644 --- a/iroh/src/node.rs +++ b/iroh/src/node.rs @@ -498,11 +498,15 @@ mod tests { use anyhow::{bail, Context}; use bytes::Bytes; use iroh_base::{node_addr::AddrInfoOptions, ticket::BlobTicket}; - use iroh_blobs::{provider::AddProgress, util::SetTagOption, BlobFormat}; + use iroh_blobs::{ + provider::AddProgress, + rpc::client::blobs::{AddOutcome, WrapOption}, + util::SetTagOption, + BlobFormat, + }; use iroh_net::{key::SecretKey, relay::RelayMode, test_utils::DnsPkarrServer, NodeAddr}; use super::*; - use crate::client::blobs::{AddOutcome, WrapOption}; #[tokio::test] async fn test_ticket_multiple_addrs() { diff --git a/iroh/src/util/fs.rs b/iroh/src/util/fs.rs index d91416ef9dc..0afbf133e1a 100644 --- a/iroh/src/util/fs.rs +++ b/iroh/src/util/fs.rs @@ -7,12 +7,11 @@ use std::{ use anyhow::{bail, Context}; use bytes::Bytes; +use iroh_blobs::rpc::client::blobs::WrapOption; use iroh_net::key::SecretKey; use tokio::io::AsyncWriteExt; use walkdir::WalkDir; -use crate::client::blobs::WrapOption; - /// A data source #[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone)] pub struct DataSource { diff --git a/iroh/tests/batch.rs b/iroh/tests/batch.rs index c3289d03e6c..a6d9fef2402 100644 --- a/iroh/tests/batch.rs +++ b/iroh/tests/batch.rs @@ -3,11 +3,11 @@ use std::{io, time::Duration}; use bao_tree::blake3; use bytes::Bytes; use futures_lite::StreamExt; -use iroh::{ - client::blobs::{AddDirOpts, WrapOption}, - node::GcPolicy, +use iroh::node::GcPolicy; +use iroh_blobs::{ + rpc::client::blobs::{AddDirOpts, WrapOption}, + store::mem::Store, }; -use iroh_blobs::store::mem::Store; async fn create_node() -> anyhow::Result<(iroh::node::Node, async_channel::Receiver<()>)> { let (gc_send, gc_recv) = async_channel::unbounded(); From 0196cab2a8cdf0f8d5a596072777d42140912ba4 Mon Sep 17 00:00:00 2001 From: Ruediger Klaehn Date: Thu, 31 Oct 2024 14:38:21 +0200 Subject: [PATCH 08/20] more reexports --- iroh/src/client.rs | 8 ++------ iroh/src/client/blobs.rs | 5 +++++ 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/iroh/src/client.rs b/iroh/src/client.rs index af637de318a..cfed055d5e7 100644 --- a/iroh/src/client.rs +++ b/iroh/src/client.rs @@ -22,7 +22,6 @@ pub mod authors; pub mod blobs; pub mod docs; pub mod net; -// pub use iroh_blobs::rpc::client::{blobs, tags}; // Keep this type exposed, otherwise every occurrence of `RpcClient` in the API // will show up as `RpcClient>` in the docs. @@ -57,11 +56,8 @@ impl Iroh { } /// Returns the blobs client. - pub fn blobs( - &self, - ) -> iroh_blobs::rpc::client::blobs::Client, RpcService> - { - iroh_blobs::rpc::client::blobs::Client::new(self.rpc.clone().map()) + pub fn blobs(&self) -> blobs::Client { + blobs::Client::new(self.rpc.clone().map()) } /// Returns the docs client. diff --git a/iroh/src/client/blobs.rs b/iroh/src/client/blobs.rs index 94a2c71608c..6a8c6f48e0d 100644 --- a/iroh/src/client/blobs.rs +++ b/iroh/src/client/blobs.rs @@ -6,3 +6,8 @@ use super::RpcService; /// Reexport of iroh-blobs rpc client pub type Client = iroh_blobs::rpc::client::blobs::Client, RpcService>; + +pub use iroh_blobs::rpc::client::blobs::{ + AddDirOpts, AddFileOpts, AddOutcome, AddProgress, AddReaderOpts, BlobInfo, DownloadMode, + DownloadOptions, DownloadOutcome, DownloadProgress, IncompleteBlobInfo, WrapOption, +}; From c46e4698dfeb21345a1a904410c5e20c87f51a57 Mon Sep 17 00:00:00 2001 From: Ruediger Klaehn Date: Thu, 31 Oct 2024 14:45:31 +0200 Subject: [PATCH 09/20] Add tags client reexports for completeness --- iroh-dns-server/src/http/rate_limiting.rs | 4 ++-- iroh/src/client.rs | 1 + iroh/src/client/blobs.rs | 2 +- iroh/src/client/docs.rs | 9 +++------ iroh/src/client/tags.rs | 9 +++++++++ 5 files changed, 16 insertions(+), 9 deletions(-) create mode 100644 iroh/src/client/tags.rs diff --git a/iroh-dns-server/src/http/rate_limiting.rs b/iroh-dns-server/src/http/rate_limiting.rs index 53952f339e8..ac9f7264daa 100644 --- a/iroh-dns-server/src/http/rate_limiting.rs +++ b/iroh-dns-server/src/http/rate_limiting.rs @@ -15,11 +15,11 @@ pub enum RateLimitConfig { /// Disable rate limit for http server. Disabled, /// Enable rate limit for http server based on the connection peer IP address. - /// https://docs.rs/tower_governor/latest/tower_governor/key_extractor/struct.PeerIpKeyExtractor.html + /// #[default] Simple, /// Enable rate limit for http server based on a smart logic for extracting the connection original IP address, useful for reverse proxies. - /// https://docs.rs/tower_governor/latest/tower_governor/key_extractor/struct.SmartIpKeyExtractor.html + /// Smart, } diff --git a/iroh/src/client.rs b/iroh/src/client.rs index cfed055d5e7..cc9dccd3b41 100644 --- a/iroh/src/client.rs +++ b/iroh/src/client.rs @@ -22,6 +22,7 @@ pub mod authors; pub mod blobs; pub mod docs; pub mod net; +pub mod tags; // Keep this type exposed, otherwise every occurrence of `RpcClient` in the API // will show up as `RpcClient>` in the docs. diff --git a/iroh/src/client/blobs.rs b/iroh/src/client/blobs.rs index 6a8c6f48e0d..f40ec74bba0 100644 --- a/iroh/src/client/blobs.rs +++ b/iroh/src/client/blobs.rs @@ -9,5 +9,5 @@ pub type Client = pub use iroh_blobs::rpc::client::blobs::{ AddDirOpts, AddFileOpts, AddOutcome, AddProgress, AddReaderOpts, BlobInfo, DownloadMode, - DownloadOptions, DownloadOutcome, DownloadProgress, IncompleteBlobInfo, WrapOption, + DownloadOptions, DownloadOutcome, DownloadProgress, IncompleteBlobInfo, Reader, WrapOption, }; diff --git a/iroh/src/client/docs.rs b/iroh/src/client/docs.rs index caff14c23b8..800589e2e90 100644 --- a/iroh/src/client/docs.rs +++ b/iroh/src/client/docs.rs @@ -31,7 +31,7 @@ use quic_rpc::message::RpcMsg; use ref_cast::RefCast; use serde::{Deserialize, Serialize}; -use super::{flatten, RpcClient}; +use super::{blobs, flatten, RpcClient}; use crate::rpc_protocol::{ docs::{ CloseRequest, CreateRequest, DelRequest, DelResponse, DocListRequest, DocSubscribeRequest, @@ -474,13 +474,10 @@ impl Entry { /// Reads the content of an [`Entry`] as a streaming [`blobs::Reader`]. /// /// You can pass either a [`Doc`] or the `Iroh` client by reference as `client`. - pub async fn content_reader( - &self, - client: impl Into<&RpcClient>, - ) -> Result { + pub async fn content_reader(&self, client: impl Into<&RpcClient>) -> Result { let client: RpcClient = client.into().clone(); let client: quic_rpc::RpcClient = client.map(); - iroh_blobs::rpc::client::blobs::Reader::from_rpc_read(&client, self.content_hash()).await + blobs::Reader::from_rpc_read(&client, self.content_hash()).await } /// Reads all content of an [`Entry`] into a buffer. diff --git a/iroh/src/client/tags.rs b/iroh/src/client/tags.rs new file mode 100644 index 00000000000..9694b85cd8f --- /dev/null +++ b/iroh/src/client/tags.rs @@ -0,0 +1,9 @@ +//! Reexport of iroh-blobs rpc client +use quic_rpc::client::BoxedServiceConnection; + +use super::RpcService; +/// Reexport of iroh-blobs rpc client +pub type Client = + iroh_blobs::rpc::client::tags::Client, RpcService>; + +pub use iroh_blobs::rpc::client::tags::TagInfo; From 261a03a175fae8ce014a52d4608f43232456b8df Mon Sep 17 00:00:00 2001 From: Ruediger Klaehn Date: Thu, 31 Oct 2024 14:49:11 +0200 Subject: [PATCH 10/20] simplify some types using the exports --- iroh/src/client.rs | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/iroh/src/client.rs b/iroh/src/client.rs index cc9dccd3b41..d9f6fc1026a 100644 --- a/iroh/src/client.rs +++ b/iroh/src/client.rs @@ -6,7 +6,6 @@ use std::collections::BTreeMap; use anyhow::Result; use futures_lite::{Stream, StreamExt}; -use quic_rpc::client::BoxedServiceConnection; use ref_cast::RefCast; use crate::rpc_protocol::node::{CounterStats, ShutdownRequest, StatsRequest, StatusRequest}; @@ -72,10 +71,8 @@ impl Iroh { } /// Returns the tags client. - pub fn tags( - &self, - ) -> iroh_blobs::rpc::client::tags::Client, RpcService> { - iroh_blobs::rpc::client::tags::Client::new(self.rpc.clone().map()) + pub fn tags(&self) -> tags::Client { + tags::Client::new(self.rpc.clone().map()) } /// Returns the gossip client. From f2660f63a1decbd98370d7821e9fbf99cf2030d1 Mon Sep 17 00:00:00 2001 From: Ruediger Klaehn Date: Thu, 31 Oct 2024 15:13:40 +0200 Subject: [PATCH 11/20] more shorter imports --- iroh-cli/src/commands/blobs.rs | 10 ++++++---- iroh-cli/src/commands/docs.rs | 3 ++- iroh/src/client/blobs.rs | 5 +++-- iroh/src/client/docs.rs | 2 +- iroh/src/node.rs | 8 ++------ iroh/src/util/fs.rs | 3 ++- iroh/tests/batch.rs | 6 ++---- 7 files changed, 18 insertions(+), 19 deletions(-) diff --git a/iroh-cli/src/commands/blobs.rs b/iroh-cli/src/commands/blobs.rs index ac3b5f6f1cb..f945f9d788b 100644 --- a/iroh-cli/src/commands/blobs.rs +++ b/iroh-cli/src/commands/blobs.rs @@ -21,16 +21,18 @@ use iroh::{ get::{db::DownloadProgress, progress::BlobProgress, Stats}, net_protocol::DownloadMode, provider::AddProgress, - rpc::client::blobs::{ - BlobInfo, BlobStatus, CollectionInfo, DownloadOptions, IncompleteBlobInfo, WrapOption, - }, store::{ ConsistencyCheckProgress, ExportFormat, ExportMode, ReportLevel, ValidateProgress, }, util::SetTagOption, BlobFormat, Hash, HashAndFormat, Tag, }, - client::Iroh, + client::{ + blobs::{ + BlobInfo, BlobStatus, CollectionInfo, DownloadOptions, IncompleteBlobInfo, WrapOption, + }, + Iroh, + }, net::{key::PublicKey, relay::RelayUrl, NodeAddr}, }; use tokio::io::AsyncWriteExt; diff --git a/iroh-cli/src/commands/docs.rs b/iroh-cli/src/commands/docs.rs index e5d8e4c1de1..29a05d361b9 100644 --- a/iroh-cli/src/commands/docs.rs +++ b/iroh-cli/src/commands/docs.rs @@ -17,7 +17,8 @@ use futures_lite::{Stream, StreamExt}; use indicatif::{HumanBytes, HumanDuration, MultiProgress, ProgressBar, ProgressStyle}; use iroh::{ base::{base32::fmt_short, node_addr::AddrInfoOptions}, - blobs::{provider::AddProgress, rpc::client::blobs::WrapOption, util::SetTagOption, Hash, Tag}, + blobs::{provider::AddProgress, util::SetTagOption, Hash, Tag}, + client::blobs::WrapOption, client::{ docs::{Doc, Entry, LiveEvent, Origin, ShareMode}, Iroh, diff --git a/iroh/src/client/blobs.rs b/iroh/src/client/blobs.rs index f40ec74bba0..8682279b56a 100644 --- a/iroh/src/client/blobs.rs +++ b/iroh/src/client/blobs.rs @@ -8,6 +8,7 @@ pub type Client = iroh_blobs::rpc::client::blobs::Client, RpcService>; pub use iroh_blobs::rpc::client::blobs::{ - AddDirOpts, AddFileOpts, AddOutcome, AddProgress, AddReaderOpts, BlobInfo, DownloadMode, - DownloadOptions, DownloadOutcome, DownloadProgress, IncompleteBlobInfo, Reader, WrapOption, + AddDirOpts, AddFileOpts, AddOutcome, AddProgress, AddReaderOpts, BlobInfo, BlobStatus, + CollectionInfo, DownloadMode, DownloadOptions, DownloadOutcome, DownloadProgress, + IncompleteBlobInfo, Reader, WrapOption, }; diff --git a/iroh/src/client/docs.rs b/iroh/src/client/docs.rs index 800589e2e90..1fa7c9e5f1a 100644 --- a/iroh/src/client/docs.rs +++ b/iroh/src/client/docs.rs @@ -487,7 +487,7 @@ impl Entry { let client: RpcClient = client.into().clone(); let client: quic_rpc::RpcClient = client.map(); - iroh_blobs::rpc::client::blobs::Reader::from_rpc_read(&client, self.content_hash()) + blobs::Reader::from_rpc_read(&client, self.content_hash()) .await? .read_to_bytes() .await diff --git a/iroh/src/node.rs b/iroh/src/node.rs index 7a5a2a717f3..5c8bdab55cc 100644 --- a/iroh/src/node.rs +++ b/iroh/src/node.rs @@ -495,15 +495,11 @@ fn node_address_for_storage(info: RemoteInfo) -> Option { #[cfg(test)] mod tests { + use crate::client::blobs::{AddOutcome, WrapOption}; use anyhow::{bail, Context}; use bytes::Bytes; use iroh_base::{node_addr::AddrInfoOptions, ticket::BlobTicket}; - use iroh_blobs::{ - provider::AddProgress, - rpc::client::blobs::{AddOutcome, WrapOption}, - util::SetTagOption, - BlobFormat, - }; + use iroh_blobs::{provider::AddProgress, util::SetTagOption, BlobFormat}; use iroh_net::{key::SecretKey, relay::RelayMode, test_utils::DnsPkarrServer, NodeAddr}; use super::*; diff --git a/iroh/src/util/fs.rs b/iroh/src/util/fs.rs index 0afbf133e1a..d91416ef9dc 100644 --- a/iroh/src/util/fs.rs +++ b/iroh/src/util/fs.rs @@ -7,11 +7,12 @@ use std::{ use anyhow::{bail, Context}; use bytes::Bytes; -use iroh_blobs::rpc::client::blobs::WrapOption; use iroh_net::key::SecretKey; use tokio::io::AsyncWriteExt; use walkdir::WalkDir; +use crate::client::blobs::WrapOption; + /// A data source #[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone)] pub struct DataSource { diff --git a/iroh/tests/batch.rs b/iroh/tests/batch.rs index a6d9fef2402..86d6ff76615 100644 --- a/iroh/tests/batch.rs +++ b/iroh/tests/batch.rs @@ -3,11 +3,9 @@ use std::{io, time::Duration}; use bao_tree::blake3; use bytes::Bytes; use futures_lite::StreamExt; +use iroh::client::blobs::{AddDirOpts, WrapOption}; use iroh::node::GcPolicy; -use iroh_blobs::{ - rpc::client::blobs::{AddDirOpts, WrapOption}, - store::mem::Store, -}; +use iroh_blobs::store::mem::Store; async fn create_node() -> anyhow::Result<(iroh::node::Node, async_channel::Receiver<()>)> { let (gc_send, gc_recv) = async_channel::unbounded(); From 445aa3587c16dfa63df3eda9b56d90734213a73a Mon Sep 17 00:00:00 2001 From: Ruediger Klaehn Date: Thu, 31 Oct 2024 15:16:26 +0200 Subject: [PATCH 12/20] fmt --- iroh-cli/src/commands/docs.rs | 2 +- iroh/src/node.rs | 2 +- iroh/tests/batch.rs | 6 ++++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/iroh-cli/src/commands/docs.rs b/iroh-cli/src/commands/docs.rs index 29a05d361b9..4846db12089 100644 --- a/iroh-cli/src/commands/docs.rs +++ b/iroh-cli/src/commands/docs.rs @@ -18,8 +18,8 @@ use indicatif::{HumanBytes, HumanDuration, MultiProgress, ProgressBar, ProgressS use iroh::{ base::{base32::fmt_short, node_addr::AddrInfoOptions}, blobs::{provider::AddProgress, util::SetTagOption, Hash, Tag}, - client::blobs::WrapOption, client::{ + blobs::WrapOption, docs::{Doc, Entry, LiveEvent, Origin, ShareMode}, Iroh, }, diff --git a/iroh/src/node.rs b/iroh/src/node.rs index 5c8bdab55cc..db07cce3e88 100644 --- a/iroh/src/node.rs +++ b/iroh/src/node.rs @@ -495,7 +495,6 @@ fn node_address_for_storage(info: RemoteInfo) -> Option { #[cfg(test)] mod tests { - use crate::client::blobs::{AddOutcome, WrapOption}; use anyhow::{bail, Context}; use bytes::Bytes; use iroh_base::{node_addr::AddrInfoOptions, ticket::BlobTicket}; @@ -503,6 +502,7 @@ mod tests { use iroh_net::{key::SecretKey, relay::RelayMode, test_utils::DnsPkarrServer, NodeAddr}; use super::*; + use crate::client::blobs::{AddOutcome, WrapOption}; #[tokio::test] async fn test_ticket_multiple_addrs() { diff --git a/iroh/tests/batch.rs b/iroh/tests/batch.rs index 86d6ff76615..c3289d03e6c 100644 --- a/iroh/tests/batch.rs +++ b/iroh/tests/batch.rs @@ -3,8 +3,10 @@ use std::{io, time::Duration}; use bao_tree::blake3; use bytes::Bytes; use futures_lite::StreamExt; -use iroh::client::blobs::{AddDirOpts, WrapOption}; -use iroh::node::GcPolicy; +use iroh::{ + client::blobs::{AddDirOpts, WrapOption}, + node::GcPolicy, +}; use iroh_blobs::store::mem::Store; async fn create_node() -> anyhow::Result<(iroh::node::Node, async_channel::Receiver<()>)> { From 8caa393aa6afd482ecc4765e1f498896da6a6b24 Mon Sep 17 00:00:00 2001 From: Ruediger Klaehn Date: Tue, 5 Nov 2024 18:40:52 +0200 Subject: [PATCH 13/20] WIP --- Cargo.lock | 1655 ++++++++++++++++++++++++-------------- Cargo.toml | 7 +- iroh-cli/Cargo.toml | 2 +- iroh/Cargo.toml | 6 +- iroh/src/client/blobs.rs | 4 +- iroh/src/client/docs.rs | 4 +- iroh/src/client/quic.rs | 6 +- iroh/src/client/tags.rs | 4 +- iroh/src/node/builder.rs | 6 +- iroh/src/node/rpc.rs | 1 - 10 files changed, 1080 insertions(+), 615 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bf4a6987461..fe977904f98 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -18,18 +18,18 @@ dependencies = [ [[package]] name = "addr2line" -version = "0.22.0" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e4503c46a5c0c7844e948c9a4d6acd9f50cccb4de1c48eb9e291ea17470c678" +checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" dependencies = [ "gimli", ] [[package]] -name = "adler" -version = "1.0.2" +name = "adler2" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" [[package]] name = "aead" @@ -92,9 +92,9 @@ checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" [[package]] name = "anstream" -version = "0.6.14" +version = "0.6.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "418c75fa768af9c03be99d17643f93f79bbba589895012a80e3452a19ddda15b" +checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b" dependencies = [ "anstyle", "anstyle-parse", @@ -107,43 +107,43 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.7" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "038dfcf04a5feb68e9c60b21c9625a54c2c0616e79b72b0fd87075a056ae1d1b" +checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" [[package]] name = "anstyle-parse" -version = "0.2.4" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c03a11a9034d92058ceb6ee011ce58af4a9bf61491aa7e1e59ecd24bd40d22d4" +checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9" dependencies = [ "utf8parse", ] [[package]] name = "anstyle-query" -version = "1.1.0" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad186efb764318d35165f1758e7dcef3b10628e26d41a44bc5550652e6804391" +checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] name = "anstyle-wincon" -version = "3.0.3" +version = "3.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61a38449feb7068f52bb06c12759005cf459ee52bb4adc1d5a7c4322d716fb19" +checksum = "2109dbce0e72be3ec00bed26e6a7479ca384ad226efdd66db8fa2e3a38c83125" dependencies = [ "anstyle", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] name = "anyhow" -version = "1.0.91" +version = "1.0.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c042108f3ed77fd83760a5fd79b53be043192bb3b9dba91d8c574c0ada7850c8" +checksum = "74f37166d7d48a0284b99dd824694c26119c700b53bf0d1540cdb147dbdaaf13" [[package]] name = "arc-swap" @@ -153,21 +153,21 @@ checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457" [[package]] name = "arrayref" -version = "0.3.8" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d151e35f61089500b617991b791fc8bfd237ae50cd5950803758a179b41e67a" +checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" [[package]] name = "arrayvec" -version = "0.7.4" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "asn1-rs" -version = "0.6.1" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22ad1373757efa0f70ec53939aabc7152e1591cb485208052993070ac8d2429d" +checksum = "5493c3bedbacf7fd7382c6346bbd66687d12bbaad3a89a2d2c303ee6cf20b048" dependencies = [ "asn1-rs-derive", "asn1-rs-impl", @@ -181,13 +181,13 @@ dependencies = [ [[package]] name = "asn1-rs-derive" -version = "0.5.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7378575ff571966e99a744addeff0bff98b8ada0dedf1956d59e634db95eaac1" +checksum = "965c2d33e53cb6b267e148a4cb0760bc01f4904c1cd4bb4002a085bb016d1490" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", "synstructure", ] @@ -199,7 +199,7 @@ checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -222,18 +222,18 @@ checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] name = "async-trait" -version = "0.1.81" +version = "0.1.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e0c28dcc82d7c8ead5cb13beb15405b57b8546e93215673ff8ca0349a028107" +checksum = "721cae7de5c34fbb2acd27e21e6d2cf7b886dce0c27388d46c4e6c47ea4318dd" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -264,15 +264,15 @@ dependencies = [ [[package]] name = "autocfg" -version = "1.3.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" +checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" [[package]] name = "axum" -version = "0.7.5" +version = "0.7.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a6c9af12842a67734c9a2e355436e5d03b22383ed60cf13cd0c18fbfe3dcbcf" +checksum = "504e3947307ac8326a5437504c517c4b56716c9d98fac0028c2acc7ca47d70ae" dependencies = [ "async-trait", "axum-core", @@ -297,7 +297,7 @@ dependencies = [ "serde_urlencoded", "sync_wrapper 1.0.1", "tokio", - "tower", + "tower 0.5.1", "tower-layer", "tower-service", "tracing", @@ -305,9 +305,9 @@ dependencies = [ [[package]] name = "axum-core" -version = "0.4.3" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a15c63fd72d41492dc4f497196f5da1fb04fb7529e631d73630d1b491e47a2e3" +checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199" dependencies = [ "async-trait", "bytes", @@ -318,7 +318,7 @@ dependencies = [ "mime", "pin-project-lite", "rustversion", - "sync_wrapper 0.1.2", + "sync_wrapper 1.0.1", "tower-layer", "tower-service", "tracing", @@ -326,14 +326,13 @@ dependencies = [ [[package]] name = "axum-macros" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00c055ee2d014ae5981ce1016374e8213682aa14d9bf40e48ab48b5f3ef20eaa" +checksum = "57d123550fa8d071b7255cb0cc04dc302baa6c8c4a79f55701552684d8399bce" dependencies = [ - "heck 0.4.1", "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -356,7 +355,7 @@ dependencies = [ "rustls-pki-types", "tokio", "tokio-rustls", - "tower", + "tower 0.4.13", "tower-service", ] @@ -373,17 +372,17 @@ dependencies = [ [[package]] name = "backtrace" -version = "0.3.73" +version = "0.3.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cc23269a4f8976d0a4d2e7109211a419fe30e8d88d677cd60b6bc79c5732e0a" +checksum = "8d82cb332cdfaed17ae235a638438ac4d4839913cc2af585c3c6746e8f8bee1a" dependencies = [ "addr2line", - "cc", "cfg-if", "libc", "miniz_oxide", "object", "rustc-demangle", + "windows-targets 0.52.6", ] [[package]] @@ -393,7 +392,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1f7a89a8ee5889d2593ae422ce6e1bb03e48a0e8a16e4fa0882dfcbe7e182ef" dependencies = [ "bytes", - "futures-lite 2.3.0", + "futures-lite 2.4.0", "genawaiter", "iroh-blake3", "iroh-io", @@ -478,18 +477,6 @@ version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" -[[package]] -name = "bitvec" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c" -dependencies = [ - "funty", - "radium", - "tap", - "wyz", -] - [[package]] name = "block-buffer" version = "0.10.4" @@ -519,18 +506,18 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.7.1" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8318a53db07bb3f8dca91a600466bdb3f2eaadeedfdbcf02e1accbad9271ba50" +checksum = "9ac0150caa2ae65ca5bd83f25c7de183dea78d4d366469f148435e2acfbad0da" dependencies = [ "serde", ] [[package]] name = "camino" -version = "1.1.7" +version = "1.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0ec6b951b160caa93cc0c7b209e5a3bff7aae9062213451ac99493cd844c239" +checksum = "8b96ec4966b5813e2c0507c1f86115c8c5abaadc3980879c3424042a02fd1ad3" dependencies = [ "serde", ] @@ -580,9 +567,12 @@ dependencies = [ [[package]] name = "cc" -version = "1.1.6" +version = "1.1.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2aba8f4e9906c7ce3c73463f62a7f0c65183ada1a2d47e397cc8810827f9694f" +checksum = "0f57c4b4da2a9d619dd035f27316d7a426305b75be93d09e92f2b9229c34feaf" +dependencies = [ + "shlex", +] [[package]] name = "cesu8" @@ -596,6 +586,12 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + [[package]] name = "chacha20" version = "0.9.1" @@ -662,9 +658,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.9" +version = "4.5.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64acc1846d54c1fe936a78dc189c34e28d3f5afc348403f28ecf53660b9b8462" +checksum = "b97f376d85a664d5837dbae44bf546e6477a679ff6610010f17276f686d867e8" dependencies = [ "clap_builder", "clap_derive", @@ -672,9 +668,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.9" +version = "4.5.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fb8393d67ba2e7bfaf28a23458e4e2b543cc73a99595511eb207fdb8aede942" +checksum = "19bc80abd44e4bed93ca373a0704ccbd1b710dc5749406201bb018272808dc54" dependencies = [ "anstream", "anstyle", @@ -684,21 +680,21 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.8" +version = "4.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bac35c6dafb060fd4d275d9a4ffae97917c13a6327903a8be2153cd964f7085" +checksum = "4ac6a0c7b1a9e9a5186361f67dfa1b88213572f427fb9ab038efb2bd8c582dab" dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] name = "clap_lex" -version = "0.7.1" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b82cf0babdbd58558212896d1a4272303a57bdb245c2bf1147185fb45640e70" +checksum = "1462739cb27611015575c0c11df5df7601141071f07518d56fcc1be504cbec97" [[package]] name = "clipboard-win" @@ -719,9 +715,9 @@ checksum = "67ba02a97a2bd10f4b59b25c7973101c79642302776489e030cd13cdab09ed15" [[package]] name = "colorchoice" -version = "1.0.1" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b6a852b24ab71dffc585bcb46eaf7959d175cb865a7152e35b348d1b2960422" +checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" [[package]] name = "colored" @@ -798,9 +794,9 @@ checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" [[package]] name = "constant_time_eq" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7144d30dcf0fafbce74250a3963025d8d52177934239851c917d29f1df280c2" +checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" [[package]] name = "cordyceps" @@ -824,15 +820,15 @@ dependencies = [ [[package]] name = "core-foundation-sys" -version = "0.8.6" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] name = "cpufeatures" -version = "0.2.12" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504" +checksum = "608697df725056feaccfa42cffdaeeec3fccc4ffc38358ecd19b243e716a78e0" dependencies = [ "libc", ] @@ -1038,7 +1034,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -1062,7 +1058,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -1073,7 +1069,7 @@ checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" dependencies = [ "darling_core", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -1128,7 +1124,7 @@ checksum = "8034092389675178f570469e6c3b0465d3d30b4505c294a6550db47f3c17ad18" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -1158,7 +1154,7 @@ checksum = "cb7330aeadfbe296029522e6c40f315320aba36fc43a5b3632f3795348f3bd22" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", "unicode-xid", ] @@ -1175,9 +1171,9 @@ dependencies = [ [[package]] name = "diatomic-waker" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af873b6853650fb206431c52fa7bbf6917146b70a8a9979d6d141f5d5394086b" +checksum = "ab03c107fafeb3ee9f5925686dbb7a73bc76e3932abb0d2b365cb64b169cf04c" [[package]] name = "diff" @@ -1247,7 +1243,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -1397,14 +1393,14 @@ checksum = "c34f04666d835ff5d62e058c3995147c06f42fe86ff053337632bca83e42702d" [[package]] name = "enum-as-inner" -version = "0.6.0" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ffccbb6966c05b32ef8fbac435df276c4ae4d3dc55a8cd0eb9745e6c12f546a" +checksum = "a1e6a265c649f3f5979b601d26f1d05ada116434c87741c9493cb56218f76cbc" dependencies = [ - "heck 0.4.1", + "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -1417,7 +1413,7 @@ dependencies = [ "num-traits", "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -1437,7 +1433,7 @@ checksum = "de0d48a183585823424a4ce1aa132d174a6a81bd540895822eb4c8373a8e49e8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -1530,9 +1526,9 @@ dependencies = [ [[package]] name = "fastrand" -version = "2.1.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fc0510504f03c51ada170672ac806f1f105a88aa97a5281117e1ddc3368e51a" +checksum = "e8c02a5121d4ea3eb16a80748c74f5549a5665e4c21333c6098f283870fbdea6" [[package]] name = "fd-lock" @@ -1561,11 +1557,17 @@ version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" +[[package]] +name = "fixedbitset" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" + [[package]] name = "flume" -version = "0.11.0" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55ac459de2512911e4b674ce33cf20befaba382d05b62b008afc1c8b57cbf181" +checksum = "da0e4dd2a88388a1f4ccc7c9ce104604dab68d9f408dc34cd45823d5a9069095" dependencies = [ "futures-core", "futures-sink", @@ -1579,6 +1581,12 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foldhash" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f81ec6369c545a7d40e4589b5597581fa1c441fe1cce96dd1de43159910a36a2" + [[package]] name = "form_urlencoded" version = "1.2.1" @@ -1598,17 +1606,11 @@ dependencies = [ "thiserror", ] -[[package]] -name = "funty" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" - [[package]] name = "futures" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0" +checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" dependencies = [ "futures-channel", "futures-core", @@ -1643,11 +1645,11 @@ dependencies = [ [[package]] name = "futures-concurrency" -version = "7.6.1" +version = "7.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b14ac911e85d57c5ea6eef76d7b4d4a3177ecd15f4bea2e61927e9e3823e19f" +checksum = "d9b724496da7c26fcce66458526ce68fc2ecf4aaaa994281cf322ded5755520c" dependencies = [ - "bitvec", + "fixedbitset", "futures-buffered", "futures-core", "futures-lite 1.13.0", @@ -1664,9 +1666,9 @@ checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" [[package]] name = "futures-executor" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d" +checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" dependencies = [ "futures-core", "futures-task", @@ -1696,11 +1698,11 @@ dependencies = [ [[package]] name = "futures-lite" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52527eb5074e35e9339c6b4e8d12600c7128b68fb25dcb9fa9dec18f7c25f3a5" +checksum = "3f1fa2f9765705486b33fd2acf1577f8ec449c2ba1f318ae5447697b7c08d210" dependencies = [ - "fastrand 2.1.0", + "fastrand 2.1.1", "futures-core", "futures-io", "parking", @@ -1715,7 +1717,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -1824,9 +1826,9 @@ dependencies = [ [[package]] name = "gimli" -version = "0.29.0" +version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40ecd4077b5ae9fd2e9e169b102c6c330d0605168eb0e8bf79952b256dbefffd" +checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" [[package]] name = "glob" @@ -1867,9 +1869,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa82e28a107a8cc405f0839610bdc9b15f1e25ec7d696aa5cf173edbcb1486ab" +checksum = "524e8ac6999421f49a846c2d4411f337e53497d8ec55d67753beffa43c5d9205" dependencies = [ "atomic-waker", "bytes", @@ -1877,7 +1879,7 @@ dependencies = [ "futures-core", "futures-sink", "http 1.1.0", - "indexmap 2.2.6", + "indexmap 2.6.0", "slab", "tokio", "tokio-util", @@ -1916,7 +1918,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" dependencies = [ "ahash", +] + +[[package]] +name = "hashbrown" +version = "0.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a9bfc1af68b1726ea47d3d5109de126281def866b33970e10fbab11b5dafab3" +dependencies = [ "allocator-api2", + "equivalent", + "foldhash", ] [[package]] @@ -1970,6 +1982,12 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" +[[package]] +name = "hermit-abi" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbf6a919d6cf397374f7dfeeea91d974c7c0a7221d0d0f4f20d859d329e53fcc" + [[package]] name = "hex" version = "0.4.3" @@ -2202,9 +2220,9 @@ dependencies = [ [[package]] name = "httparse" -version = "1.9.4" +version = "1.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fcc0b4a115bf80b728eb8ea024ad5bd707b615bfed49e0665b6e0f86fd082d9" +checksum = "7d71d3574edd2771538b901e6549113b4006ece66150fb69c0fb6d9a2adae946" [[package]] name = "httpdate" @@ -2233,9 +2251,9 @@ dependencies = [ [[package]] name = "hyper" -version = "1.4.1" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50dfd22e0e76d0f662d429a5f80fcaf3855009297eab6a0a9f8543834744ba05" +checksum = "bbbff0a806a4728c99295b254c8838933b5b082d75e3cb70c8dab21fdfbcfa9a" dependencies = [ "bytes", "futures-channel", @@ -2254,9 +2272,9 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.27.2" +version = "0.27.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ee4be2c948921a1a5320b629c4193916ed787a7f7f293fd3f7f5a6c9de74155" +checksum = "08afdbb5c31130e3034af566421053ab03787c640246a446327f550d11bcb333" dependencies = [ "futures-util", "http 1.1.0", @@ -2272,9 +2290,9 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.6" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ab92f4f49ee4fb4f997c784b7a2e0fa70050211e0b6a287f898c3c9785ca956" +checksum = "df2dcfbe0677734ab2f3ffa7fa7bfd4706bfdc1ef393f2ee30184aed67e631b4" dependencies = [ "bytes", "futures-channel", @@ -2285,16 +2303,15 @@ dependencies = [ "pin-project-lite", "socket2", "tokio", - "tower", "tower-service", "tracing", ] [[package]] name = "iana-time-zone" -version = "0.1.60" +version = "0.1.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7ffbb5a1b541ea2561f8c41c087286cc091e21e556a4f09a8f6cbf17b69b141" +checksum = "235e081f3925a06703c2d0117ea8b91f042756fd6e7a6e5d901e8ca1a996b220" dependencies = [ "android_system_properties", "core-foundation-sys", @@ -2313,6 +2330,124 @@ dependencies = [ "cc", ] +[[package]] +name = "icu_collections" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db2fa452206ebee18c4b5c2274dbf1de17008e874b4dc4f0aea9d01ca79e4526" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locid" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_locid_transform" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01d11ac35de8e40fdeda00d9e1e9d92525f3f9d887cdd7aa81d727596788b54e" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_locid_transform_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_locid_transform_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdc8ff3388f852bede6b579ad4e978ab004f139284d7b28715f773507b946f6e" + +[[package]] +name = "icu_normalizer" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19ce3e0da2ec68599d193c93d088142efd7f9c5d6fc9b803774855747dc6a84f" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "utf16_iter", + "utf8_iter", + "write16", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8cafbf7aa791e9b22bec55a167906f9e1215fd475cd22adfcf660e03e989516" + +[[package]] +name = "icu_properties" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93d6020766cfc6302c15dbbc9c8778c37e62c14427cb7f6e601d849e092aeef5" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locid_transform", + "icu_properties_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67a8effbc3dd3e4ba1afa8ad918d5684b8868b3b26500753effea8d2eed19569" + +[[package]] +name = "icu_provider" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ed421c8a8ef78d3e2dbc98a973be2f3770cb42b606e3ab18d6237c4dfde68d9" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_provider_macros", + "stable_deref_trait", + "tinystr", + "writeable", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_provider_macros" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", +] + [[package]] name = "ident_case" version = "1.0.1" @@ -2339,6 +2474,27 @@ dependencies = [ "unicode-normalization", ] +[[package]] +name = "idna" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "686f825264d630750a544639377bae737628043f20d38bbc029e8f29ea968a7e" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daca1df1c957320b2cf139ac61e7bd64fed304c5040df000a745aa1de3b4ef71" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + [[package]] name = "igd-next" version = "0.15.1" @@ -2373,12 +2529,12 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.2.6" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" +checksum = "707907fe3c25f5424cce2cb7e1cbcafee6bdbe735ca90ef77c29e84591e5b9da" dependencies = [ "equivalent", - "hashbrown 0.14.5", + "hashbrown 0.15.1", "serde", ] @@ -2437,9 +2593,9 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.9.0" +version = "2.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" +checksum = "ddc24109865250148c2e0f3d25d4f0f479571723792d3802153c60922a4fb708" dependencies = [ "serde", ] @@ -2456,21 +2612,21 @@ dependencies = [ "console", "derive_more", "futures-buffered", - "futures-lite 2.3.0", + "futures-lite 2.4.0", "futures-util", "genawaiter", "hex", "indicatif", "iroh", - "iroh-base", - "iroh-blobs", + "iroh-base 0.27.0", + "iroh-blobs 0.28.1", "iroh-docs", "iroh-gossip", "iroh-io", - "iroh-metrics", - "iroh-net", - "iroh-quinn", - "iroh-router", + "iroh-metrics 0.27.0", + "iroh-net 0.27.0", + "iroh-quinn 0.11.3", + "iroh-router 0.27.0", "iroh-test", "nested_enum_utils", "num_cpus", @@ -2478,7 +2634,7 @@ dependencies = [ "portable-atomic", "postcard", "proptest", - "quic-rpc 0.13.0", + "quic-rpc", "quic-rpc-derive", "rand", "rand_chacha", @@ -2520,7 +2676,7 @@ dependencies = [ "proptest", "rand", "rand_core", - "redb 2.1.1", + "redb 2.2.0", "serde", "serde_json", "serde_test", @@ -2531,6 +2687,34 @@ dependencies = [ "zeroize", ] +[[package]] +name = "iroh-base" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c21fd8eb71f166a172a9779c2244db992218e9a9bd929b9df6fc355d2b630c9" +dependencies = [ + "aead", + "anyhow", + "crypto_box", + "data-encoding", + "derive_more", + "ed25519-dalek", + "getrandom", + "hex", + "iroh-blake3", + "once_cell", + "postcard", + "rand", + "rand_core", + "redb 2.2.0", + "serde", + "ssh-key", + "thiserror", + "ttl_cache", + "url", + "zeroize", +] + [[package]] name = "iroh-blake3" version = "1.4.5" @@ -2547,7 +2731,50 @@ dependencies = [ [[package]] name = "iroh-blobs" version = "0.27.0" -source = "git+https://github.com/n0-computer/iroh-blobs?branch=add-rpc#b3d0c58e80f15dbe1bb148793efe3fd164d91054" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69e83475d6c8cc312a224d4c6bdf287f862d30d2b176768ef71f188a0d8aa7cd" +dependencies = [ + "anyhow", + "async-channel", + "bao-tree", + "bytes", + "chrono", + "derive_more", + "futures-buffered", + "futures-lite 2.4.0", + "genawaiter", + "hashlink", + "hex", + "iroh-base 0.27.0", + "iroh-io", + "iroh-metrics 0.27.0", + "iroh-net 0.27.0", + "iroh-quinn 0.11.3", + "num_cpus", + "oneshot", + "parking_lot", + "pin-project", + "postcard", + "rand", + "range-collections", + "redb 1.5.1", + "redb 2.2.0", + "reflink-copy", + "self_cell", + "serde", + "smallvec", + "tempfile", + "thiserror", + "tokio", + "tokio-util", + "tracing", + "tracing-futures", +] + +[[package]] +name = "iroh-blobs" +version = "0.28.1" +source = "git+https://github.com/n0-computer/iroh-blobs?branch=try-out-new-rpc#ec7967b3c92ab4596a9abb3e91090cc575ab9e37" dependencies = [ "anyhow", "async-channel", @@ -2556,17 +2783,17 @@ dependencies = [ "chrono", "derive_more", "futures-buffered", - "futures-lite 2.3.0", + "futures-lite 2.4.0", "futures-util", "genawaiter", "hashlink", "hex", - "iroh-base", + "iroh-base 0.28.0", "iroh-io", - "iroh-metrics", - "iroh-net", - "iroh-quinn", - "iroh-router", + "iroh-metrics 0.28.0", + "iroh-net 0.28.1", + "iroh-quinn 0.12.0", + "iroh-router 0.28.0", "nested_enum_utils", "num_cpus", "oneshot", @@ -2574,12 +2801,12 @@ dependencies = [ "pin-project", "portable-atomic", "postcard", - "quic-rpc 0.13.0", + "quic-rpc", "quic-rpc-derive", "rand", "range-collections", "redb 1.5.1", - "redb 2.1.1", + "redb 2.2.0", "ref-cast", "reflink-copy", "self_cell", @@ -2614,21 +2841,21 @@ dependencies = [ "dirs-next", "duct", "futures-buffered", - "futures-lite 2.3.0", + "futures-lite 2.4.0", "futures-util", "hex", "human-time", "indicatif", "iroh", "iroh-gossip", - "iroh-metrics", + "iroh-metrics 0.27.0", "nix 0.27.1", "parking_lot", "pkarr", "portable-atomic", - "portmapper", + "portmapper 0.1.0", "postcard", - "quic-rpc 0.12.0", + "quic-rpc", "rand", "rand_xorshift", "ratatui", @@ -2667,21 +2894,21 @@ dependencies = [ "clap", "derive_more", "dirs-next", - "futures-lite 2.3.0", + "futures-lite 2.4.0", "governor", "hickory-proto 0.25.0-alpha.2", "hickory-resolver", "hickory-server", "http 1.1.0", - "iroh-metrics", - "iroh-net", + "iroh-metrics 0.27.0", + "iroh-net 0.27.0", "iroh-test", "lru", "mainline", "parking_lot", "pkarr", "rcgen", - "redb 2.1.1", + "redb 2.2.0", "regex", "rustls", "rustls-pemfile", @@ -2706,7 +2933,8 @@ dependencies = [ [[package]] name = "iroh-docs" version = "0.27.0" -source = "git+https://github.com/n0-computer/iroh-docs?branch=main#16bc7fe4c7dee1b1b88f54390856c3fafa3d7656" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb2c8bc1fe680549dd9776031be30e68663884026f71a914aab7108e0b8f8ce4" dependencies = [ "anyhow", "async-channel", @@ -2714,23 +2942,22 @@ dependencies = [ "derive_more", "ed25519-dalek", "futures-buffered", - "futures-lite 2.3.0", + "futures-lite 2.4.0", "futures-util", "hex", - "iroh-base", + "iroh-base 0.27.0", "iroh-blake3", - "iroh-blobs", + "iroh-blobs 0.27.0", "iroh-gossip", - "iroh-metrics", - "iroh-net", - "iroh-router", + "iroh-metrics 0.27.0", + "iroh-net 0.27.0", "lru", "num_enum", "postcard", "rand", "rand_core", "redb 1.5.1", - "redb 2.1.1", + "redb 2.2.0", "self_cell", "serde", "strum 0.25.0", @@ -2745,7 +2972,8 @@ dependencies = [ [[package]] name = "iroh-gossip" version = "0.27.0" -source = "git+https://github.com/n0-computer/iroh-gossip?branch=main#77d392cc68a06024addb4b4162d18f0709bab3ab" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b434d455389493ff2b2ecbab035c12eb3762f24d04080855ecd4956bf7739448" dependencies = [ "anyhow", "async-channel", @@ -2753,23 +2981,17 @@ dependencies = [ "derive_more", "ed25519-dalek", "futures-concurrency", - "futures-lite 2.3.0", + "futures-lite 2.4.0", "futures-util", - "indexmap 2.2.6", - "iroh-base", + "indexmap 2.6.0", + "iroh-base 0.27.0", "iroh-blake3", - "iroh-metrics", - "iroh-net", - "iroh-router", - "nested_enum_utils", + "iroh-metrics 0.27.0", + "iroh-net 0.27.0", "postcard", - "quic-rpc 0.13.0", - "quic-rpc-derive", "rand", "rand_core", "serde", - "serde-error", - "strum 0.26.3", "tokio", "tokio-util", "tracing", @@ -2777,12 +2999,12 @@ dependencies = [ [[package]] name = "iroh-io" -version = "0.6.0" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d1047ad5ca29ab4ff316b6830d86e7ea52cea54325e4d4a849692e1274b498" +checksum = "17e302c5ad649c6a7aa9ae8468e1c4dc2469321af0c6de7341c1be1bdaab434b" dependencies = [ "bytes", - "futures-lite 2.3.0", + "futures-lite 2.4.0", "pin-project", "smallvec", "tokio", @@ -2808,15 +3030,36 @@ dependencies = [ ] [[package]] -name = "iroh-net" -version = "0.27.0" +name = "iroh-metrics" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0d40f2ee3997489d47403d204a06514ed65373d224b5b43a8ea133f543e5db1" dependencies = [ "anyhow", - "axum", - "backoff", - "base64 0.22.1", - "bytes", - "clap", + "erased_set", + "http-body-util", + "hyper", + "hyper-util", + "once_cell", + "prometheus-client", + "reqwest", + "serde", + "struct_iterable", + "time", + "tokio", + "tracing", +] + +[[package]] +name = "iroh-net" +version = "0.27.0" +dependencies = [ + "anyhow", + "axum", + "backoff", + "base64 0.22.1", + "bytes", + "clap", "criterion", "crypto_box", "der", @@ -2824,7 +3067,7 @@ dependencies = [ "duct", "futures-buffered", "futures-concurrency", - "futures-lite 2.3.0", + "futures-lite 2.4.0", "futures-sink", "futures-util", "genawaiter", @@ -2838,11 +3081,11 @@ dependencies = [ "hyper", "hyper-util", "igd-next", - "iroh-base", - "iroh-metrics", - "iroh-net", - "iroh-quinn", - "iroh-quinn-proto", + "iroh-base 0.27.0", + "iroh-metrics 0.27.0", + "iroh-net 0.27.0", + "iroh-quinn 0.11.3", + "iroh-quinn-proto 0.11.6", "iroh-quinn-udp", "iroh-test", "libc", @@ -2851,14 +3094,14 @@ dependencies = [ "netlink-packet-core", "netlink-packet-route", "netlink-sys", - "netwatch", + "netwatch 0.1.0", "ntest", "num_enum", "once_cell", "parking_lot", "pin-project", "pkarr", - "portmapper", + "portmapper 0.1.0", "postcard", "pretty_assertions", "proptest", @@ -2903,6 +3146,85 @@ dependencies = [ "z32", ] +[[package]] +name = "iroh-net" +version = "0.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b40e1f1f9029e198c6d05bd232d3239814b0a66ac4668978729b709aeb6a44e2" +dependencies = [ + "anyhow", + "backoff", + "base64 0.22.1", + "bytes", + "der", + "derive_more", + "duct", + "futures-buffered", + "futures-concurrency", + "futures-lite 2.4.0", + "futures-sink", + "futures-util", + "genawaiter", + "governor", + "hex", + "hickory-proto 0.25.0-alpha.2", + "hickory-resolver", + "hostname", + "http 1.1.0", + "http-body-util", + "hyper", + "hyper-util", + "igd-next", + "iroh-base 0.28.0", + "iroh-metrics 0.28.0", + "iroh-quinn 0.12.0", + "iroh-quinn-proto 0.12.0", + "iroh-quinn-udp", + "libc", + "netdev", + "netlink-packet-core", + "netlink-packet-route", + "netlink-sys", + "netwatch 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "num_enum", + "once_cell", + "parking_lot", + "pin-project", + "pkarr", + "portmapper 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "postcard", + "rand", + "rcgen", + "reqwest", + "ring", + "rtnetlink", + "rustls", + "rustls-webpki", + "serde", + "smallvec", + "socket2", + "strum 0.26.3", + "stun-rs", + "surge-ping", + "thiserror", + "time", + "tokio", + "tokio-rustls", + "tokio-stream", + "tokio-tungstenite", + "tokio-tungstenite-wasm", + "tokio-util", + "tracing", + "tungstenite", + "url", + "watchable", + "webpki-roots", + "windows 0.51.1", + "wmi", + "x509-parser", + "z32", +] + [[package]] name = "iroh-net-bench" version = "0.27.0" @@ -2910,11 +3232,11 @@ dependencies = [ "anyhow", "bytes", "clap", - "futures-lite 2.3.0", + "futures-lite 2.4.0", "hdrhistogram", - "iroh-metrics", - "iroh-net", - "iroh-quinn", + "iroh-metrics 0.27.0", + "iroh-net 0.27.0", + "iroh-quinn 0.11.3", "rcgen", "rustls", "socket2", @@ -2930,10 +3252,28 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4fd590a39a14cfc168efa4d894de5039d65641e62d8da4a80733018ababe3c33" dependencies = [ "bytes", - "iroh-quinn-proto", + "iroh-quinn-proto 0.11.6", + "iroh-quinn-udp", + "pin-project-lite", + "rustc-hash", + "rustls", + "socket2", + "thiserror", + "tokio", + "tracing", +] + +[[package]] +name = "iroh-quinn" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35ba75a5c57cff299d2d7ca1ddee053f66339d1756bd79ec637bcad5aa61100e" +dependencies = [ + "bytes", + "iroh-quinn-proto 0.12.0", "iroh-quinn-udp", "pin-project-lite", - "rustc-hash 2.0.0", + "rustc-hash", "rustls", "socket2", "thiserror", @@ -2950,7 +3290,25 @@ dependencies = [ "bytes", "rand", "ring", - "rustc-hash 2.0.0", + "rustc-hash", + "rustls", + "rustls-platform-verifier", + "slab", + "thiserror", + "tinyvec", + "tracing", +] + +[[package]] +name = "iroh-quinn-proto" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2c869ba52683d3d067c83ab4c00a2fda18eaf13b1434d4c1352f428674d4a5d" +dependencies = [ + "bytes", + "rand", + "ring", + "rustc-hash", "rustls", "rustls-platform-verifier", "slab", @@ -2961,9 +3319,9 @@ dependencies = [ [[package]] name = "iroh-quinn-udp" -version = "0.5.4" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0619b59471fdd393ac8a6c047f640171119c1c8b41f7d2927db91776dcdbc5f" +checksum = "bfcfc0abc2fdf8cf18a6c72893b7cbebeac2274a3b1306c1760c48c0e10ac5e0" dependencies = [ "libc", "once_cell", @@ -2979,15 +3337,31 @@ dependencies = [ "anyhow", "clap", "futures-buffered", - "futures-lite 2.3.0", + "futures-lite 2.4.0", "futures-util", - "iroh-net", + "iroh-net 0.27.0", "tokio", "tokio-util", "tracing", "tracing-subscriber", ] +[[package]] +name = "iroh-router" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1fd18ec6325dd3f01625f12c01acff50a4374ee1ab708e7b2078885fd63ad30" +dependencies = [ + "anyhow", + "futures-buffered", + "futures-lite 2.4.0", + "futures-util", + "iroh-net 0.28.1", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "iroh-test" version = "0.27.0" @@ -3000,20 +3374,20 @@ dependencies = [ [[package]] name = "is-terminal" -version = "0.4.12" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f23ff5ef2b80d608d61efee834934d862cd92461afc0560dedf493e4c033738b" +checksum = "261f68e344040fbd0edea105bef17c66edf46f984ddb1115b775ce31be948f4b" dependencies = [ - "hermit-abi", + "hermit-abi 0.4.0", "libc", "windows-sys 0.52.0", ] [[package]] name = "is_terminal_polyfill" -version = "1.70.0" +version = "1.70.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8478577c03552c21db0e2724ffb8986a5ce7af88107e6be5d2ee6e158c12800" +checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" [[package]] name = "itertools" @@ -3070,9 +3444,9 @@ checksum = "8eaf4bc02d17cbdd7ff4c7438cafcdf7fb9a4613313ad11b4f8fefe7d3fa0130" [[package]] name = "js-sys" -version = "0.3.69" +version = "0.3.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d" +checksum = "6a88f1bda2bd75b0452a14784937d796722fdebfe50df998aeb3f0b7603019a9" dependencies = [ "wasm-bindgen", ] @@ -3088,15 +3462,15 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.155" +version = "0.2.161" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" +checksum = "8e9489c2807c139ffd9c1794f4af0ebe86a828db53ecdc7fea2111d0fed085d1" [[package]] name = "libm" -version = "0.2.8" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058" +checksum = "8355be11b20d696c8f18f6cc018c4e372165b1fa8126cef092399c9951984ffa" [[package]] name = "libredox" @@ -3120,6 +3494,12 @@ version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" +[[package]] +name = "litemap" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "643cb0b8d4fcc284004d5fd0d67ccf61dfffadb7f75e1e71bc420f4688a3a704" + [[package]] name = "litrs" version = "0.4.1" @@ -3157,11 +3537,11 @@ dependencies = [ [[package]] name = "lru" -version = "0.12.3" +version = "0.12.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3262e75e648fce39813cb56ac41f3c3e3f65217ebf3844d818d1f9398cfb0dc" +checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" dependencies = [ - "hashbrown 0.14.5", + "hashbrown 0.15.1", ] [[package]] @@ -3246,11 +3626,11 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.7.4" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8a240ddb74feaf34a79a7add65a741f3167852fba007066dcac1ca548d89c08" +checksum = "e2d80299ef12ff69b16a84bb182e3b9df68b5a91574d3d4fa6e41b65deec4df1" dependencies = [ - "adler", + "adler2", ] [[package]] @@ -3271,7 +3651,7 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "80e04d1dcff3aae0704555fe5fee3bcfaf3d1fdf8a7e521d5b9d2b42acb52cec" dependencies = [ - "hermit-abi", + "hermit-abi 0.3.9", "libc", "wasi", "windows-sys 0.52.0", @@ -3387,7 +3767,7 @@ dependencies = [ "anyhow", "bytes", "derive_more", - "futures-lite 2.3.0", + "futures-lite 2.4.0", "futures-sink", "futures-util", "iroh-test", @@ -3409,6 +3789,35 @@ dependencies = [ "wmi", ] +[[package]] +name = "netwatch" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a639d52c0996ac640e2a7052a5265c8f71efdbdadc83188435ffc358b7ca931" +dependencies = [ + "anyhow", + "bytes", + "derive_more", + "futures-lite 2.4.0", + "futures-sink", + "futures-util", + "libc", + "netdev", + "netlink-packet-core", + "netlink-packet-route", + "netlink-sys", + "once_cell", + "rtnetlink", + "serde", + "socket2", + "thiserror", + "time", + "tokio", + "tracing", + "windows 0.51.1", + "wmi", +] + [[package]] name = "nibble_vec" version = "0.1.0" @@ -3595,29 +4004,29 @@ version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" dependencies = [ - "hermit-abi", + "hermit-abi 0.3.9", "libc", ] [[package]] name = "num_enum" -version = "0.7.2" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02339744ee7253741199f897151b38e72257d13802d4ee837285cc2990a90845" +checksum = "4e613fc340b2220f734a8595782c551f1250e969d87d3be1ae0579e8d4065179" dependencies = [ "num_enum_derive", ] [[package]] name = "num_enum_derive" -version = "0.7.2" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "681030a937600a36906c185595136d26abfebb4aa9c65701cefcaf8578bb982b" +checksum = "af1844ef2428cc3e1cb900be36181049ef3d3193c63e43026cfe202983b27a56" dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -3637,27 +4046,27 @@ checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" [[package]] name = "object" -version = "0.36.1" +version = "0.36.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "081b846d1d56ddfc18fdf1a922e4f6e07a11768ea1b92dec44e42b72712ccfce" +checksum = "aedf0a2d09c573ed1d8d85b30c119153926a2b36dce0ab28322c09a117a4683e" dependencies = [ "memchr", ] [[package]] name = "oid-registry" -version = "0.7.0" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c958dd45046245b9c3c2547369bb634eb461670b2e7e0de552905801a648d1d" +checksum = "a8d8034d9489cdaf79228eb9f6a3b8d7bb32ba00d6645ebd48eef4077ceb5bd9" dependencies = [ "asn1-rs", ] [[package]] name = "once_cell" -version = "1.19.0" +version = "1.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" +checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" [[package]] name = "oneshot" @@ -3691,12 +4100,12 @@ checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" [[package]] name = "os_pipe" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29d73ba8daf8fac13b0501d1abeddcfe21ba7401ada61a819144b6c2a4f32209" +checksum = "5ffd2b0a5634335b135d5728d84c5e0fd726954b87111f7506a61c502280d982" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -3745,9 +4154,9 @@ dependencies = [ [[package]] name = "parking" -version = "2.2.0" +version = "2.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb813b8af86854136c6922af0598d719255ecb2179515e6e7730d468f05c9cae" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" [[package]] name = "parking_lot" @@ -3767,7 +4176,7 @@ checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" dependencies = [ "cfg-if", "libc", - "redox_syscall 0.5.3", + "redox_syscall", "smallvec", "windows-targets 0.52.6", ] @@ -3805,9 +4214,9 @@ checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "pest" -version = "2.7.11" +version = "2.7.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd53dff83f26735fdc1ca837098ccf133605d794cdae66acfc2bfac3ec809d95" +checksum = "879952a81a83930934cbf1786752d6dedc3b1f29e8f8fb2ad1d0a36f377cf442" dependencies = [ "memchr", "thiserror", @@ -3816,9 +4225,9 @@ dependencies = [ [[package]] name = "pest_derive" -version = "2.7.11" +version = "2.7.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a548d2beca6773b1c244554d36fcf8548a8a58e74156968211567250e48e49a" +checksum = "d214365f632b123a47fd913301e14c946c61d1c183ee245fa76eb752e59a02dd" dependencies = [ "pest", "pest_generator", @@ -3826,22 +4235,22 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.7.11" +version = "2.7.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c93a82e8d145725dcbaf44e5ea887c8a869efdcc28706df2d08c69e17077183" +checksum = "eb55586734301717aea2ac313f50b2eb8f60d2fc3dc01d190eefa2e625f60c4e" dependencies = [ "pest", "pest_meta", "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] name = "pest_meta" -version = "2.7.11" +version = "2.7.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a941429fea7e08bedec25e4f6785b6ffaacc6b755da98df5ef3e7dcf4a124c4f" +checksum = "b75da2a70cf4d9cb76833c990ac9cd3923c9a8905a8929789ce347c84564d03d" dependencies = [ "once_cell", "pest", @@ -3850,29 +4259,29 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.1.5" +version = "1.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6bf43b791c5b9e34c3d182969b4abb522f9343702850a2e57f460d00d09b4b3" +checksum = "be57f64e946e500c8ee36ef6331845d40a93055567ec57e8fae13efd33759b95" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.5" +version = "1.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" +checksum = "3c0f5fad0874fc7abcd4d750e76917eaebbecaa2c20bde22e1dbeeba8beb758c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] name = "pin-project-lite" -version = "0.2.14" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bda66fc9667c18cb2758a2ac84d1167245054bcf85d5d1aaa6923f45801bdd02" +checksum = "915a1e146535de9163f3987b8944ed8cf49a18bb0056bcebcdcece385cece4ff" [[package]] name = "pin-utils" @@ -3930,9 +4339,9 @@ dependencies = [ [[package]] name = "plotters" -version = "0.3.6" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a15b6eccb8484002195a3e44fe65a4ce8e93a625797a063735536fd59cb01cf3" +checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" dependencies = [ "num-traits", "plotters-backend", @@ -3943,15 +4352,15 @@ dependencies = [ [[package]] name = "plotters-backend" -version = "0.3.6" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "414cec62c6634ae900ea1c56128dfe87cf63e7caece0852ec76aba307cebadb7" +checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" [[package]] name = "plotters-svg" -version = "0.3.6" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81b30686a7d9c3e010b84284bdd26a29f2138574f52f5eb6f794fc0ad924e705" +checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" dependencies = [ "plotters-backend", ] @@ -3974,7 +4383,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -4011,9 +4420,9 @@ dependencies = [ [[package]] name = "portable-atomic" -version = "1.7.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da544ee218f0d287a911e9c99a39a8c9bc8fcad3cb8db5959940044ecfc67265" +checksum = "cc9c68a3f6da06753e9335d63e27f6b9754dd1920d941135b7ea8224f141adb2" [[package]] name = "portmapper" @@ -4023,12 +4432,12 @@ dependencies = [ "base64 0.22.1", "bytes", "derive_more", - "futures-lite 2.3.0", + "futures-lite 2.4.0", "futures-util", "igd-next", - "iroh-metrics", + "iroh-metrics 0.27.0", "libc", - "netwatch", + "netwatch 0.1.0", "ntest", "num_enum", "rand", @@ -4044,6 +4453,35 @@ dependencies = [ "url", ] +[[package]] +name = "portmapper" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93d60045fdcfe8ff6b781cf1027fdbb08ed319d93aff7da4bedc018e3bc92226" +dependencies = [ + "anyhow", + "base64 0.22.1", + "bytes", + "derive_more", + "futures-lite 2.4.0", + "futures-util", + "igd-next", + "iroh-metrics 0.28.0", + "libc", + "netwatch 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "num_enum", + "rand", + "serde", + "smallvec", + "socket2", + "thiserror", + "time", + "tokio", + "tokio-util", + "tracing", + "url", +] + [[package]] name = "positioned-io" version = "0.3.3" @@ -4087,15 +4525,18 @@ checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" [[package]] name = "ppv-lite86" -version = "0.2.17" +version = "0.2.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" +checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04" +dependencies = [ + "zerocopy", +] [[package]] name = "precis-core" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d73e9dd26361c32e7cd13d1032bb01c4e26a23287274e8a4e2f228cf2c9ff77b" +checksum = "25a414cabc93f5f45d53463e73b3d89d3c5c0dc4a34dbf6901f0c6358f017203" dependencies = [ "precis-tools", "ucd-parse", @@ -4104,9 +4545,9 @@ dependencies = [ [[package]] name = "precis-profiles" -version = "0.1.10" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bde4bd6624c60cb0abe2bea1dbdbb9085f629a853861e64df4abb099f8076ad4" +checksum = "f58e2841ef58164e2626464d4fde67fa301d5e2c78a10300c1756312a03b169f" dependencies = [ "lazy_static", "precis-core", @@ -4116,9 +4557,9 @@ dependencies = [ [[package]] name = "precis-tools" -version = "0.1.7" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d07ecadec70b0f560f09abf815ae0ee1a940d38d2354c938ba7229ac7c9f5f52" +checksum = "016da884bc4c2c4670211641abef402d15fa2b06c6e9088ff270dac93675aee2" dependencies = [ "lazy_static", "regex", @@ -4137,9 +4578,9 @@ dependencies = [ [[package]] name = "pretty_assertions" -version = "1.4.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af7cee1a6c8a5b9208b3cb1061f10c0cb689087b3d8ce85fb9d2dd7a29b6ba66" +checksum = "3ae130e2f271fbc2ac3a40fb1d07180839cdbbe443c7a27e1e3c13c5cac0116d" dependencies = [ "diff", "yansi", @@ -4156,11 +4597,11 @@ dependencies = [ [[package]] name = "proc-macro-crate" -version = "3.1.0" +version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d37c51ca738a55da99dc0c4a34860fd675453b8b36209178c2249bb13651284" +checksum = "8ecf48c7ca261d60b74ab1a7b20da18bede46776b2e55535cb958eb595c5fa7b" dependencies = [ - "toml_edit 0.21.1", + "toml_edit", ] [[package]] @@ -4197,9 +4638,9 @@ checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068" [[package]] name = "proc-macro2" -version = "1.0.86" +version = "1.0.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" +checksum = "f139b0662de085916d1fb67d2b4169d1addddda1919e696f3252b740b629986e" dependencies = [ "unicode-ident", ] @@ -4224,7 +4665,7 @@ checksum = "440f724eba9f6996b75d63681b0a92b06947f1457076d503a4d2e2c8f56442b8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -4241,7 +4682,7 @@ dependencies = [ "rand", "rand_chacha", "rand_xorshift", - "regex-syntax 0.8.4", + "regex-syntax 0.8.5", "rusty-fork", "tempfile", "unarray", @@ -4264,45 +4705,19 @@ dependencies = [ [[package]] name = "quic-rpc" -version = "0.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87cb85690ab1688eade9a5de4d94545a9ceef60639b3370f5e1a28f525eb5589" -dependencies = [ - "anyhow", - "bincode", - "derive_more", - "educe", - "flume", - "futures-lite 2.3.0", - "futures-sink", - "futures-util", - "hex", - "iroh-quinn", - "pin-project", - "serde", - "slab", - "tokio", - "tokio-serde", - "tokio-util", - "tracing", -] - -[[package]] -name = "quic-rpc" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b0ea1bd0b3124538bb71ed8cedbe92608fd1cf227e4f5ff53fb28746737b794" +version = "0.15.0" +source = "git+https://github.com/n0-computer/quic-rpc.git?branch=map-transports-not-services#16b7c4942365b96fc750627459261e52e1190f6c" dependencies = [ "anyhow", "bincode", "derive_more", "educe", "flume", - "futures-lite 2.3.0", + "futures-lite 2.4.0", "futures-sink", "futures-util", "hex", - "iroh-quinn", + "iroh-quinn 0.12.0", "pin-project", "serde", "slab", @@ -4314,12 +4729,11 @@ dependencies = [ [[package]] name = "quic-rpc-derive" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94b91a3f7a42657cbfbd0c2499c1f037738eff45bb7f59c6ce3d3d9e890d141c" +version = "0.15.0" +source = "git+https://github.com/n0-computer/quic-rpc.git?branch=map-transports-not-services#16b7c4942365b96fc750627459261e52e1190f6c" dependencies = [ "proc-macro2", - "quic-rpc 0.13.0", + "quic-rpc", "quote", "syn 1.0.109", ] @@ -4332,16 +4746,17 @@ checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" [[package]] name = "quinn" -version = "0.11.2" +version = "0.11.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4ceeeeabace7857413798eb1ffa1e9c905a9946a57d81fb69b4b71c4d8eb3ad" +checksum = "8c7c5fdde3cdae7203427dc4f0a68fe0ed09833edc525a03456b153b79828684" dependencies = [ "bytes", "pin-project-lite", "quinn-proto", "quinn-udp", - "rustc-hash 1.1.0", + "rustc-hash", "rustls", + "socket2", "thiserror", "tokio", "tracing", @@ -4349,14 +4764,14 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.3" +version = "0.11.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddf517c03a109db8100448a4be38d498df8a210a99fe0e1b9eaf39e78c640efe" +checksum = "fadfaed2cd7f389d0161bb73eeb07b7b78f8691047a6f3e73caaeae55310a4a6" dependencies = [ "bytes", "rand", "ring", - "rustc-hash 1.1.0", + "rustc-hash", "rustls", "slab", "thiserror", @@ -4366,21 +4781,23 @@ dependencies = [ [[package]] name = "quinn-udp" -version = "0.5.3" +version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25a78e6f726d84fcf960409f509ae354a32648f090c8d32a2ea8b1a1bc3bab14" +checksum = "7d5a626c6807713b15cac82a6acaccd6043c9a5408c24baae07611fec3f243da" dependencies = [ + "cfg_aliases", "libc", "once_cell", "socket2", - "windows-sys 0.52.0", + "tracing", + "windows-sys 0.59.0", ] [[package]] name = "quote" -version = "1.0.36" +version = "1.0.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" +checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" dependencies = [ "proc-macro2", ] @@ -4395,12 +4812,6 @@ dependencies = [ "pest_derive", ] -[[package]] -name = "radium" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" - [[package]] name = "radix_trie" version = "0.2.1" @@ -4484,9 +4895,9 @@ dependencies = [ [[package]] name = "raw-cpuid" -version = "11.1.0" +version = "11.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb9ee317cfe3fbd54b36a511efc1edd42e216903c9cd575e686dd68a2ba90d8d" +checksum = "1ab240315c661615f2ee9f0f2cd32d5a7343a84d5ebcccb99d46e6637565e7b0" dependencies = [ "bitflags 2.6.0", ] @@ -4534,36 +4945,27 @@ dependencies = [ [[package]] name = "redb" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6dd20d3cdeb9c7d2366a0b16b93b35b75aec15309fbeb7ce477138c9f68c8c0" +checksum = "84b1de48a7cf7ba193e81e078d17ee2b786236eed1d3f7c60f8a09545efc4925" dependencies = [ "libc", ] [[package]] name = "redox_syscall" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" -dependencies = [ - "bitflags 1.3.2", -] - -[[package]] -name = "redox_syscall" -version = "0.5.3" +version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a908a6e00f1fdd0dfd9c0eb08ce85126f6d8bbda50017e74bc4a4b7d4a926a4" +checksum = "9b6dfecf2c74bce2466cabf93f6664d6998a69eb21e39f4207930065b27b771f" dependencies = [ "bitflags 2.6.0", ] [[package]] name = "redox_users" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd283d9651eeda4b2a83a43c1c91b266c40fd76ecd39a50a8c630ae69dc72891" +checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43" dependencies = [ "getrandom", "libredox", @@ -4587,7 +4989,7 @@ checksum = "bcc303e793d3734489387d205e9b186fac9c6cfacedd98cbb2e8a5943595f3e6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -4603,14 +5005,14 @@ dependencies = [ [[package]] name = "regex" -version = "1.10.5" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b91213439dad192326a0d7c6ee3955910425f441d7038e0d6933b0aec5c4517f" +checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.4.7", - "regex-syntax 0.8.4", + "regex-automata 0.4.8", + "regex-syntax 0.8.5", ] [[package]] @@ -4624,13 +5026,13 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.7" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38caf58cc5ef2fed281f89292ef23f6365465ed9a41b7a7754eb4e26496c92df" +checksum = "368758f23274712b504848e9d5a6f010445cc8b87a7cdb4d7cbee666c1288da3" dependencies = [ "aho-corasick", "memchr", - "regex-syntax 0.8.4", + "regex-syntax 0.8.5", ] [[package]] @@ -4647,15 +5049,15 @@ checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] name = "regex-syntax" -version = "0.8.4" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" +checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" [[package]] name = "reqwest" -version = "0.12.7" +version = "0.12.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8f4955649ef5c38cc7f9e8aa41761d48fb9677197daea9984dc54f56aad5e63" +checksum = "a77c62af46e79de0a562e1a9849205ffcb7fc1238876e9bd743357570e04046f" dependencies = [ "base64 0.22.1", "bytes", @@ -4773,12 +5175,6 @@ version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" -[[package]] -name = "rustc-hash" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" - [[package]] name = "rustc-hash" version = "2.0.0" @@ -4787,9 +5183,9 @@ checksum = "583034fd73374156e66797ed8e5b0d5690409c9226b22d87cb7f19821c05d152" [[package]] name = "rustc_version" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" dependencies = [ "semver", ] @@ -4805,9 +5201,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.34" +version = "0.38.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f" +checksum = "375116bee2be9ed569afe2154ea6a99dfdffd257f533f187498c2a8f5feaf4ee" dependencies = [ "bitflags 2.6.0", "errno", @@ -4818,9 +5214,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.11" +version = "0.23.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4828ea528154ae444e5a642dbb7d5623354030dc9822b83fd9bb79683c7399d0" +checksum = "eee87ff5d9b36712a58574e12e9f0ea80f915a5b0ac518d322b24a465617925e" dependencies = [ "log", "once_cell", @@ -4833,9 +5229,9 @@ dependencies = [ [[package]] name = "rustls-native-certs" -version = "0.7.2" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04182dffc9091a404e0fc069ea5cd60e5b866c3adf881eff99a32d048242dffa" +checksum = "e5bfb394eeed242e909609f56089eecfe5fda225042e8b171791b9c95f5931e5" dependencies = [ "openssl-probe", "rustls-pemfile", @@ -4846,25 +5242,24 @@ dependencies = [ [[package]] name = "rustls-pemfile" -version = "2.1.2" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29993a25686778eb88d4189742cd713c9bce943bc54251a33509dc63cbacf73d" +checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" dependencies = [ - "base64 0.22.1", "rustls-pki-types", ] [[package]] name = "rustls-pki-types" -version = "1.7.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "976295e77ce332211c0d24d92c0e83e50f5c5f046d11082cea19f3df13a3562d" +checksum = "16f1201b3c9a7ee8039bcadc17b7e605e2945b27eee7631788c1bd2b0643674b" [[package]] name = "rustls-platform-verifier" -version = "0.3.3" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93bda3f493b9abe5b93b3e7e3ecde0df292f2bd28c0296b90586ee0055ff5123" +checksum = "afbb878bdfdf63a336a5e63561b1835e7a8c91524f51621db870169eac84b490" dependencies = [ "core-foundation", "core-foundation-sys", @@ -4889,9 +5284,9 @@ checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" [[package]] name = "rustls-webpki" -version = "0.102.6" +version = "0.102.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e6b52d4fda176fd835fdc55a835d4a89b8499cad995885a21149d5ad62f852e" +checksum = "64ca1bc8749bd4cf37b5ce386cc146580777b4e8572c7b97baf22c83f444bee9" dependencies = [ "ring", "rustls-pki-types", @@ -4900,9 +5295,9 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.17" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "955d28af4278de8121b7ebeb796b6a45735dc01436d898801014aced2773a3d6" +checksum = "0e819f2bc632f285be6d7cd36e25940d45b2391dd6d9b939e79de557f7014248" [[package]] name = "rusty-fork" @@ -4965,11 +5360,11 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.23" +version = "0.1.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534" +checksum = "01227be5826fa0690321a2ba6c5cd57a19cf3f6a09e76973b58e61de6ab9d1c1" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -5014,9 +5409,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.11.1" +version = "2.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75da29fe9b9b08fe9d6b22b5b4bcbc75d8db3aa31e639aa56bb62e9d46bfceaf" +checksum = "ea4a292869320c0272d7bc55a5a6aafaff59b4f63404a003887b679a2e05b4b6" dependencies = [ "core-foundation-sys", "libc", @@ -5039,9 +5434,9 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.204" +version = "1.0.214" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc76f558e0cbb2a839d37354c575f1dc3fdc6546b5be373ba43d95f231bf7c12" +checksum = "f55c3193aca71c12ad7890f1785d2b73e1b9f63a0bbc353c08ef26fe03fc56b5" dependencies = [ "serde_derive", ] @@ -5076,22 +5471,23 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.204" +version = "1.0.214" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0cd7e117be63d3c3678776753929474f3b04a43a080c744d6b0ae2a8c28e222" +checksum = "de523f781f095e28fa605cdce0f8307e451cc0fd14e2eb4cd2e98a355b147766" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] name = "serde_json" -version = "1.0.120" +version = "1.0.132" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e0d21c9a8cae1235ad58a00c11cb40d4b1e5c784f1ef2c537876ed6ffd8b7c5" +checksum = "d726bfaff4b320266d395898905d0eba0345aae23b54aee3a737e260fd46db03" dependencies = [ "itoa", + "memchr", "ryu", "serde", ] @@ -5108,18 +5504,18 @@ dependencies = [ [[package]] name = "serde_spanned" -version = "0.6.6" +version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79e674e01f999af37c49f70a6ede167a8a60b2503e56c5599532a65baa5969a0" +checksum = "87607cb1398ed59d48732e575a4c28a7a8ebf2454b964fe3f224f2afc07909e1" dependencies = [ "serde", ] [[package]] name = "serde_test" -version = "1.0.176" +version = "1.0.177" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a2f49ace1498612d14f7e0b8245519584db8299541dfe31a06374a828d620ab" +checksum = "7f901ee573cab6b3060453d2d5f0bae4e6d628c23c0a962ff9b5f1d7c8d4f1ed" dependencies = [ "serde", ] @@ -5138,15 +5534,15 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.9.0" +version = "3.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69cecfa94848272156ea67b2b1a53f20fc7bc638c4a46d2f8abde08f05f4b857" +checksum = "8e28bdad6db2b8340e449f7108f020b3b092e8583a9e3fb82713e1d4e71fe817" dependencies = [ "base64 0.22.1", "chrono", "hex", "indexmap 1.9.3", - "indexmap 2.2.6", + "indexmap 2.6.0", "serde", "serde_derive", "serde_json", @@ -5156,14 +5552,14 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.9.0" +version = "3.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8fee4991ef4f274617a51ad4af30519438dacb2f56ac773b08a1922ff743350" +checksum = "9d846214a9854ef724f3da161b426242d8de7c1fc7de2f89bb1efcb154dca79d" dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -5215,12 +5611,12 @@ dependencies = [ [[package]] name = "shared_child" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0d94659ad3c2137fef23ae75b03d5241d633f8acded53d672decfa0e6e0caef" +checksum = "09fa9338aed9a1df411814a5b2252f7cd206c55ae9bf2fa763f8de84603aa60c" dependencies = [ "libc", - "winapi", + "windows-sys 0.59.0", ] [[package]] @@ -5238,6 +5634,12 @@ dependencies = [ "dirs", ] +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + [[package]] name = "signal-hook" version = "0.3.17" @@ -5250,9 +5652,9 @@ dependencies = [ [[package]] name = "signal-hook-mio" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29ad2e15f37ec9a6cc544097b78a1ec90001e9f71b81338ca39f430adaca99af" +checksum = "34db1a06d485c9142248b7a054f034b349b212551f3dfd19c94d45a754a217cd" dependencies = [ "libc", "mio 0.8.11", @@ -5372,9 +5774,9 @@ dependencies = [ [[package]] name = "ssh-key" -version = "0.6.6" +version = "0.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca9b366a80cf18bb6406f4cf4d10aebfb46140a8c0c33f666a144c5c76ecbafc" +checksum = "3b86f5297f0f04d08cabaa0f6bff7cb6aec4d9c3b49d87990d63da9d9156a8c3" dependencies = [ "ed25519-dalek", "p256", @@ -5398,7 +5800,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d904e7009df136af5297832a3ace3370cd14ff1546a232f4f185036c2736fcac" dependencies = [ "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -5445,7 +5847,7 @@ dependencies = [ "proc-macro2", "quote", "struct_iterable_internal", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -5482,7 +5884,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -5495,14 +5897,14 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] name = "stun-rs" -version = "0.1.8" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0adebf9fb8fba5c39ee34092b0383f247e4d1255b98fcffec94b4b797b85b677" +checksum = "b79cc624c9a747353810310af44f1f03f71eb4561284a894acc0396e6d0de76e" dependencies = [ "base64 0.22.1", "bounded-integer", @@ -5572,9 +5974,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.72" +version = "2.0.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc4b9b9bf2add8093d3f2c0204471e951b2285580335de42f9d2534f3ae7a8af" +checksum = "25aa4ce346d03a6dcd68dd8b4010bcb74e54e62c90c573f394c46eae99aba32d" dependencies = [ "proc-macro2", "quote", @@ -5615,7 +6017,7 @@ checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -5634,9 +6036,9 @@ dependencies = [ [[package]] name = "system-configuration" -version = "0.6.0" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "658bc6ee10a9b4fcf576e9b0819d95ec16f4d2c02d39fd83ac1c8789785c4a42" +checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b" dependencies = [ "bitflags 2.6.0", "core-foundation", @@ -5653,22 +6055,17 @@ dependencies = [ "libc", ] -[[package]] -name = "tap" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" - [[package]] name = "tempfile" -version = "3.10.1" +version = "3.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1" +checksum = "f0f2c9fc62d0beef6951ccffd757e241266a2c833136efbe35af6cd2567dca5b" dependencies = [ "cfg-if", - "fastrand 2.1.0", + "fastrand 2.1.1", + "once_cell", "rustix", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -5693,22 +6090,22 @@ checksum = "614b328ff036a4ef882c61570f72918f7e9c5bee1da33f8e7f91e01daee7e56c" [[package]] name = "thiserror" -version = "1.0.63" +version = "1.0.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0342370b38b6a11b6cc11d6a805569958d54cfa061a29969c3b5ce2ea405724" +checksum = "02dd99dc800bbb97186339685293e1cc5d9df1f8fae2d0aecd9ff1c77efea892" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.63" +version = "1.0.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4558b58466b9ad7ca0f102865eccc95938dca1a74a856f2b57b6629050da261" +checksum = "a7c61ec9a6f64d2793d8a45faba21efbe3ced62a886d44c36a009b2b519b4c7e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -5754,6 +6151,16 @@ dependencies = [ "time-core", ] +[[package]] +name = "tinystr" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f" +dependencies = [ + "displaydoc", + "zerovec", +] + [[package]] name = "tinytemplate" version = "1.2.1" @@ -5805,7 +6212,7 @@ checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -5866,9 +6273,9 @@ dependencies = [ [[package]] name = "tokio-stream" -version = "0.1.15" +version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "267ac89e0bec6e691e5813911606935d77c476ff49024f98abcea3e7b15e37af" +checksum = "4f4e6ce100d0eb49a2734f8c0812bcd324cf357d21810932c5df6b96ef2b86f1" dependencies = [ "futures-core", "pin-project-lite", @@ -5924,48 +6331,37 @@ dependencies = [ [[package]] name = "toml" -version = "0.8.15" +version = "0.8.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac2caab0bf757388c6c0ae23b3293fdb463fee59434529014f85e3263b995c28" +checksum = "a1ed1f98e3fdc28d6d910e6737ae6ab1a93bf1985935a1193e68f93eeb68d24e" dependencies = [ - "indexmap 2.2.6", + "indexmap 2.6.0", "serde", "serde_spanned", "toml_datetime", - "toml_edit 0.22.16", + "toml_edit", ] [[package]] name = "toml_datetime" -version = "0.6.6" +version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4badfd56924ae69bcc9039335b2e017639ce3f9b001c393c1b2d1ef846ce2cbf" +checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41" dependencies = [ "serde", ] [[package]] name = "toml_edit" -version = "0.21.1" +version = "0.22.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a8534fd7f78b5405e860340ad6575217ce99f38d4d5c8f2442cb5ecb50090e1" +checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5" dependencies = [ - "indexmap 2.2.6", - "toml_datetime", - "winnow 0.5.40", -] - -[[package]] -name = "toml_edit" -version = "0.22.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "278f3d518e152219c994ce877758516bca5e118eaed6996192a774fb9fbf0788" -dependencies = [ - "indexmap 2.2.6", + "indexmap 2.6.0", "serde", "serde_spanned", "toml_datetime", - "winnow 0.6.14", + "winnow", ] [[package]] @@ -5978,6 +6374,21 @@ dependencies = [ "futures-util", "pin-project", "pin-project-lite", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2873938d487c3cfb9aed7546dc9f2711d867c9f90c46b889989a2cb84eba6b4f" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper 0.1.2", "tokio", "tower-layer", "tower-service", @@ -6003,15 +6414,15 @@ dependencies = [ [[package]] name = "tower-layer" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" [[package]] name = "tower-service" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" [[package]] name = "tower_governor" @@ -6025,7 +6436,7 @@ dependencies = [ "http 1.1.0", "pin-project", "thiserror", - "tower", + "tower 0.4.13", "tracing", ] @@ -6061,7 +6472,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -6165,9 +6576,9 @@ dependencies = [ [[package]] name = "ucd-trie" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed646292ffc8188ef8ea4d1e0e0150fb15a5c2e12ad9b8fc191ae7a8a7f3c4b9" +checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971" [[package]] name = "unarray" @@ -6177,30 +6588,30 @@ checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" [[package]] name = "unicode-bidi" -version = "0.3.15" +version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" +checksum = "5ab17db44d7388991a428b2ee655ce0c212e862eff1768a455c58f9aad6e7893" [[package]] name = "unicode-ident" -version = "1.0.12" +version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" +checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe" [[package]] name = "unicode-normalization" -version = "0.1.23" +version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a56d1686db2308d901306f92a263857ef59ea39678a5458e7cb17f01415101f5" +checksum = "5033c97c4262335cded6d6fc3e5c18ab755e1a3dc96376350f3d8e9f009ad956" dependencies = [ "tinyvec", ] [[package]] name = "unicode-segmentation" -version = "1.11.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4c87d22b6e3f4a18d4d40ef354e97c90fcb14dd91d7dc0aa9d8a1172ebf7202" +checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" [[package]] name = "unicode-truncate" @@ -6215,15 +6626,15 @@ dependencies = [ [[package]] name = "unicode-width" -version = "0.1.13" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0336d538f7abc86d282a4189614dfaa90810dfc2c6f6427eaf88e16311dd225d" +checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" [[package]] name = "unicode-xid" -version = "0.2.4" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" [[package]] name = "universal-hash" @@ -6243,9 +6654,9 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "ureq" -version = "2.10.0" +version = "2.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72139d247e5f97a3eff96229a7ae85ead5328a39efe76f8bf5a06313d505b6ea" +checksum = "b74fc6b57825be3373f7054754755f03ac3a8f5d70015ccad699ba2029956f4a" dependencies = [ "base64 0.22.1", "log", @@ -6258,12 +6669,12 @@ dependencies = [ [[package]] name = "url" -version = "2.5.2" +version = "2.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22784dbdf76fdde8af1aeda5622b546b422b6fc585325248a2bf9f5e41e94d6c" +checksum = "8d157f1b96d14500ffdc1f10ba712e780825526c03d9a49b4d0324b0d9113ada" dependencies = [ "form_urlencoded", - "idna 0.5.0", + "idna 1.0.3", "percent-encoding", "serde", ] @@ -6274,6 +6685,18 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" +[[package]] +name = "utf16_iter" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + [[package]] name = "utf8parse" version = "0.2.2" @@ -6288,9 +6711,9 @@ checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" [[package]] name = "version_check" -version = "0.9.4" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" [[package]] name = "wait-timeout" @@ -6340,34 +6763,35 @@ checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" [[package]] name = "wasm-bindgen" -version = "0.2.92" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8" +checksum = "128d1e363af62632b8eb57219c8fd7877144af57558fb2ef0368d0087bddeb2e" dependencies = [ "cfg-if", + "once_cell", "wasm-bindgen-macro", ] [[package]] name = "wasm-bindgen-backend" -version = "0.2.92" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da" +checksum = "cb6dd4d3ca0ddffd1dd1c9c04f94b868c37ff5fac97c30b97cff2d74fce3a358" dependencies = [ "bumpalo", "log", "once_cell", "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.42" +version = "0.4.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76bc14366121efc8dbb487ab05bcc9d346b3b5ec0eaa76e46594cabbe51762c0" +checksum = "cc7ec4f8827a71586374db3e87abdb5a2bb3a15afed140221307c3ec06b1f63b" dependencies = [ "cfg-if", "js-sys", @@ -6377,9 +6801,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.92" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726" +checksum = "e79384be7f8f5a9dd5d7167216f022090cf1f9ec128e6e6a482a2cb5c5422c56" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -6387,22 +6811,22 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.92" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" +checksum = "26c6ab57572f7a24a4985830b120de1594465e5d500f24afe89e16b4e833ef68" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.92" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" +checksum = "65fc09f10666a9f147042251e0dda9c18f166ff7de300607007e96bdebc1068d" [[package]] name = "watchable" @@ -6418,9 +6842,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.69" +version = "0.3.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77afa9a11836342370f4817622a2f0f418b134426d91a82dfb48f532d2ec13ef" +checksum = "f6488b90108c040df0fe62fa815cbdee25124641df01814dd7282749234c6112" dependencies = [ "js-sys", "wasm-bindgen", @@ -6428,20 +6852,20 @@ dependencies = [ [[package]] name = "webpki-roots" -version = "0.26.3" +version = "0.26.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd7c23921eeb1713a4e851530e9b9756e4fb0e89978582942612524cf09f01cd" +checksum = "841c67bff177718f1d4dfefde8d8f0e78f9b6589319ba88312f567fc5841a958" dependencies = [ "rustls-pki-types", ] [[package]] name = "whoami" -version = "1.5.1" +version = "1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a44ab49fad634e88f55bf8f9bb3abd2f27d7204172a112c7c9987e01c1c94ea9" +checksum = "372d5b87f58ec45c384ba03563b03544dc5fadc3983e434b286913f5b4a9bb6d" dependencies = [ - "redox_syscall 0.4.1", + "redox_syscall", "wasite", "web-sys", ] @@ -6470,11 +6894,11 @@ checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" [[package]] name = "winapi-util" -version = "0.1.8" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d4cc384e1e73b93bafa6fb4f1df8c41695c8a91cf9c4c64358067d15a7b6c6b" +checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -6502,18 +6926,6 @@ dependencies = [ "windows-targets 0.48.5", ] -[[package]] -name = "windows" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e48a53791691ab099e5e2ad123536d0fff50652600abaf43bbf952894110d0be" -dependencies = [ - "windows-core 0.52.0", - "windows-implement 0.52.0", - "windows-interface 0.52.0", - "windows-targets 0.52.6", -] - [[package]] name = "windows" version = "0.58.0" @@ -6548,24 +6960,13 @@ version = "0.58.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ba6d44ec8c2591c134257ce647b7ea6b20335bf6379a27dac5f1641fcf59f99" dependencies = [ - "windows-implement 0.58.0", - "windows-interface 0.58.0", + "windows-implement", + "windows-interface", "windows-result", "windows-strings", "windows-targets 0.52.6", ] -[[package]] -name = "windows-implement" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12168c33176773b86799be25e2a2ba07c7aab9968b37541f1094dbd7a60c8946" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.72", -] - [[package]] name = "windows-implement" version = "0.58.0" @@ -6574,18 +6975,7 @@ checksum = "2bbd5b46c938e506ecbce286b6628a02171d56153ba733b6c741fc627ec9579b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", -] - -[[package]] -name = "windows-interface" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d8dc32e0095a7eeccebd0e3f09e9509365ecb3fc6ac4d6f5f14a3f6392942d1" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -6596,7 +6986,7 @@ checksum = "053c4c462dc91d3b1504c6fe5a726dd15e216ba718e84a0e46a88fbe5ded3515" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -6779,18 +7169,9 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "winnow" -version = "0.5.40" +version = "0.6.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f593a95398737aeed53e489c785df13f3618e41dbcd6718c6addbf1395aa6876" -dependencies = [ - "memchr", -] - -[[package]] -name = "winnow" -version = "0.6.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "374ec40a2d767a3c1b4972d9475ecd557356637be906f2cb3f7fe17a6eb5e22f" +checksum = "36c1fec1a2bb5866f07c25f68c26e565c4c200aebb96d7e55710c19d3e8ac49b" dependencies = [ "memchr", ] @@ -6807,26 +7188,30 @@ dependencies = [ [[package]] name = "wmi" -version = "0.13.3" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc2f0a4062ca522aad4705a2948fd4061b3857537990202a8ddd5af21607f79a" +checksum = "ff00ac1309d4c462be86f03a55e409509e8bf4323ec296aeb4b381dd9aabe6ec" dependencies = [ "chrono", "futures", "log", "serde", "thiserror", - "windows 0.52.0", + "windows 0.58.0", + "windows-core 0.58.0", ] [[package]] -name = "wyz" -version = "0.5.1" +name = "write16" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed" -dependencies = [ - "tap", -] +checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936" + +[[package]] +name = "writeable" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" [[package]] name = "x509-parser" @@ -6847,9 +7232,9 @@ dependencies = [ [[package]] name = "xml-rs" -version = "0.8.20" +version = "0.8.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "791978798f0597cfc70478424c2b4fdc2b7a8024aaff78497ef00f24ef674193" +checksum = "af4e2e2f7cba5a093896c1e150fbfe177d1883e7448200efb81d40b9d339ef26" [[package]] name = "xmltree" @@ -6862,9 +7247,9 @@ dependencies = [ [[package]] name = "yansi" -version = "0.5.1" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09041cd90cf85f7f8b2df60c646f853b7f535ce68f85244eb6731cf89fa498ec" +checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" [[package]] name = "yasna" @@ -6875,6 +7260,30 @@ dependencies = [ "time", ] +[[package]] +name = "yoke" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c5b1314b079b0930c31e3af543d8ee1757b1951ae1e1565ec704403a7240ca5" +dependencies = [ + "serde", + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28cc31741b18cb6f1d5ff12f5b7523e3d6eb0852bbbad19d73905511d9849b95" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", + "synstructure", +] + [[package]] name = "z32" version = "1.1.1" @@ -6887,6 +7296,7 @@ version = "0.7.35" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" dependencies = [ + "byteorder", "zerocopy-derive", ] @@ -6898,7 +7308,28 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", +] + +[[package]] +name = "zerofrom" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91ec111ce797d0e0784a1116d0ddcdbea84322cd79e5d5ad173daeba4f93ab55" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ea7b4a3637ea8669cedf0f1fd5c286a17f3de97b8dd5a70a6c167a1730e63a5" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", + "synstructure", ] [[package]] @@ -6906,3 +7337,35 @@ name = "zeroize" version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" + +[[package]] +name = "zerovec" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa2b893d79df23bfb12d5461018d408ea19dfafe76c2c7ef6d4eba614f8ff079" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", +] + +[[patch.unused]] +name = "iroh-docs" +version = "0.28.0" +source = "git+https://github.com/n0-computer/iroh-docs?branch=main#c3017de4573930fd03a05ec4aa6c7ab7a84f1890" + +[[patch.unused]] +name = "iroh-gossip" +version = "0.28.1" +source = "git+https://github.com/n0-computer/iroh-gossip?branch=try-out-new-quicrpc#6a16febc071b0673a8831a00842f0fc8d76ace03" diff --git a/Cargo.toml b/Cargo.toml index 3df379e0ad6..36ea8bd5b4a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -55,6 +55,9 @@ iroh-test = { path = "./iroh-test" } iroh-router = { path = "./iroh-router" } # iroh-blobs = { git = "https://github.com/n0-computer/iroh-blobs", branch = "main" } -iroh-gossip = { git = "https://github.com/n0-computer/iroh-gossip", branch = "main" } +iroh-gossip = { git = "https://github.com/n0-computer/iroh-gossip", branch = "try-out-new-quicrpc" } iroh-docs = { git = "https://github.com/n0-computer/iroh-docs", branch = "main" } -iroh-blobs = { git = "https://github.com/n0-computer/iroh-blobs", branch = "add-rpc" } \ No newline at end of file +iroh-blobs = { git = "https://github.com/n0-computer/iroh-blobs", branch = "try-out-new-rpc" } + +quic-rpc = { git = "https://github.com/n0-computer/quic-rpc.git", branch = "map-transports-not-services", optional = true } +quic-rpc-derive = { git = "https://github.com/n0-computer/quic-rpc.git", branch = "map-transports-not-services", optional = true } \ No newline at end of file diff --git a/iroh-cli/Cargo.toml b/iroh-cli/Cargo.toml index 71071696fc6..99b21c4a9da 100644 --- a/iroh-cli/Cargo.toml +++ b/iroh-cli/Cargo.toml @@ -48,7 +48,7 @@ pkarr = { version = "2.2.0", default-features = false } portable-atomic = "1" portmapper = { version = "0.1.0", path = "../net-tools/portmapper" } postcard = "1.0.8" -quic-rpc = { version = "0.12", features = ["flume-transport", "quinn-transport"] } +quic-rpc = { version = "0.15", features = ["flume-transport", "quinn-transport"] } rand = "0.8.5" ratatui = "0.26.2" reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] } diff --git a/iroh/Cargo.toml b/iroh/Cargo.toml index 922d99d08a5..ea876a7e349 100644 --- a/iroh/Cargo.toml +++ b/iroh/Cargo.toml @@ -26,7 +26,7 @@ futures-lite = "2.3" futures-util = "0.3" genawaiter = { version = "0.99", default-features = false, features = ["futures03"] } hex = { version = "0.4.3" } -iroh-blobs = { version = "0.27.0", features = ["downloader"] } +iroh-blobs = { version = "0.28.1", features = ["downloader"] } iroh-base = { version = "0.27.0", features = ["key"] } iroh-io = { version = "0.6.0", features = ["stats"] } iroh-metrics = { version = "0.27.0", optional = true } @@ -39,8 +39,8 @@ iroh-docs = { version = "0.27.0" } iroh-gossip = "0.27.0" parking_lot = "0.12.1" postcard = { version = "1", default-features = false, features = ["alloc", "use-std", "experimental-derive"] } -quic-rpc = { version = "0.13", default-features = false, features = ["flume-transport", "quinn-transport"] } -quic-rpc-derive = { version = "0.13" } +quic-rpc = { version = "0.15", default-features = false, features = ["flume-transport", "quinn-transport"] } +quic-rpc-derive = { version = "0.15" } quinn = { package = "iroh-quinn", version = "0.11" } rand = "0.8" serde = { version = "1", features = ["derive"] } diff --git a/iroh/src/client/blobs.rs b/iroh/src/client/blobs.rs index 8682279b56a..e97ed8b8cad 100644 --- a/iroh/src/client/blobs.rs +++ b/iroh/src/client/blobs.rs @@ -1,11 +1,11 @@ //! Reexport of iroh-blobs rpc client -use quic_rpc::client::BoxedServiceConnection; +use quic_rpc::client::BoxedConnector; use super::RpcService; /// Reexport of iroh-blobs rpc client pub type Client = - iroh_blobs::rpc::client::blobs::Client, RpcService>; + iroh_blobs::rpc::client::blobs::Client, RpcService>; pub use iroh_blobs::rpc::client::blobs::{ AddDirOpts, AddFileOpts, AddOutcome, AddProgress, AddReaderOpts, BlobInfo, BlobStatus, diff --git a/iroh/src/client/docs.rs b/iroh/src/client/docs.rs index 1fa7c9e5f1a..a0c2237260f 100644 --- a/iroh/src/client/docs.rs +++ b/iroh/src/client/docs.rs @@ -476,7 +476,7 @@ impl Entry { /// You can pass either a [`Doc`] or the `Iroh` client by reference as `client`. pub async fn content_reader(&self, client: impl Into<&RpcClient>) -> Result { let client: RpcClient = client.into().clone(); - let client: quic_rpc::RpcClient = client.map(); + let client: quic_rpc::RpcClient = client.map(); blobs::Reader::from_rpc_read(&client, self.content_hash()).await } @@ -485,7 +485,7 @@ impl Entry { /// You can pass either a [`Doc`] or the `Iroh` client by reference as `client`. pub async fn content_bytes(&self, client: impl Into<&RpcClient>) -> Result { let client: RpcClient = client.into().clone(); - let client: quic_rpc::RpcClient = client.map(); + let client: quic_rpc::RpcClient = client.map(); blobs::Reader::from_rpc_read(&client, self.content_hash()) .await? diff --git a/iroh/src/client/quic.rs b/iroh/src/client/quic.rs index a5ab2d1351e..f7634cd57df 100644 --- a/iroh/src/client/quic.rs +++ b/iroh/src/client/quic.rs @@ -8,7 +8,7 @@ use std::{ }; use anyhow::{bail, Context}; -use quic_rpc::transport::{boxed::Connection as BoxedConnection, quinn::QuinnConnection}; +use quic_rpc::transport::{boxed::BoxedConnector, quinn::QuinnConnector}; use super::{Iroh, RpcClient}; use crate::{node::RpcStatus, rpc_protocol::node::StatusRequest}; @@ -43,8 +43,8 @@ pub(crate) async fn connect_raw(addr: SocketAddr) -> anyhow::Result { let endpoint = create_quinn_client(bind_addr, vec![RPC_ALPN.to_vec()], false)?; let server_name = "localhost".to_string(); - let connection = QuinnConnection::new(endpoint, addr, server_name); - let connection = BoxedConnection::new(connection); + let connection = QuinnConnector::new(endpoint, addr, server_name); + let connection = BoxedConnector::new(connection); let client = RpcClient::new(connection); // Do a status request to check if the server is running. let _version = tokio::time::timeout(Duration::from_secs(1), client.rpc(StatusRequest)) diff --git a/iroh/src/client/tags.rs b/iroh/src/client/tags.rs index 9694b85cd8f..1988c14a906 100644 --- a/iroh/src/client/tags.rs +++ b/iroh/src/client/tags.rs @@ -1,9 +1,9 @@ //! Reexport of iroh-blobs rpc client -use quic_rpc::client::BoxedServiceConnection; +use quic_rpc::server::BoxedListener; use super::RpcService; /// Reexport of iroh-blobs rpc client pub type Client = - iroh_blobs::rpc::client::tags::Client, RpcService>; + iroh_blobs::rpc::client::tags::Client, RpcService>; pub use iroh_blobs::rpc::client::tags::TagInfo; diff --git a/iroh/src/node/builder.rs b/iroh/src/node/builder.rs index c1c505f136b..912967e9169 100644 --- a/iroh/src/node/builder.rs +++ b/iroh/src/node/builder.rs @@ -692,11 +692,11 @@ where // Initialize the internal RPC connection. let (internal_rpc, controller) = - quic_rpc::transport::flume::service_connection::(32); - let internal_rpc = quic_rpc::transport::boxed::ServerEndpoint::new(internal_rpc); + quic_rpc::transport::flume::channel(32); + let internal_rpc = quic_rpc::transport::boxed::BoxedListener::new(internal_rpc); // box the controller. Boxing has a special case for the flume channel that avoids allocations, // so this has zero overhead. - let controller = quic_rpc::transport::boxed::Connection::new(controller); + let controller = quic_rpc::transport::boxed::BoxedConnector::new(controller); let client = crate::client::Iroh::new(quic_rpc::RpcClient::new(controller.clone())); let inner = Arc::new(NodeInner { diff --git a/iroh/src/node/rpc.rs b/iroh/src/node/rpc.rs index bd10752fd4d..b20c7780fd6 100644 --- a/iroh/src/node/rpc.rs +++ b/iroh/src/node/rpc.rs @@ -161,7 +161,6 @@ impl Handler { chan: RpcChannel< iroh_blobs::rpc::proto::RpcService, IrohServerEndpoint, - crate::rpc_protocol::RpcService, >, ) -> Result<(), RpcServerError> { self.blobs().handle_rpc_request(msg, chan).await From 914452c4caf1a66ced48371d7fac4ffcc3f30ddf Mon Sep 17 00:00:00 2001 From: Ruediger Klaehn Date: Wed, 6 Nov 2024 17:49:12 +0200 Subject: [PATCH 14/20] use released 0.15 --- Cargo.lock | 51 ++++++++++++++++++++++++++++++++++++++++++--------- Cargo.toml | 3 --- 2 files changed, 42 insertions(+), 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6b2cf7f4d16..32804057f7e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2634,8 +2634,8 @@ dependencies = [ "portable-atomic", "postcard", "proptest", - "quic-rpc", - "quic-rpc-derive", + "quic-rpc 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", + "quic-rpc-derive 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", "rand", "rand_chacha", "ref-cast", @@ -2730,8 +2730,8 @@ dependencies = [ "pin-project", "portable-atomic", "postcard", - "quic-rpc", - "quic-rpc-derive", + "quic-rpc 0.15.0 (git+https://github.com/n0-computer/quic-rpc.git?branch=map-transports-not-services)", + "quic-rpc-derive 0.15.0 (git+https://github.com/n0-computer/quic-rpc.git?branch=map-transports-not-services)", "rand", "range-collections", "redb 1.5.1", @@ -2784,7 +2784,7 @@ dependencies = [ "portable-atomic", "portmapper", "postcard", - "quic-rpc", + "quic-rpc 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", "rand", "rand_xorshift", "ratatui", @@ -2919,8 +2919,8 @@ dependencies = [ "iroh-router", "nested_enum_utils", "postcard", - "quic-rpc", - "quic-rpc-derive", + "quic-rpc 0.15.0 (git+https://github.com/n0-computer/quic-rpc.git?branch=map-transports-not-services)", + "quic-rpc-derive 0.15.0 (git+https://github.com/n0-computer/quic-rpc.git?branch=map-transports-not-services)", "rand", "rand_core", "serde", @@ -4428,7 +4428,8 @@ dependencies = [ [[package]] name = "quic-rpc" version = "0.15.0" -source = "git+https://github.com/n0-computer/quic-rpc.git?branch=map-transports-not-services#16b7c4942365b96fc750627459261e52e1190f6c" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61e131f594054d27d077162815db3b5e9ddd76a28fbb9091b68095971e75c286" dependencies = [ "anyhow", "bincode", @@ -4449,13 +4450,45 @@ dependencies = [ "tracing", ] +[[package]] +name = "quic-rpc" +version = "0.15.0" +source = "git+https://github.com/n0-computer/quic-rpc.git?branch=map-transports-not-services#16b7c4942365b96fc750627459261e52e1190f6c" +dependencies = [ + "anyhow", + "derive_more", + "educe", + "flume", + "futures-lite 2.4.0", + "futures-sink", + "futures-util", + "hex", + "pin-project", + "serde", + "slab", + "tokio", + "tracing", +] + +[[package]] +name = "quic-rpc-derive" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbef4c942978f74ef296ae40d43d4375c9d730b65a582688a358108cfd5c0cf7" +dependencies = [ + "proc-macro2", + "quic-rpc 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", + "quote", + "syn 1.0.109", +] + [[package]] name = "quic-rpc-derive" version = "0.15.0" source = "git+https://github.com/n0-computer/quic-rpc.git?branch=map-transports-not-services#16b7c4942365b96fc750627459261e52e1190f6c" dependencies = [ "proc-macro2", - "quic-rpc", + "quic-rpc 0.15.0 (git+https://github.com/n0-computer/quic-rpc.git?branch=map-transports-not-services)", "quote", "syn 1.0.109", ] diff --git a/Cargo.toml b/Cargo.toml index 36ea8bd5b4a..ae4074f2c62 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -58,6 +58,3 @@ iroh-router = { path = "./iroh-router" } iroh-gossip = { git = "https://github.com/n0-computer/iroh-gossip", branch = "try-out-new-quicrpc" } iroh-docs = { git = "https://github.com/n0-computer/iroh-docs", branch = "main" } iroh-blobs = { git = "https://github.com/n0-computer/iroh-blobs", branch = "try-out-new-rpc" } - -quic-rpc = { git = "https://github.com/n0-computer/quic-rpc.git", branch = "map-transports-not-services", optional = true } -quic-rpc-derive = { git = "https://github.com/n0-computer/quic-rpc.git", branch = "map-transports-not-services", optional = true } \ No newline at end of file From 5cf5388dfad1f097de217095d6ee0c3eb23bee36 Mon Sep 17 00:00:00 2001 From: Ruediger Klaehn Date: Wed, 6 Nov 2024 17:58:23 +0200 Subject: [PATCH 15/20] use main --- Cargo.lock | 51 ++++++++++----------------------------------------- Cargo.toml | 5 ++--- 2 files changed, 12 insertions(+), 44 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 32804057f7e..bbb0fd93027 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2634,8 +2634,8 @@ dependencies = [ "portable-atomic", "postcard", "proptest", - "quic-rpc 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", - "quic-rpc-derive 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", + "quic-rpc", + "quic-rpc-derive", "rand", "rand_chacha", "ref-cast", @@ -2703,7 +2703,7 @@ dependencies = [ [[package]] name = "iroh-blobs" version = "0.28.1" -source = "git+https://github.com/n0-computer/iroh-blobs?branch=try-out-new-rpc#ec7967b3c92ab4596a9abb3e91090cc575ab9e37" +source = "git+https://github.com/n0-computer/iroh-blobs?branch=main#2337e46d89d6f2f2fda512439890ab74bab24fb4" dependencies = [ "anyhow", "async-channel", @@ -2730,8 +2730,8 @@ dependencies = [ "pin-project", "portable-atomic", "postcard", - "quic-rpc 0.15.0 (git+https://github.com/n0-computer/quic-rpc.git?branch=map-transports-not-services)", - "quic-rpc-derive 0.15.0 (git+https://github.com/n0-computer/quic-rpc.git?branch=map-transports-not-services)", + "quic-rpc", + "quic-rpc-derive", "rand", "range-collections", "redb 1.5.1", @@ -2784,7 +2784,7 @@ dependencies = [ "portable-atomic", "portmapper", "postcard", - "quic-rpc 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", + "quic-rpc", "rand", "rand_xorshift", "ratatui", @@ -2901,7 +2901,7 @@ dependencies = [ [[package]] name = "iroh-gossip" version = "0.28.1" -source = "git+https://github.com/n0-computer/iroh-gossip?branch=try-out-new-quicrpc#6a16febc071b0673a8831a00842f0fc8d76ace03" +source = "git+https://github.com/n0-computer/iroh-gossip?branch=main#3d6659daf326f57cbafec189be996b89c7f441bd" dependencies = [ "anyhow", "async-channel", @@ -2919,8 +2919,8 @@ dependencies = [ "iroh-router", "nested_enum_utils", "postcard", - "quic-rpc 0.15.0 (git+https://github.com/n0-computer/quic-rpc.git?branch=map-transports-not-services)", - "quic-rpc-derive 0.15.0 (git+https://github.com/n0-computer/quic-rpc.git?branch=map-transports-not-services)", + "quic-rpc", + "quic-rpc-derive", "rand", "rand_core", "serde", @@ -4450,26 +4450,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "quic-rpc" -version = "0.15.0" -source = "git+https://github.com/n0-computer/quic-rpc.git?branch=map-transports-not-services#16b7c4942365b96fc750627459261e52e1190f6c" -dependencies = [ - "anyhow", - "derive_more", - "educe", - "flume", - "futures-lite 2.4.0", - "futures-sink", - "futures-util", - "hex", - "pin-project", - "serde", - "slab", - "tokio", - "tracing", -] - [[package]] name = "quic-rpc-derive" version = "0.15.0" @@ -4477,18 +4457,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cbef4c942978f74ef296ae40d43d4375c9d730b65a582688a358108cfd5c0cf7" dependencies = [ "proc-macro2", - "quic-rpc 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "quic-rpc-derive" -version = "0.15.0" -source = "git+https://github.com/n0-computer/quic-rpc.git?branch=map-transports-not-services#16b7c4942365b96fc750627459261e52e1190f6c" -dependencies = [ - "proc-macro2", - "quic-rpc 0.15.0 (git+https://github.com/n0-computer/quic-rpc.git?branch=map-transports-not-services)", + "quic-rpc", "quote", "syn 1.0.109", ] diff --git a/Cargo.toml b/Cargo.toml index ae4074f2c62..a7d2a811049 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -54,7 +54,6 @@ iroh-metrics = { path = "./iroh-metrics" } iroh-test = { path = "./iroh-test" } iroh-router = { path = "./iroh-router" } -# iroh-blobs = { git = "https://github.com/n0-computer/iroh-blobs", branch = "main" } -iroh-gossip = { git = "https://github.com/n0-computer/iroh-gossip", branch = "try-out-new-quicrpc" } +iroh-gossip = { git = "https://github.com/n0-computer/iroh-gossip", branch = "main" } iroh-docs = { git = "https://github.com/n0-computer/iroh-docs", branch = "main" } -iroh-blobs = { git = "https://github.com/n0-computer/iroh-blobs", branch = "try-out-new-rpc" } +iroh-blobs = { git = "https://github.com/n0-computer/iroh-blobs", branch = "main" } From 54c92e14f0aea7ac54f1ce8f5b42ac4903e0828d Mon Sep 17 00:00:00 2001 From: Ruediger Klaehn Date: Wed, 6 Nov 2024 18:01:47 +0200 Subject: [PATCH 16/20] remove commented out code --- iroh/src/node/rpc.rs | 64 -------------------------------------------- 1 file changed, 64 deletions(-) diff --git a/iroh/src/node/rpc.rs b/iroh/src/node/rpc.rs index df39c7b17a8..d18b88a0248 100644 --- a/iroh/src/node/rpc.rs +++ b/iroh/src/node/rpc.rs @@ -166,70 +166,6 @@ impl Handler { .map_err(|e| e.errors_into()) } - // async fn handle_blobs_request( - // self, - // msg: blobs::Request, - // chan: RpcChannel< - // iroh_blobs::rpc::proto::RpcService, - // IrohServerEndpoint, - // crate::rpc_protocol::RpcService, - // >, - // ) -> Result<(), RpcServerError> { - // use blobs::Request::*; - // debug!("handling blob request: {msg}"); - // match msg { - // List(msg) => chan.server_streaming(msg, self, Self::blob_list).await, - // ListIncomplete(msg) => { - // chan.server_streaming(msg, self, Self::blob_list_incomplete) - // .await - // } - // CreateCollection(msg) => chan.rpc(msg, self, Self::create_collection).await, - // Delete(msg) => chan.rpc(msg, self, Self::blob_delete_blob).await, - // AddPath(msg) => { - // chan.server_streaming(msg, self, Self::blob_add_from_path) - // .await - // } - // Download(msg) => chan.server_streaming(msg, self, Self::blob_download).await, - // Export(msg) => chan.server_streaming(msg, self, Self::blob_export).await, - // Validate(msg) => chan.server_streaming(msg, self, Self::blob_validate).await, - // Fsck(msg) => { - // chan.server_streaming(msg, self, Self::blob_consistency_check) - // .await - // } - // ReadAt(msg) => chan.server_streaming(msg, self, Self::blob_read_at).await, - // AddStream(msg) => chan.bidi_streaming(msg, self, Self::blob_add_stream).await, - // AddStreamUpdate(_msg) => Err(RpcServerError::UnexpectedUpdateMessage), - // BlobStatus(msg) => chan.rpc(msg, self, Self::blob_status).await, - // BatchCreate(msg) => chan.bidi_streaming(msg, self, Self::batch_create).await, - // BatchUpdate(_) => Err(RpcServerError::UnexpectedStartMessage), - // BatchAddStream(msg) => chan.bidi_streaming(msg, self, Self::batch_add_stream).await, - // BatchAddStreamUpdate(_) => Err(RpcServerError::UnexpectedStartMessage), - // BatchAddPath(msg) => { - // chan.server_streaming(msg, self, Self::batch_add_from_path) - // .await - // } - // BatchCreateTempTag(msg) => chan.rpc(msg, self, Self::batch_create_temp_tag).await, - // } - // } - - // async fn handle_tags_request( - // self, - // msg: tags::Request, - // chan: RpcChannel< - // iroh_blobs::rpc::proto::RpcService, - // IrohServerEndpoint, - // crate::rpc_protocol::RpcService, - // >, - // ) -> Result<(), RpcServerError> { - // use tags::Request::*; - // match msg { - // ListTags(msg) => chan.server_streaming(msg, self, Self::blob_list_tags).await, - // DeleteTag(msg) => chan.rpc(msg, self, Self::blob_delete_tag).await, - // Create(msg) => chan.rpc(msg, self, Self::tags_create).await, - // Set(msg) => chan.rpc(msg, self, Self::tags_set).await, - // } - // } - async fn handle_gossip_request( self, msg: iroh_gossip::RpcRequest, From 55848ae717e4de1a88d6e974adfc4441d6e46e5d Mon Sep 17 00:00:00 2001 From: Ruediger Klaehn Date: Wed, 6 Nov 2024 18:04:38 +0200 Subject: [PATCH 17/20] remove some fs utils --- Cargo.lock | 1 - iroh/Cargo.toml | 1 - iroh/src/util/fs.rs | 110 -------------------------------------------- 3 files changed, 112 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bbb0fd93027..40417ede1b7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2654,7 +2654,6 @@ dependencies = [ "tracing", "tracing-subscriber", "url", - "walkdir", ] [[package]] diff --git a/iroh/Cargo.toml b/iroh/Cargo.toml index 30545d800ba..d0d1062352a 100644 --- a/iroh/Cargo.toml +++ b/iroh/Cargo.toml @@ -51,7 +51,6 @@ tokio = { version = "1", features = ["io-util", "rt"] } tokio-stream = "0.1" tokio-util = { version = "0.7", features = ["codec", "io-util", "io", "time"] } tracing = "0.1" -walkdir = "2" # Examples clap = { version = "4", features = ["derive"], optional = true } diff --git a/iroh/src/util/fs.rs b/iroh/src/util/fs.rs index df548964b5f..061be9a7627 100644 --- a/iroh/src/util/fs.rs +++ b/iroh/src/util/fs.rs @@ -1,6 +1,5 @@ //! Utilities for filesystem operations. use std::{ - borrow::Cow, fs::read_dir, path::{Component, Path, PathBuf}, }; @@ -9,115 +8,6 @@ use anyhow::{bail, Context}; use bytes::Bytes; use iroh_net::key::SecretKey; use tokio::io::AsyncWriteExt; -use walkdir::WalkDir; - -use crate::client::blobs::WrapOption; - -/// A data source -#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone)] -pub struct DataSource { - /// Custom name - name: String, - /// Path to the file - path: PathBuf, -} - -impl DataSource { - /// Creates a new [`DataSource`] from a [`PathBuf`]. - pub fn new(path: PathBuf) -> Self { - let name = path - .file_name() - .map(|s| s.to_string_lossy().to_string()) - .unwrap_or_default(); - DataSource { path, name } - } - /// Creates a new [`DataSource`] from a [`PathBuf`] and a custom name. - pub fn with_name(path: PathBuf, name: String) -> Self { - DataSource { path, name } - } - - /// Returns blob name for this data source. - /// - /// If no name was provided when created it is derived from the path name. - pub fn name(&self) -> Cow<'_, str> { - Cow::Borrowed(&self.name) - } - - /// Returns the path of this data source. - pub fn path(&self) -> &Path { - &self.path - } -} - -impl From for DataSource { - fn from(value: PathBuf) -> Self { - DataSource::new(value) - } -} - -impl From<&std::path::Path> for DataSource { - fn from(value: &std::path::Path) -> Self { - DataSource::new(value.to_path_buf()) - } -} - -/// Create data sources from a path. -pub fn scan_path(path: PathBuf, wrap: WrapOption) -> anyhow::Result> { - if path.is_dir() { - scan_dir(path, wrap) - } else { - let name = match wrap { - WrapOption::NoWrap => bail!("Cannot scan a file without wrapping"), - WrapOption::Wrap { name: None } => file_name(&path)?, - WrapOption::Wrap { name: Some(name) } => name, - }; - Ok(vec![DataSource { name, path }]) - } -} - -fn file_name(path: &Path) -> anyhow::Result { - relative_canonicalized_path_to_string(path.file_name().context("path is invalid")?) -} - -/// Create data sources from a directory. -pub fn scan_dir(root: PathBuf, wrap: WrapOption) -> anyhow::Result> { - if !root.is_dir() { - bail!("Expected {} to be a file", root.to_string_lossy()); - } - let prefix = match wrap { - WrapOption::NoWrap => None, - WrapOption::Wrap { name: None } => Some(file_name(&root)?), - WrapOption::Wrap { name: Some(name) } => Some(name), - }; - let files = WalkDir::new(&root).into_iter(); - let data_sources = files - .map(|entry| { - let entry = entry?; - if !entry.file_type().is_file() { - // Skip symlinks. Directories are handled by WalkDir. - return Ok(None); - } - let path = entry.into_path(); - let mut name = relative_canonicalized_path_to_string(path.strip_prefix(&root)?)?; - if let Some(prefix) = &prefix { - name = format!("{prefix}/{name}"); - } - anyhow::Ok(Some(DataSource { name, path })) - }) - .filter_map(Result::transpose); - let data_sources: Vec> = data_sources.collect::>(); - data_sources.into_iter().collect::>>() -} - -/// This function converts a canonicalized relative path to a string, returning -/// an error if the path is not valid unicode. -/// -/// This function will also fail if the path is non canonical, i.e. contains -/// `..` or `.`, or if the path components contain any windows or unix path -/// separators. -pub fn relative_canonicalized_path_to_string(path: impl AsRef) -> anyhow::Result { - canonicalized_path_to_string(path, true) -} /// Loads a [`SecretKey`] from the provided file, or stores a newly generated one /// at the given location. From 1f824b07212e00c36597850e277ce74ef42737fc Mon Sep 17 00:00:00 2001 From: Ruediger Klaehn Date: Wed, 6 Nov 2024 18:05:28 +0200 Subject: [PATCH 18/20] clippy --- iroh-net/src/relay/client.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iroh-net/src/relay/client.rs b/iroh-net/src/relay/client.rs index a5ee9b645ac..e7c449b7739 100644 --- a/iroh-net/src/relay/client.rs +++ b/iroh-net/src/relay/client.rs @@ -1029,7 +1029,7 @@ fn host_header_value(relay_url: RelayUrl) -> Result { .host_str() .ok_or_else(|| ClientError::InvalidUrl(relay_url.to_string()))?; // strip the trailing dot, if present: example.com. -> example.com - let relay_url_host = relay_url_host.strip_suffix(".").unwrap_or(relay_url_host); + let relay_url_host = relay_url_host.strip_suffix('.').unwrap_or(relay_url_host); // build the host header value (reserve up to 6 chars for the ":" and port digits): let mut host_header_value = String::with_capacity(relay_url_host.len() + 6); host_header_value += relay_url_host; From 4a9d90d9488809831a0f91e836ce828febf50405 Mon Sep 17 00:00:00 2001 From: Ruediger Klaehn Date: Wed, 6 Nov 2024 18:10:23 +0200 Subject: [PATCH 19/20] replace individual reexports with module reexport --- iroh/src/client.rs | 9 ++++----- iroh/src/client/blobs.rs | 6 ------ 2 files changed, 4 insertions(+), 11 deletions(-) delete mode 100644 iroh/src/client/blobs.rs diff --git a/iroh/src/client.rs b/iroh/src/client.rs index a5a01145c62..ca6e9096ad3 100644 --- a/iroh/src/client.rs +++ b/iroh/src/client.rs @@ -6,7 +6,6 @@ use std::collections::BTreeMap; use anyhow::Result; use futures_lite::{Stream, StreamExt}; -use quic_rpc::client::BoxedConnector; use ref_cast::RefCast; use crate::rpc_protocol::node::{CounterStats, ShutdownRequest, StatsRequest, StatusRequest}; @@ -19,7 +18,8 @@ pub(crate) use self::quic::{connect_raw as quic_connect_raw, RPC_ALPN}; pub use self::{docs::Doc, net::NodeStatus}; pub mod authors; -pub mod blobs; +pub use iroh_blobs::rpc::client::blobs; +pub use iroh_gossip::rpc::client as gossip; pub mod docs; pub mod net; pub mod tags; @@ -77,9 +77,8 @@ impl Iroh { } /// Returns the gossip client. - pub fn gossip(&self) -> iroh_gossip::RpcClient { - let channel = self.rpc.clone().map().boxed(); - iroh_gossip::RpcClient::>::new(channel) + pub fn gossip(&self) -> gossip::Client { + gossip::Client::new(self.rpc.clone().map().boxed()) } /// Returns the net client. diff --git a/iroh/src/client/blobs.rs b/iroh/src/client/blobs.rs deleted file mode 100644 index 372942d8710..00000000000 --- a/iroh/src/client/blobs.rs +++ /dev/null @@ -1,6 +0,0 @@ -//! Reexport of iroh-blobs rpc client -pub use iroh_blobs::rpc::client::blobs::{ - AddDirOpts, AddFileOpts, AddOutcome, AddProgress, AddReaderOpts, BlobInfo, BlobStatus, Client, - CollectionInfo, DownloadMode, DownloadOptions, DownloadOutcome, DownloadProgress, - IncompleteBlobInfo, Reader, WrapOption, -}; From 58ae428e70703b1bfd3f5d28f03398975d234940 Mon Sep 17 00:00:00 2001 From: Ruediger Klaehn Date: Wed, 6 Nov 2024 18:11:51 +0200 Subject: [PATCH 20/20] same for tags --- iroh/src/client.rs | 3 +-- iroh/src/client/tags.rs | 3 --- 2 files changed, 1 insertion(+), 5 deletions(-) delete mode 100644 iroh/src/client/tags.rs diff --git a/iroh/src/client.rs b/iroh/src/client.rs index ca6e9096ad3..c74d2082814 100644 --- a/iroh/src/client.rs +++ b/iroh/src/client.rs @@ -18,11 +18,10 @@ pub(crate) use self::quic::{connect_raw as quic_connect_raw, RPC_ALPN}; pub use self::{docs::Doc, net::NodeStatus}; pub mod authors; -pub use iroh_blobs::rpc::client::blobs; +pub use iroh_blobs::rpc::client::{blobs, tags}; pub use iroh_gossip::rpc::client as gossip; pub mod docs; pub mod net; -pub mod tags; // Keep this type exposed, otherwise every occurrence of `RpcClient` in the API // will show up as `RpcClient>` in the docs. diff --git a/iroh/src/client/tags.rs b/iroh/src/client/tags.rs deleted file mode 100644 index 194cbfc7de5..00000000000 --- a/iroh/src/client/tags.rs +++ /dev/null @@ -1,3 +0,0 @@ -//! Tags api -/// Reexport of iroh-blobs rpc client -pub use iroh_blobs::rpc::client::tags::{Client, TagInfo};