From 0110d6dffb8828ee309c5edbaac9cb7270b20cbb Mon Sep 17 00:00:00 2001 From: Simon Laux Date: Sat, 1 Nov 2025 00:16:45 +0100 Subject: [PATCH 01/41] fix python lint errors --- deltachat-rpc-client/tests/test_something.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deltachat-rpc-client/tests/test_something.py b/deltachat-rpc-client/tests/test_something.py index ad30f5eac7..027840a1d4 100644 --- a/deltachat-rpc-client/tests/test_something.py +++ b/deltachat-rpc-client/tests/test_something.py @@ -11,7 +11,7 @@ import pytest from deltachat_rpc_client import Contact, EventType, Message, events -from deltachat_rpc_client.const import DownloadState, MessageState +from deltachat_rpc_client.const import MessageState from deltachat_rpc_client.pytestplugin import E2EE_INFO_MSGS from deltachat_rpc_client.rpc import JsonRpcError From e7751ce24360b1dce93ab30c1eea7d824188a4de Mon Sep 17 00:00:00 2001 From: Simon Laux Date: Sun, 9 Nov 2025 15:29:26 +0100 Subject: [PATCH 02/41] receive pre-mesages, start with changes to imap loop. --- src/context.rs | 8 +- src/download.rs | 27 +++---- src/imap.rs | 108 ++++++++++++++++++++++++-- src/imap/scan_folders.rs | 2 +- src/imap/session.rs | 2 + src/message.rs | 11 ++- src/scheduler.rs | 162 +++++++++++++++++++++++++++++++++------ src/sql/migrations.rs | 20 +++++ 8 files changed, 293 insertions(+), 47 deletions(-) diff --git a/src/context.rs b/src/context.rs index 93e5d0ce07..70172822bf 100644 --- a/src/context.rs +++ b/src/context.rs @@ -591,7 +591,7 @@ impl Context { convert_folder_meaning(self, folder_meaning).await? { connection - .fetch_move_delete(self, &mut session, &watch_folder, folder_meaning) + .fetch_move_delete(self, &mut session, true, &watch_folder, folder_meaning) .await?; } } @@ -605,6 +605,12 @@ impl Context { warn!(self, "Failed to update quota: {err:#}."); } } + + // OPTIONAL TODO: if time left start downloading messages + // while (msg = download_when_normal_starts) { + // if not time_left {break;} + // connection.download_message(msg) } + // } } info!( diff --git a/src/download.rs b/src/download.rs index fa075886ef..df67f8ca1a 100644 --- a/src/download.rs +++ b/src/download.rs @@ -28,7 +28,7 @@ pub(crate) const PRE_MSG_ATTACHMENT_SIZE_THRESHOLD: u64 = 140_000; /// This limit defines what messages are fully fetched in the background. /// This is for all messages that don't have the Chat-Is-Full-Message header. #[allow(unused)] -pub(crate) const MAX_FETCH_MSG_SIZE: usize = 1_000_000; +pub(crate) const MAX_FETCH_MSG_SIZE: u32 = 1_000_000; /// Max size for pre messages. A warning is emitted when this is exceeded. /// Should be well below `MAX_FETCH_MSG_SIZE` @@ -79,11 +79,17 @@ impl MsgId { } DownloadState::InProgress => return Err(anyhow!("Download already in progress.")), DownloadState::Available | DownloadState::Failure => { + if msg.rfc724_mid().is_empty() { + return Err(anyhow!("Download not possible, message has no rfc724_mid")); + } self.update_download_state(context, DownloadState::InProgress) .await?; context .sql - .execute("INSERT INTO download (msg_id) VALUES (?)", (self,)) + .execute( + "INSERT INTO download (msg_id) VALUES (?)", + (msg.rfc724_mid(),), + ) .await?; context.scheduler.interrupt_inbox().await; } @@ -132,25 +138,14 @@ impl Message { /// Most messages are downloaded automatically on fetch instead. pub(crate) async fn download_msg( context: &Context, - msg_id: MsgId, + rfc724_mid: String, session: &mut Session, ) -> Result<()> { - let Some(msg) = Message::load_from_db_optional(context, msg_id).await? else { - // If partially downloaded message was already deleted - // we do not know its Message-ID anymore - // so cannot download it. - // - // Probably the message expired due to `delete_device_after` - // setting or was otherwise removed from the device, - // so we don't want it to reappear anyway. - return Ok(()); - }; - let row = context .sql .query_row_optional( "SELECT uid, folder FROM imap WHERE rfc724_mid=? AND target!=''", - (&msg.rfc724_mid,), + (&rfc724_mid,), |row| { let server_uid: u32 = row.get(0)?; let server_folder: String = row.get(1)?; @@ -165,7 +160,7 @@ pub(crate) async fn download_msg( }; session - .fetch_single_msg(context, &server_folder, server_uid, msg.rfc724_mid.clone()) + .fetch_single_msg(context, &server_folder, server_uid, rfc724_mid) .await?; Ok(()) } diff --git a/src/imap.rs b/src/imap.rs index 1258d944a0..8b5c310a8c 100644 --- a/src/imap.rs +++ b/src/imap.rs @@ -23,7 +23,6 @@ use num_traits::FromPrimitive; use ratelimit::Ratelimit; use url::Url; -use crate::calls::{create_fallback_ice_servers, create_ice_servers_from_metadata}; use crate::chat::{self, ChatId, ChatIdBlocked, add_device_msg}; use crate::chatlist_events; use crate::config::Config; @@ -48,6 +47,10 @@ use crate::tools::{self, create_id, duration_to_str, time}; use crate::transport::{ ConfiguredLoginParam, ConfiguredServerLoginParam, prioritize_server_login_params, }; +use crate::{ + calls::{create_fallback_ice_servers, create_ice_servers_from_metadata}, + download::MAX_FETCH_MSG_SIZE, +}; pub(crate) mod capabilities; mod client; @@ -503,6 +506,7 @@ impl Imap { &mut self, context: &Context, session: &mut Session, + is_background_fetch: bool, watch_folder: &str, folder_meaning: FolderMeaning, ) -> Result<()> { @@ -512,7 +516,13 @@ impl Imap { } let msgs_fetched = self - .fetch_new_messages(context, session, watch_folder, folder_meaning) + .fetch_new_messages( + context, + session, + is_background_fetch, + watch_folder, + folder_meaning, + ) .await .context("fetch_new_messages")?; if msgs_fetched && context.get_config_delete_device_after().await?.is_some() { @@ -538,6 +548,7 @@ impl Imap { &mut self, context: &Context, session: &mut Session, + is_background_fetch: bool, folder: &str, folder_meaning: FolderMeaning, ) -> Result { @@ -565,7 +576,13 @@ impl Imap { let mut read_cnt = 0; loop { let (n, fetch_more) = self - .fetch_new_msg_batch(context, session, folder, folder_meaning) + .fetch_new_msg_batch( + context, + session, + is_background_fetch, + folder, + folder_meaning, + ) .await?; read_cnt += n; if !fetch_more { @@ -579,6 +596,7 @@ impl Imap { &mut self, context: &Context, session: &mut Session, + is_background_fetch: bool, folder: &str, folder_meaning: FolderMeaning, ) -> Result<(usize, bool)> { @@ -597,10 +615,22 @@ impl Imap { let read_cnt = msgs.len(); let mut uids_fetch = Vec::::with_capacity(msgs.len() + 1); + let mut available_full_msgs = Vec::::with_capacity(msgs.len()); + let mut download_when_normal_starts = Vec::::with_capacity(msgs.len()); let mut uid_message_ids = BTreeMap::new(); let mut largest_uid_skipped = None; let delete_target = context.get_delete_msgs_target().await?; + let download_limit = { + let download_limit: Option = + context.get_config_parsed(Config::DownloadLimit).await?; + if download_limit == Some(0) { + None + } else { + download_limit + } + }; + // Store the info about IMAP messages in the database. for (uid, ref fetch_response) in msgs { let headers = match get_fetch_headers(fetch_response) { @@ -679,8 +709,57 @@ impl Imap { ) .await.context("prefetch_should_download")? { - uids_fetch.push(uid); - uid_message_ids.insert(uid, message_id); + let fetch_now: bool = if headers + .get_header_value(HeaderDef::ChatIsFullMessage) + .is_some() + { + // This is a full-message + available_full_msgs.push(message_id.clone()); + + //TODO simplify this code + let fits_download_size_limit = download_limit.is_none() + || if let (Some(size), Some(download_limit)) = + (fetch_response.size, download_limit) + && size < download_limit + { + true + } else { + false + }; + + if fits_download_size_limit { + if is_background_fetch { + download_when_normal_starts.push(message_id.clone()); + false + } else { + true + } + } else { + false + } + } else { + // This is not a full message + if is_background_fetch { + if let Some(size) = fetch_response.size + && size < MAX_FETCH_MSG_SIZE + { + // may be a pre-message or a pure-text message, fetch now + true + } else { + // This is e.g. a classical email + // Queue for full download, in order to prevent missing messages + download_when_normal_starts.push(message_id.clone()); + false + } + } else { + true + } + }; + + if fetch_now { + uids_fetch.push(uid); + uid_message_ids.insert(uid, message_id); + } } else { largest_uid_skipped = Some(uid); } @@ -752,6 +831,25 @@ impl Imap { chat::mark_old_messages_as_noticed(context, received_msgs).await?; + // TODO: is there correct place for this? + if fetch_res.is_ok() { + for rfc724_mid in available_full_msgs { + context + .sql + .insert("INSERT INTO available_full_msgs VALUES (?)", (rfc724_mid,)) + .await?; + } + for rfc724_mid in download_when_normal_starts { + context + .sql + .insert( + "INSERT INTO download (rfc724_mid) VALUES (?)", + (rfc724_mid,), + ) + .await?; + } + } + // Now fail if fetching failed, so we will // establish a new session if this one is broken. fetch_res?; diff --git a/src/imap/scan_folders.rs b/src/imap/scan_folders.rs index d3208c8158..72f142e06f 100644 --- a/src/imap/scan_folders.rs +++ b/src/imap/scan_folders.rs @@ -76,7 +76,7 @@ impl Imap { && folder_meaning != FolderMeaning::Trash && folder_meaning != FolderMeaning::Unknown { - self.fetch_move_delete(context, session, folder.name(), folder_meaning) + self.fetch_move_delete(context, session, false, folder.name(), folder_meaning) .await .context("Can't fetch new msgs in scanned folder") .log_err(context) diff --git a/src/imap/session.rs b/src/imap/session.rs index a633974d4b..f426c25a26 100644 --- a/src/imap/session.rs +++ b/src/imap/session.rs @@ -17,6 +17,7 @@ use crate::tools; /// - Chat-Version to check if a message is a chat message /// - Autocrypt-Setup-Message to check if a message is an autocrypt setup message, /// not necessarily sent by Delta Chat. +/// - Chat-Is-Full-Message to skip it in background fetch or when it is too large const PREFETCH_FLAGS: &str = "(UID INTERNALDATE RFC822.SIZE BODY.PEEK[HEADER.FIELDS (\ MESSAGE-ID \ DATE \ @@ -24,6 +25,7 @@ const PREFETCH_FLAGS: &str = "(UID INTERNALDATE RFC822.SIZE BODY.PEEK[HEADER.FIE FROM \ IN-REPLY-TO REFERENCES \ CHAT-VERSION \ + CHAT-IS-FULL-MESSAGE \ AUTO-SUBMITTED \ AUTOCRYPT-SETUP-MESSAGE\ )])"; diff --git a/src/message.rs b/src/message.rs index b42b162fcb..3b04e5fcaa 100644 --- a/src/message.rs +++ b/src/message.rs @@ -1671,9 +1671,18 @@ pub async fn delete_msgs_ex( let update_db = |trans: &mut rusqlite::Transaction| { trans.execute( "UPDATE imap SET target=? WHERE rfc724_mid=?", - (target, msg.rfc724_mid), + (target, &msg.rfc724_mid), )?; trans.execute("DELETE FROM smtp WHERE msg_id=?", (msg_id,))?; + trans.execute( + "DELETE FROM download WHERE rfc724_mid=?", + (&msg.rfc724_mid,), + )?; + // TODO: is the following nessesary? + trans.execute( + "DELETE FROM available_full_msgs WHERE rfc724_mid=?", + (&msg.rfc724_mid,), + )?; Ok(()) }; if let Err(e) = context.sql.transaction(update_db).await { diff --git a/src/scheduler.rs b/src/scheduler.rs index 2a3537daf4..7b791c3209 100644 --- a/src/scheduler.rs +++ b/src/scheduler.rs @@ -345,38 +345,149 @@ pub(crate) struct Scheduler { recently_seen_loop: RecentlySeenLoop, } +// #region todo-move-to-download.rs + +async fn get_msg_id_by_rfc724_mid(context: &Context, rfc724_mid: &str) -> Result> { + context + .sql + .query_get_value::("SELECT id FROM msgs WHERE rfc724_mid=?", (&rfc724_mid,)) + .await +} + +async fn available_full_msgs_contains_rfc724_mid( + context: &Context, + rfc724_mid: &str, +) -> Result { + Ok(context + .sql + .query_get_value::( + "SELECT rfc724_mid FROM available_full_msgs WHERE rfc724_mid=?", + (&rfc724_mid,), + ) + .await? + .is_some()) +} + +async fn set_msg_state_to_failed(context: &Context, rfc724_mid: &str) -> Result<()> { + if let Some(msg_id) = get_msg_id_by_rfc724_mid(context, rfc724_mid).await? { + // Update download state to failure + // so it can be retried. + // + // On success update_download_state() is not needed + // as receive_imf() already + // set the state and emitted the event. + msg_id + .update_download_state(context, DownloadState::Failure) + .await?; + } + Ok(()) +} + +async fn remove_from_download_table(context: &Context, rfc724_mid: &str) -> Result<()> { + context + .sql + .execute("DELETE FROM download WHERE rfc724_mid=?", (&rfc724_mid,)) + .await?; + Ok(()) +} + +async fn remove_from_available_full_msgs_table(context: &Context, rfc724_mid: &str) -> Result<()> { + context + .sql + .execute( + "DELETE FROM available_full_msgs WHERE rfc724_mid=?", + (&rfc724_mid,), + ) + .await?; + Ok(()) +} + +async fn premessage_is_downloaded_for(context: &Context, rfc724_mid: &str) -> Result { + Ok(get_msg_id_by_rfc724_mid(context, rfc724_mid) + .await? + .is_some()) +} + async fn download_msgs(context: &Context, session: &mut Session) -> Result<()> { - let msg_ids = context + let rfc724_mids = context .sql - .query_map_vec("SELECT msg_id FROM download", (), |row| { - let msg_id: MsgId = row.get(0)?; - Ok(msg_id) + .query_map_vec("SELECT rfc724_mid FROM download", (), |row| { + let rfc724_mid: String = row.get(0)?; + Ok(rfc724_mid) }) .await?; - for msg_id in msg_ids { - if let Err(err) = download_msg(context, msg_id, session).await { - warn!(context, "Failed to download message {msg_id}: {:#}.", err); - - // Update download state to failure - // so it can be retried. - // - // On success update_download_state() is not needed - // as receive_imf() already - // set the state and emitted the event. - msg_id - .update_download_state(context, DownloadState::Failure) - .await?; + for rfc724_mid in &rfc724_mids { + let res = download_msg(context, rfc724_mid.clone(), session).await; + if res.is_ok() { + remove_from_download_table(context, rfc724_mid).await?; + remove_from_available_full_msgs_table(context, rfc724_mid).await?; + } + if let Err(err) = res { + warn!( + context, + "Failed to download message rfc724_mid={rfc724_mid}: {:#}.", err + ); + if !premessage_is_downloaded_for(context, rfc724_mid).await? { + // This is probably a classical email that vanished before we could download it + warn!( + context, + "{rfc724_mid} is probably a classical email that vanished before we could download it" + ); + remove_from_download_table(context, rfc724_mid).await?; + } else if available_full_msgs_contains_rfc724_mid(context, rfc724_mid).await? { + // set the message to DownloadState::Failure - probably it was deleted on the server in the meantime + set_msg_state_to_failed(context, rfc724_mid).await?; + remove_from_download_table(context, rfc724_mid).await?; + remove_from_available_full_msgs_table(context, rfc724_mid).await?; + } else { + // leave the message in DownloadState::InProgress; + // it will be downloaded once it arrives. + } } - context - .sql - .execute("DELETE FROM download WHERE msg_id=?", (msg_id,)) - .await?; } Ok(()) } +/// Download known full messages without pre_message +/// in order to guard against lost pre-messages: +// TODO better fn name +pub async fn download_known_full_messages_without_pre_message( + context: &Context, + session: &mut Session, +) -> Result<()> { + let rfc724_mids = context + .sql + .query_map_vec("SELECT rfc724_mid FROM available_full_msgs", (), |row| { + let rfc724_mid: String = row.get(0)?; + Ok(rfc724_mid) + }) + .await?; + for rfc724_mid in &rfc724_mids { + if !premessage_is_downloaded_for(context, rfc724_mid).await? { + // Download the full-message unconditionally, + // because the pre-message got lost. + // The message may be in the wrong order, + // but at least we have it at all. + let res = download_msg(context, rfc724_mid.clone(), session).await; + if res.is_ok() { + remove_from_available_full_msgs_table(context, rfc724_mid).await?; + } + if let Err(err) = res { + warn!( + context, + "download_known_full_messages_without_pre_message: Failed to download message rfc724_mid={rfc724_mid}: {:#}.", + err + ); + } + } + } + Ok(()) +} + +// #endregion + async fn inbox_loop( ctx: Context, started: oneshot::Sender<()>, @@ -596,7 +707,7 @@ async fn fetch_idle( { // Fetch the watched folder. connection - .fetch_move_delete(ctx, &mut session, &watch_folder, folder_meaning) + .fetch_move_delete(ctx, &mut session, true, &watch_folder, folder_meaning) .await .context("fetch_move_delete")?; @@ -607,6 +718,11 @@ async fn fetch_idle( delete_expired_imap_messages(ctx) .await .context("delete_expired_imap_messages")?; + + //------- + // TODO: verify that this is the correct position for this call + // in order to guard against lost pre-messages: + download_known_full_messages_without_pre_message(ctx, &mut session).await?; } else if folder_config == Config::ConfiguredInboxFolder { session.last_full_folder_scan.lock().await.take(); } @@ -635,7 +751,7 @@ async fn fetch_idle( // no new messages. We want to select the watched folder anyway before going IDLE // there, so this does not take additional protocol round-trip. connection - .fetch_move_delete(ctx, &mut session, &watch_folder, folder_meaning) + .fetch_move_delete(ctx, &mut session, true, &watch_folder, folder_meaning) .await .context("fetch_move_delete after scan_folders")?; } diff --git a/src/sql/migrations.rs b/src/sql/migrations.rs index f2e0bbb291..8ae8661145 100644 --- a/src/sql/migrations.rs +++ b/src/sql/migrations.rs @@ -1339,6 +1339,26 @@ CREATE INDEX gossip_timestamp_index ON gossip_timestamp (chat_id, fingerprint); .await?; } + inc_and_check(&mut migration_version, 139)?; + if dbversion < migration_version { + sql.execute_migration( + "CREATE TABLE download_new ( + rfc724_mid TEXT NOT NULL + ); + INSERT INTO download_new (rfc724_mid) + SELECT m.rfc724_mid FROM download d + JOIN msgs m ON d.msg_id = m.id + WHERE m.rfc724_mid IS NOT NULL AND m.rfc724_mid != ''; + DROP TABLE download; + ALTER TABLE download_new RENAME TO download; + CREATE TABLE available_full_msgs ( + rfc724_mid TEXT NOT NULL + );", + migration_version, + ) + .await?; + } + let new_version = sql .get_raw_config_int(VERSION_CFG) .await? From 31a91d2a22de38a42b21b52e4aaa3f0e24705d53 Mon Sep 17 00:00:00 2001 From: Simon Laux Date: Wed, 12 Nov 2025 15:44:52 +0100 Subject: [PATCH 03/41] simplify code a bit --- src/imap.rs | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/src/imap.rs b/src/imap.rs index 8b5c310a8c..519267211e 100644 --- a/src/imap.rs +++ b/src/imap.rs @@ -642,6 +642,9 @@ impl Imap { }; let message_id = prefetch_get_message_id(&headers); + let size = fetch_response + .size + .context("imap fetch response does not contain size")?; // Determine the target folder where the message should be moved to. // @@ -716,18 +719,8 @@ impl Imap { // This is a full-message available_full_msgs.push(message_id.clone()); - //TODO simplify this code - let fits_download_size_limit = download_limit.is_none() - || if let (Some(size), Some(download_limit)) = - (fetch_response.size, download_limit) - && size < download_limit - { - true - } else { - false - }; - - if fits_download_size_limit { + // whether it fits download size limit + if download_limit.is_none_or(|download_limit| size < download_limit) { if is_background_fetch { download_when_normal_starts.push(message_id.clone()); false @@ -740,9 +733,7 @@ impl Imap { } else { // This is not a full message if is_background_fetch { - if let Some(size) = fetch_response.size - && size < MAX_FETCH_MSG_SIZE - { + if size < MAX_FETCH_MSG_SIZE { // may be a pre-message or a pure-text message, fetch now true } else { From c928c4b48f226a0cb8ca9a32dfb4e9111db0bffb Mon Sep 17 00:00:00 2001 From: Simon Laux Date: Wed, 12 Nov 2025 18:30:20 +0100 Subject: [PATCH 04/41] refactor: move download code from `scheduler.rs` to `download.rs`, also move `get_msg_id_by_rfc724_mid` to `MsgId::get_by_rfc724_mid` --- src/download.rs | 135 ++++++++++++++++++++++++++++++++++++++++++- src/message.rs | 10 ++++ src/scheduler.rs | 146 +---------------------------------------------- 3 files changed, 145 insertions(+), 146 deletions(-) diff --git a/src/download.rs b/src/download.rs index df67f8ca1a..c40419059f 100644 --- a/src/download.rs +++ b/src/download.rs @@ -8,7 +8,7 @@ use serde::{Deserialize, Serialize}; use crate::context::Context; use crate::imap::session::Session; -use crate::log::info; +use crate::log::{info, warn}; use crate::message::{Message, MsgId}; use crate::{EventType, chatlist_events}; @@ -202,6 +202,139 @@ impl Session { } } +async fn set_msg_state_to_failed(context: &Context, rfc724_mid: &str) -> Result<()> { + if let Some(msg_id) = MsgId::get_by_rfc724_mid(context, rfc724_mid).await? { + // Update download state to failure + // so it can be retried. + // + // On success update_download_state() is not needed + // as receive_imf() already + // set the state and emitted the event. + msg_id + .update_download_state(context, DownloadState::Failure) + .await?; + } + Ok(()) +} + +async fn available_full_msgs_contains_rfc724_mid( + context: &Context, + rfc724_mid: &str, +) -> Result { + Ok(context + .sql + .query_get_value::( + "SELECT rfc724_mid FROM available_full_msgs WHERE rfc724_mid=?", + (&rfc724_mid,), + ) + .await? + .is_some()) +} + +async fn remove_from_available_full_msgs_table(context: &Context, rfc724_mid: &str) -> Result<()> { + context + .sql + .execute( + "DELETE FROM available_full_msgs WHERE rfc724_mid=?", + (&rfc724_mid,), + ) + .await?; + Ok(()) +} + +async fn remove_from_download_table(context: &Context, rfc724_mid: &str) -> Result<()> { + context + .sql + .execute("DELETE FROM download WHERE rfc724_mid=?", (&rfc724_mid,)) + .await?; + Ok(()) +} + +// this is a dedicated method because it is used in multiple places. +async fn premessage_is_downloaded_for(context: &Context, rfc724_mid: &str) -> Result { + Ok(MsgId::get_by_rfc724_mid(context, rfc724_mid) + .await? + .is_some()) +} + +pub(crate) async fn download_msgs(context: &Context, session: &mut Session) -> Result<()> { + let rfc724_mids = context + .sql + .query_map_vec("SELECT rfc724_mid FROM download", (), |row| { + let rfc724_mid: String = row.get(0)?; + Ok(rfc724_mid) + }) + .await?; + + for rfc724_mid in &rfc724_mids { + let res = download_msg(context, rfc724_mid.clone(), session).await; + if res.is_ok() { + remove_from_download_table(context, rfc724_mid).await?; + remove_from_available_full_msgs_table(context, rfc724_mid).await?; + } + if let Err(err) = res { + warn!( + context, + "Failed to download message rfc724_mid={rfc724_mid}: {:#}.", err + ); + if !premessage_is_downloaded_for(context, rfc724_mid).await? { + // This is probably a classical email that vanished before we could download it + warn!( + context, + "{rfc724_mid} is probably a classical email that vanished before we could download it" + ); + remove_from_download_table(context, rfc724_mid).await?; + } else if available_full_msgs_contains_rfc724_mid(context, rfc724_mid).await? { + // set the message to DownloadState::Failure - probably it was deleted on the server in the meantime + set_msg_state_to_failed(context, rfc724_mid).await?; + remove_from_download_table(context, rfc724_mid).await?; + remove_from_available_full_msgs_table(context, rfc724_mid).await?; + } else { + // leave the message in DownloadState::InProgress; + // it will be downloaded once it arrives. + } + } + } + + Ok(()) +} + +/// Download known full messages without pre_message +/// in order to guard against lost pre-messages: +// TODO better fn name +pub(crate) async fn download_known_full_messages_without_pre_message( + context: &Context, + session: &mut Session, +) -> Result<()> { + let rfc724_mids = context + .sql + .query_map_vec("SELECT rfc724_mid FROM available_full_msgs", (), |row| { + let rfc724_mid: String = row.get(0)?; + Ok(rfc724_mid) + }) + .await?; + for rfc724_mid in &rfc724_mids { + if !premessage_is_downloaded_for(context, rfc724_mid).await? { + // Download the full-message unconditionally, + // because the pre-message got lost. + // The message may be in the wrong order, + // but at least we have it at all. + let res = download_msg(context, rfc724_mid.clone(), session).await; + if res.is_ok() { + remove_from_available_full_msgs_table(context, rfc724_mid).await?; + } + if let Err(err) = res { + warn!( + context, + "download_known_full_messages_without_pre_message: Failed to download message rfc724_mid={rfc724_mid}: {:#}.", + err + ); + } + } + } + Ok(()) +} + #[cfg(test)] mod tests { use mailparse::MailHeaderMap; diff --git a/src/message.rs b/src/message.rs index 3b04e5fcaa..e950d1475b 100644 --- a/src/message.rs +++ b/src/message.rs @@ -163,6 +163,16 @@ impl MsgId { self.0 } + pub(crate) async fn get_by_rfc724_mid( + context: &Context, + rfc724_mid: &str, + ) -> Result> { + context + .sql + .query_get_value::("SELECT id FROM msgs WHERE rfc724_mid=?", (&rfc724_mid,)) + .await + } + /// Returns server foldernames and UIDs of a message, used for message info pub async fn get_info_server_urls( context: &Context, diff --git a/src/scheduler.rs b/src/scheduler.rs index 7b791c3209..0a064c415c 100644 --- a/src/scheduler.rs +++ b/src/scheduler.rs @@ -16,13 +16,12 @@ pub(crate) use self::connectivity::ConnectivityStore; use crate::config::{self, Config}; use crate::contact::{ContactId, RecentlySeenLoop}; use crate::context::Context; -use crate::download::{DownloadState, download_msg}; +use crate::download::{download_known_full_messages_without_pre_message, download_msgs}; use crate::ephemeral::{self, delete_expired_imap_messages}; use crate::events::EventType; use crate::imap::{FolderMeaning, Imap, session::Session}; use crate::location; use crate::log::{LogExt, error, info, warn}; -use crate::message::MsgId; use crate::smtp::{Smtp, send_smtp_messages}; use crate::sql; use crate::stats::maybe_send_stats; @@ -345,149 +344,6 @@ pub(crate) struct Scheduler { recently_seen_loop: RecentlySeenLoop, } -// #region todo-move-to-download.rs - -async fn get_msg_id_by_rfc724_mid(context: &Context, rfc724_mid: &str) -> Result> { - context - .sql - .query_get_value::("SELECT id FROM msgs WHERE rfc724_mid=?", (&rfc724_mid,)) - .await -} - -async fn available_full_msgs_contains_rfc724_mid( - context: &Context, - rfc724_mid: &str, -) -> Result { - Ok(context - .sql - .query_get_value::( - "SELECT rfc724_mid FROM available_full_msgs WHERE rfc724_mid=?", - (&rfc724_mid,), - ) - .await? - .is_some()) -} - -async fn set_msg_state_to_failed(context: &Context, rfc724_mid: &str) -> Result<()> { - if let Some(msg_id) = get_msg_id_by_rfc724_mid(context, rfc724_mid).await? { - // Update download state to failure - // so it can be retried. - // - // On success update_download_state() is not needed - // as receive_imf() already - // set the state and emitted the event. - msg_id - .update_download_state(context, DownloadState::Failure) - .await?; - } - Ok(()) -} - -async fn remove_from_download_table(context: &Context, rfc724_mid: &str) -> Result<()> { - context - .sql - .execute("DELETE FROM download WHERE rfc724_mid=?", (&rfc724_mid,)) - .await?; - Ok(()) -} - -async fn remove_from_available_full_msgs_table(context: &Context, rfc724_mid: &str) -> Result<()> { - context - .sql - .execute( - "DELETE FROM available_full_msgs WHERE rfc724_mid=?", - (&rfc724_mid,), - ) - .await?; - Ok(()) -} - -async fn premessage_is_downloaded_for(context: &Context, rfc724_mid: &str) -> Result { - Ok(get_msg_id_by_rfc724_mid(context, rfc724_mid) - .await? - .is_some()) -} - -async fn download_msgs(context: &Context, session: &mut Session) -> Result<()> { - let rfc724_mids = context - .sql - .query_map_vec("SELECT rfc724_mid FROM download", (), |row| { - let rfc724_mid: String = row.get(0)?; - Ok(rfc724_mid) - }) - .await?; - - for rfc724_mid in &rfc724_mids { - let res = download_msg(context, rfc724_mid.clone(), session).await; - if res.is_ok() { - remove_from_download_table(context, rfc724_mid).await?; - remove_from_available_full_msgs_table(context, rfc724_mid).await?; - } - if let Err(err) = res { - warn!( - context, - "Failed to download message rfc724_mid={rfc724_mid}: {:#}.", err - ); - if !premessage_is_downloaded_for(context, rfc724_mid).await? { - // This is probably a classical email that vanished before we could download it - warn!( - context, - "{rfc724_mid} is probably a classical email that vanished before we could download it" - ); - remove_from_download_table(context, rfc724_mid).await?; - } else if available_full_msgs_contains_rfc724_mid(context, rfc724_mid).await? { - // set the message to DownloadState::Failure - probably it was deleted on the server in the meantime - set_msg_state_to_failed(context, rfc724_mid).await?; - remove_from_download_table(context, rfc724_mid).await?; - remove_from_available_full_msgs_table(context, rfc724_mid).await?; - } else { - // leave the message in DownloadState::InProgress; - // it will be downloaded once it arrives. - } - } - } - - Ok(()) -} - -/// Download known full messages without pre_message -/// in order to guard against lost pre-messages: -// TODO better fn name -pub async fn download_known_full_messages_without_pre_message( - context: &Context, - session: &mut Session, -) -> Result<()> { - let rfc724_mids = context - .sql - .query_map_vec("SELECT rfc724_mid FROM available_full_msgs", (), |row| { - let rfc724_mid: String = row.get(0)?; - Ok(rfc724_mid) - }) - .await?; - for rfc724_mid in &rfc724_mids { - if !premessage_is_downloaded_for(context, rfc724_mid).await? { - // Download the full-message unconditionally, - // because the pre-message got lost. - // The message may be in the wrong order, - // but at least we have it at all. - let res = download_msg(context, rfc724_mid.clone(), session).await; - if res.is_ok() { - remove_from_available_full_msgs_table(context, rfc724_mid).await?; - } - if let Err(err) = res { - warn!( - context, - "download_known_full_messages_without_pre_message: Failed to download message rfc724_mid={rfc724_mid}: {:#}.", - err - ); - } - } - } - Ok(()) -} - -// #endregion - async fn inbox_loop( ctx: Context, started: oneshot::Sender<()>, From 6985536be5311181f0eea8a922946cf0198df03c Mon Sep 17 00:00:00 2001 From: Simon Laux Date: Wed, 12 Nov 2025 20:39:54 +0100 Subject: [PATCH 05/41] `MAX_FETCH_MSG_SIZE` is no longer unused --- src/download.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/download.rs b/src/download.rs index c40419059f..d5e42de762 100644 --- a/src/download.rs +++ b/src/download.rs @@ -26,8 +26,7 @@ pub(crate) const PRE_MSG_ATTACHMENT_SIZE_THRESHOLD: u64 = 140_000; /// Max message size to be fetched in the background. /// This limit defines what messages are fully fetched in the background. -/// This is for all messages that don't have the Chat-Is-Full-Message header. -#[allow(unused)] +/// This is for all messages that don't have the full message header. pub(crate) const MAX_FETCH_MSG_SIZE: u32 = 1_000_000; /// Max size for pre messages. A warning is emitted when this is exceeded. From 45de7cd06dc9bbc7fa48ce59dc3230de81c66f21 Mon Sep 17 00:00:00 2001 From: Simon Laux Date: Fri, 14 Nov 2025 23:56:49 +0100 Subject: [PATCH 06/41] Parse if it is a pre-message or full-message --- src/mimeparser.rs | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/src/mimeparser.rs b/src/mimeparser.rs index 3698db2932..a84b7dbae5 100644 --- a/src/mimeparser.rs +++ b/src/mimeparser.rs @@ -147,6 +147,20 @@ pub(crate) struct MimeMessage { /// Sender timestamp in secs since epoch. Allowed to be in the future due to unsynchronized /// clocks, but not too much. pub(crate) timestamp_sent: i64, + + pub(crate) pre_message: Option, +} + +#[derive(Debug, Clone, PartialEq)] +pub enum PreMessageMode { + /// This is full messages + /// it replaces it's pre-message attachment if it exists already, + /// and if the pre-message does not exist it is treated as normal message + FullMessage, + /// This is a pre-message, + /// it adds a message preview for a full message + /// and it is ignored if the full message was downloaded already + PreMessage { full_msg_rfc724_mid: String }, } #[derive(Debug, PartialEq)] @@ -346,6 +360,22 @@ impl MimeMessage { let mut aheader_values = mail.headers.get_all_values(HeaderDef::Autocrypt.into()); + let pre_message = if let Some(full_msg_rfc724_mid) = + mail.headers.get_header_value(HeaderDef::ChatFullMessageId) + { + Some(PreMessageMode::PreMessage { + full_msg_rfc724_mid, + }) + } else if mail + .headers + .get_header_value(HeaderDef::ChatIsFullMessage) + .is_some() + { + Some(PreMessageMode::FullMessage) + } else { + None + }; + let mail_raw; // Memory location for a possible decrypted message. let decrypted_msg; // Decrypted signed OpenPGP message. let secrets: Vec = context @@ -609,6 +639,7 @@ impl MimeMessage { is_bot: None, timestamp_rcvd, timestamp_sent, + pre_message, }; match mail { From fa063da1b62bf9ffe675608ed25beaa4a5845c15 Mon Sep 17 00:00:00 2001 From: Simon Laux Date: Sat, 15 Nov 2025 00:38:36 +0100 Subject: [PATCH 07/41] start with receiving logic --- src/download.rs | 5 ++++- src/receive_imf.rs | 42 +++++++++++++++++++++++++++++++++++++++++- 2 files changed, 45 insertions(+), 2 deletions(-) diff --git a/src/download.rs b/src/download.rs index d5e42de762..0764b2e97a 100644 --- a/src/download.rs +++ b/src/download.rs @@ -250,7 +250,10 @@ async fn remove_from_download_table(context: &Context, rfc724_mid: &str) -> Resu } // this is a dedicated method because it is used in multiple places. -async fn premessage_is_downloaded_for(context: &Context, rfc724_mid: &str) -> Result { +pub(crate) async fn premessage_is_downloaded_for( + context: &Context, + rfc724_mid: &str, +) -> Result { Ok(MsgId::get_by_rfc724_mid(context, rfc724_mid) .await? .is_some()) diff --git a/src/receive_imf.rs b/src/receive_imf.rs index 460736d4fe..04880f2036 100644 --- a/src/receive_imf.rs +++ b/src/receive_imf.rs @@ -20,7 +20,7 @@ use crate::constants::{self, Blocked, Chattype, DC_CHAT_ID_TRASH, EDITED_PREFIX, use crate::contact::{self, Contact, ContactId, Origin, mark_contact_id_as_verified}; use crate::context::Context; use crate::debug_logging::maybe_set_logging_xdc_inner; -use crate::download::DownloadState; +use crate::download::{DownloadState, premessage_is_downloaded_for}; use crate::ephemeral::{Timer as EphemeralTimer, stock_ephemeral_timer_changed}; use crate::events::EventType; use crate::headerdef::{HeaderDef, HeaderDefMap}; @@ -1087,6 +1087,37 @@ async fn decide_chat_assignment( { info!(context, "Chat edit/delete/iroh/sync message (TRASH)."); true + } else if let Some(pre_message) = &mime_parser.pre_message { + use crate::mimeparser::PreMessageMode::*; + match pre_message { + FullMessage => { + // if pre message exist, then trash after replacing, otherwise treat as normal message + let pre_message_exists = premessage_is_downloaded_for(context, rfc724_mid).await?; + info!( + context, + "Message is a Full Message ({}).", + if pre_message_exists { + "pre-message exists already, so trash after replacing attachment" + } else { + "no pre-message -> Keep" + } + ); + pre_message_exists + } + PreMessage { + full_msg_rfc724_mid, + } => { + // if full message already exists, then trash/ignore + let full_msg_exists = + premessage_is_downloaded_for(context, full_msg_rfc724_mid).await?; + info!( + context, + "Message is a Pre-Message (full_msg_exists:{full_msg_exists})." + ); + full_msg_exists + // TODO find out if trashing affects multi device usage? + } + } } else if mime_parser.is_system_message == SystemMessage::CallAccepted || mime_parser.is_system_message == SystemMessage::CallEnded { @@ -2253,6 +2284,15 @@ async fn handle_edit_delete( Ok(()) } +async fn handle_full_message_replace( + context: &Context, + mime_parser: &MimeMessage, + from_id: ContactId, +) -> Result<()> { + rfc724_mid_exists + todo!(); +} + async fn tweak_sort_timestamp( context: &Context, mime_parser: &mut MimeMessage, From a5d9ed23d16c8076d37fe931b4cad7addd6fa692 Mon Sep 17 00:00:00 2001 From: Simon Laux Date: Sat, 15 Nov 2025 00:43:29 +0100 Subject: [PATCH 08/41] get rid of `MsgId::get_by_rfc724_mid` because it was a duplicate of `message::rfc724_mid_exists` --- src/download.rs | 6 +++--- src/message.rs | 10 ---------- src/receive_imf.rs | 2 +- 3 files changed, 4 insertions(+), 14 deletions(-) diff --git a/src/download.rs b/src/download.rs index 0764b2e97a..7efec91180 100644 --- a/src/download.rs +++ b/src/download.rs @@ -9,7 +9,7 @@ use serde::{Deserialize, Serialize}; use crate::context::Context; use crate::imap::session::Session; use crate::log::{info, warn}; -use crate::message::{Message, MsgId}; +use crate::message::{self, Message, MsgId, rfc724_mid_exists}; use crate::{EventType, chatlist_events}; /// If a message is downloaded only partially @@ -202,7 +202,7 @@ impl Session { } async fn set_msg_state_to_failed(context: &Context, rfc724_mid: &str) -> Result<()> { - if let Some(msg_id) = MsgId::get_by_rfc724_mid(context, rfc724_mid).await? { + if let Some(msg_id) = rfc724_mid_exists(context, rfc724_mid).await? { // Update download state to failure // so it can be retried. // @@ -254,7 +254,7 @@ pub(crate) async fn premessage_is_downloaded_for( context: &Context, rfc724_mid: &str, ) -> Result { - Ok(MsgId::get_by_rfc724_mid(context, rfc724_mid) + Ok(message::rfc724_mid_exists(context, rfc724_mid) .await? .is_some()) } diff --git a/src/message.rs b/src/message.rs index e950d1475b..3b04e5fcaa 100644 --- a/src/message.rs +++ b/src/message.rs @@ -163,16 +163,6 @@ impl MsgId { self.0 } - pub(crate) async fn get_by_rfc724_mid( - context: &Context, - rfc724_mid: &str, - ) -> Result> { - context - .sql - .query_get_value::("SELECT id FROM msgs WHERE rfc724_mid=?", (&rfc724_mid,)) - .await - } - /// Returns server foldernames and UIDs of a message, used for message info pub async fn get_info_server_urls( context: &Context, diff --git a/src/receive_imf.rs b/src/receive_imf.rs index 04880f2036..975e05bfde 100644 --- a/src/receive_imf.rs +++ b/src/receive_imf.rs @@ -2289,7 +2289,7 @@ async fn handle_full_message_replace( mime_parser: &MimeMessage, from_id: ContactId, ) -> Result<()> { - rfc724_mid_exists + // rfc724_mid_exists todo!(); } From 511d4d284c9f21c2cac121f598a4dd8827d99c71 Mon Sep 17 00:00:00 2001 From: Simon Laux Date: Sun, 16 Nov 2025 17:31:12 +0100 Subject: [PATCH 09/41] docs: add hint to `MimeMessage::from_bytes` stating that it has side-effects. --- src/mimeparser.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/mimeparser.rs b/src/mimeparser.rs index a84b7dbae5..61d752f7f6 100644 --- a/src/mimeparser.rs +++ b/src/mimeparser.rs @@ -253,6 +253,9 @@ const MIME_AC_SETUP_FILE: &str = "application/autocrypt-setup"; impl MimeMessage { /// Parse a mime message. + /// + /// This method has some side-effects, + /// such as saving blobs and saving found public keys to the database. pub(crate) async fn from_bytes(context: &Context, body: &[u8]) -> Result { let mail = mailparse::parse_mail(body)?; From 38e2745fb525020e255bfdf8c75423c38aef45a1 Mon Sep 17 00:00:00 2001 From: Simon Laux Date: Sun, 16 Nov 2025 18:49:56 +0100 Subject: [PATCH 10/41] receiving full message --- src/receive_imf.rs | 70 +++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 66 insertions(+), 4 deletions(-) diff --git a/src/receive_imf.rs b/src/receive_imf.rs index 975e05bfde..66d2d7e254 100644 --- a/src/receive_imf.rs +++ b/src/receive_imf.rs @@ -29,7 +29,6 @@ use crate::key::{DcKey, Fingerprint}; use crate::key::{self_fingerprint, self_fingerprint_opt}; use crate::log::LogExt; use crate::log::{info, warn}; -use crate::logged_debug_assert; use crate::message::{ self, Message, MessageState, MessengerMessage, MsgId, Viewtype, rfc724_mid_exists, }; @@ -45,6 +44,7 @@ use crate::stock_str; use crate::sync::Sync::*; use crate::tools::{self, buf_compress, remove_subject_prefix, validate_broadcast_secret}; use crate::{chatlist_events, ensure_and_debug_assert, ensure_and_debug_assert_eq, location}; +use crate::{logged_debug_assert, mimeparser}; /// This is the struct that is returned after receiving one email (aka MIME message). /// @@ -1931,6 +1931,7 @@ async fn add_parts( } handle_edit_delete(context, mime_parser, from_id).await?; + handle_full_message(context, mime_parser, from_id).await?; if mime_parser.is_system_message == SystemMessage::CallAccepted || mime_parser.is_system_message == SystemMessage::CallEnded @@ -2284,13 +2285,74 @@ async fn handle_edit_delete( Ok(()) } -async fn handle_full_message_replace( +async fn handle_full_message( context: &Context, mime_parser: &MimeMessage, from_id: ContactId, ) -> Result<()> { - // rfc724_mid_exists - todo!(); + if let Some(mimeparser::PreMessageMode::FullMessage) = &mime_parser.pre_message { + // if pre message exist, replace attachment + // only replacing attachment ensures that doesn't overwrite the text if it was edited before. + let rfc724_mid = mime_parser + .get_rfc724_mid() + .context("expected full message to have a message id")?; + + let Some(msg_id) = message::rfc724_mid_exists(context, &rfc724_mid).await? else { + warn!( + context, + "Download Full-Message: Database entry does not exist." + ); + return Ok(()); + }; + let Some(original_msg) = Message::load_from_db_optional(context, msg_id).await? else { + // else: message is processed like a normal message + warn!( + context, + "Download Full-Message: pre message was not downloaded, yet so treat as normal message" + ); + return Ok(()); + }; + + if original_msg.from_id != from_id { + warn!(context, "Download Full-Message: Bad sender."); + return Ok(()); + } + if let Some(part) = mime_parser.parts.first() { + if !part.typ.has_file() { + warn!( + context, + "Download Full-Message: First mime part's message-viewtype has no file" + ); + return Ok(()); + } + + let edit_msg_showpadlock = part + .param + .get_bool(Param::GuaranteeE2ee) + .unwrap_or_default(); + if edit_msg_showpadlock || !original_msg.get_showpadlock() { + context + .sql + .execute( + "UPDATE msgs SET param=?, type=?, bytes=?, error=?, download_state=? WHERE id=?", + ( + part.param.to_string(), + part.typ, + part.bytes as isize, + part.error.as_deref().unwrap_or_default(), + DownloadState::Done as u32, + original_msg.id, + ), + ) + .await?; + context.emit_msgs_changed(original_msg.chat_id, original_msg.id); + } else { + warn!(context, "Download Full-Message: Not encrypted."); + } + } + } + + Ok(()) } async fn tweak_sort_timestamp( From 07a8d2f91a71fb9cb063b03417529a34515a1682 Mon Sep 17 00:00:00 2001 From: Simon Laux Date: Sun, 16 Nov 2025 20:37:30 +0100 Subject: [PATCH 11/41] send and receive `attachment_size` and set viewtype to text in pre_message --- src/headerdef.rs | 4 ++++ src/mimefactory.rs | 9 +++++++++ src/mimeparser.rs | 14 ++++++++++++-- src/receive_imf.rs | 18 ++++++++++++++++-- 4 files changed, 41 insertions(+), 4 deletions(-) diff --git a/src/headerdef.rs b/src/headerdef.rs index 510c2d9e29..0fa5e78f91 100644 --- a/src/headerdef.rs +++ b/src/headerdef.rs @@ -109,6 +109,10 @@ pub enum HeaderDef { /// referencing the full-message's rfc724_mid. ChatFullMessageId, + /// Announce full message attachment size inside of a pre-message. + #[strum(serialize = "Chat-Full-Message-Size")] // correct casing + ChatFullMessageSize, + /// This message is preceded by a pre-message /// and thus this message can be skipped while fetching messages. /// This is a cleartext / unproteced header. diff --git a/src/mimefactory.rs b/src/mimefactory.rs index cf71bb329c..ce9263b511 100644 --- a/src/mimefactory.rs +++ b/src/mimefactory.rs @@ -1875,6 +1875,15 @@ impl MimeFactory { // add attachment part if msg.viewtype.has_file() { if let Some(PreMessageMode::PreMessage { .. }) = self.pre_message_mode { + let attachment_size = msg + .get_filebytes(context) + .await? + .context("attachment exists, but get_filebytes returned nothing")? + .to_string(); + headers.push(( + HeaderDef::ChatFullMessageSize.get_headername(), + mail_builder::headers::raw::Raw::new(attachment_size).into(), + )); // TODO: generate thumbnail and attach it instead (if it makes sense) } else { let file_part = build_body_file(context, &msg).await?; diff --git a/src/mimeparser.rs b/src/mimeparser.rs index 61d752f7f6..143ad43190 100644 --- a/src/mimeparser.rs +++ b/src/mimeparser.rs @@ -152,7 +152,7 @@ pub(crate) struct MimeMessage { } #[derive(Debug, Clone, PartialEq)] -pub enum PreMessageMode { +pub(crate) enum PreMessageMode { /// This is full messages /// it replaces it's pre-message attachment if it exists already, /// and if the pre-message does not exist it is treated as normal message @@ -160,7 +160,10 @@ pub enum PreMessageMode { /// This is a pre-message, /// it adds a message preview for a full message /// and it is ignored if the full message was downloaded already - PreMessage { full_msg_rfc724_mid: String }, + PreMessage { + full_msg_rfc724_mid: String, + attachment_size: u64, + }, } #[derive(Debug, PartialEq)] @@ -366,8 +369,15 @@ impl MimeMessage { let pre_message = if let Some(full_msg_rfc724_mid) = mail.headers.get_header_value(HeaderDef::ChatFullMessageId) { + let attachment_size: u64 = mail + .headers + .get_header_value(HeaderDef::ChatFullMessageSize) + .unwrap_or_default() + .parse() + .unwrap_or_default(); Some(PreMessageMode::PreMessage { full_msg_rfc724_mid, + attachment_size, }) } else if mail .headers diff --git a/src/receive_imf.rs b/src/receive_imf.rs index 66d2d7e254..1708219a06 100644 --- a/src/receive_imf.rs +++ b/src/receive_imf.rs @@ -1106,6 +1106,7 @@ async fn decide_chat_assignment( } PreMessage { full_msg_rfc724_mid, + .. } => { // if full message already exists, then trash/ignore let full_msg_exists = @@ -2065,7 +2066,11 @@ RETURNING id sort_timestamp, if trash { 0 } else { mime_parser.timestamp_sent }, if trash { 0 } else { mime_parser.timestamp_rcvd }, - if trash { Viewtype::Unknown } else { typ }, + if trash { + Viewtype::Unknown + } else if let Some(mimeparser::PreMessageMode::PreMessage {..}) = mime_parser.pre_message { + Viewtype::Text + } else { typ }, if trash { MessageState::Undefined } else { state }, if trash { MessengerMessage::No } else { is_dc_message }, if trash || hidden { "" } else { msg }, @@ -2077,7 +2082,16 @@ RETURNING id param.to_string() }, !trash && hidden, - if trash { 0 } else { part.bytes as isize }, + if trash { + 0 + } else if let Some(mimeparser::PreMessageMode::PreMessage { + attachment_size, + .. + }) = mime_parser.pre_message { + attachment_size as isize + } else { + part.bytes as isize + }, if save_mime_modified && !(trash || hidden) { mime_headers.clone() } else { From c26057684a2bf3d0631a24011d63bfaf79b74dfc Mon Sep 17 00:00:00 2001 From: Simon Laux Date: Sat, 22 Nov 2025 03:13:58 +0100 Subject: [PATCH 12/41] metadata as struct in pre-message in header. And fill params that we can already fill from the metadata. Also add a new api to check what viewtype the message will have once downloaded. --- deltachat-ffi/deltachat.h | 3 +- src/download.rs | 2 + src/download/pre_msg_metadata.rs | 106 +++++++++++++++++++++++++++++++ src/headerdef.rs | 6 +- src/message.rs | 28 ++++++++ src/mimefactory.rs | 14 ++-- src/mimeparser.rs | 28 ++++++-- src/param.rs | 7 ++ src/receive_imf.rs | 15 +++-- 9 files changed, 185 insertions(+), 24 deletions(-) create mode 100644 src/download/pre_msg_metadata.rs diff --git a/deltachat-ffi/deltachat.h b/deltachat-ffi/deltachat.h index 866e6a7ce0..e3b2cd17a5 100644 --- a/deltachat-ffi/deltachat.h +++ b/deltachat-ffi/deltachat.h @@ -4283,7 +4283,8 @@ char* dc_msg_get_webxdc_info (const dc_msg_t* msg); /** * Get the size of the file. Returns the size of the file associated with a - * message, if applicable. + * message, if applicable. + * If message is a pre-message, then this returns size of the to be downloaded file. * * Typically, this is used to show the size of document files, e.g. a PDF. * diff --git a/src/download.rs b/src/download.rs index 7efec91180..6820c675bd 100644 --- a/src/download.rs +++ b/src/download.rs @@ -12,6 +12,8 @@ use crate::log::{info, warn}; use crate::message::{self, Message, MsgId, rfc724_mid_exists}; use crate::{EventType, chatlist_events}; +pub(crate) mod pre_msg_metadata; + /// If a message is downloaded only partially /// and `delete_server_after` is set to small timeouts (eg. "at once"), /// the user might have no chance to actually download that message. diff --git a/src/download/pre_msg_metadata.rs b/src/download/pre_msg_metadata.rs new file mode 100644 index 0000000000..13cc0b869b --- /dev/null +++ b/src/download/pre_msg_metadata.rs @@ -0,0 +1,106 @@ +use anyhow::{Context as _, Result}; +use num_traits::ToPrimitive; +use serde::{Deserialize, Serialize}; + +use crate::context::Context; +use crate::log::warn; +use crate::message::Message; +use crate::message::Viewtype; +use crate::param::{Param, Params}; + +/// Metadata contained in PreMessage that describes the Full Message. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct PreMsgMetadata { + /// size of the attachment in bytes + pub(crate) size: u64, + /// Real viewtype of message + pub(crate) viewtype: Viewtype, + /// the original file name + pub(crate) filename: String, + /// Dimensions: width and height of image or video + pub(crate) dimensions: Option<(i32, i32)>, + /// Duration of audio file or video in milliseconds + pub(crate) duration: Option, +} + +impl PreMsgMetadata { + // Returns PreMsgMetadata for messages with files and None for messages without file attachment + pub(crate) async fn from_msg(context: &Context, message: &Message) -> Result> { + if !message.viewtype.has_file() { + return Ok(None); + } + + let size = message + .get_filebytes(context) + .await? + .context("unexpected: file has no size")?; + let filename = message + .param + .get(Param::Filename) + .unwrap_or_default() + .to_owned(); + let dimensions = { + match ( + message.param.get_int(Param::Width), + message.param.get_int(Param::Height), + ) { + (None, None) => None, + (Some(width), Some(height)) => Some((width, height)), + _ => { + warn!(context, "Message has misses either width or height"); + None + } + } + }; + let duration = message.param.get_int(Param::Duration); + + Ok(Some(Self { + size, + filename, + viewtype: message.viewtype, + dimensions, + duration, + })) + } + + pub(crate) fn to_header_value(self) -> Result { + Ok(serde_json::to_string(&self)?) + } + + pub(crate) fn try_from_header_value(value: &str) -> Result { + Ok(serde_json::from_str(value)?) + } +} + +impl Params { + /// Applies data from pre_msg_metadata to Params + pub(crate) fn apply_from_pre_msg_metadata( + &mut self, + pre_msg_metadata: &PreMsgMetadata, + ) -> &mut Self { + self.set(Param::FullMessageFileBytes, pre_msg_metadata.size); + if !pre_msg_metadata.filename.is_empty() { + self.set(Param::Filename, &pre_msg_metadata.filename); + } + self.set_i64( + Param::FullMessageViewtype, + pre_msg_metadata.viewtype.to_i64().unwrap_or_default(), + ); + if let Some((width, height)) = pre_msg_metadata.dimensions { + self.set(Param::Width, width); + self.set(Param::Height, height); + } + if let Some(duration) = pre_msg_metadata.duration { + self.set(Param::Duration, duration); + } + + self + } +} + +#[cfg(test)] +mod tests { + // todo build from message + // todo create artifically and serialize to header + // todo deserialize from header +} diff --git a/src/headerdef.rs b/src/headerdef.rs index 0fa5e78f91..ab217a0e19 100644 --- a/src/headerdef.rs +++ b/src/headerdef.rs @@ -109,9 +109,9 @@ pub enum HeaderDef { /// referencing the full-message's rfc724_mid. ChatFullMessageId, - /// Announce full message attachment size inside of a pre-message. - #[strum(serialize = "Chat-Full-Message-Size")] // correct casing - ChatFullMessageSize, + /// Announce full message metadata in a pre-message. + /// contains serialized PreMsgMetadata struct + ChatFullMessageMetadata, /// This message is preceded by a pre-message /// and thus this message can be skipped while fetching messages. diff --git a/src/message.rs b/src/message.rs index 3b04e5fcaa..895a94517b 100644 --- a/src/message.rs +++ b/src/message.rs @@ -8,6 +8,7 @@ use std::str; use anyhow::{Context as _, Result, ensure, format_err}; use deltachat_contact_tools::{VcardContact, parse_vcard}; use deltachat_derive::{FromSql, ToSql}; +use num_traits::FromPrimitive; use serde::{Deserialize, Serialize}; use tokio::{fs, io}; @@ -786,7 +787,19 @@ impl Message { } /// Returns the size of the file in bytes, if applicable. + /// If message is a pre-message, then this returns size of the to be downloaded file. pub async fn get_filebytes(&self, context: &Context) -> Result> { + // if download state is not downloaded then return value from from params metadata + if self.download_state != DownloadState::Done { + if let Some(file_size) = self + .param + .get(Param::FullMessageFileBytes) + .and_then(|s| s.parse().ok()) + { + return Ok(Some(file_size)); + } + } + // TODO: also modify update docs in all places that use this (cffi and jsonrpc, possibly also in python) if let Some(path) = self.param.get_file_path(context)? { Ok(Some(get_filebytes(context, &path).await.with_context( || format!("failed to get {} size in bytes", path.display()), @@ -796,6 +809,21 @@ impl Message { } } + /// If message is a pre-message, + /// then this returns the viewtype it will have when it is downloaded. + pub fn get_full_message_viewtype(&self) -> Option { + if self.download_state != DownloadState::Done { + if let Some(viewtype) = self + .param + .get_i64(Param::FullMessageViewtype) + .and_then(|v| Viewtype::from_i64(v)) + { + return Some(viewtype); + } + } + None + } + /// Returns width of associated image or video file. pub fn get_width(&self) -> i32 { self.param.get_int(Param::Width).unwrap_or_default() diff --git a/src/mimefactory.rs b/src/mimefactory.rs index ce9263b511..4215075e25 100644 --- a/src/mimefactory.rs +++ b/src/mimefactory.rs @@ -21,6 +21,7 @@ use crate::constants::{ASM_SUBJECT, BROADCAST_INCOMPATIBILITY_MSG}; use crate::constants::{Chattype, DC_FROM_HANDSHAKE}; use crate::contact::{Contact, ContactId, Origin}; use crate::context::Context; +use crate::download::pre_msg_metadata::PreMsgMetadata; use crate::e2ee::EncryptHelper; use crate::ensure_and_debug_assert; use crate::ephemeral::Timer as EphemeralTimer; @@ -1875,14 +1876,13 @@ impl MimeFactory { // add attachment part if msg.viewtype.has_file() { if let Some(PreMessageMode::PreMessage { .. }) = self.pre_message_mode { - let attachment_size = msg - .get_filebytes(context) - .await? - .context("attachment exists, but get_filebytes returned nothing")? - .to_string(); + let Some(metadata) = PreMsgMetadata::from_msg(context, &msg).await? else { + bail!("failed to generate metadata for pre-message") + }; + headers.push(( - HeaderDef::ChatFullMessageSize.get_headername(), - mail_builder::headers::raw::Raw::new(attachment_size).into(), + HeaderDef::ChatFullMessageMetadata.into(), + mail_builder::headers::raw::Raw::new(metadata.to_header_value()?).into(), )); // TODO: generate thumbnail and attach it instead (if it makes sense) } else { diff --git a/src/mimeparser.rs b/src/mimeparser.rs index 143ad43190..3bf606e4ec 100644 --- a/src/mimeparser.rs +++ b/src/mimeparser.rs @@ -23,6 +23,7 @@ use crate::contact::ContactId; use crate::context::Context; use crate::decrypt::{try_decrypt, validate_detached_signature}; use crate::dehtml::dehtml; +use crate::download::pre_msg_metadata::PreMsgMetadata; use crate::events::EventType; use crate::headerdef::{HeaderDef, HeaderDefMap}; use crate::key::{self, DcKey, Fingerprint, SignedPublicKey, load_self_secret_keyring}; @@ -162,7 +163,7 @@ pub(crate) enum PreMessageMode { /// and it is ignored if the full message was downloaded already PreMessage { full_msg_rfc724_mid: String, - attachment_size: u64, + metadata: Option, }, } @@ -369,15 +370,28 @@ impl MimeMessage { let pre_message = if let Some(full_msg_rfc724_mid) = mail.headers.get_header_value(HeaderDef::ChatFullMessageId) { - let attachment_size: u64 = mail + let metadata = if let Some(value) = mail .headers - .get_header_value(HeaderDef::ChatFullMessageSize) - .unwrap_or_default() - .parse() - .unwrap_or_default(); + .get_header_value(HeaderDef::ChatFullMessageMetadata) + { + match PreMsgMetadata::try_from_header_value(&value) { + Ok(metadata) => Some(metadata), + Err(error) => { + error!( + context, + "failed to parse metadata header in pre-message: {error:#?}" + ); + None + } + } + } else { + warn!(context, "expected pre-message to have metadata header"); + None + }; + Some(PreMessageMode::PreMessage { full_msg_rfc724_mid, - attachment_size, + metadata, }) } else if mail .headers diff --git a/src/param.rs b/src/param.rs index 16c10b033e..f0291d7e57 100644 --- a/src/param.rs +++ b/src/param.rs @@ -251,6 +251,13 @@ pub enum Param { /// For info messages: Contact ID in added or removed to a group. ContactAddedRemoved = b'5', + + /// For (pre-)Message: ViewType of the Full Message, + /// because pre message is always `Viewtype::Text`. + FullMessageViewtype = b'8', + + /// For (pre-)Message: File byte size of Full Message attachment + FullMessageFileBytes = b'9', } /// An object for handling key=value parameter lists. diff --git a/src/receive_imf.rs b/src/receive_imf.rs index 1708219a06..400c648bc3 100644 --- a/src/receive_imf.rs +++ b/src/receive_imf.rs @@ -2016,6 +2016,14 @@ async fn add_parts( } }; + if let Some(mimeparser::PreMessageMode::PreMessage { + metadata: Some(metadata), + .. + }) = &mime_parser.pre_message + { + param.apply_from_pre_msg_metadata(metadata); + }; + // If you change which information is skipped if the message is trashed, // also change `MsgId::trash()` and `delete_expired_messages()` let trash = chat_id.is_trash() || (is_location_kml && part_is_empty && !save_mime_modified); @@ -2084,13 +2092,8 @@ RETURNING id !trash && hidden, if trash { 0 - } else if let Some(mimeparser::PreMessageMode::PreMessage { - attachment_size, - .. - }) = mime_parser.pre_message { - attachment_size as isize } else { - part.bytes as isize + part.bytes as isize }, if save_mime_modified && !(trash || hidden) { mime_headers.clone() From 7f3d2f5fb54ed59c89cf5a80ff06a35cadc04a9d Mon Sep 17 00:00:00 2001 From: Simon Laux Date: Sat, 22 Nov 2025 03:14:39 +0100 Subject: [PATCH 13/41] api: jsonrpc: add `full_message_view_type` to `Message` and `MessageInfo` --- deltachat-jsonrpc/src/api/types/message.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/deltachat-jsonrpc/src/api/types/message.rs b/deltachat-jsonrpc/src/api/types/message.rs index 567e3ffdc9..7ebadb14f4 100644 --- a/deltachat-jsonrpc/src/api/types/message.rs +++ b/deltachat-jsonrpc/src/api/types/message.rs @@ -48,6 +48,9 @@ pub struct MessageObject { has_location: bool, has_html: bool, view_type: MessageViewtype, + /// If message is a pre-message, + /// then this returns the viewtype it will have when it is downloaded. + full_message_view_type: Option, state: u32, /// An error text, if there is one. @@ -92,6 +95,8 @@ pub struct MessageObject { file: Option, file_mime: Option, + /// Returns the size of the file in bytes, if applicable. + /// If message is a pre-message, then this returns size of the to be downloaded file. file_bytes: u64, file_name: Option, @@ -210,6 +215,7 @@ impl MessageObject { has_location: message.has_location(), has_html: message.has_html(), view_type: message.get_viewtype().into(), + full_message_view_type: message.get_full_message_viewtype().map(Into::into), state: message .get_state() .to_u32() @@ -681,6 +687,9 @@ pub struct MessageInfo { rfc724_mid: String, server_urls: Vec, hop_info: String, + /// If message is a pre-message, + /// then this returns the viewtype it will have when it is downloaded. + full_message_view_type: Option, } impl MessageInfo { @@ -704,6 +713,7 @@ impl MessageInfo { rfc724_mid: message.rfc724_mid().to_owned(), server_urls, hop_info, + full_message_view_type: message.get_full_message_viewtype().map(Into::into), }) } } From ea511d4709f40a5a7b3540a4e637299528a17fa9 Mon Sep 17 00:00:00 2001 From: Simon Laux Date: Sat, 22 Nov 2025 03:19:11 +0100 Subject: [PATCH 14/41] apply clippy suggestion --- src/message.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/message.rs b/src/message.rs index 895a94517b..f970041b4b 100644 --- a/src/message.rs +++ b/src/message.rs @@ -816,7 +816,7 @@ impl Message { if let Some(viewtype) = self .param .get_i64(Param::FullMessageViewtype) - .and_then(|v| Viewtype::from_i64(v)) + .and_then(Viewtype::from_i64) { return Some(viewtype); } From ce7e62e9176d6f568ffe40b7d094614419dfe5bd Mon Sep 17 00:00:00 2001 From: Simon Laux Date: Sat, 22 Nov 2025 03:24:29 +0100 Subject: [PATCH 15/41] make PreMsgMetadata.to_header_value not consume self/PreMsgMetadata --- src/download/pre_msg_metadata.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/download/pre_msg_metadata.rs b/src/download/pre_msg_metadata.rs index 13cc0b869b..8bb7df4a0f 100644 --- a/src/download/pre_msg_metadata.rs +++ b/src/download/pre_msg_metadata.rs @@ -63,7 +63,7 @@ impl PreMsgMetadata { })) } - pub(crate) fn to_header_value(self) -> Result { + pub(crate) fn to_header_value(&self) -> Result { Ok(serde_json::to_string(&self)?) } From f5577d960ecd39bfb6878ac17250f795b02e287b Mon Sep 17 00:00:00 2001 From: Simon Laux Date: Sun, 23 Nov 2025 17:16:25 +0100 Subject: [PATCH 16/41] add api to merge params --- src/param.rs | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/src/param.rs b/src/param.rs index f0291d7e57..236fe98a83 100644 --- a/src/param.rs +++ b/src/param.rs @@ -440,6 +440,15 @@ impl Params { self.set(key, format!("{value}")); self } + + /// Merge in parameters from other Params struct, + /// overwriting the keys that are in both + /// with the values from the new Params struct. + pub fn merge_in_from_params(&mut self, new_params: Self) -> &mut Self { + let mut new_params = new_params; + self.inner.append(&mut new_params.inner); + self + } } #[cfg(test)] @@ -502,4 +511,18 @@ mod tests { assert_eq!(p.get(Param::Height), Some("14")); Ok(()) } + + #[test] + fn test_merge() -> Result<()> { + let mut p = Params::from_str("w=12\na=5\nh=14")?; + let p2 = Params::from_str("L=1\nh=17")?; + assert_eq!(p.len(), 3); + p.merge_in_from_params(p2); + assert_eq!(p.len(), 4); + assert_eq!(p.get(Param::Width), Some("12")); + assert_eq!(p.get(Param::Height), Some("17")); + assert_eq!(p.get(Param::Forwarded), Some("5")); + assert_eq!(p.get(Param::IsEdited), Some("1")); + Ok(()) + } } From ca6bf656e9fc8b1b3c5236266baf83d7e4cd28af Mon Sep 17 00:00:00 2001 From: Simon Laux Date: Sun, 23 Nov 2025 17:21:51 +0100 Subject: [PATCH 17/41] on download full message: merge new params into old params and remove full-message metadata params --- src/receive_imf.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/receive_imf.rs b/src/receive_imf.rs index 400c648bc3..602f4526c7 100644 --- a/src/receive_imf.rs +++ b/src/receive_imf.rs @@ -2347,13 +2347,19 @@ async fn handle_full_message( .param .get_bool(Param::GuaranteeE2ee) .unwrap_or_default(); + if edit_msg_showpadlock || !original_msg.get_showpadlock() { + let mut new_params = original_msg.param.clone(); + new_params + .merge_in_from_params(part.param.clone()) + .remove(Param::FullMessageFileBytes) + .remove(Param::FullMessageViewtype); context .sql .execute( "UPDATE msgs SET param=?, type=?, bytes=?, error=?, download_state=? WHERE id=?", ( - part.param.to_string(), + new_params.to_string(), part.typ, part.bytes as isize, part.error.as_deref().unwrap_or_default(), From b9a4c43a9773462daab0a8d6ba427eff2bcc06d4 Mon Sep 17 00:00:00 2001 From: Simon Laux Date: Sat, 29 Nov 2025 14:51:28 +0100 Subject: [PATCH 18/41] move tests to `src/tests/pre_messages.rs` --- src/download.rs | 381 +---------------------------------- src/tests.rs | 1 + src/tests/pre_messages.rs | 406 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 410 insertions(+), 378 deletions(-) create mode 100644 src/tests/pre_messages.rs diff --git a/src/download.rs b/src/download.rs index 6820c675bd..7d5acb71a2 100644 --- a/src/download.rs +++ b/src/download.rs @@ -341,17 +341,11 @@ pub(crate) async fn download_known_full_messages_without_pre_message( #[cfg(test)] mod tests { - use mailparse::MailHeaderMap; use num_traits::FromPrimitive; - use tokio::fs; use super::*; - use crate::chat::{self, create_group, send_msg}; - use crate::config::Config; - use crate::headerdef::{HeaderDef, HeaderDefMap}; - use crate::message::Viewtype; - use crate::receive_imf::receive_imf_from_inbox; - use crate::test_utils::{self, TestContext, TestContextManager}; + use crate::chat::send_msg; + use crate::test_utils::TestContext; #[test] fn test_downloadstate_values() { @@ -401,374 +395,5 @@ mod tests { Ok(()) } - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn test_download_stub_message() -> Result<()> { - let t = TestContext::new_alice().await; - - let header = "Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\ - From: bob@example.com\n\ - To: alice@example.org\n\ - Subject: foo\n\ - Message-ID: \n\ - Chat-Version: 1.0\n\ - Date: Sun, 22 Mar 2020 22:37:57 +0000\ - Content-Type: text/plain"; - - t.sql - .execute( - r#"INSERT INTO chats VALUES( - 11001,100,'bob@example.com',0,'',2,'', - replace('C=1763151754\nt=foo','\n',char(10)),0,0,0,0,0,1763151754,0,NULL,0); - "#, - (), - ) - .await?; - t.sql.execute(r#"INSERT INTO msgs VALUES( - 11001,'Mr.12345678901@example.com','',0, - 11001,11001,1,1763151754,10,10,1,0, - '[97.66 KiB message]','','',0,1763151754,1763151754,0,X'', - '','',1,0,'',0,0,0,'foo',10,replace('Hop: From: userid; Date: Mon, 4 Dec 2006 13:51:39 +0000\n\nDKIM Results: Passed=true','\n',char(10)),1,NULL,0); - "#, ()).await?; - let msg = t.get_last_msg().await; - assert_eq!(msg.download_state(), DownloadState::Available); - assert_eq!(msg.get_subject(), "foo"); - assert!(msg.get_text().contains("[97.66 KiB message]")); - - receive_imf_from_inbox( - &t, - "Mr.12345678901@example.com", - format!("{header}\n\n100k text...").as_bytes(), - false, - ) - .await?; - let msg = t.get_last_msg().await; - assert_eq!(msg.download_state(), DownloadState::Done); - assert_eq!(msg.get_subject(), "foo"); - assert_eq!(msg.get_text(), "100k text..."); - - Ok(()) - } - /// Tests that pre message is sent for attachment larger than `PRE_MSG_ATTACHMENT_SIZE_THRESHOLD` - /// Also test that pre message is sent first, before the full message - /// And that Autocrypt-gossip and selfavatar never go into full-messages - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn test_sending_pre_message() -> Result<()> { - let mut tcm = TestContextManager::new(); - let alice = &tcm.alice().await; - let bob = &tcm.bob().await; - let fiona = &tcm.fiona().await; - let group_id = alice - .create_group_with_members("test group", &[bob, fiona]) - .await; - - let mut msg = Message::new(Viewtype::File); - msg.set_file_from_bytes(alice, "test.bin", &[0u8; 300_000], None)?; - msg.set_text("test".to_owned()); - - // assert that test attachment is bigger than limit - assert!(msg.get_filebytes(alice).await?.unwrap() > PRE_MSG_ATTACHMENT_SIZE_THRESHOLD); - - let msg_id = chat::send_msg(alice, group_id, &mut msg).await?; - let smtp_rows = alice.get_smtp_rows_for_msg(msg_id).await; - - // pre-message and full message should be present - // and test that correct headers are present on both messages - assert_eq!(smtp_rows.len(), 2); - let pre_message = smtp_rows.first().expect("first element exists"); - let pre_message_parsed = mailparse::parse_mail(pre_message.payload.as_bytes())?; - let full_message = smtp_rows.get(1).expect("second element exists"); - let full_message_parsed = mailparse::parse_mail(full_message.payload.as_bytes())?; - - assert!( - pre_message_parsed - .headers - .get_first_header(HeaderDef::ChatIsFullMessage.get_headername()) - .is_none() - ); - assert!( - full_message_parsed - .headers - .get_first_header(HeaderDef::ChatIsFullMessage.get_headername()) - .is_some() - ); - - assert_eq!( - full_message_parsed - .headers - .get_header_value(HeaderDef::MessageId), - Some(format!("<{}>", msg.rfc724_mid)), - "full message should have the rfc message id of the database message" - ); - - assert_ne!( - pre_message_parsed - .headers - .get_header_value(HeaderDef::MessageId), - full_message_parsed - .headers - .get_header_value(HeaderDef::MessageId), - "message ids of pre message and full message should be different" - ); - - let decrypted_full_message = bob.parse_msg(full_message).await; - assert_eq!(decrypted_full_message.decrypting_failed, false); - assert_eq!( - decrypted_full_message.header_exists(HeaderDef::ChatFullMessageId), - false - ); - - let decrypted_pre_message = bob.parse_msg(pre_message).await; - assert_eq!( - decrypted_pre_message - .get_header(HeaderDef::ChatFullMessageId) - .map(String::from), - full_message_parsed - .headers - .get_header_value(HeaderDef::MessageId) - ); - assert!( - pre_message_parsed - .headers - .get_header_value(HeaderDef::ChatFullMessageId) - .is_none(), - "no Chat-Full-Message-ID header in unprotected headers of Pre-Message" - ); - - Ok(()) - } - - /// Tests that pre message has autocrypt gossip headers and self avatar - /// and full message doesn't have these headers - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn test_selfavatar_and_autocrypt_gossip_goto_pre_message() -> Result<()> { - let mut tcm = TestContextManager::new(); - let alice = &tcm.alice().await; - let bob = &tcm.bob().await; - let fiona = &tcm.fiona().await; - let group_id = alice - .create_group_with_members("test group", &[bob, fiona]) - .await; - - let mut msg = Message::new(Viewtype::File); - msg.set_file_from_bytes(alice, "test.bin", &[0u8; 300_000], None)?; - msg.set_text("test".to_owned()); - - // assert that test attachment is bigger than limit - assert!(msg.get_filebytes(alice).await?.unwrap() > PRE_MSG_ATTACHMENT_SIZE_THRESHOLD); - - // simulate conditions for sending self avatar - let avatar_src = alice.get_blobdir().join("avatar.png"); - fs::write(&avatar_src, test_utils::AVATAR_900x900_BYTES).await?; - alice - .set_config(Config::Selfavatar, Some(avatar_src.to_str().unwrap())) - .await?; - - let msg_id = chat::send_msg(alice, group_id, &mut msg).await?; - let smtp_rows = alice.get_smtp_rows_for_msg(msg_id).await; - - assert_eq!(smtp_rows.len(), 2); - let pre_message = smtp_rows.first().expect("first element exists"); - let full_message = smtp_rows.get(1).expect("second element exists"); - let full_message_parsed = mailparse::parse_mail(full_message.payload.as_bytes())?; - - let decrypted_pre_message = bob.parse_msg(pre_message).await; - assert!( - decrypted_pre_message - .get_header(HeaderDef::ChatFullMessageId) - .is_some(), - "tested message is not a pre-message, sending order may be broken" - ); - assert_ne!(decrypted_pre_message.gossiped_keys.len(), 0); - assert_ne!(decrypted_pre_message.user_avatar, None); - - let decrypted_full_message = bob.parse_msg(full_message).await; - assert!( - full_message_parsed - .headers - .get_first_header(HeaderDef::ChatIsFullMessage.get_headername()) - .is_some(), - "tested message is not a full-message, sending order may be broken" - ); - assert_eq!(decrypted_full_message.gossiped_keys.len(), 0); - assert_eq!(decrypted_full_message.user_avatar, None); - Ok(()) - } - - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn test_unecrypted_gets_no_pre_message() -> Result<()> { - let mut tcm = TestContextManager::new(); - let alice = &tcm.alice().await; - - let chat = alice - .create_chat_with_contact("example", "email@example.org") - .await; - - let mut msg = Message::new(Viewtype::File); - msg.set_file_from_bytes(alice, "test.bin", &[0u8; 300_000], None)?; - msg.set_text("test".to_owned()); - - let msg_id = chat::send_msg(alice, chat.id, &mut msg).await?; - let smtp_rows = alice.get_smtp_rows_for_msg(msg_id).await; - - assert_eq!(smtp_rows.len(), 1); - let message_bytes = smtp_rows - .first() - .expect("first element exists") - .payload - .as_bytes(); - let message = mailparse::parse_mail(message_bytes)?; - assert!( - message - .headers - .get_first_header(HeaderDef::ChatIsFullMessage.get_headername()) - .is_none(), - ); - Ok(()) - } - - /// Tests that no pre message is sent for normal message - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn test_not_sending_pre_message_no_attachment() -> Result<()> { - let mut tcm = TestContextManager::new(); - let alice = &tcm.alice().await; - let bob = &tcm.bob().await; - let chat = alice.create_chat(bob).await; - - // send normal text message - let mut msg = Message::new(Viewtype::Text); - msg.set_text("test".to_owned()); - let msg_id = chat::send_msg(alice, chat.id, &mut msg).await.unwrap(); - let smtp_rows = alice.get_smtp_rows_for_msg(msg_id).await; - - assert_eq!(smtp_rows.len(), 1, "only one message should be sent"); - - let msg = smtp_rows.first().expect("first element exists"); - let mail = mailparse::parse_mail(msg.payload.as_bytes())?; - - assert!( - mail.headers - .get_first_header(HeaderDef::ChatIsFullMessage.get_headername()) - .is_none(), - "no 'Chat-Is-Full-Message'-header should be present" - ); - assert!( - mail.headers - .get_first_header(HeaderDef::ChatFullMessageId.get_headername()) - .is_none(), - "no 'Chat-Full-Message-ID'-header should be present in clear text headers" - ); - let decrypted_message = bob.parse_msg(msg).await; - assert!( - !decrypted_message.header_exists(HeaderDef::ChatFullMessageId), - "no 'Chat-Full-Message-ID'-header should be present" - ); - - // test that pre message is not send for large large text - let mut msg = Message::new(Viewtype::Text); - let long_text = String::from_utf8(vec![b'a'; 300_000])?; - assert!(long_text.len() > PRE_MSG_ATTACHMENT_SIZE_THRESHOLD.try_into().unwrap()); - msg.set_text(long_text); - let msg_id = chat::send_msg(alice, chat.id, &mut msg).await.unwrap(); - let smtp_rows = alice.get_smtp_rows_for_msg(msg_id).await; - - assert_eq!(smtp_rows.len(), 1, "only one message should be sent"); - - let msg = smtp_rows.first().expect("first element exists"); - let mail = mailparse::parse_mail(msg.payload.as_bytes())?; - - assert!( - mail.headers - .get_first_header(HeaderDef::ChatIsFullMessage.get_headername()) - .is_none() - ); - assert!( - mail.headers - .get_first_header(HeaderDef::ChatFullMessageId.get_headername()) - .is_none(), - "no 'Chat-Full-Message-ID'-header should be present in clear text headers" - ); - let decrypted_message = bob.parse_msg(msg).await; - assert!( - !decrypted_message.header_exists(HeaderDef::ChatFullMessageId), - "no 'Chat-Full-Message-ID'-header should be present" - ); - Ok(()) - } - - /// Tests that no pre message is sent for attachment smaller than `PRE_MSG_ATTACHMENT_SIZE_THRESHOLD` - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn test_not_sending_pre_message_for_small_attachment() -> Result<()> { - let mut tcm = TestContextManager::new(); - let alice = &tcm.alice().await; - let bob = &tcm.bob().await; - let chat = alice.create_chat(bob).await; - - let mut msg = Message::new(Viewtype::File); - msg.set_file_from_bytes(alice, "test.bin", &[0u8; 100_000], None)?; - msg.set_text("test".to_owned()); - - // assert that test attachment is smaller than limit - assert!(msg.get_filebytes(alice).await?.unwrap() < PRE_MSG_ATTACHMENT_SIZE_THRESHOLD); - - let msg_id = chat::send_msg(alice, chat.id, &mut msg).await.unwrap(); - let smtp_rows = alice.get_smtp_rows_for_msg(msg_id).await; - - // only one message and no "is full message" header should be present - assert_eq!(smtp_rows.len(), 1); - - let msg = smtp_rows.first().expect("first element exists"); - let mail = mailparse::parse_mail(msg.payload.as_bytes())?; - - assert!( - mail.headers - .get_first_header(HeaderDef::ChatIsFullMessage.get_headername()) - .is_none() - ); - assert!( - mail.headers - .get_first_header(HeaderDef::ChatFullMessageId.get_headername()) - .is_none(), - "no 'Chat-Full-Message-ID'-header should be present in clear text headers" - ); - let decrypted_message = bob.parse_msg(msg).await; - assert!( - !decrypted_message.header_exists(HeaderDef::ChatFullMessageId), - "no 'Chat-Full-Message-ID'-header should be present" - ); - - Ok(()) - } - - /// Tests that pre message is not send for large webxdc updates - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn test_render_webxdc_status_update_object_range() -> Result<()> { - let t = TestContext::new_alice().await; - let chat_id = create_group(&t, "a chat").await?; - - let instance = { - let mut instance = Message::new(Viewtype::File); - instance.set_file_from_bytes( - &t, - "minimal.xdc", - include_bytes!("../test-data/webxdc/minimal.xdc"), - None, - )?; - let instance_msg_id = send_msg(&t, chat_id, &mut instance).await?; - assert_eq!(instance.viewtype, Viewtype::Webxdc); - Message::load_from_db(&t, instance_msg_id).await - } - .unwrap(); - - t.pop_sent_msg().await; - assert_eq!(t.sql.count("SELECT COUNT(*) FROM smtp", ()).await?, 0); - - let long_text = String::from_utf8(vec![b'a'; 300_000])?; - assert!(long_text.len() > PRE_MSG_ATTACHMENT_SIZE_THRESHOLD.try_into().unwrap()); - t.send_webxdc_status_update(instance.id, &format!("{{\"payload\": \"{long_text}\"}}")) - .await?; - t.flush_status_updates().await?; - - assert_eq!(t.sql.count("SELECT COUNT(*) FROM smtp", ()).await?, 1); - Ok(()) - } + // NOTE: The download tests for pre-messages are in src/tests/pre_messages.rs } diff --git a/src/tests.rs b/src/tests.rs index 6e642dce74..b7ae08fbad 100644 --- a/src/tests.rs +++ b/src/tests.rs @@ -1,3 +1,4 @@ mod account_events; mod aeap; +mod pre_messages; mod verified_chats; diff --git a/src/tests/pre_messages.rs b/src/tests/pre_messages.rs new file mode 100644 index 0000000000..a8ee846eaf --- /dev/null +++ b/src/tests/pre_messages.rs @@ -0,0 +1,406 @@ +use crate::download::DownloadState; +use crate::test_utils::{self, TestContext, TestContextManager}; +use anyhow::Result; + +/// Test that downloading old stub messages still works +mod legacy { + use super::*; + use crate::receive_imf::receive_imf_from_inbox; + + // The code for downloading stub messages stays + // during the transition perios to pre-messages + // so people can still download their files shortly after they updated. + // After there are a few release with pre-message rolled out, + // we will remove the ability to download stub messages and replace the following test + // so it checks that it doesn't crash or that the messages are replaced by sth. + // like "download failed/expired, please ask sender to send it again" + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn test_download_stub_message() -> Result<()> { + let t = TestContext::new_alice().await; + + let header = "Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\ + From: bob@example.com\n\ + To: alice@example.org\n\ + Subject: foo\n\ + Message-ID: \n\ + Chat-Version: 1.0\n\ + Date: Sun, 22 Mar 2020 22:37:57 +0000\ + Content-Type: text/plain"; + + t.sql + .execute( + r#"INSERT INTO chats VALUES( + 11001,100,'bob@example.com',0,'',2,'', + replace('C=1763151754\nt=foo','\n',char(10)),0,0,0,0,0,1763151754,0,NULL,0); + "#, + (), + ) + .await?; + t.sql.execute(r#"INSERT INTO msgs VALUES( + 11001,'Mr.12345678901@example.com','',0, + 11001,11001,1,1763151754,10,10,1,0, + '[97.66 KiB message]','','',0,1763151754,1763151754,0,X'', + '','',1,0,'',0,0,0,'foo',10,replace('Hop: From: userid; Date: Mon, 4 Dec 2006 13:51:39 +0000\n\nDKIM Results: Passed=true','\n',char(10)),1,NULL,0); + "#, ()).await?; + let msg = t.get_last_msg().await; + assert_eq!(msg.download_state(), DownloadState::Available); + assert_eq!(msg.get_subject(), "foo"); + assert!(msg.get_text().contains("[97.66 KiB message]")); + + receive_imf_from_inbox( + &t, + "Mr.12345678901@example.com", + format!("{header}\n\n100k text...").as_bytes(), + false, + ) + .await?; + let msg = t.get_last_msg().await; + assert_eq!(msg.download_state(), DownloadState::Done); + assert_eq!(msg.get_subject(), "foo"); + assert_eq!(msg.get_text(), "100k text..."); + + Ok(()) + } +} + +/// Tests about sending pre-messages +/// - When to send a pre-message and full-message instead of a normal message +/// - Test that sent pre- and full-message contain the right Headers +/// and that they are send in the correct order (pre-message is sent first.) +mod sending { + use super::*; + use mailparse::MailHeaderMap; + use tokio::fs; + + use crate::chat::{self, create_group, send_msg}; + use crate::config::Config; + use crate::download::PRE_MSG_ATTACHMENT_SIZE_THRESHOLD; + use crate::headerdef::{HeaderDef, HeaderDefMap}; + use crate::message::{Message, Viewtype}; + /// Tests that pre message is sent for attachment larger than `PRE_MSG_ATTACHMENT_SIZE_THRESHOLD` + /// Also test that pre message is sent first, before the full message + /// And that Autocrypt-gossip and selfavatar never go into full-messages + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn test_sending_pre_message() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + let fiona = &tcm.fiona().await; + let group_id = alice + .create_group_with_members("test group", &[bob, fiona]) + .await; + + let mut msg = Message::new(Viewtype::File); + msg.set_file_from_bytes(alice, "test.bin", &[0u8; 300_000], None)?; + msg.set_text("test".to_owned()); + + // assert that test attachment is bigger than limit + assert!(msg.get_filebytes(alice).await?.unwrap() > PRE_MSG_ATTACHMENT_SIZE_THRESHOLD); + + let msg_id = chat::send_msg(alice, group_id, &mut msg).await?; + let smtp_rows = alice.get_smtp_rows_for_msg(msg_id).await; + + // pre-message and full message should be present + // and test that correct headers are present on both messages + assert_eq!(smtp_rows.len(), 2); + let pre_message = smtp_rows.first().expect("first element exists"); + let pre_message_parsed = mailparse::parse_mail(pre_message.payload.as_bytes())?; + let full_message = smtp_rows.get(1).expect("second element exists"); + let full_message_parsed = mailparse::parse_mail(full_message.payload.as_bytes())?; + + assert!( + pre_message_parsed + .headers + .get_first_header(HeaderDef::ChatIsFullMessage.get_headername()) + .is_none() + ); + assert!( + full_message_parsed + .headers + .get_first_header(HeaderDef::ChatIsFullMessage.get_headername()) + .is_some() + ); + + assert_eq!( + full_message_parsed + .headers + .get_header_value(HeaderDef::MessageId), + Some(format!("<{}>", msg.rfc724_mid)), + "full message should have the rfc message id of the database message" + ); + + assert_ne!( + pre_message_parsed + .headers + .get_header_value(HeaderDef::MessageId), + full_message_parsed + .headers + .get_header_value(HeaderDef::MessageId), + "message ids of pre message and full message should be different" + ); + + let decrypted_full_message = bob.parse_msg(full_message).await; + assert_eq!(decrypted_full_message.decrypting_failed, false); + assert_eq!( + decrypted_full_message.header_exists(HeaderDef::ChatFullMessageId), + false + ); + + let decrypted_pre_message = bob.parse_msg(pre_message).await; + assert_eq!( + decrypted_pre_message + .get_header(HeaderDef::ChatFullMessageId) + .map(String::from), + full_message_parsed + .headers + .get_header_value(HeaderDef::MessageId) + ); + assert!( + pre_message_parsed + .headers + .get_header_value(HeaderDef::ChatFullMessageId) + .is_none(), + "no Chat-Full-Message-ID header in unprotected headers of Pre-Message" + ); + + Ok(()) + } + + /// Tests that pre message has autocrypt gossip headers and self avatar + /// and full message doesn't have these headers + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn test_selfavatar_and_autocrypt_gossip_goto_pre_message() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + let fiona = &tcm.fiona().await; + let group_id = alice + .create_group_with_members("test group", &[bob, fiona]) + .await; + + let mut msg = Message::new(Viewtype::File); + msg.set_file_from_bytes(alice, "test.bin", &[0u8; 300_000], None)?; + msg.set_text("test".to_owned()); + + // assert that test attachment is bigger than limit + assert!(msg.get_filebytes(alice).await?.unwrap() > PRE_MSG_ATTACHMENT_SIZE_THRESHOLD); + + // simulate conditions for sending self avatar + let avatar_src = alice.get_blobdir().join("avatar.png"); + fs::write(&avatar_src, test_utils::AVATAR_900x900_BYTES).await?; + alice + .set_config(Config::Selfavatar, Some(avatar_src.to_str().unwrap())) + .await?; + + let msg_id = chat::send_msg(alice, group_id, &mut msg).await?; + let smtp_rows = alice.get_smtp_rows_for_msg(msg_id).await; + + assert_eq!(smtp_rows.len(), 2); + let pre_message = smtp_rows.first().expect("first element exists"); + let full_message = smtp_rows.get(1).expect("second element exists"); + let full_message_parsed = mailparse::parse_mail(full_message.payload.as_bytes())?; + + let decrypted_pre_message = bob.parse_msg(pre_message).await; + assert!( + decrypted_pre_message + .get_header(HeaderDef::ChatFullMessageId) + .is_some(), + "tested message is not a pre-message, sending order may be broken" + ); + assert_ne!(decrypted_pre_message.gossiped_keys.len(), 0); + assert_ne!(decrypted_pre_message.user_avatar, None); + + let decrypted_full_message = bob.parse_msg(full_message).await; + assert!( + full_message_parsed + .headers + .get_first_header(HeaderDef::ChatIsFullMessage.get_headername()) + .is_some(), + "tested message is not a full-message, sending order may be broken" + ); + assert_eq!(decrypted_full_message.gossiped_keys.len(), 0); + assert_eq!(decrypted_full_message.user_avatar, None); + Ok(()) + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn test_unecrypted_gets_no_pre_message() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + + let chat = alice + .create_chat_with_contact("example", "email@example.org") + .await; + + let mut msg = Message::new(Viewtype::File); + msg.set_file_from_bytes(alice, "test.bin", &[0u8; 300_000], None)?; + msg.set_text("test".to_owned()); + + let msg_id = chat::send_msg(alice, chat.id, &mut msg).await?; + let smtp_rows = alice.get_smtp_rows_for_msg(msg_id).await; + + assert_eq!(smtp_rows.len(), 1); + let message_bytes = smtp_rows + .first() + .expect("first element exists") + .payload + .as_bytes(); + let message = mailparse::parse_mail(message_bytes)?; + assert!( + message + .headers + .get_first_header(HeaderDef::ChatIsFullMessage.get_headername()) + .is_none(), + ); + Ok(()) + } + + /// Tests that no pre message is sent for normal message + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn test_not_sending_pre_message_no_attachment() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + let chat = alice.create_chat(bob).await; + + // send normal text message + let mut msg = Message::new(Viewtype::Text); + msg.set_text("test".to_owned()); + let msg_id = chat::send_msg(alice, chat.id, &mut msg).await.unwrap(); + let smtp_rows = alice.get_smtp_rows_for_msg(msg_id).await; + + assert_eq!(smtp_rows.len(), 1, "only one message should be sent"); + + let msg = smtp_rows.first().expect("first element exists"); + let mail = mailparse::parse_mail(msg.payload.as_bytes())?; + + assert!( + mail.headers + .get_first_header(HeaderDef::ChatIsFullMessage.get_headername()) + .is_none(), + "no 'Chat-Is-Full-Message'-header should be present" + ); + assert!( + mail.headers + .get_first_header(HeaderDef::ChatFullMessageId.get_headername()) + .is_none(), + "no 'Chat-Full-Message-ID'-header should be present in clear text headers" + ); + let decrypted_message = bob.parse_msg(msg).await; + assert!( + !decrypted_message.header_exists(HeaderDef::ChatFullMessageId), + "no 'Chat-Full-Message-ID'-header should be present" + ); + + // test that pre message is not send for large large text + let mut msg = Message::new(Viewtype::Text); + let long_text = String::from_utf8(vec![b'a'; 300_000])?; + assert!(long_text.len() > PRE_MSG_ATTACHMENT_SIZE_THRESHOLD.try_into().unwrap()); + msg.set_text(long_text); + let msg_id = chat::send_msg(alice, chat.id, &mut msg).await.unwrap(); + let smtp_rows = alice.get_smtp_rows_for_msg(msg_id).await; + + assert_eq!(smtp_rows.len(), 1, "only one message should be sent"); + + let msg = smtp_rows.first().expect("first element exists"); + let mail = mailparse::parse_mail(msg.payload.as_bytes())?; + + assert!( + mail.headers + .get_first_header(HeaderDef::ChatIsFullMessage.get_headername()) + .is_none() + ); + assert!( + mail.headers + .get_first_header(HeaderDef::ChatFullMessageId.get_headername()) + .is_none(), + "no 'Chat-Full-Message-ID'-header should be present in clear text headers" + ); + let decrypted_message = bob.parse_msg(msg).await; + assert!( + !decrypted_message.header_exists(HeaderDef::ChatFullMessageId), + "no 'Chat-Full-Message-ID'-header should be present" + ); + Ok(()) + } + + /// Tests that no pre message is sent for attachment smaller than `PRE_MSG_ATTACHMENT_SIZE_THRESHOLD` + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn test_not_sending_pre_message_for_small_attachment() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + let chat = alice.create_chat(bob).await; + + let mut msg = Message::new(Viewtype::File); + msg.set_file_from_bytes(alice, "test.bin", &[0u8; 100_000], None)?; + msg.set_text("test".to_owned()); + + // assert that test attachment is smaller than limit + assert!(msg.get_filebytes(alice).await?.unwrap() < PRE_MSG_ATTACHMENT_SIZE_THRESHOLD); + + let msg_id = chat::send_msg(alice, chat.id, &mut msg).await.unwrap(); + let smtp_rows = alice.get_smtp_rows_for_msg(msg_id).await; + + // only one message and no "is full message" header should be present + assert_eq!(smtp_rows.len(), 1); + + let msg = smtp_rows.first().expect("first element exists"); + let mail = mailparse::parse_mail(msg.payload.as_bytes())?; + + assert!( + mail.headers + .get_first_header(HeaderDef::ChatIsFullMessage.get_headername()) + .is_none() + ); + assert!( + mail.headers + .get_first_header(HeaderDef::ChatFullMessageId.get_headername()) + .is_none(), + "no 'Chat-Full-Message-ID'-header should be present in clear text headers" + ); + let decrypted_message = bob.parse_msg(msg).await; + assert!( + !decrypted_message.header_exists(HeaderDef::ChatFullMessageId), + "no 'Chat-Full-Message-ID'-header should be present" + ); + + Ok(()) + } + + /// Tests that pre message is not send for large webxdc updates + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn test_render_webxdc_status_update_object_range() -> Result<()> { + let t = TestContext::new_alice().await; + let chat_id = create_group(&t, "a chat").await?; + + let instance = { + let mut instance = Message::new(Viewtype::File); + instance.set_file_from_bytes( + &t, + "minimal.xdc", + include_bytes!("../../test-data/webxdc/minimal.xdc"), + None, + )?; + let instance_msg_id = send_msg(&t, chat_id, &mut instance).await?; + assert_eq!(instance.viewtype, Viewtype::Webxdc); + Message::load_from_db(&t, instance_msg_id).await + } + .unwrap(); + + t.pop_sent_msg().await; + assert_eq!(t.sql.count("SELECT COUNT(*) FROM smtp", ()).await?, 0); + + let long_text = String::from_utf8(vec![b'a'; 300_000])?; + assert!(long_text.len() > PRE_MSG_ATTACHMENT_SIZE_THRESHOLD.try_into().unwrap()); + t.send_webxdc_status_update(instance.id, &format!("{{\"payload\": \"{long_text}\"}}")) + .await?; + t.flush_status_updates().await?; + + assert_eq!(t.sql.count("SELECT COUNT(*) FROM smtp", ()).await?, 1); + Ok(()) + } +} + +/// Tests about receiving pre-messages and full messages +mod receiving {} From f1f98687ad821a2c805d87a7e01a5a58125e2df9 Mon Sep 17 00:00:00 2001 From: Simon Laux Date: Sun, 30 Nov 2025 04:52:16 +0100 Subject: [PATCH 19/41] dynamically allocate test attachment bytes --- src/tests/pre_messages.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tests/pre_messages.rs b/src/tests/pre_messages.rs index a8ee846eaf..aeffd9eb9c 100644 --- a/src/tests/pre_messages.rs +++ b/src/tests/pre_messages.rs @@ -233,7 +233,7 @@ mod sending { .await; let mut msg = Message::new(Viewtype::File); - msg.set_file_from_bytes(alice, "test.bin", &[0u8; 300_000], None)?; + msg.set_file_from_bytes(alice, "test.bin", &vec![0u8; 300_000], None)?; msg.set_text("test".to_owned()); let msg_id = chat::send_msg(alice, chat.id, &mut msg).await?; @@ -333,7 +333,7 @@ mod sending { let chat = alice.create_chat(bob).await; let mut msg = Message::new(Viewtype::File); - msg.set_file_from_bytes(alice, "test.bin", &[0u8; 100_000], None)?; + msg.set_file_from_bytes(alice, "test.bin", &vec![0u8; 100_000], None)?; msg.set_text("test".to_owned()); // assert that test attachment is smaller than limit From 01cee7e5b431a938a2bef21dc3cf75e489fa0704 Mon Sep 17 00:00:00 2001 From: Simon Laux Date: Sun, 30 Nov 2025 05:52:12 +0100 Subject: [PATCH 20/41] fix detection of pre-messages. (it looked for the ChatFullMessageId header in the unencrypted headers before) --- src/mimeparser.rs | 60 ++++++++++++++++++++++++++--------------------- 1 file changed, 33 insertions(+), 27 deletions(-) diff --git a/src/mimeparser.rs b/src/mimeparser.rs index 3bf606e4ec..4a1deb0f42 100644 --- a/src/mimeparser.rs +++ b/src/mimeparser.rs @@ -367,33 +367,7 @@ impl MimeMessage { let mut aheader_values = mail.headers.get_all_values(HeaderDef::Autocrypt.into()); - let pre_message = if let Some(full_msg_rfc724_mid) = - mail.headers.get_header_value(HeaderDef::ChatFullMessageId) - { - let metadata = if let Some(value) = mail - .headers - .get_header_value(HeaderDef::ChatFullMessageMetadata) - { - match PreMsgMetadata::try_from_header_value(&value) { - Ok(metadata) => Some(metadata), - Err(error) => { - error!( - context, - "failed to parse metadata header in pre-message: {error:#?}" - ); - None - } - } - } else { - warn!(context, "expected pre-message to have metadata header"); - None - }; - - Some(PreMessageMode::PreMessage { - full_msg_rfc724_mid, - metadata, - }) - } else if mail + let mut pre_message = if mail .headers .get_header_value(HeaderDef::ChatIsFullMessage) .is_some() @@ -631,6 +605,38 @@ impl MimeMessage { signatures.clear(); } + if let (Ok(mail), true) = (mail, is_encrypted) { + if let Some(full_msg_rfc724_mid) = + mail.headers.get_header_value(HeaderDef::ChatFullMessageId) + { + // TODO: is there a better method for this task? (removing `<>` delimiters of RFC message ID) + let full_msg_rfc724_mid = full_msg_rfc724_mid.replace("<", "").replace(">", ""); + let metadata = if let Some(value) = mail + .headers + .get_header_value(HeaderDef::ChatFullMessageMetadata) + { + match PreMsgMetadata::try_from_header_value(&value) { + Ok(metadata) => Some(metadata), + Err(error) => { + error!( + context, + "failed to parse metadata header in pre-message: {error:#?}" + ); + None + } + } + } else { + warn!(context, "expected pre-message to have metadata header"); + None + }; + + pre_message = Some(PreMessageMode::PreMessage { + full_msg_rfc724_mid, + metadata, + }); + } + } + let mut parser = MimeMessage { parts: Vec::new(), headers, From 59b55e30b1b1946caa114eee67105ea7a882b955 Mon Sep 17 00:00:00 2001 From: Simon Laux Date: Sun, 30 Nov 2025 05:53:52 +0100 Subject: [PATCH 21/41] fix setting dl state to avaiable on pre-messages --- src/receive_imf.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/receive_imf.rs b/src/receive_imf.rs index 602f4526c7..80373ed4df 100644 --- a/src/receive_imf.rs +++ b/src/receive_imf.rs @@ -2110,6 +2110,8 @@ RETURNING id DownloadState::Done } else if mime_parser.decrypting_failed { DownloadState::Undecipherable + } else if let Some(mimeparser::PreMessageMode::PreMessage {..}) = mime_parser.pre_message { + DownloadState::Available } else { DownloadState::Done }, From 91e8f6732587ad295e04a3e1c1a309c3334dc4d9 Mon Sep 17 00:00:00 2001 From: Simon Laux Date: Sun, 30 Nov 2025 06:40:26 +0100 Subject: [PATCH 22/41] fix: save pre message with rfc724_mid of full message als disable replacement for full messages --- src/receive_imf.rs | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/receive_imf.rs b/src/receive_imf.rs index 80373ed4df..80cbade189 100644 --- a/src/receive_imf.rs +++ b/src/receive_imf.rs @@ -515,7 +515,15 @@ pub(crate) async fn receive_imf_inner( // check, if the mail is already in our database. // make sure, this check is done eg. before securejoin-processing. let (replace_msg_id, replace_chat_id); - if let Some(old_msg_id) = message::rfc724_mid_exists(context, rfc724_mid).await? { + if mime_parser.pre_message == Some(mimeparser::PreMessageMode::FullMessage) { + // Full Message just replace the attachment and mofified Params, not the whole message + // This is done in the `handle_full_message` method. + replace_msg_id = None; + replace_chat_id = None; + } else if let Some(old_msg_id) = message::rfc724_mid_exists(context, rfc724_mid).await? { + // This code handles the download of old partial download stub messages + // It will be removed after a transitioning period, + // after we have released a few versions with pre-messages replace_msg_id = Some(old_msg_id); replace_chat_id = if let Some(msg) = Message::load_from_db_optional(context, old_msg_id) .await? @@ -524,8 +532,6 @@ pub(crate) async fn receive_imf_inner( // the message was partially downloaded before and is fully downloaded now. info!(context, "Message already partly in DB, replacing."); Some(msg.chat_id) - - // TODO: look at this place } else { // The message was already fully downloaded // or cannot be loaded because it is deleted. @@ -2067,7 +2073,9 @@ RETURNING id "#)?; let row_id: MsgId = stmt.query_row(params![ replace_msg_id, - rfc724_mid_orig, + if let Some(mimeparser::PreMessageMode::PreMessage {full_msg_rfc724_mid, .. }) = &mime_parser.pre_message { + full_msg_rfc724_mid + } else { rfc724_mid_orig }, if trash { DC_CHAT_ID_TRASH } else { chat_id }, if trash { ContactId::UNDEFINED } else { from_id }, if trash { ContactId::UNDEFINED } else { to_id }, From d361eeb8f9421afe9a327b404557653b8a5ac016 Mon Sep 17 00:00:00 2001 From: Simon Laux Date: Sun, 30 Nov 2025 06:42:28 +0100 Subject: [PATCH 23/41] add some receiving tests and update test todo for premessage metadata --- src/download/pre_msg_metadata.rs | 2 +- src/tests/pre_messages.rs | 127 ++++++++++++++++++++++++++++++- 2 files changed, 127 insertions(+), 2 deletions(-) diff --git a/src/download/pre_msg_metadata.rs b/src/download/pre_msg_metadata.rs index 8bb7df4a0f..d9924b09bb 100644 --- a/src/download/pre_msg_metadata.rs +++ b/src/download/pre_msg_metadata.rs @@ -100,7 +100,7 @@ impl Params { #[cfg(test)] mod tests { - // todo build from message + // todo build from message (different types: file, image, audio) // todo create artifically and serialize to header // todo deserialize from header } diff --git a/src/tests/pre_messages.rs b/src/tests/pre_messages.rs index aeffd9eb9c..b6b4a152c8 100644 --- a/src/tests/pre_messages.rs +++ b/src/tests/pre_messages.rs @@ -403,4 +403,129 @@ mod sending { } /// Tests about receiving pre-messages and full messages -mod receiving {} +mod receiving { + use super::*; + use pretty_assertions::assert_eq; + + use crate::chat::{self, ChatId}; + use crate::download::PRE_MSG_ATTACHMENT_SIZE_THRESHOLD; + use crate::download::pre_msg_metadata::PreMsgMetadata; + use crate::message::{Message, Viewtype}; + use crate::mimeparser::MimeMessage; + use crate::param::Param; + use crate::test_utils::SentMessage; + + async fn send_large_file_message<'a>( + sender: &'a TestContext, + target_chat: ChatId, + attachment_size: u64, + ) -> Result<(SentMessage<'a>, SentMessage<'a>)> { + let mut msg = Message::new(Viewtype::File); + msg.set_file_from_bytes( + sender, + "test.bin", + &vec![0u8; attachment_size as usize], + None, + )?; + msg.set_text("test".to_owned()); + + // assert that test attachment is bigger than limit + assert!(msg.get_filebytes(sender).await?.unwrap() > PRE_MSG_ATTACHMENT_SIZE_THRESHOLD); + + let msg_id = chat::send_msg(sender, target_chat, &mut msg).await?; + let smtp_rows = sender.get_smtp_rows_for_msg(msg_id).await; + + assert_eq!(smtp_rows.len(), 2); + let pre_message = smtp_rows.first().expect("pre-message exists"); + let full_message = smtp_rows.get(1).expect("full message exists"); + Ok((pre_message.to_owned(), full_message.to_owned())) + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn test_mimeparser_pre_message_and_full_message() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + let alice_group_id = alice.create_group_with_members("test group", &[bob]).await; + + let (pre_message, full_message) = + send_large_file_message(alice, alice_group_id, 1_000_000).await?; + + let parsed_pre_message = + MimeMessage::from_bytes(bob, pre_message.payload.as_bytes()).await?; + let parsed_full_message = + MimeMessage::from_bytes(bob, full_message.payload.as_bytes()).await?; + + assert_eq!( + parsed_full_message.pre_message, + Some(crate::mimeparser::PreMessageMode::FullMessage) + ); + + assert_eq!( + parsed_pre_message.pre_message, + Some(crate::mimeparser::PreMessageMode::PreMessage { + full_msg_rfc724_mid: parsed_full_message.get_rfc724_mid().unwrap(), + metadata: Some(PreMsgMetadata { + size: 1_000_000, + viewtype: Viewtype::File, + filename: "test.bin".to_string(), + dimensions: None, + duration: None + }) + }) + ); + + Ok(()) + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn test_receive_pre_message() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + let alice_group_id = alice.create_group_with_members("test group", &[bob]).await; + + let (pre_message, _full_message) = + send_large_file_message(alice, alice_group_id, 1_000_000).await?; + + let msg = bob.recv_msg(&pre_message).await; + + assert_eq!(msg.download_state(), DownloadState::Available); + assert_eq!(msg.viewtype, Viewtype::Text); + + // test that metadata is correctly returned by methods + assert_eq!(msg.get_filebytes(bob).await?, Some(1_000_000)); + assert_eq!(msg.get_full_message_viewtype(), Some(Viewtype::File)); + assert_eq!(msg.get_filename(), Some("test.bin".to_owned())); + + Ok(()) + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn test_recive_pre_message_and_dl_full_message() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + let alice_group_id = alice.create_group_with_members("test group", &[bob]).await; + + let (pre_message, full_message) = + send_large_file_message(alice, alice_group_id, 1_000_000).await?; + + let msg = bob.recv_msg(&pre_message).await; + assert_eq!(msg.download_state(), DownloadState::Available); + assert_eq!(msg.viewtype, Viewtype::Text); + assert!(msg.param.exists(Param::FullMessageViewtype)); + assert!(msg.param.exists(Param::FullMessageFileBytes)); + let _ = bob.recv_msg_trash(&full_message).await; + let msg = Message::load_from_db(bob, msg.id).await?; + assert_eq!(msg.download_state(), DownloadState::Done); + assert_eq!(msg.viewtype, Viewtype::File); + assert_eq!(msg.param.exists(Param::FullMessageViewtype), false); + assert_eq!(msg.param.exists(Param::FullMessageFileBytes), false); + Ok(()) + } + + // TODO: dl full message before pre message + + // TODO: dl normal message +} From 5230aba461781fa0090967c49d800ebf1d87595d Mon Sep 17 00:00:00 2001 From: Simon Laux Date: Sun, 30 Nov 2025 07:01:36 +0100 Subject: [PATCH 24/41] test: process full message before pre-message --- src/tests/pre_messages.rs | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/src/tests/pre_messages.rs b/src/tests/pre_messages.rs index b6b4a152c8..890b8dddfb 100644 --- a/src/tests/pre_messages.rs +++ b/src/tests/pre_messages.rs @@ -441,6 +441,7 @@ mod receiving { Ok((pre_message.to_owned(), full_message.to_owned())) } + /// Test that mimeparser can correctly detect and parse pre-messages and full-messages #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_mimeparser_pre_message_and_full_message() -> Result<()> { let mut tcm = TestContextManager::new(); @@ -478,6 +479,8 @@ mod receiving { Ok(()) } + /// Test receiving pre-messages and creation of the placeholder message with the metadata + /// for file attachment #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_receive_pre_message() -> Result<()> { let mut tcm = TestContextManager::new(); @@ -501,8 +504,10 @@ mod receiving { Ok(()) } + /// Test receiving the full message after receiving the pre-message + /// for file attachment #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn test_recive_pre_message_and_dl_full_message() -> Result<()> { + async fn test_receive_pre_message_and_dl_full_message() -> Result<()> { let mut tcm = TestContextManager::new(); let alice = &tcm.alice().await; let bob = &tcm.bob().await; @@ -525,7 +530,24 @@ mod receiving { Ok(()) } - // TODO: dl full message before pre message + /// Test out of order receiving. Full message is received & downloaded before pre-message. + /// In that case pre-message shall be trashed. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn test_out_of_order_receiving() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + let alice_group_id = alice.create_group_with_members("test group", &[bob]).await; + + let (pre_message, full_message) = + send_large_file_message(alice, alice_group_id, 1_000_000).await?; + + let msg = bob.recv_msg(&full_message).await; + assert_eq!(msg.download_state(), DownloadState::Done); + assert_eq!(msg.viewtype, Viewtype::File); + let _ = bob.recv_msg_trash(&pre_message).await; + Ok(()) + } - // TODO: dl normal message + // TODO: process normal message (neither full nor pre message) } From 99610b2e4499ab2cf0591d25958159dc23ca0d09 Mon Sep 17 00:00:00 2001 From: Simon Laux Date: Sun, 30 Nov 2025 07:07:48 +0100 Subject: [PATCH 25/41] test receive normal message --- src/tests/pre_messages.rs | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/src/tests/pre_messages.rs b/src/tests/pre_messages.rs index 890b8dddfb..f202a91ef2 100644 --- a/src/tests/pre_messages.rs +++ b/src/tests/pre_messages.rs @@ -549,5 +549,31 @@ mod receiving { Ok(()) } - // TODO: process normal message (neither full nor pre message) + /// Process normal message with file attachment (neither full nor pre message) + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn test_receive_normal_message() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + let alice_group_id = alice.create_group_with_members("test group", &[bob]).await; + + let mut msg = Message::new(Viewtype::File); + msg.set_file_from_bytes( + alice, + "test.bin", + &vec![0u8; (PRE_MSG_ATTACHMENT_SIZE_THRESHOLD - 10_000) as usize], + None, + )?; + msg.set_text("test".to_owned()); + let msg_id = chat::send_msg(alice, alice_group_id, &mut msg).await?; + + let smtp_rows = alice.get_smtp_rows_for_msg(msg_id).await; + assert_eq!(smtp_rows.len(), 1); + let message = smtp_rows.first().expect("message exists"); + + let msg = bob.recv_msg(message).await; + assert_eq!(msg.download_state(), DownloadState::Done); + assert_eq!(msg.viewtype, Viewtype::File); + Ok(()) + } } From 15fa49bb890695aa2bc1456b52bea95cb9fdd60e Mon Sep 17 00:00:00 2001 From: Simon Laux Date: Sun, 30 Nov 2025 07:34:00 +0100 Subject: [PATCH 26/41] some serialization tests for PreMsgMetadata --- src/download/pre_msg_metadata.rs | 97 +++++++++++++++++++++++++++++++- 1 file changed, 94 insertions(+), 3 deletions(-) diff --git a/src/download/pre_msg_metadata.rs b/src/download/pre_msg_metadata.rs index d9924b09bb..0903ce0c26 100644 --- a/src/download/pre_msg_metadata.rs +++ b/src/download/pre_msg_metadata.rs @@ -18,8 +18,10 @@ pub struct PreMsgMetadata { /// the original file name pub(crate) filename: String, /// Dimensions: width and height of image or video + #[serde(skip_serializing_if = "Option::is_none")] pub(crate) dimensions: Option<(i32, i32)>, /// Duration of audio file or video in milliseconds + #[serde(skip_serializing_if = "Option::is_none")] pub(crate) duration: Option, } @@ -100,7 +102,96 @@ impl Params { #[cfg(test)] mod tests { - // todo build from message (different types: file, image, audio) - // todo create artifically and serialize to header - // todo deserialize from header + use anyhow::Result; + use pretty_assertions::assert_eq; + + use crate::message::Viewtype; + + use super::PreMsgMetadata; + + // TODO build from message (different types: file, image, audio) + + /// Test that serialisation results in expected format + #[test] + fn serialize_to_header() -> Result<()> { + assert_eq!( + PreMsgMetadata { + size: 1_000_000, + viewtype: Viewtype::File, + filename: "test.bin".to_string(), + dimensions: None, + duration: None, + } + .to_header_value()?, + "{\"size\":1000000,\"viewtype\":\"File\",\"filename\":\"test.bin\"}" + ); + assert_eq!( + PreMsgMetadata { + size: 5_342_765, + viewtype: Viewtype::Image, + filename: "vacation.png".to_string(), + dimensions: Some((1080, 1920)), + duration: None, + } + .to_header_value()?, + "{\"size\":5342765,\"viewtype\":\"Image\",\"filename\":\"vacation.png\",\"dimensions\":[1080,1920]}" + ); + assert_eq!( + PreMsgMetadata { + size: 5_000, + viewtype: Viewtype::Audio, + filename: "audio-DD-MM-YY.ogg".to_string(), + dimensions: None, + duration: Some(152_310), + } + .to_header_value()?, + "{\"size\":5000,\"viewtype\":\"Audio\",\"filename\":\"audio-DD-MM-YY.ogg\",\"duration\":152310}" + ); + + Ok(()) + } + + /// Test that deserialisation from expected format works + /// This test will become important for compatibility between versions in the future + #[test] + fn deserialize_from_header() -> Result<()> { + assert_eq!( + serde_json::from_str::( + "{\"size\":1000000,\"viewtype\":\"File\",\"filename\":\"test.bin\",\"dimensions\":null,\"duration\":null}" + )?, + PreMsgMetadata { + size: 1_000_000, + viewtype: Viewtype::File, + filename: "test.bin".to_string(), + dimensions: None, + duration: None, + } + ); + assert_eq!( + serde_json::from_str::( + "{\"size\":5342765,\"viewtype\":\"Image\",\"filename\":\"vacation.png\",\"dimensions\":[1080,1920]}" + )?, + PreMsgMetadata { + size: 5_342_765, + viewtype: Viewtype::Image, + filename: "vacation.png".to_string(), + dimensions: Some((1080, 1920)), + duration: None, + } + ); + assert_eq!( + serde_json::from_str::( + "{\"size\":5000,\"viewtype\":\"Audio\",\"filename\":\"audio-DD-MM-YY.ogg\",\"duration\":152310}" + )?, + PreMsgMetadata { + size: 5_000, + viewtype: Viewtype::Audio, + filename: "audio-DD-MM-YY.ogg".to_string(), + dimensions: None, + duration: Some(152_310), + } + ); + + Ok(()) + } } From 6042d171666132a4203827a4217796b93e355999 Mon Sep 17 00:00:00 2001 From: Simon Laux Date: Sun, 30 Nov 2025 07:54:45 +0100 Subject: [PATCH 27/41] remove outdated todo comment --- src/message.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/message.rs b/src/message.rs index f970041b4b..cc41c009ea 100644 --- a/src/message.rs +++ b/src/message.rs @@ -799,7 +799,6 @@ impl Message { return Ok(Some(file_size)); } } - // TODO: also modify update docs in all places that use this (cffi and jsonrpc, possibly also in python) if let Some(path) = self.param.get_file_path(context)? { Ok(Some(get_filebytes(context, &path).await.with_context( || format!("failed to get {} size in bytes", path.display()), From 9cff662377ed5fb408df007d6a0826f3621c6b64 Mon Sep 17 00:00:00 2001 From: Simon Laux Date: Sun, 30 Nov 2025 07:59:57 +0100 Subject: [PATCH 28/41] test that pre-message contains message text --- src/tests/pre_messages.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/tests/pre_messages.rs b/src/tests/pre_messages.rs index f202a91ef2..f5fa927ace 100644 --- a/src/tests/pre_messages.rs +++ b/src/tests/pre_messages.rs @@ -495,6 +495,7 @@ mod receiving { assert_eq!(msg.download_state(), DownloadState::Available); assert_eq!(msg.viewtype, Viewtype::Text); + assert_eq!(msg.text, "test".to_owned()); // test that metadata is correctly returned by methods assert_eq!(msg.get_filebytes(bob).await?, Some(1_000_000)); @@ -521,12 +522,14 @@ mod receiving { assert_eq!(msg.viewtype, Viewtype::Text); assert!(msg.param.exists(Param::FullMessageViewtype)); assert!(msg.param.exists(Param::FullMessageFileBytes)); + assert_eq!(msg.text, "test".to_owned()); let _ = bob.recv_msg_trash(&full_message).await; let msg = Message::load_from_db(bob, msg.id).await?; assert_eq!(msg.download_state(), DownloadState::Done); assert_eq!(msg.viewtype, Viewtype::File); assert_eq!(msg.param.exists(Param::FullMessageViewtype), false); assert_eq!(msg.param.exists(Param::FullMessageFileBytes), false); + assert_eq!(msg.text, "test".to_owned()); Ok(()) } @@ -574,6 +577,7 @@ mod receiving { let msg = bob.recv_msg(message).await; assert_eq!(msg.download_state(), DownloadState::Done); assert_eq!(msg.viewtype, Viewtype::File); + assert_eq!(msg.text, "test".to_owned()); Ok(()) } } From 1ae1a3413de276d59aa30a39e40570949ba06ffe Mon Sep 17 00:00:00 2001 From: Simon Laux Date: Sun, 30 Nov 2025 08:42:36 +0100 Subject: [PATCH 29/41] PreMsgMetadata: test_build_from_file_msg and test_build_from_file_msg --- src/download/pre_msg_metadata.rs | 59 +++++++++++++++++++++++++++++--- src/test_utils.rs | 15 ++++++++ 2 files changed, 70 insertions(+), 4 deletions(-) diff --git a/src/download/pre_msg_metadata.rs b/src/download/pre_msg_metadata.rs index 0903ce0c26..db25def5fd 100644 --- a/src/download/pre_msg_metadata.rs +++ b/src/download/pre_msg_metadata.rs @@ -105,15 +105,66 @@ mod tests { use anyhow::Result; use pretty_assertions::assert_eq; - use crate::message::Viewtype; + use crate::{ + message::{Message, Viewtype}, + test_utils::{TestContextManager, create_test_image}, + }; use super::PreMsgMetadata; - // TODO build from message (different types: file, image, audio) + /// Build from message with file attachment + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn test_build_from_file_msg() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + + let mut file_msg = Message::new(Viewtype::File); + file_msg.set_file_from_bytes(alice, "test.bin", &vec![0u8; 1_000_000], None)?; + let pre_mesage_metadata = PreMsgMetadata::from_msg(alice, &file_msg).await?; + assert_eq!( + pre_mesage_metadata, + Some(PreMsgMetadata { + size: 1_000_000, + viewtype: Viewtype::File, + filename: "test.bin".to_string(), + dimensions: None, + duration: None, + }) + ); + Ok(()) + } + + /// Build from message with image attachment + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn test_build_from_image_msg() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let mut image_msg = Message::new(Viewtype::Image); + + let (width, height) = (1080, 1920); + let test_img = create_test_image(width, height)?; + image_msg.set_file_from_bytes(alice, "vacation.png", &test_img, None)?; + // this is usually done while sending, + // but we don't send it here, so we need to call it ourself + image_msg.try_calc_and_set_dimensions(alice).await?; + let pre_mesage_metadata = PreMsgMetadata::from_msg(alice, &image_msg).await?; + assert_eq!( + pre_mesage_metadata, + Some(PreMsgMetadata { + size: 1816098, + viewtype: Viewtype::Image, + filename: "vacation.png".to_string(), + dimensions: Some((width as i32, height as i32)), + duration: None, + }) + ); + + Ok(()) + } /// Test that serialisation results in expected format #[test] - fn serialize_to_header() -> Result<()> { + fn test_serialize_to_header() -> Result<()> { assert_eq!( PreMsgMetadata { size: 1_000_000, @@ -154,7 +205,7 @@ mod tests { /// Test that deserialisation from expected format works /// This test will become important for compatibility between versions in the future #[test] - fn deserialize_from_header() -> Result<()> { + fn test_deserialize_from_header() -> Result<()> { assert_eq!( serde_json::from_str::( "{\"size\":1000000,\"viewtype\":\"File\",\"filename\":\"test.bin\",\"dimensions\":null,\"duration\":null}" diff --git a/src/test_utils.rs b/src/test_utils.rs index f397ad1d5a..ebe35ba926 100644 --- a/src/test_utils.rs +++ b/src/test_utils.rs @@ -1705,6 +1705,21 @@ Until the false-positive is fixed: } } +/// Method to create a test image file +pub(crate) fn create_test_image(width: u32, height: u32) -> anyhow::Result> { + use image::{ImageBuffer, Rgb, RgbImage}; + use std::io::Cursor; + + let mut img: RgbImage = ImageBuffer::new(width, height); + // fill with some pattern so it stays large after compression + for (x, y, pixel) in img.enumerate_pixels_mut() { + *pixel = Rgb([(x % 255) as u8, (x + y % 255) as u8, (y % 255) as u8]); + } + let mut bytes: Vec = Vec::new(); + img.write_to(&mut Cursor::new(&mut bytes), image::ImageFormat::Png)?; + Ok(bytes) +} + #[cfg(test)] mod tests { use super::*; From b5629e933a9d47d26554f9070b8db23651ac6562 Mon Sep 17 00:00:00 2001 From: Simon Laux Date: Sun, 30 Nov 2025 09:32:53 +0100 Subject: [PATCH 30/41] test: test_receive_pre_message_image --- src/tests/pre_messages.rs | 57 ++++++++++++++++++++++++++++++--------- 1 file changed, 44 insertions(+), 13 deletions(-) diff --git a/src/tests/pre_messages.rs b/src/tests/pre_messages.rs index f5fa927ace..819c65eceb 100644 --- a/src/tests/pre_messages.rs +++ b/src/tests/pre_messages.rs @@ -413,20 +413,16 @@ mod receiving { use crate::message::{Message, Viewtype}; use crate::mimeparser::MimeMessage; use crate::param::Param; - use crate::test_utils::SentMessage; + use crate::test_utils::{SentMessage, create_test_image}; async fn send_large_file_message<'a>( sender: &'a TestContext, target_chat: ChatId, - attachment_size: u64, + view_type: Viewtype, + content: &[u8], ) -> Result<(SentMessage<'a>, SentMessage<'a>)> { - let mut msg = Message::new(Viewtype::File); - msg.set_file_from_bytes( - sender, - "test.bin", - &vec![0u8; attachment_size as usize], - None, - )?; + let mut msg = Message::new(view_type); + msg.set_file_from_bytes(sender, "test.bin", content, None)?; msg.set_text("test".to_owned()); // assert that test attachment is bigger than limit @@ -450,7 +446,8 @@ mod receiving { let alice_group_id = alice.create_group_with_members("test group", &[bob]).await; let (pre_message, full_message) = - send_large_file_message(alice, alice_group_id, 1_000_000).await?; + send_large_file_message(alice, alice_group_id, Viewtype::File, &vec![0u8; 1_000_000]) + .await?; let parsed_pre_message = MimeMessage::from_bytes(bob, pre_message.payload.as_bytes()).await?; @@ -489,7 +486,8 @@ mod receiving { let alice_group_id = alice.create_group_with_members("test group", &[bob]).await; let (pre_message, _full_message) = - send_large_file_message(alice, alice_group_id, 1_000_000).await?; + send_large_file_message(alice, alice_group_id, Viewtype::File, &vec![0u8; 1_000_000]) + .await?; let msg = bob.recv_msg(&pre_message).await; @@ -515,7 +513,8 @@ mod receiving { let alice_group_id = alice.create_group_with_members("test group", &[bob]).await; let (pre_message, full_message) = - send_large_file_message(alice, alice_group_id, 1_000_000).await?; + send_large_file_message(alice, alice_group_id, Viewtype::File, &vec![0u8; 1_000_000]) + .await?; let msg = bob.recv_msg(&pre_message).await; assert_eq!(msg.download_state(), DownloadState::Available); @@ -543,7 +542,8 @@ mod receiving { let alice_group_id = alice.create_group_with_members("test group", &[bob]).await; let (pre_message, full_message) = - send_large_file_message(alice, alice_group_id, 1_000_000).await?; + send_large_file_message(alice, alice_group_id, Viewtype::File, &vec![0u8; 1_000_000]) + .await?; let msg = bob.recv_msg(&full_message).await; assert_eq!(msg.download_state(), DownloadState::Done); @@ -580,4 +580,35 @@ mod receiving { assert_eq!(msg.text, "test".to_owned()); Ok(()) } + + /// Test receiving pre-messages and creation of the placeholder message with the metadata + /// for image attachment + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn test_receive_pre_message_image() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + let alice_group_id = alice.create_group_with_members("test group", &[bob]).await; + + let (width, height) = (1080, 1920); + let test_img = create_test_image(width, height)?; + + let (pre_message, _full_message) = + send_large_file_message(alice, alice_group_id, Viewtype::Image, &test_img).await?; + + let msg = bob.recv_msg(&pre_message).await; + + assert_eq!(msg.download_state(), DownloadState::Available); + assert_eq!(msg.viewtype, Viewtype::Text); + assert_eq!(msg.text, "test".to_owned()); + + // test that metadata is correctly returned by methods + assert_eq!(msg.get_full_message_viewtype(), Some(Viewtype::Image)); + // recoded image dimensions + assert_eq!(msg.get_filebytes(bob).await?, Some(149632)); + assert_eq!(msg.get_height(), 1280); + assert_eq!(msg.get_width(), 720); + + Ok(()) + } } From 156d5b8116e7f75d5a651e7b82c5e0a4974ce4f9 Mon Sep 17 00:00:00 2001 From: Simon Laux Date: Sun, 30 Nov 2025 11:26:13 +0100 Subject: [PATCH 31/41] Test receiving the full message after receiving an edit after receiving the pre-message --- src/tests/pre_messages.rs | 47 ++++++++++++++++++++++++++++++++------- 1 file changed, 39 insertions(+), 8 deletions(-) diff --git a/src/tests/pre_messages.rs b/src/tests/pre_messages.rs index 819c65eceb..d2996fce54 100644 --- a/src/tests/pre_messages.rs +++ b/src/tests/pre_messages.rs @@ -410,7 +410,7 @@ mod receiving { use crate::chat::{self, ChatId}; use crate::download::PRE_MSG_ATTACHMENT_SIZE_THRESHOLD; use crate::download::pre_msg_metadata::PreMsgMetadata; - use crate::message::{Message, Viewtype}; + use crate::message::{Message, MsgId, Viewtype}; use crate::mimeparser::MimeMessage; use crate::param::Param; use crate::test_utils::{SentMessage, create_test_image}; @@ -420,7 +420,7 @@ mod receiving { target_chat: ChatId, view_type: Viewtype, content: &[u8], - ) -> Result<(SentMessage<'a>, SentMessage<'a>)> { + ) -> Result<(SentMessage<'a>, SentMessage<'a>, MsgId)> { let mut msg = Message::new(view_type); msg.set_file_from_bytes(sender, "test.bin", content, None)?; msg.set_text("test".to_owned()); @@ -434,7 +434,7 @@ mod receiving { assert_eq!(smtp_rows.len(), 2); let pre_message = smtp_rows.first().expect("pre-message exists"); let full_message = smtp_rows.get(1).expect("full message exists"); - Ok((pre_message.to_owned(), full_message.to_owned())) + Ok((pre_message.to_owned(), full_message.to_owned(), msg_id)) } /// Test that mimeparser can correctly detect and parse pre-messages and full-messages @@ -445,7 +445,7 @@ mod receiving { let bob = &tcm.bob().await; let alice_group_id = alice.create_group_with_members("test group", &[bob]).await; - let (pre_message, full_message) = + let (pre_message, full_message, _alice_msg_id) = send_large_file_message(alice, alice_group_id, Viewtype::File, &vec![0u8; 1_000_000]) .await?; @@ -485,7 +485,7 @@ mod receiving { let bob = &tcm.bob().await; let alice_group_id = alice.create_group_with_members("test group", &[bob]).await; - let (pre_message, _full_message) = + let (pre_message, _full_message, _alice_msg_id) = send_large_file_message(alice, alice_group_id, Viewtype::File, &vec![0u8; 1_000_000]) .await?; @@ -512,7 +512,7 @@ mod receiving { let bob = &tcm.bob().await; let alice_group_id = alice.create_group_with_members("test group", &[bob]).await; - let (pre_message, full_message) = + let (pre_message, full_message, _alice_msg_id) = send_large_file_message(alice, alice_group_id, Viewtype::File, &vec![0u8; 1_000_000]) .await?; @@ -541,7 +541,7 @@ mod receiving { let bob = &tcm.bob().await; let alice_group_id = alice.create_group_with_members("test group", &[bob]).await; - let (pre_message, full_message) = + let (pre_message, full_message, _alice_msg_id) = send_large_file_message(alice, alice_group_id, Viewtype::File, &vec![0u8; 1_000_000]) .await?; @@ -552,6 +552,37 @@ mod receiving { Ok(()) } + /// Test receiving the full message after receiving an edit after receiving the pre-message + /// for file attachment + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn test_receive_pre_message_then_edit_and_then_dl_full_message() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + let alice_group_id = alice.create_group_with_members("test group", &[bob]).await; + + let (pre_message, full_message, alice_msg_id) = + send_large_file_message(alice, alice_group_id, Viewtype::File, &vec![0u8; 1_000_000]) + .await?; + + chat::send_edit_request(alice, alice_msg_id, "new_text".to_owned()).await?; + let edit_request = alice.pop_sent_msg().await; + + let msg = bob.recv_msg(&pre_message).await; + assert_eq!(msg.download_state(), DownloadState::Available); + assert_eq!(msg.text, "test".to_owned()); + let _ = bob.recv_msg_trash(&edit_request).await; + let msg = Message::load_from_db(bob, msg.id).await?; + assert_eq!(msg.download_state(), DownloadState::Available); + assert_eq!(msg.text, "new_text".to_owned()); + let _ = bob.recv_msg_trash(&full_message).await; + let msg = Message::load_from_db(bob, msg.id).await?; + assert_eq!(msg.download_state(), DownloadState::Done); + assert_eq!(msg.viewtype, Viewtype::File); + assert_eq!(msg.text, "new_text".to_owned()); + Ok(()) + } + /// Process normal message with file attachment (neither full nor pre message) #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_receive_normal_message() -> Result<()> { @@ -593,7 +624,7 @@ mod receiving { let (width, height) = (1080, 1920); let test_img = create_test_image(width, height)?; - let (pre_message, _full_message) = + let (pre_message, _full_message, _alice_msg_id) = send_large_file_message(alice, alice_group_id, Viewtype::Image, &test_img).await?; let msg = bob.recv_msg(&pre_message).await; From 957e967601fa84ea58f45bb8a46ebe540b5e08a2 Mon Sep 17 00:00:00 2001 From: Simon Laux Date: Sun, 30 Nov 2025 12:50:46 +0100 Subject: [PATCH 32/41] test_reaction_on_pre_message --- src/tests/pre_messages.rs | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/src/tests/pre_messages.rs b/src/tests/pre_messages.rs index d2996fce54..9e1d30dd48 100644 --- a/src/tests/pre_messages.rs +++ b/src/tests/pre_messages.rs @@ -413,6 +413,7 @@ mod receiving { use crate::message::{Message, MsgId, Viewtype}; use crate::mimeparser::MimeMessage; use crate::param::Param; + use crate::reaction::{get_msg_reactions, send_reaction}; use crate::test_utils::{SentMessage, create_test_image}; async fn send_large_file_message<'a>( @@ -642,4 +643,42 @@ mod receiving { Ok(()) } + + /// Test receiving reaction on pre-message + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn test_reaction_on_pre_message() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + let alice_group_id = alice.create_group_with_members("test group", &[bob]).await; + + let (pre_message, full_message, alice_msg_id) = + send_large_file_message(alice, alice_group_id, Viewtype::File, &vec![0u8; 1_000_000]) + .await?; + + // Bob receives pre-message + let bob_msg = bob.recv_msg(&pre_message).await; + assert_eq!(bob_msg.download_state(), DownloadState::Available); + + // Alice sends reaction to her own message + send_reaction(alice, alice_msg_id, "👍").await?; + + // Bob receives the reaction + bob.recv_msg_hidden(&alice.pop_sent_msg().await).await; + + // Test if Bob sees reaction + let reactions = get_msg_reactions(bob, bob_msg.id).await?; + assert_eq!(reactions.to_string(), "👍1"); + + // Bob downloads full message + bob.recv_msg_trash(&full_message).await; + let msg = Message::load_from_db(bob, bob_msg.id).await?; + assert_eq!(msg.download_state(), DownloadState::Done); + + // Test if Bob still sees reaction + let reactions = get_msg_reactions(bob, bob_msg.id).await?; + assert_eq!(reactions.to_string(), "👍1"); + + Ok(()) + } } From 4bbe0c167cb091454a94167d4ff4db112209cafb Mon Sep 17 00:00:00 2001 From: Simon Laux Date: Sun, 30 Nov 2025 13:50:33 +0100 Subject: [PATCH 33/41] test_full_download_after_trashed --- src/tests/pre_messages.rs | 34 +++++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/src/tests/pre_messages.rs b/src/tests/pre_messages.rs index 9e1d30dd48..1a686cc04e 100644 --- a/src/tests/pre_messages.rs +++ b/src/tests/pre_messages.rs @@ -410,7 +410,7 @@ mod receiving { use crate::chat::{self, ChatId}; use crate::download::PRE_MSG_ATTACHMENT_SIZE_THRESHOLD; use crate::download::pre_msg_metadata::PreMsgMetadata; - use crate::message::{Message, MsgId, Viewtype}; + use crate::message::{Message, MsgId, Viewtype, delete_msgs}; use crate::mimeparser::MimeMessage; use crate::param::Param; use crate::reaction::{get_msg_reactions, send_reaction}; @@ -681,4 +681,36 @@ mod receiving { Ok(()) } + + /// Tests that fully downloading the message + /// works but does not reappear when it was already deleted + /// (as in the Message-ID already exists in the database + /// and is assigned to the trash chat). + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn test_full_download_after_trashed() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + let bob_group_id = bob.create_group_with_members("test group", &[alice]).await; + + let (pre_message, full_message, _bob_msg_id) = + send_large_file_message(bob, bob_group_id, Viewtype::File, &vec![0u8; 1_000_000]) + .await?; + + // Download message from Bob partially. + let alice_msg = alice.recv_msg(&pre_message).await; + + // Delete the received message. + // Note that it remains in the database in the trash chat. + delete_msgs(alice, &[alice_msg.id]).await?; + + // Fully download message after deletion. + alice.recv_msg_trash(&full_message).await; + + // The message does not reappear. + let msg = Message::load_from_db_optional(bob, alice_msg.id).await?; + assert!(msg.is_none()); + + Ok(()) + } } From 59ca831bba1ea9be480a708c1fa4caac3b6e6921 Mon Sep 17 00:00:00 2001 From: Simon Laux Date: Sun, 30 Nov 2025 18:45:43 +0100 Subject: [PATCH 34/41] test_webxdc_update_for_not_downloaded_instance --- src/tests/pre_messages.rs | 75 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 74 insertions(+), 1 deletion(-) diff --git a/src/tests/pre_messages.rs b/src/tests/pre_messages.rs index 1a686cc04e..3039c088d7 100644 --- a/src/tests/pre_messages.rs +++ b/src/tests/pre_messages.rs @@ -405,7 +405,11 @@ mod sending { /// Tests about receiving pre-messages and full messages mod receiving { use super::*; + use async_zip::tokio::write::ZipFileWriter; + use async_zip::{Compression, ZipEntryBuilder}; + use futures::io::Cursor as FuturesCursor; use pretty_assertions::assert_eq; + use tokio_util::compat::FuturesAsyncWriteCompatExt; use crate::chat::{self, ChatId}; use crate::download::PRE_MSG_ATTACHMENT_SIZE_THRESHOLD; @@ -415,6 +419,7 @@ mod receiving { use crate::param::Param; use crate::reaction::{get_msg_reactions, send_reaction}; use crate::test_utils::{SentMessage, create_test_image}; + use crate::webxdc::StatusUpdateSerial; async fn send_large_file_message<'a>( sender: &'a TestContext, @@ -423,7 +428,12 @@ mod receiving { content: &[u8], ) -> Result<(SentMessage<'a>, SentMessage<'a>, MsgId)> { let mut msg = Message::new(view_type); - msg.set_file_from_bytes(sender, "test.bin", content, None)?; + let file_name = if view_type == Viewtype::Webxdc { + "test.xdc" + } else { + "test.bin" + }; + msg.set_file_from_bytes(sender, file_name, content, None)?; msg.set_text("test".to_owned()); // assert that test attachment is bigger than limit @@ -713,4 +723,67 @@ mod receiving { Ok(()) } + + /// Test that webxdc updates are received for pre-messages + /// and available when the full-message is downloaded + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn test_webxdc_update_for_not_downloaded_instance() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + let alice_group_id = alice.create_group_with_members("test group", &[bob]).await; + + let futures_cursor = FuturesCursor::new(Vec::new()); + let mut buffer = futures_cursor.compat_write(); + let mut writer = ZipFileWriter::with_tokio(&mut buffer); + writer + .write_entry_whole( + ZipEntryBuilder::new("padding.bin".into(), Compression::Stored), + &[0u8; 1_000_000], + ) + .await?; + writer + .write_entry_whole( + ZipEntryBuilder::new("index.html".into(), Compression::Stored), + &[0u8; 100], + ) + .await?; + writer.close().await?; + let big_webxdc_app = buffer.into_inner().into_inner(); + + // Alice sends a larger instance and an update + let (pre_message, full_message, alice_sent_instance_msg_id) = + send_large_file_message(alice, alice_group_id, Viewtype::Webxdc, &big_webxdc_app) + .await?; + alice + .send_webxdc_status_update( + alice_sent_instance_msg_id, + r#"{"payload": 7, "summary":"sum", "document":"doc"}"#, + ) + .await?; + alice.flush_status_updates().await?; + let webxdc_update = alice.pop_sent_msg().await; + + // Bob does not download instance but already receives update + let bob_instance = bob.recv_msg(&pre_message).await; + assert_eq!(bob_instance.download_state, DownloadState::Available); + bob.recv_msg_trash(&webxdc_update).await; + + // Bob downloads instance, updates should be assigned correctly + bob.recv_msg_trash(&full_message).await; + + let bob_instance = bob.get_last_msg().await; + assert_eq!(bob_instance.viewtype, Viewtype::Webxdc); + assert_eq!(bob_instance.download_state, DownloadState::Done); + assert_eq!( + bob.get_webxdc_status_updates(bob_instance.id, StatusUpdateSerial::new(0)) + .await?, + r#"[{"payload":7,"document":"doc","summary":"sum","serial":1,"max_serial":1}]"# + ); + let info = bob_instance.get_webxdc_info(bob).await?; + assert_eq!(info.document, "doc"); + assert_eq!(info.summary, "sum"); + + Ok(()) + } } From 477e2ee1749af342d3b5e9b234a7642fbfc376aa Mon Sep 17 00:00:00 2001 From: Simon Laux Date: Sun, 30 Nov 2025 18:46:59 +0100 Subject: [PATCH 35/41] simplify fake webxdc generation in test_webxdc_update_for_not_downloaded_instance --- src/tests/pre_messages.rs | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/src/tests/pre_messages.rs b/src/tests/pre_messages.rs index 3039c088d7..12a14597fb 100644 --- a/src/tests/pre_messages.rs +++ b/src/tests/pre_messages.rs @@ -736,16 +736,10 @@ mod receiving { let futures_cursor = FuturesCursor::new(Vec::new()); let mut buffer = futures_cursor.compat_write(); let mut writer = ZipFileWriter::with_tokio(&mut buffer); - writer - .write_entry_whole( - ZipEntryBuilder::new("padding.bin".into(), Compression::Stored), - &[0u8; 1_000_000], - ) - .await?; writer .write_entry_whole( ZipEntryBuilder::new("index.html".into(), Compression::Stored), - &[0u8; 100], + &[0u8; 1_000_000], ) .await?; writer.close().await?; From 5e499d8ace1185a8f80dbf114a9ba34dc1600441 Mon Sep 17 00:00:00 2001 From: Simon Laux Date: Sun, 30 Nov 2025 19:18:04 +0100 Subject: [PATCH 36/41] test_markseen_pre_msg --- src/tests/pre_messages.rs | 47 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) diff --git a/src/tests/pre_messages.rs b/src/tests/pre_messages.rs index 12a14597fb..c01193a4f7 100644 --- a/src/tests/pre_messages.rs +++ b/src/tests/pre_messages.rs @@ -414,7 +414,7 @@ mod receiving { use crate::chat::{self, ChatId}; use crate::download::PRE_MSG_ATTACHMENT_SIZE_THRESHOLD; use crate::download::pre_msg_metadata::PreMsgMetadata; - use crate::message::{Message, MsgId, Viewtype, delete_msgs}; + use crate::message::{Message, MessageState, MsgId, Viewtype, delete_msgs, markseen_msgs}; use crate::mimeparser::MimeMessage; use crate::param::Param; use crate::reaction::{get_msg_reactions, send_reaction}; @@ -780,4 +780,49 @@ mod receiving { Ok(()) } + + /// Test mark seen pre-message + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn test_markseen_pre_msg() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + let bob_chat_id = bob.create_chat(alice).await.id; + alice.create_chat(bob).await; // Make sure the chat is accepted. + + tcm.section("Bob sends a large message to Alice"); + let (pre_message, full_message, _bob_msg_id) = + send_large_file_message(bob, bob_chat_id, Viewtype::File, &vec![0u8; 1_000_000]) + .await?; + + tcm.section("Alice receives a pre-message message from Bob"); + let msg = alice.recv_msg(&pre_message).await; + assert_eq!(msg.download_state, DownloadState::Available); + assert!(msg.param.get_bool(Param::WantsMdn).unwrap_or_default()); + assert_eq!(msg.state, MessageState::InFresh); + + tcm.section("Alice marks the pre-message as read and sends a MDN"); + markseen_msgs(alice, vec![msg.id]).await?; + assert_eq!(msg.id.get_state(alice).await?, MessageState::InSeen); + assert_eq!( + alice + .sql + .count("SELECT COUNT(*) FROM smtp_mdns", ()) + .await?, + 1 + ); + + tcm.section("Alice downloads message"); + alice.recv_msg_trash(&full_message).await; + let msg = Message::load_from_db(alice, msg.id).await?; + assert_eq!(msg.download_state, DownloadState::Done); + assert!(msg.param.get_bool(Param::WantsMdn).unwrap_or_default()); + assert_eq!( + msg.state, + MessageState::InSeen, + "The message state mustn't be downgraded to `InFresh`" + ); + + Ok(()) + } } From 9803b30f359ed420e163bf3b7bec7e1acfe8750c Mon Sep 17 00:00:00 2001 From: Simon Laux Date: Sun, 30 Nov 2025 20:05:20 +0100 Subject: [PATCH 37/41] test_pre_msg_can_start_chat and test_full_msg_can_start_chat --- src/tests/pre_messages.rs | 63 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/src/tests/pre_messages.rs b/src/tests/pre_messages.rs index c01193a4f7..6da4e58125 100644 --- a/src/tests/pre_messages.rs +++ b/src/tests/pre_messages.rs @@ -412,6 +412,7 @@ mod receiving { use tokio_util::compat::FuturesAsyncWriteCompatExt; use crate::chat::{self, ChatId}; + use crate::contact::{self}; use crate::download::PRE_MSG_ATTACHMENT_SIZE_THRESHOLD; use crate::download::pre_msg_metadata::PreMsgMetadata; use crate::message::{Message, MessageState, MsgId, Viewtype, delete_msgs, markseen_msgs}; @@ -825,4 +826,66 @@ mod receiving { Ok(()) } + + /// Test that pre-message can start a chat + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn test_pre_msg_can_start_chat() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + + tcm.section("establishing a DM chat between alice and bob"); + let bob_alice_dm_chat_id = bob.create_chat(alice).await.id; + alice.create_chat(bob).await; // Make sure the chat is accepted. + + tcm.section("Alice prepares chat"); + let chat_id = chat::create_group(alice, "my group").await?; + let contacts = contact::Contact::get_all(alice, 0, None).await?; + let alice_bob_id = contacts.first().expect("contact exists"); + chat::add_contact_to_chat(alice, chat_id, *alice_bob_id).await?; + + tcm.section("Alice sends large message to promote/start chat"); + let (pre_message, _full_message, _alice_msg_id) = + send_large_file_message(alice, chat_id, Viewtype::File, &vec![0u8; 1_000_000]).await?; + + tcm.section("Bob receives the pre-message message from Alice"); + let msg = bob.recv_msg(&pre_message).await; + assert_eq!(msg.download_state, DownloadState::Available); + assert_ne!(msg.chat_id, bob_alice_dm_chat_id); + let chat = chat::Chat::load_from_db(bob, msg.chat_id).await?; + assert_eq!(chat.name, "my group"); + + Ok(()) + } + + /// Test that full-message can start a chat + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn test_full_msg_can_start_chat() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + + tcm.section("establishing a DM chat between alice and bob"); + let bob_alice_dm_chat_id = bob.create_chat(alice).await.id; + alice.create_chat(bob).await; // Make sure the chat is accepted. + + tcm.section("Alice prepares chat"); + let chat_id = chat::create_group(alice, "my group").await?; + let contacts = contact::Contact::get_all(alice, 0, None).await?; + let alice_bob_id = contacts.first().expect("contact exists"); + chat::add_contact_to_chat(alice, chat_id, *alice_bob_id).await?; + + tcm.section("Alice sends large message to promote/start chat"); + let (_pre_message, full_message, _bob_msg_id) = + send_large_file_message(alice, chat_id, Viewtype::File, &vec![0u8; 1_000_000]).await?; + + tcm.section("Bob receives the pre-message message from Alice"); + let msg = bob.recv_msg(&full_message).await; + assert_eq!(msg.download_state, DownloadState::Done); + assert_ne!(msg.chat_id, bob_alice_dm_chat_id); + let chat = chat::Chat::load_from_db(bob, msg.chat_id).await?; + assert_eq!(chat.name, "my group"); + + Ok(()) + } } From 0934dafda21cb1cd1fb0024ac1af17fad0af3eed Mon Sep 17 00:00:00 2001 From: Simon Laux Date: Sun, 30 Nov 2025 20:32:28 +0100 Subject: [PATCH 38/41] test_download_later_keeps_message_order --- src/tests/pre_messages.rs | 40 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/src/tests/pre_messages.rs b/src/tests/pre_messages.rs index 6da4e58125..df23196548 100644 --- a/src/tests/pre_messages.rs +++ b/src/tests/pre_messages.rs @@ -888,4 +888,44 @@ mod receiving { Ok(()) } + + /// Test that message ordering is still correct after downloading + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn test_download_later_keeps_message_order() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + + tcm.section( + "establishing a DM chat between alice and bob and bob sends large message to alice", + ); + let bob_alice_dm_chat = bob.create_chat(alice).await.id; + alice.create_chat(bob).await; // Make sure the chat is accepted. + let (pre_message, full_message, _bob_msg_id) = send_large_file_message( + bob, + bob_alice_dm_chat, + Viewtype::File, + &vec![0u8; 1_000_000], + ) + .await?; + + tcm.section("Alice downloads pre-message"); + let msg = alice.recv_msg(&pre_message).await; + assert_eq!(msg.download_state, DownloadState::Available); + assert_eq!(msg.state, MessageState::InFresh); + assert_eq!(alice.get_last_msg_in(msg.chat_id).await.id, msg.id); + + tcm.section("Bob sends hi to Alice"); + let hi_msg = tcm.send_recv(bob, alice, "hi").await; + assert_eq!(alice.get_last_msg_in(msg.chat_id).await.id, hi_msg.id); + + tcm.section("Alice downloads full-message"); + alice.recv_msg_trash(&full_message).await; + let msg = Message::load_from_db(alice, msg.id).await?; + assert_eq!(msg.download_state, DownloadState::Done); + assert_eq!(alice.get_last_msg_in(msg.chat_id).await.id, hi_msg.id); + assert!(msg.timestamp_sort <= hi_msg.timestamp_sort); + + Ok(()) + } } From d70bfcdd57b7095171b4cae1a85ffed734440819 Mon Sep 17 00:00:00 2001 From: Simon Laux Date: Sun, 30 Nov 2025 20:40:22 +0100 Subject: [PATCH 39/41] test_chatlist_event_on_full_msg_download --- src/tests/pre_messages.rs | 44 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/src/tests/pre_messages.rs b/src/tests/pre_messages.rs index df23196548..af99e6659e 100644 --- a/src/tests/pre_messages.rs +++ b/src/tests/pre_messages.rs @@ -411,6 +411,7 @@ mod receiving { use pretty_assertions::assert_eq; use tokio_util::compat::FuturesAsyncWriteCompatExt; + use crate::EventType; use crate::chat::{self, ChatId}; use crate::contact::{self}; use crate::download::PRE_MSG_ATTACHMENT_SIZE_THRESHOLD; @@ -928,4 +929,47 @@ mod receiving { Ok(()) } + + /// Test that ChatlistItemChanged event is emitted when downloading full-message + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn test_chatlist_event_on_full_msg_download() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + + tcm.section( + "establishing a DM chat between alice and bob and bob sends large message to alice", + ); + let bob_alice_dm_chat = bob.create_chat(alice).await.id; + alice.create_chat(bob).await; // Make sure the chat is accepted. + let (pre_message, full_message, _bob_msg_id) = send_large_file_message( + bob, + bob_alice_dm_chat, + Viewtype::File, + &vec![0u8; 1_000_000], + ) + .await?; + + tcm.section("Alice downloads pre-message"); + let msg = alice.recv_msg(&pre_message).await; + assert_eq!(msg.download_state, DownloadState::Available); + assert_eq!(msg.state, MessageState::InFresh); + assert_eq!(alice.get_last_msg_in(msg.chat_id).await.id, msg.id); + + tcm.section("Alice downloads full-message and waits for ChatlistItemChanged event "); + alice.evtracker.clear_events(); + alice.recv_msg_trash(&full_message).await; + let msg = Message::load_from_db(alice, msg.id).await?; + assert_eq!(msg.download_state, DownloadState::Done); + alice + .evtracker + .get_matching(|e| { + e == &EventType::ChatlistItemChanged { + chat_id: Some(msg.chat_id), + } + }) + .await; + + Ok(()) + } } From ad429a89b4b5fbc3741603dcc2e0a298b561db43 Mon Sep 17 00:00:00 2001 From: Simon Laux Date: Tue, 2 Dec 2025 20:49:54 +0100 Subject: [PATCH 40/41] fix download not working --- src/download.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/download.rs b/src/download.rs index 7d5acb71a2..f94cf530de 100644 --- a/src/download.rs +++ b/src/download.rs @@ -88,7 +88,7 @@ impl MsgId { context .sql .execute( - "INSERT INTO download (msg_id) VALUES (?)", + "INSERT INTO download (rfc724_mid) VALUES (?)", (msg.rfc724_mid(),), ) .await?; From 5ebcc71fa649f5d02a4637f0a2e1050b464ff8c4 Mon Sep 17 00:00:00 2001 From: Simon Laux Date: Tue, 2 Dec 2025 21:56:28 +0100 Subject: [PATCH 41/41] log splitting into pre-message --- src/chat.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/chat.rs b/src/chat.rs index 280af205da..b856b32971 100644 --- a/src/chat.rs +++ b/src/chat.rs @@ -2755,6 +2755,11 @@ async fn render_mime_message_and_pre_message( > PRE_MSG_ATTACHMENT_SIZE_THRESHOLD; if needs_pre_message { + info!( + context, + "Message is large and will be split into a pre- and a post-message.", + ); + let mut mimefactory_full_msg = mimefactory.clone(); mimefactory_full_msg.set_as_full_message(); let rendered_msg = mimefactory_full_msg.render(context).await?; @@ -2777,6 +2782,10 @@ async fn render_mime_message_and_pre_message( Ok((rendered_msg, Some(rendered_pre_msg))) } else { + info!( + context, + "Message will be sent as normal message (no pre- and post message)", + ); Ok((mimefactory.render(context).await?, None)) } }