From ba573bb334a40c661867bb03490ef01bee569ec2 Mon Sep 17 00:00:00 2001 From: Simon Laux Date: Sat, 1 Nov 2025 00:16:45 +0100 Subject: [PATCH 01/14] fix python lint errors --- deltachat-rpc-client/tests/test_something.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deltachat-rpc-client/tests/test_something.py b/deltachat-rpc-client/tests/test_something.py index ad30f5eac7..027840a1d4 100644 --- a/deltachat-rpc-client/tests/test_something.py +++ b/deltachat-rpc-client/tests/test_something.py @@ -11,7 +11,7 @@ import pytest from deltachat_rpc_client import Contact, EventType, Message, events -from deltachat_rpc_client.const import DownloadState, MessageState +from deltachat_rpc_client.const import MessageState from deltachat_rpc_client.pytestplugin import E2EE_INFO_MSGS from deltachat_rpc_client.rpc import JsonRpcError From b69082df0a03cffb5683f84974cc55ce6e4bed7b Mon Sep 17 00:00:00 2001 From: Simon Laux Date: Thu, 6 Nov 2025 16:05:20 +0100 Subject: [PATCH 02/14] fix lint of python test --- deltachat-rpc-client/tests/test_something.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deltachat-rpc-client/tests/test_something.py b/deltachat-rpc-client/tests/test_something.py index 027840a1d4..ad30f5eac7 100644 --- a/deltachat-rpc-client/tests/test_something.py +++ b/deltachat-rpc-client/tests/test_something.py @@ -11,7 +11,7 @@ import pytest from deltachat_rpc_client import Contact, EventType, Message, events -from deltachat_rpc_client.const import MessageState +from deltachat_rpc_client.const import DownloadState, MessageState from deltachat_rpc_client.pytestplugin import E2EE_INFO_MSGS from deltachat_rpc_client.rpc import JsonRpcError From 17df902e3fb09732f11f29edb2456be7fb164b4b Mon Sep 17 00:00:00 2001 From: Simon Laux Date: Fri, 14 Nov 2025 20:54:15 +0100 Subject: [PATCH 03/14] restore test `test_webxdc_update_for_not_downloaded_instance` and rename and modify it to employ message reordering instead of a partially downloaded instance --- src/webxdc/webxdc_tests.rs | 40 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/src/webxdc/webxdc_tests.rs b/src/webxdc/webxdc_tests.rs index acd15cb480..6aa51b2b1a 100644 --- a/src/webxdc/webxdc_tests.rs +++ b/src/webxdc/webxdc_tests.rs @@ -328,6 +328,46 @@ async fn test_webxdc_contact_request() -> Result<()> { Ok(()) } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_webxdc_update_for_not_yet_received_instance() -> Result<()> { + // Alice sends an instance and an update + let alice = TestContext::new_alice().await; + let bob = TestContext::new_bob().await; + let chat = alice.create_chat(&bob).await; + let mut alice_instance = create_webxdc_instance( + &alice, + "chess.xdc", + include_bytes!("../../test-data/webxdc/chess.xdc"), + )?; + let sent1 = alice.send_msg(chat.id, &mut alice_instance).await; + let alice_instance = sent1.load_from_db().await; + alice + .send_webxdc_status_update( + alice_instance.id, + r#"{"payload": 7, "summary":"sum", "document":"doc"}"#, + ) + .await?; + alice.flush_status_updates().await?; + let sent2 = alice.pop_sent_msg().await; + + // Bob receives a status update before instance + bob.recv_msg_trash(&sent2).await; + // Bob downloads instance, updates should be assigned correctly + let _ = bob.recv_msg(&sent1).await; + let bob_instance = bob.get_last_msg().await; + assert_eq!(bob_instance.viewtype, Viewtype::Webxdc); + assert_eq!( + bob.get_webxdc_status_updates(bob_instance.id, StatusUpdateSerial(0)) + .await?, + r#"[{"payload":7,"document":"doc","summary":"sum","serial":1,"max_serial":1}]"# + ); + let info = bob_instance.get_webxdc_info(&bob).await?; + assert_eq!(info.document, "doc"); + assert_eq!(info.summary, "sum"); + + Ok(()) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_delete_webxdc_instance() -> Result<()> { let t = TestContext::new_alice().await; From 6f9823f9bb51e5d1253aacd46cd3952d67f4fb49 Mon Sep 17 00:00:00 2001 From: Simon Laux Date: Fri, 14 Nov 2025 21:57:24 +0100 Subject: [PATCH 04/14] remove `test_webxdc_update_for_not_yet_received_instance` --- src/webxdc/webxdc_tests.rs | 40 -------------------------------------- 1 file changed, 40 deletions(-) diff --git a/src/webxdc/webxdc_tests.rs b/src/webxdc/webxdc_tests.rs index 6aa51b2b1a..acd15cb480 100644 --- a/src/webxdc/webxdc_tests.rs +++ b/src/webxdc/webxdc_tests.rs @@ -328,46 +328,6 @@ async fn test_webxdc_contact_request() -> Result<()> { Ok(()) } -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn test_webxdc_update_for_not_yet_received_instance() -> Result<()> { - // Alice sends an instance and an update - let alice = TestContext::new_alice().await; - let bob = TestContext::new_bob().await; - let chat = alice.create_chat(&bob).await; - let mut alice_instance = create_webxdc_instance( - &alice, - "chess.xdc", - include_bytes!("../../test-data/webxdc/chess.xdc"), - )?; - let sent1 = alice.send_msg(chat.id, &mut alice_instance).await; - let alice_instance = sent1.load_from_db().await; - alice - .send_webxdc_status_update( - alice_instance.id, - r#"{"payload": 7, "summary":"sum", "document":"doc"}"#, - ) - .await?; - alice.flush_status_updates().await?; - let sent2 = alice.pop_sent_msg().await; - - // Bob receives a status update before instance - bob.recv_msg_trash(&sent2).await; - // Bob downloads instance, updates should be assigned correctly - let _ = bob.recv_msg(&sent1).await; - let bob_instance = bob.get_last_msg().await; - assert_eq!(bob_instance.viewtype, Viewtype::Webxdc); - assert_eq!( - bob.get_webxdc_status_updates(bob_instance.id, StatusUpdateSerial(0)) - .await?, - r#"[{"payload":7,"document":"doc","summary":"sum","serial":1,"max_serial":1}]"# - ); - let info = bob_instance.get_webxdc_info(&bob).await?; - assert_eq!(info.document, "doc"); - assert_eq!(info.summary, "sum"); - - Ok(()) -} - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_delete_webxdc_instance() -> Result<()> { let t = TestContext::new_alice().await; From 6042fd0faada589032c79c6d8c0d66b247ab7213 Mon Sep 17 00:00:00 2001 From: Simon Laux Date: Sun, 2 Nov 2025 21:49:30 +0100 Subject: [PATCH 05/14] feat: send pre-message on messages with large attachments --- src/chat.rs | 66 ++++++++++++++++++++++++++++++++++++++++++---- src/download.rs | 14 ++++++++++ src/headerdef.rs | 22 ++++++++++++++++ src/mimefactory.rs | 60 +++++++++++++++++++++++++++++++++++++---- 4 files changed, 152 insertions(+), 10 deletions(-) diff --git a/src/chat.rs b/src/chat.rs index e68ae7c61f..7876307088 100644 --- a/src/chat.rs +++ b/src/chat.rs @@ -27,7 +27,9 @@ use crate::constants::{ use crate::contact::{self, Contact, ContactId, Origin}; use crate::context::Context; use crate::debug_logging::maybe_set_logging_xdc; -use crate::download::DownloadState; +use crate::download::{ + DownloadState, PRE_MESSAGE_ATTACHMENT_SIZE_THRESHOLD, PRE_MESSAGE_SIZE_WARNING_THRESHOLD, +}; use crate::ephemeral::{Timer as EphemeralTimer, start_chat_ephemeral_timers}; use crate::events::EventType; use crate::key::self_fingerprint; @@ -35,7 +37,7 @@ use crate::location; use crate::log::{LogExt, error, info, warn}; use crate::logged_debug_assert; use crate::message::{self, Message, MessageState, MsgId, Viewtype}; -use crate::mimefactory::MimeFactory; +use crate::mimefactory::{MimeFactory, RenderedEmail}; use crate::mimeparser::SystemMessage; use crate::param::{Param, Params}; use crate::receive_imf::ReceivedMsg; @@ -2804,7 +2806,47 @@ pub(crate) async fn create_send_msg_jobs(context: &Context, msg: &mut Message) - return Ok(Vec::new()); } - let rendered_msg = match mimefactory.render(context).await { + // render message and pre message. + // pre message is a small message with metadata + // which announces a larger message. Large messages are not downloaded in the background. + + let needs_pre_message = msg.viewtype.has_file() + && msg + .get_filebytes(context) + .await? + .context("filebytes not available, even though message has attachment")? + > PRE_MESSAGE_ATTACHMENT_SIZE_THRESHOLD; + + let render_result: Result<(RenderedEmail, Option)> = async { + if needs_pre_message { + let mut mimefactory_full_msg = mimefactory.clone(); + mimefactory_full_msg.set_as_full_message(); + let rendered_msg = mimefactory_full_msg.render(context).await?; + + let mut mimefactory_pre_msg = mimefactory; + mimefactory_pre_msg.set_as_pre_message_for(&rendered_msg); + let rendered_pre_msg = mimefactory_pre_msg + .render(context) + .await + .context("pre-message failed to render")?; + + if rendered_pre_msg.message.len() > PRE_MESSAGE_SIZE_WARNING_THRESHOLD { + warn!( + context, + "pre message for message (MsgId={}) is larger than expected: {}", + msg.id, + rendered_pre_msg.message.len() + ); + } + + Ok((rendered_msg, Some(rendered_pre_msg))) + } else { + Ok((mimefactory.render(context).await?, None)) + } + } + .await; + + let (rendered_msg, rendered_pre_msg) = match render_result { Ok(res) => Ok(res), Err(err) => { message::set_msg_failed(context, msg, &err.to_string()).await?; @@ -2825,7 +2867,7 @@ pub(crate) async fn create_send_msg_jobs(context: &Context, msg: &mut Message) - msg.id, needs_encryption ); - } + }; let now = smeared_time(context); @@ -2870,12 +2912,26 @@ pub(crate) async fn create_send_msg_jobs(context: &Context, msg: &mut Message) - } else { for recipients_chunk in recipients.chunks(chunk_size) { let recipients_chunk = recipients_chunk.join(" "); + // send pre-message before actual message + if let Some(pre_msg) = &rendered_pre_msg { + let row_id = t.execute( + "INSERT INTO smtp (rfc724_mid, recipients, mime, msg_id) \ + VALUES (?1, ?2, ?3, ?4)", + ( + &pre_msg.rfc724_mid, + &recipients_chunk, + &pre_msg.message, + msg.id, // TODO: check if this is correct or we need another id here? + ), + )?; + row_ids.push(row_id.try_into()?); + } let row_id = t.execute( "INSERT INTO smtp (rfc724_mid, recipients, mime, msg_id) \ VALUES (?1, ?2, ?3, ?4)", ( &rendered_msg.rfc724_mid, - recipients_chunk, + &recipients_chunk, &rendered_msg.message, msg.id, ), diff --git a/src/download.rs b/src/download.rs index bf54662175..0b2003dd02 100644 --- a/src/download.rs +++ b/src/download.rs @@ -18,6 +18,20 @@ use crate::{EventType, chatlist_events}; /// `MIN_DELETE_SERVER_AFTER` increases the timeout in this case. pub(crate) const MIN_DELETE_SERVER_AFTER: i64 = 48 * 60 * 60; +/// From this point onward outgoing messages are considered large +/// and get a pre-message, which announces the full message. +// this is only about sending so we can modify it any time. +pub(crate) const PRE_MESSAGE_ATTACHMENT_SIZE_THRESHOLD: u64 = 140_000; + +/// Max message size to be fetched in the background. +/// This limit defines what messages are fully fetched in the background. +/// This is for all messages that don't have the full message header. +pub(crate) const MAX_FETCH_MSG_SIZE: usize = 1_000_000; + +/// Max size for pre messages. A warning is emitted when this is exceeded. +/// Should be well below `MAX_FETCH_MSG_SIZE` +pub(crate) const PRE_MESSAGE_SIZE_WARNING_THRESHOLD: usize = 150_000; + /// Download state of the message. #[derive( Debug, diff --git a/src/headerdef.rs b/src/headerdef.rs index 32c2281b58..eca9bc2fb7 100644 --- a/src/headerdef.rs +++ b/src/headerdef.rs @@ -102,6 +102,15 @@ pub enum HeaderDef { /// used to encrypt and decrypt messages. /// This secret is sent to a new member in the member-addition message. ChatBroadcastSecret, + /// This message announces a bigger message with attachment that is refereced by rfc724_mid. + #[strum(serialize = "Chat-Full-Message-ID")] // correct casing + ChatFullMessageId, + + /// This message has a pre-message + /// and thus this message can be skipped while fetching messages. + /// This is a cleartext / unproteced header. + #[strum(serialize = "Chat-Is-Full-Message")] // correct casing + ChatIsFullMessage, /// [Autocrypt](https://autocrypt.org/) header. Autocrypt, @@ -194,4 +203,17 @@ mod tests { ); assert_eq!(headers.get_header_value(HeaderDef::Autocrypt), None); } + + #[test] + /// Tests that headers have correct casing for sending. + fn header_name_correct_casing_for_sending() { + assert_eq!( + HeaderDef::ChatFullMessageId.get_headername(), + "Chat-Full-Message-ID" + ); + assert_eq!( + HeaderDef::ChatIsFullMessage.get_headername(), + "Chat-Is-Full-Message" + ); + } } diff --git a/src/mimefactory.rs b/src/mimefactory.rs index 3c7a2df638..a0289cb889 100644 --- a/src/mimefactory.rs +++ b/src/mimefactory.rs @@ -58,6 +58,15 @@ pub enum Loaded { }, } +#[derive(Debug, Clone, PartialEq)] +pub enum PreMessageMode { + /// adds a is full message header in unpretected part + FullMessage, + /// adds reference to full message to protected part + /// also adds metadata and hashes and explicitly excludes attachment + PreMessage { full_msg_rfc724_mid: String }, +} + /// Helper to construct mime messages. #[derive(Debug, Clone)] pub struct MimeFactory { @@ -145,6 +154,9 @@ pub struct MimeFactory { /// This field is used to sustain the topic id of webxdcs needed for peer channels. webxdc_topic: Option, + + /// This field is used when this is either a pre-message or a full-message. + pre_message_mode: Option, } /// Result of rendering a message, ready to be submitted to a send job. @@ -498,6 +510,7 @@ impl MimeFactory { sync_ids_to_delete: None, attach_selfavatar, webxdc_topic, + pre_message_mode: None, }; Ok(factory) } @@ -546,6 +559,7 @@ impl MimeFactory { sync_ids_to_delete: None, attach_selfavatar: false, webxdc_topic: None, + pre_message_mode: None, }; Ok(res) @@ -778,7 +792,10 @@ impl MimeFactory { headers.push(("Date", mail_builder::headers::raw::Raw::new(date).into())); let rfc724_mid = match &self.loaded { - Loaded::Message { msg, .. } => msg.rfc724_mid.clone(), + Loaded::Message { msg, .. } => match &self.pre_message_mode { + Some(PreMessageMode::PreMessage { .. }) => create_outgoing_rfc724_mid(), + _ => msg.rfc724_mid.clone(), + }, Loaded::Mdn { .. } => create_outgoing_rfc724_mid(), }; headers.push(( @@ -980,6 +997,22 @@ impl MimeFactory { "MIME-Version", mail_builder::headers::raw::Raw::new("1.0").into(), )); + + if self.pre_message_mode == Some(PreMessageMode::FullMessage) { + unprotected_headers.push(( + HeaderDef::ChatIsFullMessage.get_headername(), + mail_builder::headers::raw::Raw::new("1").into(), + )); + } else if let Some(PreMessageMode::PreMessage { + full_msg_rfc724_mid, + }) = self.pre_message_mode.clone() + { + unprotected_headers.push(( + HeaderDef::ChatFullMessageId.get_headername(), + mail_builder::headers::raw::Raw::new(full_msg_rfc724_mid).into(), + )); + } + for header @ (original_header_name, _header_value) in &headers { let header_name = original_header_name.to_lowercase(); if header_name == "message-id" { @@ -1111,7 +1144,10 @@ impl MimeFactory { for (addr, key) in &encryption_pubkeys { let fingerprint = key.dc_fingerprint().hex(); let cmd = msg.param.get_cmd(); - let should_do_gossip = cmd == SystemMessage::MemberAddedToGroup + let is_full_msg = + self.pre_message_mode != Some(PreMessageMode::FullMessage); + let should_do_gossip = is_full_msg + && cmd == SystemMessage::MemberAddedToGroup || cmd == SystemMessage::SecurejoinMessage || multiple_recipients && { let gossiped_timestamp: Option = context @@ -1837,8 +1873,12 @@ impl MimeFactory { // add attachment part if msg.viewtype.has_file() { - let file_part = build_body_file(context, &msg).await?; - parts.push(file_part); + if let Some(PreMessageMode::PreMessage { .. }) = self.pre_message_mode { + // TODO: generate thumbnail and attach it instead (if it makes sense) + } else { + let file_part = build_body_file(context, &msg).await?; + parts.push(file_part); + } } if let Some(msg_kml_part) = self.get_message_kml_part() { @@ -1883,7 +1923,7 @@ impl MimeFactory { } } - if self.attach_selfavatar { + if self.attach_selfavatar && self.pre_message_mode != Some(PreMessageMode::FullMessage) { match context.get_config(Config::Selfavatar).await? { Some(path) => match build_avatar_file(context, &path).await { Ok(avatar) => headers.push(( @@ -1952,6 +1992,16 @@ impl MimeFactory { Ok(message) } + + pub fn set_as_full_message(&mut self) { + self.pre_message_mode = Some(PreMessageMode::FullMessage); + } + + pub fn set_as_pre_message_for(&mut self, full_message: &RenderedEmail) { + self.pre_message_mode = Some(PreMessageMode::PreMessage { + full_msg_rfc724_mid: full_message.rfc724_mid.clone(), + }); + } } fn hidden_recipients() -> Address<'static> { From 19545067f4e4942dd0f7bddbcaad0e5e1da46583 Mon Sep 17 00:00:00 2001 From: Simon Laux Date: Thu, 6 Nov 2025 18:01:22 +0100 Subject: [PATCH 06/14] allow unused for `MAX_FETCH_MSG_SIZE` --- src/download.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/download.rs b/src/download.rs index 0b2003dd02..fb730a9e57 100644 --- a/src/download.rs +++ b/src/download.rs @@ -26,6 +26,7 @@ pub(crate) const PRE_MESSAGE_ATTACHMENT_SIZE_THRESHOLD: u64 = 140_000; /// Max message size to be fetched in the background. /// This limit defines what messages are fully fetched in the background. /// This is for all messages that don't have the full message header. +#[allow(unused)] pub(crate) const MAX_FETCH_MSG_SIZE: usize = 1_000_000; /// Max size for pre messages. A warning is emitted when this is exceeded. From f49ecfa2c88ad59abd28d9d08746a7a06ff3ba5a Mon Sep 17 00:00:00 2001 From: Simon Laux Date: Thu, 6 Nov 2025 21:15:10 +0100 Subject: [PATCH 07/14] remove todo comment --- src/chat.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/chat.rs b/src/chat.rs index 7876307088..a1e4c04a4b 100644 --- a/src/chat.rs +++ b/src/chat.rs @@ -2921,7 +2921,7 @@ pub(crate) async fn create_send_msg_jobs(context: &Context, msg: &mut Message) - &pre_msg.rfc724_mid, &recipients_chunk, &pre_msg.message, - msg.id, // TODO: check if this is correct or we need another id here? + msg.id, ), )?; row_ids.push(row_id.try_into()?); From bf5d08130ac45704e126c79bf8c60d8462af8d61 Mon Sep 17 00:00:00 2001 From: Simon Laux Date: Sat, 8 Nov 2025 19:27:19 +0100 Subject: [PATCH 08/14] add tests for sending pre-messages --- src/download.rs | 223 +++++++++++++++++++++++++++++++++++++++++++++- src/test_utils.rs | 21 +++++ 2 files changed, 243 insertions(+), 1 deletion(-) diff --git a/src/download.rs b/src/download.rs index fb730a9e57..260a6b86bc 100644 --- a/src/download.rs +++ b/src/download.rs @@ -208,10 +208,13 @@ impl Session { #[cfg(test)] mod tests { + use mailparse::MailHeaderMap; use num_traits::FromPrimitive; use super::*; - use crate::chat::send_msg; + use crate::chat::{self, create_group, send_msg}; + use crate::headerdef::{HeaderDef, HeaderDefMap}; + use crate::message::Viewtype; use crate::receive_imf::receive_imf_from_inbox; use crate::test_utils::TestContext; @@ -310,4 +313,222 @@ mod tests { Ok(()) } + /// Tests that pre message is sent for attachment larger than `PRE_MESSAGE_ATTACHMENT_SIZE_THRESHOLD` + /// Also test that pre message is sent first, before the full message + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn test_sending_pre_message() -> Result<()> { + let alice = TestContext::new_alice().await; + let bob = TestContext::new_bob().await; + let chat = alice.create_chat(&bob).await; + + let mut msg = Message::new(Viewtype::File); + msg.set_file_from_bytes(&alice.ctx, "test.bin", &[0u8; 300_000], None)?; + msg.set_text("test".to_owned()); + + // assert that test attachment is bigger than limit + assert!( + msg.get_filebytes(&alice.ctx).await?.unwrap() > PRE_MESSAGE_ATTACHMENT_SIZE_THRESHOLD + ); + + let msg_id = chat::send_msg(&alice.ctx, chat.id, &mut msg).await.unwrap(); + let smtp_rows = alice.get_smtp_rows_for_msg(msg_id).await; + + // pre-message and full message should be present + // and test that correct headers are present on both messages + assert_eq!(smtp_rows.len(), 2); + let pre_message = mailparse::parse_mail( + smtp_rows + .first() + .expect("first element exists") + .2 + .as_bytes(), + )?; + let full_message = mailparse::parse_mail( + smtp_rows + .get(1) + .expect("second element exists") + .2 + .as_bytes(), + )?; + + assert!( + pre_message + .get_headers() + .get_first_header(HeaderDef::ChatIsFullMessage.get_headername()) + .is_none() + ); + assert!( + full_message + .get_headers() + .get_first_header(HeaderDef::ChatIsFullMessage.get_headername()) + .is_some() + ); + + assert_eq!( + pre_message + .headers + .get_header_value(HeaderDef::ChatFullMessageId), + full_message.headers.get_header_value(HeaderDef::MessageId) + ); + assert!( + full_message + .headers + .get_header_value(HeaderDef::ChatFullMessageId) + .is_none() + ); + + // full message should have the rfc message id + assert_eq!( + full_message.headers.get_header_value(HeaderDef::MessageId), + Some(msg.rfc724_mid) + ); + + // test that message ids are different + assert_ne!( + pre_message.headers.get_header_value(HeaderDef::MessageId), + full_message.headers.get_header_value(HeaderDef::MessageId) + ); + + // also test that Autocrypt-gossip and selfavatar should never go into full-messages + // TODO: (this needs decryption, right?) + Ok(()) + } + + /// Tests that no pre message is sent for normal message + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn test_not_sending_pre_message_no_attachment() -> Result<()> { + let alice = TestContext::new_alice().await; + let bob = TestContext::new_bob().await; + let chat = alice.create_chat(&bob).await; + + // send normal text message + let mut msg = Message::new(Viewtype::Text); + msg.set_text("test".to_owned()); + let msg_id = chat::send_msg(&alice.ctx, chat.id, &mut msg).await.unwrap(); + let smtp_rows = alice.get_smtp_rows_for_msg(msg_id).await; + + // only one message and no "is full message" header should be present + assert_eq!(smtp_rows.len(), 1); + + let mime = smtp_rows.first().expect("first element exists").2.clone(); + let mail = mailparse::parse_mail(mime.as_bytes())?; + + assert!( + mail.get_headers() + .get_first_header(HeaderDef::ChatIsFullMessage.get_headername()) + .is_none() + ); + assert!( + mail.get_headers() + .get_first_header(HeaderDef::ChatFullMessageId.get_headername()) + .is_none() + ); + Ok(()) + } + + /// Tests that no pre message is sent for attachment smaller than `PRE_MESSAGE_ATTACHMENT_SIZE_THRESHOLD` + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn test_not_sending_pre_message_for_small_attachment() -> Result<()> { + let alice = TestContext::new_alice().await; + let bob = TestContext::new_bob().await; + let chat = alice.create_chat(&bob).await; + + let mut msg = Message::new(Viewtype::File); + msg.set_file_from_bytes(&alice.ctx, "test.bin", &[0u8; 100_000], None)?; + msg.set_text("test".to_owned()); + + // assert that test attachment is smaller than limit + assert!( + msg.get_filebytes(&alice.ctx).await?.unwrap() < PRE_MESSAGE_ATTACHMENT_SIZE_THRESHOLD + ); + + let msg_id = chat::send_msg(&alice.ctx, chat.id, &mut msg).await.unwrap(); + let smtp_rows = alice.get_smtp_rows_for_msg(msg_id).await; + + // only one message and no "is full message" header should be present + assert_eq!(smtp_rows.len(), 1); + + let mime = smtp_rows.first().expect("first element exists").2.clone(); + let mail = mailparse::parse_mail(mime.as_bytes())?; + + assert!( + mail.get_headers() + .get_first_header(HeaderDef::ChatIsFullMessage.get_headername()) + .is_none() + ); + assert!( + mail.get_headers() + .get_first_header(HeaderDef::ChatFullMessageId.get_headername()) + .is_none() + ); + + Ok(()) + } + + /// Tests that pre message is not send for large webxdc updates + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn test_render_webxdc_status_update_object_range() -> Result<()> { + let t = TestContext::new_alice().await; + let chat_id = create_group(&t, "a chat").await?; + + let instance = { + let mut instance = Message::new(Viewtype::File); + instance.set_file_from_bytes( + &t, + "minimal.xdc", + include_bytes!("../test-data/webxdc/minimal.xdc"), + None, + )?; + let instance_msg_id = send_msg(&t, chat_id, &mut instance).await?; + assert_eq!(instance.viewtype, Viewtype::Webxdc); + Message::load_from_db(&t, instance_msg_id).await + } + .unwrap(); + + t.pop_sent_msg().await; + assert_eq!(t.sql.count("SELECT COUNT(*) FROM smtp", ()).await?, 0); + + let long_text = String::from_utf8(vec![b'a'; 300_000])?; + assert!(long_text.len() > PRE_MESSAGE_ATTACHMENT_SIZE_THRESHOLD.try_into().unwrap()); + t.send_webxdc_status_update(instance.id, &format!("{{\"payload\": \"{long_text}\"}}")) + .await?; + t.flush_status_updates().await?; + + assert_eq!(t.sql.count("SELECT COUNT(*) FROM smtp", ()).await?, 1); + Ok(()) + } + + // test that pre message is not send for large large text + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn test_not_sending_pre_message_for_large_text() -> Result<()> { + let alice = TestContext::new_alice().await; + let bob = TestContext::new_bob().await; + let chat = alice.create_chat(&bob).await; + + // send normal text message + let mut msg = Message::new(Viewtype::Text); + let long_text = String::from_utf8(vec![b'a'; 300_000])?; + assert!(long_text.len() > PRE_MESSAGE_ATTACHMENT_SIZE_THRESHOLD.try_into().unwrap()); + msg.set_text(long_text); + let msg_id = chat::send_msg(&alice.ctx, chat.id, &mut msg).await.unwrap(); + let smtp_rows = alice.get_smtp_rows_for_msg(msg_id).await; + + // only one message and no "is full message" header should be present + assert_eq!(smtp_rows.len(), 1); + + let mime = smtp_rows.first().expect("first element exists").2.clone(); + let mail = mailparse::parse_mail(mime.as_bytes())?; + + assert!( + mail.get_headers() + .get_first_header(HeaderDef::ChatIsFullMessage.get_headername()) + .is_none() + ); + assert!( + mail.get_headers() + .get_first_header(HeaderDef::ChatFullMessageId.get_headername()) + .is_none() + ); + Ok(()) + } } diff --git a/src/test_utils.rs b/src/test_utils.rs index 73e9875f13..f3b7dc836b 100644 --- a/src/test_utils.rs +++ b/src/test_utils.rs @@ -697,6 +697,27 @@ impl TestContext { }) } + pub async fn get_smtp_rows_for_msg(&self, msg_id: MsgId) -> Vec<(i64, MsgId, String, String)> { + self.ctx + .sql + .query_map_vec( + r#" + SELECT id, msg_id, mime, recipients + FROM smtp + WHERE msg_id=?"#, + (msg_id,), + |row| { + let rowid: i64 = row.get(0)?; + let msg_id: MsgId = row.get(1)?; + let mime: String = row.get(2)?; + let recipients: String = row.get(3)?; + Ok((rowid, msg_id, mime, recipients)) + }, + ) + .await + .unwrap() + } + /// Retrieves a sent sync message from the db. /// /// This retrieves and removes a sync message which has been scheduled to send from the jobs From 709a9656603799dc1f8df0a59867d00fe655ccdb Mon Sep 17 00:00:00 2001 From: Simon Laux Date: Sat, 8 Nov 2025 19:44:54 +0100 Subject: [PATCH 09/14] delimit `Chat-Full-Message-ID` with `<>` like other message ids. --- src/mimefactory.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mimefactory.rs b/src/mimefactory.rs index a0289cb889..e8e17795ea 100644 --- a/src/mimefactory.rs +++ b/src/mimefactory.rs @@ -1009,7 +1009,7 @@ impl MimeFactory { { unprotected_headers.push(( HeaderDef::ChatFullMessageId.get_headername(), - mail_builder::headers::raw::Raw::new(full_msg_rfc724_mid).into(), + mail_builder::headers::message_id::MessageId::new(full_msg_rfc724_mid).into(), )); } From 9ffe3002f45479080382197352c371c657104941 Mon Sep 17 00:00:00 2001 From: Simon Laux Date: Sat, 8 Nov 2025 19:56:35 +0100 Subject: [PATCH 10/14] fix test --- src/download.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/download.rs b/src/download.rs index 260a6b86bc..c86b50d814 100644 --- a/src/download.rs +++ b/src/download.rs @@ -380,7 +380,7 @@ mod tests { // full message should have the rfc message id assert_eq!( full_message.headers.get_header_value(HeaderDef::MessageId), - Some(msg.rfc724_mid) + Some(format!("<{}>", msg.rfc724_mid)) ); // test that message ids are different From a3846ee45eac538d77efd8d2b8013abee4625e2d Mon Sep 17 00:00:00 2001 From: Simon Laux Date: Sat, 8 Nov 2025 20:41:36 +0100 Subject: [PATCH 11/14] test that Autocrypt-gossip and selfavatar should never go into full-messages and fix exclusion of gossip header, which was broken --- src/download.rs | 29 +++++++++++++++++++---------- src/mimefactory.rs | 15 ++++++++++----- 2 files changed, 29 insertions(+), 15 deletions(-) diff --git a/src/download.rs b/src/download.rs index c86b50d814..af479d4313 100644 --- a/src/download.rs +++ b/src/download.rs @@ -216,6 +216,7 @@ mod tests { use crate::headerdef::{HeaderDef, HeaderDefMap}; use crate::message::Viewtype; use crate::receive_imf::receive_imf_from_inbox; + use crate::mimeparser::MimeMessage; use crate::test_utils::TestContext; #[test] @@ -315,11 +316,15 @@ mod tests { } /// Tests that pre message is sent for attachment larger than `PRE_MESSAGE_ATTACHMENT_SIZE_THRESHOLD` /// Also test that pre message is sent first, before the full message + /// And that Autocrypt-gossip and selfavatar never go into full-messages #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_sending_pre_message() -> Result<()> { let alice = TestContext::new_alice().await; let bob = TestContext::new_bob().await; - let chat = alice.create_chat(&bob).await; + let fiona = TestContext::new_fiona().await; + let group_id = alice + .create_group_with_members("test group", &[&bob, &fiona]) + .await; let mut msg = Message::new(Viewtype::File); msg.set_file_from_bytes(&alice.ctx, "test.bin", &[0u8; 300_000], None)?; @@ -330,7 +335,9 @@ mod tests { msg.get_filebytes(&alice.ctx).await?.unwrap() > PRE_MESSAGE_ATTACHMENT_SIZE_THRESHOLD ); - let msg_id = chat::send_msg(&alice.ctx, chat.id, &mut msg).await.unwrap(); + let msg_id = chat::send_msg(&alice.ctx, group_id, &mut msg) + .await + .unwrap(); let smtp_rows = alice.get_smtp_rows_for_msg(msg_id).await; // pre-message and full message should be present @@ -343,13 +350,12 @@ mod tests { .2 .as_bytes(), )?; - let full_message = mailparse::parse_mail( - smtp_rows - .get(1) - .expect("second element exists") - .2 - .as_bytes(), - )?; + let full_message_bytes = smtp_rows + .get(1) + .expect("second element exists") + .2 + .as_bytes(); + let full_message = mailparse::parse_mail(full_message_bytes)?; assert!( pre_message @@ -390,7 +396,10 @@ mod tests { ); // also test that Autocrypt-gossip and selfavatar should never go into full-messages - // TODO: (this needs decryption, right?) + let decrypted_full_message = MimeMessage::from_bytes(&bob.ctx, full_message_bytes).await?; + assert!(!decrypted_full_message.decrypting_failed); + assert_eq!(decrypted_full_message.gossiped_keys.len(), 0); + assert_eq!(decrypted_full_message.user_avatar, None); Ok(()) } diff --git a/src/mimefactory.rs b/src/mimefactory.rs index e8e17795ea..b2d814cdab 100644 --- a/src/mimefactory.rs +++ b/src/mimefactory.rs @@ -1145,11 +1145,12 @@ impl MimeFactory { let fingerprint = key.dc_fingerprint().hex(); let cmd = msg.param.get_cmd(); let is_full_msg = - self.pre_message_mode != Some(PreMessageMode::FullMessage); - let should_do_gossip = is_full_msg - && cmd == SystemMessage::MemberAddedToGroup - || cmd == SystemMessage::SecurejoinMessage - || multiple_recipients && { + self.pre_message_mode == Some(PreMessageMode::FullMessage); + let should_do_gossip = !is_full_msg + && (cmd == SystemMessage::MemberAddedToGroup + || cmd == SystemMessage::SecurejoinMessage + || multiple_recipients) + && { let gossiped_timestamp: Option = context .sql .query_get_value( @@ -1186,6 +1187,10 @@ impl MimeFactory { continue; } + debug_assert!( + self.pre_message_mode != Some(PreMessageMode::FullMessage) + ); + let header = Aheader { addr: addr.clone(), public_key: key.clone(), From 74a1fc95496205db98cdeb024206bbcc5f9409e1 Mon Sep 17 00:00:00 2001 From: Simon Laux Date: Sat, 8 Nov 2025 21:21:16 +0100 Subject: [PATCH 12/14] fix bug that broke `receive_imf::receive_imf_tests::test_dont_reverify_by_self_on_outgoing_msg` --- src/mimefactory.rs | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/src/mimefactory.rs b/src/mimefactory.rs index b2d814cdab..c9be47db7f 100644 --- a/src/mimefactory.rs +++ b/src/mimefactory.rs @@ -1144,13 +1144,9 @@ impl MimeFactory { for (addr, key) in &encryption_pubkeys { let fingerprint = key.dc_fingerprint().hex(); let cmd = msg.param.get_cmd(); - let is_full_msg = - self.pre_message_mode == Some(PreMessageMode::FullMessage); - let should_do_gossip = !is_full_msg - && (cmd == SystemMessage::MemberAddedToGroup - || cmd == SystemMessage::SecurejoinMessage - || multiple_recipients) - && { + let should_do_gossip = cmd == SystemMessage::MemberAddedToGroup + || cmd == SystemMessage::SecurejoinMessage + || multiple_recipients && { let gossiped_timestamp: Option = context .sql .query_get_value( @@ -1183,7 +1179,10 @@ impl MimeFactory { let is_verified = verifier_id.is_some_and(|verifier_id| verifier_id != 0); - if !should_do_gossip { + let is_full_msg = + self.pre_message_mode == Some(PreMessageMode::FullMessage); + + if !should_do_gossip || is_full_msg { continue; } From 2d32a5882be55f1e2bdaef3ac1662515b9cc022d Mon Sep 17 00:00:00 2001 From: Simon Laux Date: Fri, 14 Nov 2025 22:37:02 +0100 Subject: [PATCH 13/14] cargo fmt after rebase --- src/download.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/download.rs b/src/download.rs index af479d4313..1cff3a3c66 100644 --- a/src/download.rs +++ b/src/download.rs @@ -215,8 +215,8 @@ mod tests { use crate::chat::{self, create_group, send_msg}; use crate::headerdef::{HeaderDef, HeaderDefMap}; use crate::message::Viewtype; - use crate::receive_imf::receive_imf_from_inbox; use crate::mimeparser::MimeMessage; + use crate::receive_imf::receive_imf_from_inbox; use crate::test_utils::TestContext; #[test] From 305498d577124c5480b31c2033144ae746dedee0 Mon Sep 17 00:00:00 2001 From: Simon Laux Date: Fri, 14 Nov 2025 23:10:22 +0100 Subject: [PATCH 14/14] fix typo --- src/mimefactory.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mimefactory.rs b/src/mimefactory.rs index c9be47db7f..355b6e8d49 100644 --- a/src/mimefactory.rs +++ b/src/mimefactory.rs @@ -60,7 +60,7 @@ pub enum Loaded { #[derive(Debug, Clone, PartialEq)] pub enum PreMessageMode { - /// adds a is full message header in unpretected part + /// adds a is full message header in unprotected part FullMessage, /// adds reference to full message to protected part /// also adds metadata and hashes and explicitly excludes attachment