diff --git a/src/chat.rs b/src/chat.rs index e68ae7c61f..a1e4c04a4b 100644 --- a/src/chat.rs +++ b/src/chat.rs @@ -27,7 +27,9 @@ use crate::constants::{ use crate::contact::{self, Contact, ContactId, Origin}; use crate::context::Context; use crate::debug_logging::maybe_set_logging_xdc; -use crate::download::DownloadState; +use crate::download::{ + DownloadState, PRE_MESSAGE_ATTACHMENT_SIZE_THRESHOLD, PRE_MESSAGE_SIZE_WARNING_THRESHOLD, +}; use crate::ephemeral::{Timer as EphemeralTimer, start_chat_ephemeral_timers}; use crate::events::EventType; use crate::key::self_fingerprint; @@ -35,7 +37,7 @@ use crate::location; use crate::log::{LogExt, error, info, warn}; use crate::logged_debug_assert; use crate::message::{self, Message, MessageState, MsgId, Viewtype}; -use crate::mimefactory::MimeFactory; +use crate::mimefactory::{MimeFactory, RenderedEmail}; use crate::mimeparser::SystemMessage; use crate::param::{Param, Params}; use crate::receive_imf::ReceivedMsg; @@ -2804,7 +2806,47 @@ pub(crate) async fn create_send_msg_jobs(context: &Context, msg: &mut Message) - return Ok(Vec::new()); } - let rendered_msg = match mimefactory.render(context).await { + // render message and pre message. + // pre message is a small message with metadata + // which announces a larger message. Large messages are not downloaded in the background. + + let needs_pre_message = msg.viewtype.has_file() + && msg + .get_filebytes(context) + .await? + .context("filebytes not available, even though message has attachment")? + > PRE_MESSAGE_ATTACHMENT_SIZE_THRESHOLD; + + let render_result: Result<(RenderedEmail, Option)> = async { + if needs_pre_message { + let mut mimefactory_full_msg = mimefactory.clone(); + mimefactory_full_msg.set_as_full_message(); + let rendered_msg = mimefactory_full_msg.render(context).await?; + + let mut mimefactory_pre_msg = mimefactory; + mimefactory_pre_msg.set_as_pre_message_for(&rendered_msg); + let rendered_pre_msg = mimefactory_pre_msg + .render(context) + .await + .context("pre-message failed to render")?; + + if rendered_pre_msg.message.len() > PRE_MESSAGE_SIZE_WARNING_THRESHOLD { + warn!( + context, + "pre message for message (MsgId={}) is larger than expected: {}", + msg.id, + rendered_pre_msg.message.len() + ); + } + + Ok((rendered_msg, Some(rendered_pre_msg))) + } else { + Ok((mimefactory.render(context).await?, None)) + } + } + .await; + + let (rendered_msg, rendered_pre_msg) = match render_result { Ok(res) => Ok(res), Err(err) => { message::set_msg_failed(context, msg, &err.to_string()).await?; @@ -2825,7 +2867,7 @@ pub(crate) async fn create_send_msg_jobs(context: &Context, msg: &mut Message) - msg.id, needs_encryption ); - } + }; let now = smeared_time(context); @@ -2870,12 +2912,26 @@ pub(crate) async fn create_send_msg_jobs(context: &Context, msg: &mut Message) - } else { for recipients_chunk in recipients.chunks(chunk_size) { let recipients_chunk = recipients_chunk.join(" "); + // send pre-message before actual message + if let Some(pre_msg) = &rendered_pre_msg { + let row_id = t.execute( + "INSERT INTO smtp (rfc724_mid, recipients, mime, msg_id) \ + VALUES (?1, ?2, ?3, ?4)", + ( + &pre_msg.rfc724_mid, + &recipients_chunk, + &pre_msg.message, + msg.id, + ), + )?; + row_ids.push(row_id.try_into()?); + } let row_id = t.execute( "INSERT INTO smtp (rfc724_mid, recipients, mime, msg_id) \ VALUES (?1, ?2, ?3, ?4)", ( &rendered_msg.rfc724_mid, - recipients_chunk, + &recipients_chunk, &rendered_msg.message, msg.id, ), diff --git a/src/download.rs b/src/download.rs index bf54662175..1cff3a3c66 100644 --- a/src/download.rs +++ b/src/download.rs @@ -18,6 +18,21 @@ use crate::{EventType, chatlist_events}; /// `MIN_DELETE_SERVER_AFTER` increases the timeout in this case. pub(crate) const MIN_DELETE_SERVER_AFTER: i64 = 48 * 60 * 60; +/// From this point onward outgoing messages are considered large +/// and get a pre-message, which announces the full message. +// this is only about sending so we can modify it any time. +pub(crate) const PRE_MESSAGE_ATTACHMENT_SIZE_THRESHOLD: u64 = 140_000; + +/// Max message size to be fetched in the background. +/// This limit defines what messages are fully fetched in the background. +/// This is for all messages that don't have the full message header. +#[allow(unused)] +pub(crate) const MAX_FETCH_MSG_SIZE: usize = 1_000_000; + +/// Max size for pre messages. A warning is emitted when this is exceeded. +/// Should be well below `MAX_FETCH_MSG_SIZE` +pub(crate) const PRE_MESSAGE_SIZE_WARNING_THRESHOLD: usize = 150_000; + /// Download state of the message. #[derive( Debug, @@ -193,10 +208,14 @@ impl Session { #[cfg(test)] mod tests { + use mailparse::MailHeaderMap; use num_traits::FromPrimitive; use super::*; - use crate::chat::send_msg; + use crate::chat::{self, create_group, send_msg}; + use crate::headerdef::{HeaderDef, HeaderDefMap}; + use crate::message::Viewtype; + use crate::mimeparser::MimeMessage; use crate::receive_imf::receive_imf_from_inbox; use crate::test_utils::TestContext; @@ -295,4 +314,230 @@ mod tests { Ok(()) } + /// Tests that pre message is sent for attachment larger than `PRE_MESSAGE_ATTACHMENT_SIZE_THRESHOLD` + /// Also test that pre message is sent first, before the full message + /// And that Autocrypt-gossip and selfavatar never go into full-messages + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn test_sending_pre_message() -> Result<()> { + let alice = TestContext::new_alice().await; + let bob = TestContext::new_bob().await; + let fiona = TestContext::new_fiona().await; + let group_id = alice + .create_group_with_members("test group", &[&bob, &fiona]) + .await; + + let mut msg = Message::new(Viewtype::File); + msg.set_file_from_bytes(&alice.ctx, "test.bin", &[0u8; 300_000], None)?; + msg.set_text("test".to_owned()); + + // assert that test attachment is bigger than limit + assert!( + msg.get_filebytes(&alice.ctx).await?.unwrap() > PRE_MESSAGE_ATTACHMENT_SIZE_THRESHOLD + ); + + let msg_id = chat::send_msg(&alice.ctx, group_id, &mut msg) + .await + .unwrap(); + let smtp_rows = alice.get_smtp_rows_for_msg(msg_id).await; + + // pre-message and full message should be present + // and test that correct headers are present on both messages + assert_eq!(smtp_rows.len(), 2); + let pre_message = mailparse::parse_mail( + smtp_rows + .first() + .expect("first element exists") + .2 + .as_bytes(), + )?; + let full_message_bytes = smtp_rows + .get(1) + .expect("second element exists") + .2 + .as_bytes(); + let full_message = mailparse::parse_mail(full_message_bytes)?; + + assert!( + pre_message + .get_headers() + .get_first_header(HeaderDef::ChatIsFullMessage.get_headername()) + .is_none() + ); + assert!( + full_message + .get_headers() + .get_first_header(HeaderDef::ChatIsFullMessage.get_headername()) + .is_some() + ); + + assert_eq!( + pre_message + .headers + .get_header_value(HeaderDef::ChatFullMessageId), + full_message.headers.get_header_value(HeaderDef::MessageId) + ); + assert!( + full_message + .headers + .get_header_value(HeaderDef::ChatFullMessageId) + .is_none() + ); + + // full message should have the rfc message id + assert_eq!( + full_message.headers.get_header_value(HeaderDef::MessageId), + Some(format!("<{}>", msg.rfc724_mid)) + ); + + // test that message ids are different + assert_ne!( + pre_message.headers.get_header_value(HeaderDef::MessageId), + full_message.headers.get_header_value(HeaderDef::MessageId) + ); + + // also test that Autocrypt-gossip and selfavatar should never go into full-messages + let decrypted_full_message = MimeMessage::from_bytes(&bob.ctx, full_message_bytes).await?; + assert!(!decrypted_full_message.decrypting_failed); + assert_eq!(decrypted_full_message.gossiped_keys.len(), 0); + assert_eq!(decrypted_full_message.user_avatar, None); + Ok(()) + } + + /// Tests that no pre message is sent for normal message + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn test_not_sending_pre_message_no_attachment() -> Result<()> { + let alice = TestContext::new_alice().await; + let bob = TestContext::new_bob().await; + let chat = alice.create_chat(&bob).await; + + // send normal text message + let mut msg = Message::new(Viewtype::Text); + msg.set_text("test".to_owned()); + let msg_id = chat::send_msg(&alice.ctx, chat.id, &mut msg).await.unwrap(); + let smtp_rows = alice.get_smtp_rows_for_msg(msg_id).await; + + // only one message and no "is full message" header should be present + assert_eq!(smtp_rows.len(), 1); + + let mime = smtp_rows.first().expect("first element exists").2.clone(); + let mail = mailparse::parse_mail(mime.as_bytes())?; + + assert!( + mail.get_headers() + .get_first_header(HeaderDef::ChatIsFullMessage.get_headername()) + .is_none() + ); + assert!( + mail.get_headers() + .get_first_header(HeaderDef::ChatFullMessageId.get_headername()) + .is_none() + ); + Ok(()) + } + + /// Tests that no pre message is sent for attachment smaller than `PRE_MESSAGE_ATTACHMENT_SIZE_THRESHOLD` + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn test_not_sending_pre_message_for_small_attachment() -> Result<()> { + let alice = TestContext::new_alice().await; + let bob = TestContext::new_bob().await; + let chat = alice.create_chat(&bob).await; + + let mut msg = Message::new(Viewtype::File); + msg.set_file_from_bytes(&alice.ctx, "test.bin", &[0u8; 100_000], None)?; + msg.set_text("test".to_owned()); + + // assert that test attachment is smaller than limit + assert!( + msg.get_filebytes(&alice.ctx).await?.unwrap() < PRE_MESSAGE_ATTACHMENT_SIZE_THRESHOLD + ); + + let msg_id = chat::send_msg(&alice.ctx, chat.id, &mut msg).await.unwrap(); + let smtp_rows = alice.get_smtp_rows_for_msg(msg_id).await; + + // only one message and no "is full message" header should be present + assert_eq!(smtp_rows.len(), 1); + + let mime = smtp_rows.first().expect("first element exists").2.clone(); + let mail = mailparse::parse_mail(mime.as_bytes())?; + + assert!( + mail.get_headers() + .get_first_header(HeaderDef::ChatIsFullMessage.get_headername()) + .is_none() + ); + assert!( + mail.get_headers() + .get_first_header(HeaderDef::ChatFullMessageId.get_headername()) + .is_none() + ); + + Ok(()) + } + + /// Tests that pre message is not send for large webxdc updates + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn test_render_webxdc_status_update_object_range() -> Result<()> { + let t = TestContext::new_alice().await; + let chat_id = create_group(&t, "a chat").await?; + + let instance = { + let mut instance = Message::new(Viewtype::File); + instance.set_file_from_bytes( + &t, + "minimal.xdc", + include_bytes!("../test-data/webxdc/minimal.xdc"), + None, + )?; + let instance_msg_id = send_msg(&t, chat_id, &mut instance).await?; + assert_eq!(instance.viewtype, Viewtype::Webxdc); + Message::load_from_db(&t, instance_msg_id).await + } + .unwrap(); + + t.pop_sent_msg().await; + assert_eq!(t.sql.count("SELECT COUNT(*) FROM smtp", ()).await?, 0); + + let long_text = String::from_utf8(vec![b'a'; 300_000])?; + assert!(long_text.len() > PRE_MESSAGE_ATTACHMENT_SIZE_THRESHOLD.try_into().unwrap()); + t.send_webxdc_status_update(instance.id, &format!("{{\"payload\": \"{long_text}\"}}")) + .await?; + t.flush_status_updates().await?; + + assert_eq!(t.sql.count("SELECT COUNT(*) FROM smtp", ()).await?, 1); + Ok(()) + } + + // test that pre message is not send for large large text + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn test_not_sending_pre_message_for_large_text() -> Result<()> { + let alice = TestContext::new_alice().await; + let bob = TestContext::new_bob().await; + let chat = alice.create_chat(&bob).await; + + // send normal text message + let mut msg = Message::new(Viewtype::Text); + let long_text = String::from_utf8(vec![b'a'; 300_000])?; + assert!(long_text.len() > PRE_MESSAGE_ATTACHMENT_SIZE_THRESHOLD.try_into().unwrap()); + msg.set_text(long_text); + let msg_id = chat::send_msg(&alice.ctx, chat.id, &mut msg).await.unwrap(); + let smtp_rows = alice.get_smtp_rows_for_msg(msg_id).await; + + // only one message and no "is full message" header should be present + assert_eq!(smtp_rows.len(), 1); + + let mime = smtp_rows.first().expect("first element exists").2.clone(); + let mail = mailparse::parse_mail(mime.as_bytes())?; + + assert!( + mail.get_headers() + .get_first_header(HeaderDef::ChatIsFullMessage.get_headername()) + .is_none() + ); + assert!( + mail.get_headers() + .get_first_header(HeaderDef::ChatFullMessageId.get_headername()) + .is_none() + ); + Ok(()) + } } diff --git a/src/headerdef.rs b/src/headerdef.rs index 32c2281b58..eca9bc2fb7 100644 --- a/src/headerdef.rs +++ b/src/headerdef.rs @@ -102,6 +102,15 @@ pub enum HeaderDef { /// used to encrypt and decrypt messages. /// This secret is sent to a new member in the member-addition message. ChatBroadcastSecret, + /// This message announces a bigger message with attachment that is refereced by rfc724_mid. + #[strum(serialize = "Chat-Full-Message-ID")] // correct casing + ChatFullMessageId, + + /// This message has a pre-message + /// and thus this message can be skipped while fetching messages. + /// This is a cleartext / unproteced header. + #[strum(serialize = "Chat-Is-Full-Message")] // correct casing + ChatIsFullMessage, /// [Autocrypt](https://autocrypt.org/) header. Autocrypt, @@ -194,4 +203,17 @@ mod tests { ); assert_eq!(headers.get_header_value(HeaderDef::Autocrypt), None); } + + #[test] + /// Tests that headers have correct casing for sending. + fn header_name_correct_casing_for_sending() { + assert_eq!( + HeaderDef::ChatFullMessageId.get_headername(), + "Chat-Full-Message-ID" + ); + assert_eq!( + HeaderDef::ChatIsFullMessage.get_headername(), + "Chat-Is-Full-Message" + ); + } } diff --git a/src/mimefactory.rs b/src/mimefactory.rs index 3c7a2df638..355b6e8d49 100644 --- a/src/mimefactory.rs +++ b/src/mimefactory.rs @@ -58,6 +58,15 @@ pub enum Loaded { }, } +#[derive(Debug, Clone, PartialEq)] +pub enum PreMessageMode { + /// adds a is full message header in unprotected part + FullMessage, + /// adds reference to full message to protected part + /// also adds metadata and hashes and explicitly excludes attachment + PreMessage { full_msg_rfc724_mid: String }, +} + /// Helper to construct mime messages. #[derive(Debug, Clone)] pub struct MimeFactory { @@ -145,6 +154,9 @@ pub struct MimeFactory { /// This field is used to sustain the topic id of webxdcs needed for peer channels. webxdc_topic: Option, + + /// This field is used when this is either a pre-message or a full-message. + pre_message_mode: Option, } /// Result of rendering a message, ready to be submitted to a send job. @@ -498,6 +510,7 @@ impl MimeFactory { sync_ids_to_delete: None, attach_selfavatar, webxdc_topic, + pre_message_mode: None, }; Ok(factory) } @@ -546,6 +559,7 @@ impl MimeFactory { sync_ids_to_delete: None, attach_selfavatar: false, webxdc_topic: None, + pre_message_mode: None, }; Ok(res) @@ -778,7 +792,10 @@ impl MimeFactory { headers.push(("Date", mail_builder::headers::raw::Raw::new(date).into())); let rfc724_mid = match &self.loaded { - Loaded::Message { msg, .. } => msg.rfc724_mid.clone(), + Loaded::Message { msg, .. } => match &self.pre_message_mode { + Some(PreMessageMode::PreMessage { .. }) => create_outgoing_rfc724_mid(), + _ => msg.rfc724_mid.clone(), + }, Loaded::Mdn { .. } => create_outgoing_rfc724_mid(), }; headers.push(( @@ -980,6 +997,22 @@ impl MimeFactory { "MIME-Version", mail_builder::headers::raw::Raw::new("1.0").into(), )); + + if self.pre_message_mode == Some(PreMessageMode::FullMessage) { + unprotected_headers.push(( + HeaderDef::ChatIsFullMessage.get_headername(), + mail_builder::headers::raw::Raw::new("1").into(), + )); + } else if let Some(PreMessageMode::PreMessage { + full_msg_rfc724_mid, + }) = self.pre_message_mode.clone() + { + unprotected_headers.push(( + HeaderDef::ChatFullMessageId.get_headername(), + mail_builder::headers::message_id::MessageId::new(full_msg_rfc724_mid).into(), + )); + } + for header @ (original_header_name, _header_value) in &headers { let header_name = original_header_name.to_lowercase(); if header_name == "message-id" { @@ -1146,10 +1179,17 @@ impl MimeFactory { let is_verified = verifier_id.is_some_and(|verifier_id| verifier_id != 0); - if !should_do_gossip { + let is_full_msg = + self.pre_message_mode == Some(PreMessageMode::FullMessage); + + if !should_do_gossip || is_full_msg { continue; } + debug_assert!( + self.pre_message_mode != Some(PreMessageMode::FullMessage) + ); + let header = Aheader { addr: addr.clone(), public_key: key.clone(), @@ -1837,8 +1877,12 @@ impl MimeFactory { // add attachment part if msg.viewtype.has_file() { - let file_part = build_body_file(context, &msg).await?; - parts.push(file_part); + if let Some(PreMessageMode::PreMessage { .. }) = self.pre_message_mode { + // TODO: generate thumbnail and attach it instead (if it makes sense) + } else { + let file_part = build_body_file(context, &msg).await?; + parts.push(file_part); + } } if let Some(msg_kml_part) = self.get_message_kml_part() { @@ -1883,7 +1927,7 @@ impl MimeFactory { } } - if self.attach_selfavatar { + if self.attach_selfavatar && self.pre_message_mode != Some(PreMessageMode::FullMessage) { match context.get_config(Config::Selfavatar).await? { Some(path) => match build_avatar_file(context, &path).await { Ok(avatar) => headers.push(( @@ -1952,6 +1996,16 @@ impl MimeFactory { Ok(message) } + + pub fn set_as_full_message(&mut self) { + self.pre_message_mode = Some(PreMessageMode::FullMessage); + } + + pub fn set_as_pre_message_for(&mut self, full_message: &RenderedEmail) { + self.pre_message_mode = Some(PreMessageMode::PreMessage { + full_msg_rfc724_mid: full_message.rfc724_mid.clone(), + }); + } } fn hidden_recipients() -> Address<'static> { diff --git a/src/test_utils.rs b/src/test_utils.rs index 73e9875f13..f3b7dc836b 100644 --- a/src/test_utils.rs +++ b/src/test_utils.rs @@ -697,6 +697,27 @@ impl TestContext { }) } + pub async fn get_smtp_rows_for_msg(&self, msg_id: MsgId) -> Vec<(i64, MsgId, String, String)> { + self.ctx + .sql + .query_map_vec( + r#" + SELECT id, msg_id, mime, recipients + FROM smtp + WHERE msg_id=?"#, + (msg_id,), + |row| { + let rowid: i64 = row.get(0)?; + let msg_id: MsgId = row.get(1)?; + let mime: String = row.get(2)?; + let recipients: String = row.get(3)?; + Ok((rowid, msg_id, mime, recipients)) + }, + ) + .await + .unwrap() + } + /// Retrieves a sent sync message from the db. /// /// This retrieves and removes a sync message which has been scheduled to send from the jobs