feat: send pre-message on messages with large attachments

src/chat.rs

@@ -27,15 +27,17 @@ use crate::constants::{
 use crate::contact::{self, Contact, ContactId, Origin};
 use crate::context::Context;
 use crate::debug_logging::maybe_set_logging_xdc;
-use crate::download::DownloadState;
+use crate::download::{
+    DownloadState, PRE_MESSAGE_ATTACHMENT_SIZE_THRESHOLD, PRE_MESSAGE_SIZE_WARNING_THRESHOLD,
+};
 use crate::ephemeral::{Timer as EphemeralTimer, start_chat_ephemeral_timers};
 use crate::events::EventType;
 use crate::key::self_fingerprint;
 use crate::location;
 use crate::log::{LogExt, error, info, warn};
 use crate::logged_debug_assert;
 use crate::message::{self, Message, MessageState, MsgId, Viewtype};
-use crate::mimefactory::MimeFactory;
+use crate::mimefactory::{MimeFactory, RenderedEmail};
 use crate::mimeparser::SystemMessage;
 use crate::param::{Param, Params};
 use crate::receive_imf::ReceivedMsg;
@@ -2804,7 +2806,47 @@ pub(crate) async fn create_send_msg_jobs(context: &Context, msg: &mut Message) -
         return Ok(Vec::new());
     }
 
-    let rendered_msg = match mimefactory.render(context).await {
+    // render message and pre message.
+    // pre message is a small message with metadata
+    // which announces a larger message. Large messages are not downloaded in the background.
+
+    let needs_pre_message = msg.viewtype.has_file()
+        && msg
+            .get_filebytes(context)
+            .await?
+            .context("filebytes not available, even though message has attachment")?
+            > PRE_MESSAGE_ATTACHMENT_SIZE_THRESHOLD;
+
+    let render_result: Result<(RenderedEmail, Option<RenderedEmail>)> = async {
+        if needs_pre_message {
+            let mut mimefactory_full_msg = mimefactory.clone();
+            mimefactory_full_msg.set_as_full_message();
+            let rendered_msg = mimefactory_full_msg.render(context).await?;
+
+            let mut mimefactory_pre_msg = mimefactory;
+            mimefactory_pre_msg.set_as_pre_message_for(&rendered_msg);
+            let rendered_pre_msg = mimefactory_pre_msg
+                .render(context)
+                .await
+                .context("pre-message failed to render")?;
+
+            if rendered_pre_msg.message.len() > PRE_MESSAGE_SIZE_WARNING_THRESHOLD {
+                warn!(
+                    context,
+                    "pre message for message (MsgId={}) is larger than expected: {}",
+                    msg.id,
+                    rendered_pre_msg.message.len()
+                );
+            }
+
+            Ok((rendered_msg, Some(rendered_pre_msg)))
+        } else {
+            Ok((mimefactory.render(context).await?, None))
+        }
+    }
+    .await;
+
+    let (rendered_msg, rendered_pre_msg) = match render_result {
         Ok(res) => Ok(res),
         Err(err) => {
             message::set_msg_failed(context, msg, &err.to_string()).await?;
@@ -2825,7 +2867,7 @@ pub(crate) async fn create_send_msg_jobs(context: &Context, msg: &mut Message) -
             msg.id,
             needs_encryption
         );
-    }
+    };
 
     let now = smeared_time(context);
 
@@ -2870,12 +2912,26 @@ pub(crate) async fn create_send_msg_jobs(context: &Context, msg: &mut Message) -
         } else {
             for recipients_chunk in recipients.chunks(chunk_size) {
                 let recipients_chunk = recipients_chunk.join(" ");
+                // send pre-message before actual message
+                if let Some(pre_msg) = &rendered_pre_msg {
+                    let row_id = t.execute(
+                        "INSERT INTO smtp (rfc724_mid, recipients, mime, msg_id) \
+                    VALUES            (?1,         ?2,         ?3,   ?4)",
+                        (
+                            &pre_msg.rfc724_mid,
+                            &recipients_chunk,
+                            &pre_msg.message,
+                            msg.id,
+                        ),
+                    )?;
+                    row_ids.push(row_id.try_into()?);
+                }
                 let row_id = t.execute(
                     "INSERT INTO smtp (rfc724_mid, recipients, mime, msg_id) \
                     VALUES            (?1,         ?2,         ?3,   ?4)",
                     (
                         &rendered_msg.rfc724_mid,
-                        recipients_chunk,
+                        &recipients_chunk,
                         &rendered_msg.message,
                         msg.id,
                     ),

src/download.rs

@@ -18,6 +18,21 @@ use crate::{EventType, chatlist_events};
 /// `MIN_DELETE_SERVER_AFTER` increases the timeout in this case.
 pub(crate) const MIN_DELETE_SERVER_AFTER: i64 = 48 * 60 * 60;
 
+/// From this point onward outgoing messages are considered large
+/// and get a pre-message, which announces the full message.
+// this is only about sending so we can modify it any time.
+pub(crate) const PRE_MESSAGE_ATTACHMENT_SIZE_THRESHOLD: u64 = 140_000;
+
+/// Max message size to be fetched in the background.
+/// This limit defines what messages are fully fetched in the background.
+/// This is for all messages that don't have the full message header.
+#[allow(unused)]
+pub(crate) const MAX_FETCH_MSG_SIZE: usize = 1_000_000;
+
+/// Max size for pre messages. A warning is emitted when this is exceeded.
+/// Should be well below `MAX_FETCH_MSG_SIZE`
+pub(crate) const PRE_MESSAGE_SIZE_WARNING_THRESHOLD: usize = 150_000;
+
 /// Download state of the message.
 #[derive(
     Debug,
@@ -193,10 +208,14 @@ impl Session {
 
 #[cfg(test)]
 mod tests {
+    use mailparse::MailHeaderMap;
     use num_traits::FromPrimitive;
 
     use super::*;
-    use crate::chat::send_msg;
+    use crate::chat::{self, create_group, send_msg};
+    use crate::headerdef::{HeaderDef, HeaderDefMap};
+    use crate::message::Viewtype;
+    use crate::mimeparser::MimeMessage;
     use crate::receive_imf::receive_imf_from_inbox;
     use crate::test_utils::TestContext;
 
@@ -295,4 +314,230 @@ mod tests {
 
         Ok(())
     }
+    /// Tests that pre message is sent for attachment larger than `PRE_MESSAGE_ATTACHMENT_SIZE_THRESHOLD`
+    /// Also test that pre message is sent first, before the full message
+    /// And that Autocrypt-gossip and selfavatar never go into full-messages
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_sending_pre_message() -> Result<()> {
+        let alice = TestContext::new_alice().await;
+        let bob = TestContext::new_bob().await;
+        let fiona = TestContext::new_fiona().await;
+        let group_id = alice
+            .create_group_with_members("test group", &[&bob, &fiona])
+            .await;
+
+        let mut msg = Message::new(Viewtype::File);
+        msg.set_file_from_bytes(&alice.ctx, "test.bin", &[0u8; 300_000], None)?;
+        msg.set_text("test".to_owned());
+
+        // assert that test attachment is bigger than limit
+        assert!(
+            msg.get_filebytes(&alice.ctx).await?.unwrap() > PRE_MESSAGE_ATTACHMENT_SIZE_THRESHOLD
+        );
+
+        let msg_id = chat::send_msg(&alice.ctx, group_id, &mut msg)
+            .await
+            .unwrap();
+        let smtp_rows = alice.get_smtp_rows_for_msg(msg_id).await;
+
+        //   pre-message and full message should be present
+        //   and test that correct headers are present on both messages
+        assert_eq!(smtp_rows.len(), 2);
+        let pre_message = mailparse::parse_mail(
+            smtp_rows
+                .first()
+                .expect("first element exists")
+                .2
+                .as_bytes(),
+        )?;
+        let full_message_bytes = smtp_rows
+            .get(1)
+            .expect("second element exists")
+            .2
+            .as_bytes();
+        let full_message = mailparse::parse_mail(full_message_bytes)?;
+
+        assert!(
+            pre_message
+                .get_headers()
+                .get_first_header(HeaderDef::ChatIsFullMessage.get_headername())
+                .is_none()
+        );
+        assert!(
+            full_message
+                .get_headers()
+                .get_first_header(HeaderDef::ChatIsFullMessage.get_headername())
+                .is_some()
+        );
+
+        assert_eq!(
+            pre_message
+                .headers
+                .get_header_value(HeaderDef::ChatFullMessageId),
+            full_message.headers.get_header_value(HeaderDef::MessageId)
+        );
+        assert!(
+            full_message
+                .headers
+                .get_header_value(HeaderDef::ChatFullMessageId)
+                .is_none()
+        );
+
+        // full message should have the rfc message id
+        assert_eq!(
+            full_message.headers.get_header_value(HeaderDef::MessageId),
+            Some(format!("<{}>", msg.rfc724_mid))
+        );
+
+        //  test that message ids are different
+        assert_ne!(
+            pre_message.headers.get_header_value(HeaderDef::MessageId),
+            full_message.headers.get_header_value(HeaderDef::MessageId)
+        );
+
+        // also test that Autocrypt-gossip and selfavatar should never go into full-messages
+        let decrypted_full_message = MimeMessage::from_bytes(&bob.ctx, full_message_bytes).await?;
+        assert!(!decrypted_full_message.decrypting_failed);
+        assert_eq!(decrypted_full_message.gossiped_keys.len(), 0);
+        assert_eq!(decrypted_full_message.user_avatar, None);
+        Ok(())
+    }
+
+    /// Tests that no pre message is sent for normal message
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_not_sending_pre_message_no_attachment() -> Result<()> {
+        let alice = TestContext::new_alice().await;
+        let bob = TestContext::new_bob().await;
+        let chat = alice.create_chat(&bob).await;
+
+        // send normal text message
+        let mut msg = Message::new(Viewtype::Text);
+        msg.set_text("test".to_owned());
+        let msg_id = chat::send_msg(&alice.ctx, chat.id, &mut msg).await.unwrap();
+        let smtp_rows = alice.get_smtp_rows_for_msg(msg_id).await;
+
+        //   only one message and no "is full message" header should be present
+        assert_eq!(smtp_rows.len(), 1);
+
+        let mime = smtp_rows.first().expect("first element exists").2.clone();
+        let mail = mailparse::parse_mail(mime.as_bytes())?;
+
+        assert!(
+            mail.get_headers()
+                .get_first_header(HeaderDef::ChatIsFullMessage.get_headername())
+                .is_none()
+        );
+        assert!(
+            mail.get_headers()
+                .get_first_header(HeaderDef::ChatFullMessageId.get_headername())
+                .is_none()
+        );
+        Ok(())
+    }
+
+    /// Tests that no pre message is sent for attachment smaller than `PRE_MESSAGE_ATTACHMENT_SIZE_THRESHOLD`
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_not_sending_pre_message_for_small_attachment() -> Result<()> {
+        let alice = TestContext::new_alice().await;
+        let bob = TestContext::new_bob().await;
+        let chat = alice.create_chat(&bob).await;
+
+        let mut msg = Message::new(Viewtype::File);
+        msg.set_file_from_bytes(&alice.ctx, "test.bin", &[0u8; 100_000], None)?;
+        msg.set_text("test".to_owned());
+
+        // assert that test attachment is smaller than limit
+        assert!(
+            msg.get_filebytes(&alice.ctx).await?.unwrap() < PRE_MESSAGE_ATTACHMENT_SIZE_THRESHOLD
+        );
+
+        let msg_id = chat::send_msg(&alice.ctx, chat.id, &mut msg).await.unwrap();
+        let smtp_rows = alice.get_smtp_rows_for_msg(msg_id).await;
+
+        //   only one message and no "is full message" header should be present
+        assert_eq!(smtp_rows.len(), 1);
+
+        let mime = smtp_rows.first().expect("first element exists").2.clone();
+        let mail = mailparse::parse_mail(mime.as_bytes())?;
+
+        assert!(
+            mail.get_headers()
+                .get_first_header(HeaderDef::ChatIsFullMessage.get_headername())
+                .is_none()
+        );
+        assert!(
+            mail.get_headers()
+                .get_first_header(HeaderDef::ChatFullMessageId.get_headername())
+                .is_none()
+        );
+
+        Ok(())
+    }
+
+    /// Tests that pre message is not send for large webxdc updates
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_render_webxdc_status_update_object_range() -> Result<()> {
+        let t = TestContext::new_alice().await;
+        let chat_id = create_group(&t, "a chat").await?;
+
+        let instance = {
+            let mut instance = Message::new(Viewtype::File);
+            instance.set_file_from_bytes(
+                &t,
+                "minimal.xdc",
+                include_bytes!("../test-data/webxdc/minimal.xdc"),
+                None,
+            )?;
+            let instance_msg_id = send_msg(&t, chat_id, &mut instance).await?;
+            assert_eq!(instance.viewtype, Viewtype::Webxdc);
+            Message::load_from_db(&t, instance_msg_id).await
+        }
+        .unwrap();
+
+        t.pop_sent_msg().await;
+        assert_eq!(t.sql.count("SELECT COUNT(*) FROM smtp", ()).await?, 0);
+
+        let long_text = String::from_utf8(vec![b'a'; 300_000])?;
+        assert!(long_text.len() > PRE_MESSAGE_ATTACHMENT_SIZE_THRESHOLD.try_into().unwrap());
+        t.send_webxdc_status_update(instance.id, &format!("{{\"payload\": \"{long_text}\"}}"))
+            .await?;
+        t.flush_status_updates().await?;
+
+        assert_eq!(t.sql.count("SELECT COUNT(*) FROM smtp", ()).await?, 1);
+        Ok(())
+    }
+
+    // test that pre message is not send for large large text
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_not_sending_pre_message_for_large_text() -> Result<()> {
+        let alice = TestContext::new_alice().await;
+        let bob = TestContext::new_bob().await;
+        let chat = alice.create_chat(&bob).await;
+
+        // send normal text message
+        let mut msg = Message::new(Viewtype::Text);
+        let long_text = String::from_utf8(vec![b'a'; 300_000])?;
+        assert!(long_text.len() > PRE_MESSAGE_ATTACHMENT_SIZE_THRESHOLD.try_into().unwrap());
+        msg.set_text(long_text);
+        let msg_id = chat::send_msg(&alice.ctx, chat.id, &mut msg).await.unwrap();
+        let smtp_rows = alice.get_smtp_rows_for_msg(msg_id).await;
+
+        //   only one message and no "is full message" header should be present
+        assert_eq!(smtp_rows.len(), 1);
+
+        let mime = smtp_rows.first().expect("first element exists").2.clone();
+        let mail = mailparse::parse_mail(mime.as_bytes())?;
+
+        assert!(
+            mail.get_headers()
+                .get_first_header(HeaderDef::ChatIsFullMessage.get_headername())
+                .is_none()
+        );
+        assert!(
+            mail.get_headers()
+                .get_first_header(HeaderDef::ChatFullMessageId.get_headername())
+                .is_none()
+        );
+        Ok(())
+    }
 }