feat: send pre-message on messages with large attachments

Author: Simon-Laux

src/chat.rs

@@ -27,15 +27,17 @@ use crate::constants::{
 use crate::contact::{self, Contact, ContactId, Origin};
 use crate::context::Context;
 use crate::debug_logging::maybe_set_logging_xdc;
-use crate::download::DownloadState;
+use crate::download::{
+    DownloadState, PRE_MESSAGE_ATTACHMENT_SIZE_THRESHOLD, PRE_MESSAGE_SIZE_WARNING_THRESHOLD,
+};
 use crate::ephemeral::{Timer as EphemeralTimer, start_chat_ephemeral_timers};
 use crate::events::EventType;
 use crate::key::self_fingerprint;
 use crate::location;
 use crate::log::{LogExt, error, info, warn};
 use crate::logged_debug_assert;
 use crate::message::{self, Message, MessageState, MsgId, Viewtype};
-use crate::mimefactory::MimeFactory;
+use crate::mimefactory::{MimeFactory, RenderedEmail};
 use crate::mimeparser::SystemMessage;
 use crate::param::{Param, Params};
 use crate::receive_imf::ReceivedMsg;
@@ -2804,7 +2806,47 @@ pub(crate) async fn create_send_msg_jobs(context: &Context, msg: &mut Message) -
         return Ok(Vec::new());
     }
 
-    let rendered_msg = match mimefactory.render(context).await {
+    // render message and pre message.
+    // pre message is a small message with metadata
+    // which announces a larger message. Large messages are not downloaded in the background.
+
+    let needs_pre_message = msg.viewtype.has_file()
+        && msg
+            .get_filebytes(context)
+            .await?
+            .context("filebytes not available, even though message has attachment")?
+            > PRE_MESSAGE_ATTACHMENT_SIZE_THRESHOLD;
+
+    let render_result: Result<(RenderedEmail, Option<RenderedEmail>)> = async {
+        if needs_pre_message {
+            let mut mimefactory_full_msg = mimefactory.clone();
+            mimefactory_full_msg.set_as_full_message();
+            let rendered_msg = mimefactory_full_msg.render(context).await?;
+
+            let mut mimefactory_pre_msg = mimefactory;
+            mimefactory_pre_msg.set_as_pre_message_for(&rendered_msg);
+            let rendered_pre_msg = mimefactory_pre_msg
+                .render(context)
+                .await
+                .context("pre-message failed to render")?;
+
+            if rendered_pre_msg.message.len() > PRE_MESSAGE_SIZE_WARNING_THRESHOLD {
+                warn!(
+                    context,
+                    "pre message for message (MsgId={}) is larger than expected: {}",
+                    msg.id,
+                    rendered_pre_msg.message.len()
+                );
+            }
+
+            Ok((rendered_msg, Some(rendered_pre_msg)))
+        } else {
+            Ok((mimefactory.render(context).await?, None))
+        }
+    }
+    .await;
+
+    let (rendered_msg, rendered_pre_msg) = match render_result {
         Ok(res) => Ok(res),
         Err(err) => {
             message::set_msg_failed(context, msg, &err.to_string()).await?;
@@ -2825,7 +2867,7 @@ pub(crate) async fn create_send_msg_jobs(context: &Context, msg: &mut Message) -
             msg.id,
             needs_encryption
         );
-    }
+    };
 
     let now = smeared_time(context);
 
@@ -2870,12 +2912,26 @@ pub(crate) async fn create_send_msg_jobs(context: &Context, msg: &mut Message) -
         } else {
             for recipients_chunk in recipients.chunks(chunk_size) {
                 let recipients_chunk = recipients_chunk.join(" ");
+                // send pre-message before actual message
+                if let Some(pre_msg) = &rendered_pre_msg {
+                    let row_id = t.execute(
+                        "INSERT INTO smtp (rfc724_mid, recipients, mime, msg_id) \
+                    VALUES            (?1,         ?2,         ?3,   ?4)",
+                        (
+                            &pre_msg.rfc724_mid,
+                            &recipients_chunk,
+                            &pre_msg.message,
+                            msg.id, // TODO: check if this is correct or we need another id here?
+                        ),
+                    )?;
+                    row_ids.push(row_id.try_into()?);
+                }
                 let row_id = t.execute(
                     "INSERT INTO smtp (rfc724_mid, recipients, mime, msg_id) \
                     VALUES            (?1,         ?2,         ?3,   ?4)",
                     (
                         &rendered_msg.rfc724_mid,
-                        recipients_chunk,
+                        &recipients_chunk,
                         &rendered_msg.message,
                         msg.id,
                     ),

src/download.rs

@@ -18,6 +18,20 @@ use crate::{EventType, chatlist_events};
 /// `MIN_DELETE_SERVER_AFTER` increases the timeout in this case.
 pub(crate) const MIN_DELETE_SERVER_AFTER: i64 = 48 * 60 * 60;
 
+/// From this point onward outgoing messages are considered large
+/// and get a pre-message, which announces the full message.
+// this is only about sending so we can modify it any time.
+pub(crate) const PRE_MESSAGE_ATTACHMENT_SIZE_THRESHOLD: u64 = 140_000;
+
+/// Max message size to be fetched in the background.
+/// This limit defines what messages are fully fetched in the background.
+/// This is for all messages that don't have the full message header.
+pub(crate) const MAX_FETCH_MSG_SIZE: usize = 1_000_000;
+
+/// Max size for pre messages. A warning is emitted when this is exceeded.
+/// Should be well below `MAX_FETCH_MSG_SIZE`
+pub(crate) const PRE_MESSAGE_SIZE_WARNING_THRESHOLD: usize = 150_000;
+
 /// Download state of the message.
 #[derive(
     Debug,

src/headerdef.rs

@@ -102,6 +102,15 @@ pub enum HeaderDef {
     /// used to encrypt and decrypt messages.
     /// This secret is sent to a new member in the member-addition message.
     ChatBroadcastSecret,
+    /// This message announces a bigger message with attachment that is refereced by rfc724_mid.
+    #[strum(serialize = "Chat-Full-Message-ID")] // correct casing
+    ChatFullMessageId,
+
+    /// This message has a pre-message
+    /// and thus this message can be skipped while fetching messages.
+    /// This is a cleartext / unproteced header.
+    #[strum(serialize = "Chat-Is-Full-Message")] // correct casing
+    ChatIsFullMessage,
 
     /// [Autocrypt](https://autocrypt.org/) header.
     Autocrypt,
@@ -194,4 +203,17 @@ mod tests {
         );
         assert_eq!(headers.get_header_value(HeaderDef::Autocrypt), None);
     }
+
+    #[test]
+    /// Tests that headers have correct casing for sending.
+    fn header_name_correct_casing_for_sending() {
+        assert_eq!(
+            HeaderDef::ChatFullMessageId.get_headername(),
+            "Chat-Full-Message-ID"
+        );
+        assert_eq!(
+            HeaderDef::ChatIsFullMessage.get_headername(),
+            "Chat-Is-Full-Message"
+        );
+    }
 }

src/mimefactory.rs

@@ -58,6 +58,15 @@ pub enum Loaded {
     },
 }
 
+#[derive(Debug, Clone, PartialEq)]
+pub enum PreMessageMode {
+    /// adds a is full message header in unpretected part
+    FullMessage,
+    /// adds reference to full message to protected part
+    /// also adds metadata and hashes and explicitly excludes attachment
+    PreMessage { full_msg_rfc724_mid: String },
+}
+
 /// Helper to construct mime messages.
 #[derive(Debug, Clone)]
 pub struct MimeFactory {
@@ -145,6 +154,9 @@ pub struct MimeFactory {
 
     /// This field is used to sustain the topic id of webxdcs needed for peer channels.
     webxdc_topic: Option<TopicId>,
+
+    /// This field is used when this is either a pre-message or a full-message.
+    pre_message_mode: Option<PreMessageMode>,
 }
 
 /// Result of rendering a message, ready to be submitted to a send job.
@@ -498,6 +510,7 @@ impl MimeFactory {
             sync_ids_to_delete: None,
             attach_selfavatar,
             webxdc_topic,
+            pre_message_mode: None,
         };
         Ok(factory)
     }
@@ -546,6 +559,7 @@ impl MimeFactory {
             sync_ids_to_delete: None,
             attach_selfavatar: false,
             webxdc_topic: None,
+            pre_message_mode: None,
         };
 
         Ok(res)
@@ -778,7 +792,10 @@ impl MimeFactory {
         headers.push(("Date", mail_builder::headers::raw::Raw::new(date).into()));
 
         let rfc724_mid = match &self.loaded {
-            Loaded::Message { msg, .. } => msg.rfc724_mid.clone(),
+            Loaded::Message { msg, .. } => match &self.pre_message_mode {
+                Some(PreMessageMode::PreMessage { .. }) => create_outgoing_rfc724_mid(),
+                _ => msg.rfc724_mid.clone(),
+            },
             Loaded::Mdn { .. } => create_outgoing_rfc724_mid(),
         };
         headers.push((
@@ -980,6 +997,22 @@ impl MimeFactory {
             "MIME-Version",
             mail_builder::headers::raw::Raw::new("1.0").into(),
         ));
+
+        if self.pre_message_mode == Some(PreMessageMode::FullMessage) {
+            unprotected_headers.push((
+                HeaderDef::ChatIsFullMessage.get_headername(),
+                mail_builder::headers::raw::Raw::new("1").into(),
+            ));
+        } else if let Some(PreMessageMode::PreMessage {
+            full_msg_rfc724_mid,
+        }) = self.pre_message_mode.clone()
+        {
+            unprotected_headers.push((
+                HeaderDef::ChatFullMessageId.get_headername(),
+                mail_builder::headers::raw::Raw::new(full_msg_rfc724_mid).into(),
+            ));
+        }
+
         for header @ (original_header_name, _header_value) in &headers {
             let header_name = original_header_name.to_lowercase();
             if header_name == "message-id" {
@@ -1111,7 +1144,10 @@ impl MimeFactory {
                         for (addr, key) in &encryption_pubkeys {
                             let fingerprint = key.dc_fingerprint().hex();
                             let cmd = msg.param.get_cmd();
-                            let should_do_gossip = cmd == SystemMessage::MemberAddedToGroup
+                            let is_full_msg =
+                                self.pre_message_mode != Some(PreMessageMode::FullMessage);
+                            let should_do_gossip = is_full_msg
+                                && cmd == SystemMessage::MemberAddedToGroup
                                 || cmd == SystemMessage::SecurejoinMessage
                                 || multiple_recipients && {
                                     let gossiped_timestamp: Option<i64> = context
@@ -1837,8 +1873,12 @@ impl MimeFactory {
 
         // add attachment part
         if msg.viewtype.has_file() {
-            let file_part = build_body_file(context, &msg).await?;
-            parts.push(file_part);
+            if let Some(PreMessageMode::PreMessage { .. }) = self.pre_message_mode {
+                // TODO: generate thumbnail and attach it instead (if it makes sense)
+            } else {
+                let file_part = build_body_file(context, &msg).await?;
+                parts.push(file_part);
+            }
         }
 
         if let Some(msg_kml_part) = self.get_message_kml_part() {
@@ -1883,7 +1923,7 @@ impl MimeFactory {
             }
         }
 
-        if self.attach_selfavatar {
+        if self.attach_selfavatar && self.pre_message_mode != Some(PreMessageMode::FullMessage) {
             match context.get_config(Config::Selfavatar).await? {
                 Some(path) => match build_avatar_file(context, &path).await {
                     Ok(avatar) => headers.push((
@@ -1952,6 +1992,16 @@ impl MimeFactory {
 
         Ok(message)
     }
+
+    pub fn set_as_full_message(&mut self) {
+        self.pre_message_mode = Some(PreMessageMode::FullMessage);
+    }
+
+    pub fn set_as_pre_message_for(&mut self, full_message: &RenderedEmail) {
+        self.pre_message_mode = Some(PreMessageMode::PreMessage {
+            full_msg_rfc724_mid: full_message.rfc724_mid.clone(),
+        });
+    }
 }
 
 fn hidden_recipients() -> Address<'static> {