feat: mimeparser: Omit Legacy Display Elements (#7130)

Omit Legacy Display Elements from "text/plain" and "text/html" (implement 4.5.3.{2,3} of
https://www.rfc-editor.org/rfc/rfc9788 "Header Protection for Cryptographically Protected Email").

Author: iequidoo

src/config.rs

@@ -438,6 +438,11 @@ pub enum Config {
     /// storing the same token multiple times on the server.
     EncryptedDeviceToken,
 
+    /// Enables running test hooks, e.g. see `InnerContext::pre_encrypt_mime_hook`.
+    /// This way is better than conditional compilation, i.e. `#[cfg(test)]`, because tests not
+    /// using this still run unmodified code.
+    TestHooks,
+
     /// Return an error from `receive_imf_inner()` for a fully downloaded message. For tests.
     FailOnReceivingFullMsg,
 }

src/context.rs

@@ -303,6 +303,17 @@ pub struct InnerContext {
     /// `Connectivity` values for mailboxes, unordered. Used to compute the aggregate connectivity,
     /// see [`Context::get_connectivity()`].
     pub(crate) connectivities: parking_lot::Mutex<Vec<ConnectivityStore>>,
+
+    #[expect(clippy::type_complexity)]
+    /// Transforms the root of the cryptographic payload before encryption.
+    pub(crate) pre_encrypt_mime_hook: parking_lot::Mutex<
+        Option<
+            for<'a> fn(
+                &Context,
+                mail_builder::mime::MimePart<'a>,
+            ) -> mail_builder::mime::MimePart<'a>,
+        >,
+    >,
 }
 
 /// The state of ongoing process.
@@ -467,6 +478,7 @@ impl Context {
             iroh: Arc::new(RwLock::new(None)),
             self_fingerprint: OnceLock::new(),
             connectivities: parking_lot::Mutex::new(Vec::new()),
+            pre_encrypt_mime_hook: None.into(),
         };
 
         let ctx = Context {
@@ -1051,6 +1063,13 @@ impl Context {
                 .await?
                 .to_string(),
         );
+        res.insert(
+            "test_hooks",
+            self.sql
+                .get_raw_config("test_hooks")
+                .await?
+                .unwrap_or_default(),
+        );
         res.insert(
             "fail_on_receiving_full_msg",
             self.sql

src/dehtml.rs

@@ -13,6 +13,7 @@ use quick_xml::{
 
 use crate::simplify::{SimplifiedText, simplify_quote};
 
+#[derive(Default)]
 struct Dehtml {
     strbuilder: String,
     quote: String,
@@ -25,6 +26,9 @@ struct Dehtml {
     /// Everything between `<div name="quote">` and `<div name="quoted-content">` is usually metadata
     /// If this is > `0`, then we are inside a `<div name="quoted-content">`.
     divs_since_quoted_content_div: u32,
+    /// `<div class="header-protection-legacy-display">` elements should be omitted, see
+    /// <https://www.rfc-editor.org/rfc/rfc9788.html#section-4.5.3.3>.
+    divs_since_hp_legacy_display: u32,
     /// All-Inkl just puts the quote into `<blockquote> </blockquote>`. This count is
     /// increased at each `<blockquote>` and decreased at each `</blockquote>`.
     blockquotes_since_blockquote: u32,
@@ -48,20 +52,25 @@ impl Dehtml {
     }
 
     fn get_add_text(&self) -> AddText {
-        if self.divs_since_quote_div > 0 && self.divs_since_quoted_content_div == 0 {
-            AddText::No // Everything between `<div name="quoted">` and `<div name="quoted_content">` is metadata which we don't want
+        // Everything between `<div name="quoted">` and `<div name="quoted_content">` is
+        // metadata which we don't want.
+        if self.divs_since_quote_div > 0 && self.divs_since_quoted_content_div == 0
+            || self.divs_since_hp_legacy_display > 0
+        {
+            AddText::No
         } else {
             self.add_text
         }
     }
 }
 
-#[derive(Debug, PartialEq, Clone, Copy)]
+#[derive(Debug, Default, PartialEq, Clone, Copy)]
 enum AddText {
     /// Inside `<script>`, `<style>` and similar tags
     /// which contents should not be displayed.
     No,
 
+    #[default]
     YesRemoveLineEnds,
 
     /// Inside `<pre>`.
@@ -121,12 +130,7 @@ fn dehtml_quick_xml(buf: &str) -> (String, String) {
 
     let mut dehtml = Dehtml {
         strbuilder: String::with_capacity(buf.len()),
-        quote: String::new(),
-        add_text: AddText::YesRemoveLineEnds,
-        last_href: None,
-        divs_since_quote_div: 0,
-        divs_since_quoted_content_div: 0,
-        blockquotes_since_blockquote: 0,
+        ..Default::default()
     };
 
     let mut reader = quick_xml::Reader::from_str(buf);
@@ -244,6 +248,7 @@ fn dehtml_endtag_cb(event: &BytesEnd, dehtml: &mut Dehtml) {
         "div" => {
             pop_tag(&mut dehtml.divs_since_quote_div);
             pop_tag(&mut dehtml.divs_since_quoted_content_div);
+            pop_tag(&mut dehtml.divs_since_hp_legacy_display);
 
             *dehtml.get_buf() += "\n\n";
             dehtml.add_text = AddText::YesRemoveLineEnds;
@@ -295,6 +300,8 @@ fn dehtml_starttag_cb<B: std::io::BufRead>(
         "div" => {
             maybe_push_tag(event, reader, "quote", &mut dehtml.divs_since_quote_div);
             maybe_push_tag(event, reader, "quoted-content", &mut dehtml.divs_since_quoted_content_div);
+            maybe_push_tag(event, reader, "header-protection-legacy-display",
+                &mut dehtml.divs_since_hp_legacy_display);
 
             *dehtml.get_buf() += "\n\n";
             dehtml.add_text = AddText::YesRemoveLineEnds;
@@ -539,6 +546,27 @@ mod tests {
         assert_eq!(txt.text.trim(), "two\nlines");
     }
 
+    #[test]
+    fn test_hp_legacy_display() {
+        let input = r#"
+<html><head><title></title></head><body>
+<div class="header-protection-legacy-display">
+<pre>Subject: Dinner plans</pre>
+</div>
+<p>
+Let's meet at Rama's Roti Shop at 8pm and go to the park
+from there.
+</p>
+</body>
+</html>
+        "#;
+        let txt = dehtml(input).unwrap();
+        assert_eq!(
+            txt.text.trim(),
+            "Let's meet at Rama's Roti Shop at 8pm and go to the park from there."
+        );
+    }
+
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_quote_div() {
         let input = include_str!("../test-data/message/gmx-quote-body.eml");

src/mimefactory.rs

@@ -1232,6 +1232,12 @@ impl MimeFactory {
             // once new core versions are sufficiently deployed.
             let anonymous_recipients = false;
 
+            if context.get_config_bool(Config::TestHooks).await? {
+                if let Some(hook) = &*context.pre_encrypt_mime_hook.lock() {
+                    message = hook(context, message);
+                }
+            }
+
             let encrypted = if let Some(shared_secret) = shared_secret {
                 encrypt_helper
                     .encrypt_symmetrically(context, &shared_secret, message, compress)

src/mimeparser.rs

@@ -1323,6 +1323,10 @@ impl MimeMessage {
                             let is_html = mime_type == mime::TEXT_HTML;
                             if is_html {
                                 self.is_mime_modified = true;
+                                // NB: This unconditionally removes Legacy Display Elements (see
+                                // <https://www.rfc-editor.org/rfc/rfc9788.html#section-4.5.3.3>). We
+                                // don't check for the "hp-legacy-display" Content-Type parameter
+                                // for simplicity.
                                 if let Some(text) = dehtml(&decoded_data) {
                                     text
                                 } else {
@@ -1350,16 +1354,30 @@ impl MimeMessage {
 
                         let (simplified_txt, simplified_quote) = if mime_type.type_() == mime::TEXT
                             && mime_type.subtype() == mime::PLAIN
-                            && is_format_flowed
                         {
-                            let delsp = if let Some(delsp) = mail.ctype.params.get("delsp") {
-                                delsp.as_str().eq_ignore_ascii_case("yes")
-                            } else {
-                                false
+                            // Don't check that we're inside an encrypted or signed part for
+                            // simplicity.
+                            let simplified_txt = match mail
+                                .ctype
+                                .params
+                                .get("hp-legacy-display")
+                                .is_some_and(|v| v == "1")
+                            {
+                                false => simplified_txt,
+                                true => rm_legacy_display_elements(&simplified_txt),
                             };
-                            let unflowed_text = unformat_flowed(&simplified_txt, delsp);
-                            let unflowed_quote = top_quote.map(|q| unformat_flowed(&q, delsp));
-                            (unflowed_text, unflowed_quote)
+                            if is_format_flowed {
+                                let delsp = if let Some(delsp) = mail.ctype.params.get("delsp") {
+                                    delsp.as_str().eq_ignore_ascii_case("yes")
+                                } else {
+                                    false
+                                };
+                                let unflowed_text = unformat_flowed(&simplified_txt, delsp);
+                                let unflowed_quote = top_quote.map(|q| unformat_flowed(&q, delsp));
+                                (unflowed_text, unflowed_quote)
+                            } else {
+                                (simplified_txt, top_quote)
+                            }
                         } else {
                             (simplified_txt, top_quote)
                         };
@@ -1981,6 +1999,20 @@ impl MimeMessage {
     }
 }
 
+fn rm_legacy_display_elements(text: &str) -> String {
+    let mut res = None;
+    for l in text.lines() {
+        res = res.map(|r: String| match r.is_empty() {
+            true => l.to_string(),
+            false => r + "\r\n" + l,
+        });
+        if l.is_empty() {
+            res = Some(String::new());
+        }
+    }
+    res.unwrap_or_default()
+}
+
 fn remove_header(
     headers: &mut HashMap<String, String>,
     key: &str,

src/mimeparser/mimeparser_tests.rs

@@ -1751,6 +1751,39 @@ async fn test_time_in_future() -> Result<()> {
     Ok(())
 }
 
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_hp_legacy_display() -> Result<()> {
+    let mut tcm = TestContextManager::new();
+    let alice = &tcm.alice().await;
+    let bob = &tcm.bob().await;
+
+    let mut msg = Message::new_text(
+        "Subject: Dinner plans\n\
+        \n\
+        Let's eat"
+            .to_string(),
+    );
+    msg.set_subject("Dinner plans".to_string());
+    let chat_id = alice.create_chat(bob).await.id;
+    alice.set_config_bool(Config::TestHooks, true).await?;
+    *alice.pre_encrypt_mime_hook.lock() = Some(|_, mut mime| {
+        for (h, v) in &mut mime.headers {
+            if h == "Content-Type" {
+                if let mail_builder::headers::HeaderType::ContentType(ct) = v {
+                    *ct = ct.clone().attribute("hp-legacy-display", "1");
+                }
+            }
+        }
+        mime
+    });
+    let sent_msg = alice.send_msg(chat_id, &mut msg).await;
+
+    let msg_bob = bob.recv_msg(&sent_msg).await;
+    assert_eq!(msg_bob.subject, "Dinner plans");
+    assert_eq!(msg_bob.text, "Let's eat");
+    Ok(())
+}
+
 /// Tests that subject is not prepended to the message
 /// when bot receives it.
 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]

src/test_utils.rs

@@ -1679,7 +1679,6 @@ Until the false-positive is fixed:
     }
 }
 
-#[cfg(test)]
 mod tests {
     use super::*;