rss-bot: Add option to convert body to Markdown

Author: i-ky

pyproject.toml

@@ -64,6 +64,7 @@ module = [
     "gitlint.*",
     "googleapiclient.*",
     "irc.*",
+    "markdownify.*",
     "mercurial.*",
     "nio.*",
     "oauth2client.*",

zulip/integrations/rss/requirements.txt

@@ -1 +1,2 @@
 feedparser>=6.0.10
+markdownify>=0.11.6

zulip/integrations/rss/rss-bot

@@ -13,10 +13,12 @@ import re
 import sys
 import time
 import urllib.parse
+from collections.abc import Callable
 from html.parser import HTMLParser
 from typing import Any, Dict, List, Optional, Tuple
 
 import feedparser
+from markdownify import markdownify
 from typing_extensions import override
 
 import zulip
@@ -92,6 +94,19 @@ parser.add_argument(
     help="Convert $ to $$ (for KaTeX processing)",
     default=False,
 )
+body = parser.add_mutually_exclusive_group()
+body.add_argument(
+    "--strip",
+    dest="strip",
+    action="store_true",
+    help="Strip HTML tags from body",
+)
+body.add_argument(
+    "--markdownify",
+    dest="strip",
+    action="store_false",
+    help="Convert body from HTML to Markdown",
+)
 
 opts = parser.parse_args()
 
@@ -177,7 +192,11 @@ def send_zulip(entry: Any, feed_name: str) -> Dict[str, Any]:
     if opts.unwrap:
         body = unwrap_text(body)
 
-    content = f"**[{entry.title}]({entry.link})**\n{strip_tags(body)}\n{entry.link}"
+    def md(html: str) -> str:
+        return markdownify(html, escape_underscores=False)
+
+    convert: Callable[[str], str] = strip_tags if opts.strip else md
+    content = f"**[{entry.title}]({entry.link})**\n{convert(body)}\n{entry.link}"
 
     if opts.math:
         content = content.replace("$", "$$")