Use BeautifulSoup with html5lib for better results

Author: i-ky

pyproject.toml

@@ -60,6 +60,7 @@ warn_unreachable = true
 [[tool.mypy.overrides]]
 module = [
     "apiai.*",
+    "bs4.*",
     "feedparser.*",
     "gitlint.*",
     "googleapiclient.*",

zulip/integrations/rss/requirements.txt

@@ -1,2 +1,3 @@
+bs4>=4.12.2
 feedparser>=6.0.10
 markdownify>=0.11.6

zulip/integrations/rss/rss-bot

@@ -18,7 +18,8 @@ from html.parser import HTMLParser
 from typing import Any, Dict, List, Optional, Tuple
 
 import feedparser
-from markdownify import markdownify
+from bs4 import BeautifulSoup
+from markdownify import MarkdownConverter
 from typing_extensions import override
 
 import zulip
@@ -193,7 +194,8 @@ def send_zulip(entry: Any, feed_name: str) -> Dict[str, Any]:
         body = unwrap_text(body)
 
     def md(html: str) -> str:
-        return markdownify(html, escape_underscores=False)
+        soup: BeautifulSoup = BeautifulSoup(html, "html5lib")
+        return MarkdownConverter(escape_underscores=False).convert_soup(soup)
 
     convert: Callable[[str], str] = strip_tags if opts.strip else md
     content = f"**[{entry.title}]({entry.link})**\n{convert(body)}\n{entry.link}"