[widgets] Conversation widget render markdown (#657)

### Description

Make conversational widget render markdown. Pretty much cop paste of
[chat-ui implementation
here](https://github.com/huggingface/chat-ui/blob/4805941429bb3dccbf95944b469c733cfe81f708/src/lib/components/chat/ChatMessage.svelte#L221)
with a difference:
1. rendering code in `<div class="font-mono whitespace-pre-wrap">`
rather than specific CodeBlock.svelte (that has copy-code feature)
2. No support for latex in this PR. Should I add it?

| main    | this PR |
| -------- | ------- |
|<img
src="https://github.com/huggingface/huggingface.js/assets/11827707/af35cc4e-a75c-426b-8425-c7afde3ef692">
| <img
src="https://github.com/huggingface/huggingface.js/assets/11827707/da1ca4d9-4ef3-4da8-9112-16013d7cbdad">
|


### testing locally
`pnpm install` to get `marked` (new dep; fyi, we also use marked for
chat-ui & moon)

Author: Mishig

packages/widgets/package.json

@@ -45,9 +45,10 @@
 		"static/audioProcessor.js"
 	],
 	"dependencies": {
-		"@huggingface/tasks": "workspace:^",
+		"@huggingface/inference": "workspace:^",
 		"@huggingface/jinja": "workspace:^",
-		"@huggingface/inference": "workspace:^"
+		"@huggingface/tasks": "workspace:^",
+		"marked": "^12.0.2"
 	},
 	"peerDependencies": {
 		"svelte": "^3.59.2"

packages/widgets/pnpm-lock.yaml

@@ -1,14 +1,64 @@
 <script lang="ts">
+	import type { SpecialTokensMap } from "@huggingface/tasks";
+	import { marked, type MarkedOptions } from "marked";
+
 	export let position: "left" | "right" | "center";
 	export let text: string;
+	export let specialTokensMap: SpecialTokensMap | undefined = undefined;
+
+	$: tokens = marked.lexer(sanitizeMd(text));
 
 	const CLASSES: Record<typeof position, string> = {
 		left: "rounded-2xl mr-7 place-self-start bg-gray-50 dark:bg-gray-850 dark:text-gray-200",
 		right: "rounded-2xl ml-7 bg-blue-600 text-white",
 		center: "self-center justify-center border-b text-center pb-4",
 	};
+
+	function sanitizeMd(md: string) {
+		const PUBLIC_SEP_TOKEN = "</s>";
+		let ret = md
+			.replace(/<\|[a-z]*$/, "")
+			.replace(/<\|[a-z]+\|$/, "")
+			.replace(/<$/, "")
+			.replaceAll(PUBLIC_SEP_TOKEN, " ")
+			.replaceAll(/<\|[a-z]+\|>/g, " ")
+			.replaceAll(/<br\s?\/?>/gi, "\n")
+			.replaceAll("<", "&lt;")
+			.trim();
+
+		const stopTokens = ["<|endoftext|>"];
+		if (typeof specialTokensMap?.eos_token === "string") {
+			stopTokens.push(specialTokensMap.eos_token);
+		}
+		for (const stop of stopTokens) {
+			if (ret.endsWith(stop)) {
+				ret = ret.slice(0, -stop.length).trim();
+			}
+		}
+
+		return ret;
+	}
+
+	const renderer = new marked.Renderer();
+	renderer.codespan = (code) => `<div class="font-mono">${code.replaceAll("&amp;", "&")}</div>`;
+	renderer.code = (code) => `<div class="font-mono whitespace-pre-wrap">${code.replaceAll("&amp;", "&")}</div>`;
+
+	renderer.link = (href, title, text) =>
+		`<a href="${href?.replace(/>$/, "")}" target="_blank" rel="noreferrer">${text}</a>`;
+
+	const { extensions, ...defaults } = marked.getDefaults() as MarkedOptions & {
+		extensions: any;
+	};
+	const options: MarkedOptions = {
+		...defaults,
+		gfm: true,
+		breaks: true,
+		renderer,
+	};
 </script>
 
 <div class="px-3 py-2 text-smd {CLASSES[position]}">
-	{text}
+	{#each tokens as token}
+		{@html marked.parse(token.raw, options)}
+	{/each}
 </div>

packages/widgets/src/lib/components/InferenceWidget/shared/WidgetOuputConvoBubble/WidgetOutputConvoBubble.svelte

@@ -3,11 +3,12 @@
 
 	import { isFullyScrolled, scrollToMax } from "../../../../utils/ViewUtils.js";
 	import WidgetOutputConvoBubble from "../WidgetOuputConvoBubble/WidgetOutputConvoBubble.svelte";
-	import type { ChatMessage } from "@huggingface/tasks";
+	import type { ChatMessage, SpecialTokensMap } from "@huggingface/tasks";
 	import { widgetStates } from "../../stores.js";
 
 	export let modelId: string;
 	export let messages: ChatMessage[];
+	export let specialTokensMap: SpecialTokensMap | undefined = undefined;
 
 	let wrapperEl: HTMLElement;
 	$: isMaximized = $widgetStates?.[modelId]?.isMaximized;
@@ -30,7 +31,7 @@
 	<div class="flex flex-col items-end space-y-4 p-3">
 		{#each messages as message}
 			{@const position = message.role === "user" ? "right" : message.role === "assistant" ? "left" : "center"}
-			<WidgetOutputConvoBubble {position} text={message.content} />
+			<WidgetOutputConvoBubble {position} {specialTokensMap} text={message.content} />
 		{/each}
 	</div>
 </div>

packages/widgets/src/lib/components/InferenceWidget/shared/WidgetOutputConvo/WidgetOutputConvo.svelte

@@ -48,6 +48,7 @@
 
 	let compiledTemplate: Template;
 	let tokenizerConfig: TokenizerConfig;
+	let specialTokensMap: SpecialTokensMap | undefined = undefined;
 	let inferenceClient: HfInference | undefined = undefined;
 	let abort: AbortController | undefined = undefined;
 
@@ -132,14 +133,14 @@
 			return;
 		}
 		// Render chat template
-		const special_tokens_map = extractSpecialTokensMap(tokenizerConfig);
+		specialTokensMap = extractSpecialTokensMap(tokenizerConfig);
 
 		let chatText;
 		try {
 			chatText = compiledTemplate.render({
 				messages,
 				add_generation_prompt: true,
-				...special_tokens_map,
+				...specialTokensMap,
 			});
 		} catch (e) {
 			error = `An error occurred while rendering the chat template: "${(e as Error).message}"`;
@@ -273,7 +274,7 @@
 		on:reset={clearConversation}
 		showReset={!!messages.length}
 	/>
-	<WidgetOutputConvo modelId={model.id} {messages} />
+	<WidgetOutputConvo modelId={model.id} {messages} {specialTokensMap} />
 
 	<WidgetQuickInput
 		bind:value={text}