Add image-text-to-text + multimodal fixes (#477)

Tentatively

* We align better what multimodal is
* We add a new type for llava

Author: osanseviero

packages/tasks/src/pipelines.ts

@@ -238,7 +238,7 @@ export const PIPELINE_DATA = {
 	},
 	"feature-extraction": {
 		name: "Feature Extraction",
-		modality: "multimodal",
+		modality: "nlp",
 		color: "red",
 	},
 	"text-generation": {
@@ -419,7 +419,7 @@ export const PIPELINE_DATA = {
 	},
 	"text-to-image": {
 		name: "Text-to-Image",
-		modality: "multimodal",
+		modality: "cv",
 		color: "yellow",
 	},
 	"image-to-text": {
@@ -430,7 +430,7 @@ export const PIPELINE_DATA = {
 				name: "Image Captioning",
 			},
 		],
-		modality: "multimodal",
+		modality: "cv",
 		color: "red",
 	},
 	"image-to-image": {
@@ -454,7 +454,7 @@ export const PIPELINE_DATA = {
 	},
 	"image-to-video": {
 		name: "Image-to-Video",
-		modality: "multimodal",
+		modality: "cv",
 		color: "indigo",
 	},
 	"unconditional-image-generation": {
@@ -589,9 +589,15 @@ export const PIPELINE_DATA = {
 	},
 	"text-to-video": {
 		name: "Text-to-Video",
-		modality: "multimodal",
+		modality: "cv",
 		color: "green",
 	},
+	"image-text-to-text": {
+		name: "Image + Text to Image (VLLMs)",
+		modality: "multimodal",
+		color: "red",
+		hideInDatasets: true,
+	},
 	"visual-question-answering": {
 		name: "Visual Question Answering",
 		subtasks: [
@@ -622,7 +628,7 @@ export const PIPELINE_DATA = {
 	},
 	"graph-ml": {
 		name: "Graph Machine Learning",
-		modality: "multimodal",
+		modality: "other",
 		color: "green",
 	},
 	"mask-generation": {
@@ -637,12 +643,12 @@ export const PIPELINE_DATA = {
 	},
 	"text-to-3d": {
 		name: "Text-to-3D",
-		modality: "multimodal",
+		modality: "cv",
 		color: "yellow",
 	},
 	"image-to-3d": {
 		name: "Image-to-3D",
-		modality: "multimodal",
+		modality: "cv",
 		color: "green",
 	},
 	other: {

packages/tasks/src/tasks/index.ts

@@ -53,6 +53,7 @@ export const TASKS_MODEL_LIBRARIES: Record<PipelineType, ModelLibraryKey[]> = {
 	"graph-ml": ["transformers"],
 	"image-classification": ["keras", "timm", "transformers", "transformers.js"],
 	"image-segmentation": ["transformers", "transformers.js"],
+	"image-text-to-text": ["transformers"],
 	"image-to-image": ["diffusers", "transformers", "transformers.js"],
 	"image-to-text": ["transformers", "transformers.js"],
 	"image-to-video": ["diffusers"],
@@ -130,6 +131,7 @@ export const TASKS_DATA: Record<PipelineType, TaskData | undefined> = {
 	"graph-ml": undefined,
 	"image-classification": getData("image-classification", imageClassification),
 	"image-segmentation": getData("image-segmentation", imageSegmentation),
+	"image-text-to-text": undefined,
 	"image-to-image": getData("image-to-image", imageToImage),
 	"image-to-text": getData("image-to-text", imageToText),
 	"image-to-video": undefined,

packages/widgets/src/lib/components/Icons/IconImageAndTextToText.svelte

@@ -0,0 +1,28 @@
+<script lang="ts">
+	export let classNames = "";
+</script>
+
+<svg
+	class={classNames}
+	xmlns="http://www.w3.org/2000/svg"
+	xmlns:xlink="http://www.w3.org/1999/xlink"
+	aria-hidden="true"
+	role="img"
+	width="1em"
+	height="1em"
+	preserveAspectRatio="xMidYMid meet"
+	viewBox="0 0 32 32"
+>
+	<path
+		d="M29.707 19.293l-3-3a1 1 0 0 0-1.414 0L16 25.586V30h4.414l9.293-9.293a1 1 0 0 0 0-1.414zM19.586 28H18v-1.586l5-5L24.586 23zM26 21.586L24.414 20L26 18.414L27.586 20z"
+		fill="currentColor"
+	/>
+	<path
+		d="M20 13v-2h-2.142a3.94 3.94 0 0 0-.425-1.019l1.517-1.517l-1.414-1.414l-1.517 1.517A3.944 3.944 0 0 0 15 8.142V6h-2v2.142a3.944 3.944 0 0 0-1.019.425L10.464 7.05L9.05 8.464l1.517 1.517A3.94 3.94 0 0 0 10.142 11H8v2h2.142a3.94 3.94 0 0 0 .425 1.019L9.05 15.536l1.414 1.414l1.517-1.517a3.944 3.944 0 0 0 1.019.425V18h2v-2.142a3.944 3.944 0 0 0 1.019-.425l1.517 1.517l1.414-1.414l-1.517-1.517A3.94 3.94 0 0 0 17.858 13zm-6 1a2 2 0 1 1 2-2a2.002 2.002 0 0 1-2 2z"
+		fill="currentColor"
+	/>
+	<path
+		d="M12 30H6a2.002 2.002 0 0 1-2-2V4a2.002 2.002 0 0 1 2-2h16a2.002 2.002 0 0 1 2 2v10h-2V4H6v24h6z"
+		fill="currentColor"
+	/>
+</svg>

packages/widgets/src/lib/components/PipelineIcon/PipelineIcon.svelte

@@ -7,6 +7,7 @@
 	import IconConversational from "../Icons/IconConversational.svelte";
 	import IconFeatureExtraction from "../Icons/IconFeatureExtraction.svelte";
 	import IconFillMask from "../Icons/IconFillMask.svelte";
+	import IconImageAndTextToText from "../Icons/IconImageAndTextToText.svelte";
 	import IconImageClassification from "../Icons/IconImageClassification.svelte";
 	import IconImageSegmentation from "../Icons/IconImageSegmentation.svelte";
 	import IconObjectDetection from "../Icons/IconObjectDetection.svelte";
@@ -75,6 +76,7 @@
 		"video-classification": IconVideoClassification,
 		"image-segmentation": IconImageSegmentation,
 		"text-to-image": IconTextToImage,
+		"image-text-to-text": IconImageAndTextToText,
 		"image-to-text": IconImageToText,
 		"image-to-image": IconImageToImage,
 		"image-to-video": IconImageToVideo,