Render images

Author: SignalRT

LLama.Examples/Examples/MtmdInteractiveModeExecute.cs

@@ -1,3 +1,5 @@
+using System.Collections.Generic;
+using System.IO;
 using System.Text.RegularExpressions;
 using LLama.Common;
 using Spectre.Console;
@@ -6,7 +8,7 @@
 
 namespace LLama.Examples.Examples
 {
-    // This example shows how to chat with LLaVA model with both image and text as input.
+    // This example shows how to chat with Mtmd model with both image and text as input.
     // It uses the interactive executor to inference.
     public class MtmdInteractiveModeExecute
     {
@@ -15,7 +17,7 @@ public static async Task Run()
             string multiModalProj = UserSettings.GetMMProjPath();
             string modelPath = UserSettings.GetModelPath();
             string modelImage = UserSettings.GetImagePath();
-            const int maxTokens = 1024;
+            const int maxTokens = 2048;
 
             var prompt = $"{{{modelImage}}}\nUSER:\nProvide a full description of the image.\nASSISTANT:\n";
 
@@ -44,39 +46,57 @@ public static async Task Run()
                     Temperature = 0.1f
                 },
 
-                AntiPrompts = new List<string> { "\nUSER:" },
+                AntiPrompts = new List<string> { "\nASSISTANT:" },
                 MaxTokens = maxTokens
 
             };
 
             do
             {
 
-                // Evaluate if we have images
+                // Evaluate if we have media
                 //
-                var imageMatches = Regex.Matches(prompt, "{([^}]*)}").Select(m => m.Value);
-                var imageCount = imageMatches.Count();
-                var hasImages = imageCount > 0;
+                var mediaMatches = Regex.Matches(prompt, "{([^}]*)}").Select(m => m.Value);
+                var mediaCount = mediaMatches.Count();
+                var hasMedia = mediaCount > 0;
 
-                if (hasImages)
+                if (hasMedia)
                 {
-                    var imagePathsWithCurlyBraces = Regex.Matches(prompt, "{([^}]*)}").Select(m => m.Value);
-                    var imagePaths = Regex.Matches(prompt, "{([^}]*)}").Select(m => m.Groups[1].Value).ToList();
+                    var mediaPathsWithCurlyBraces = Regex.Matches(prompt, "{([^}]*)}").Select(m => m.Value);
+                    var mediaPaths = Regex.Matches(prompt, "{([^}]*)}").Select(m => m.Groups[1].Value).ToList();
 
-                    var bitmaps = new List<SafeMtmdEmbed>();
+                    var embeds = new List<SafeMtmdEmbed>();
+                    var imageList = new List<byte[]>();
+                    var imageExtensions = new HashSet<string>(StringComparer.OrdinalIgnoreCase)
+                    {
+                        ".png",
+                        ".jpg",
+                        ".jpeg",
+                        ".bmp",
+                        ".gif",
+                        ".webp"
+                    };
+                    
                     try
                     {
-                        foreach (var imagePath in imagePaths)
+                        foreach (var mediaPath in mediaPaths)
                         {
-                            var bitmap = clipModel.LoadMedia(imagePath);
-                            bitmaps.Add(bitmap);
+                            var extension = Path.GetExtension(mediaPath);
+                            if (!string.IsNullOrEmpty(extension) && imageExtensions.Contains(extension))
+                            {
+                                // Keep the raw image data so the caller can reuse or inspect the images later.
+                                imageList.Add(File.ReadAllBytes(mediaPath));
+                            }
+
+                            var embed = clipModel.LoadMedia(mediaPath);
+                            embeds.Add(embed);
                         }
                     }
                     catch (IOException exception)
                     {
                         Console.ForegroundColor = ConsoleColor.Red;
                         Console.Write(
-                            $"Could not load your {(imageCount == 1 ? "image" : "images")}:");
+                            $"Could not load your {(mediaCount == 1 ? "media" : "medias")}:");
                         Console.Write($"{exception.Message}");
                         Console.ForegroundColor = ConsoleColor.Yellow;
                         Console.WriteLine("Please try again.");
@@ -91,7 +111,7 @@ public static async Task Run()
                     ex.Context.NativeHandle.MemorySequenceRemove( LLamaSeqId.Zero, -1, -1 );
 
                     // Replace placeholders with media markers (one marker per image)
-                    foreach (var path in imagePathsWithCurlyBraces)
+                    foreach (var path in mediaPathsWithCurlyBraces)
                     {
                         prompt = prompt.Replace(path, mediaMarker, StringComparison.Ordinal);
                     }
@@ -100,11 +120,11 @@ public static async Task Run()
                     Console.WriteLine($"Here are the images, that are sent to the chat model in addition to your message.");
                     Console.WriteLine();
 
-                    // foreach (var consoleImage in bitmaps.Select(embed => new CanvasImage(embed.GetDataSpan().ToArray())))
-                    // {
-                    //     consoleImage.MaxWidth = 50;
-                    //     AnsiConsole.Write(consoleImage);
-                    // }
+                    foreach (var consoleImage in imageList.Select(image => new CanvasImage(image.ToArray())))
+                    {
+                        consoleImage.MaxWidth = 50;
+                        AnsiConsole.Write(consoleImage);
+                    }
 
                     Console.WriteLine();
                     Console.ForegroundColor = ConsoleColor.Yellow;
@@ -116,7 +136,7 @@ public static async Task Run()
                     // Initialize Images in executor
                     //
                     ex.Embeds.Clear();
-                    foreach (var embed in bitmaps)
+                    foreach (var embed in embeds)
                         ex.Embeds.Add(embed);
                 }