Review logic

Author: SignalRT

LLama.Examples/Examples/BatchedExecutorMtmd.cs

@@ -63,26 +63,27 @@ public static async Task Run()
             Console.ResetColor();
 
             var remaining = TokenCount;
-            while (remaining > 0)
+
+            async Task<bool> ProcessNextAsync()
             {
                 var decodeResult = await executor.Infer();
                 if (decodeResult == DecodeResult.NoKvSlot)
                 {
                     Console.ForegroundColor = ConsoleColor.Red;
                     Console.WriteLine("Insufficient KV cache space for multimodal evaluation.");
                     Console.ResetColor();
-                    break;
+                    return false;
                 }
 
                 if (decodeResult != DecodeResult.Ok)
                     throw new RuntimeError($"Failed to evaluate batch: {decodeResult}.");
 
                 if (!conversation.RequiresSampling)
-                    continue;
+                    return true;
 
                 var token = conversation.Sample(sampler);
                 if (token.IsEndOfGeneration(vocab))
-                    break;
+                    return false;
 
                 decoder.Add(token);
                 var delta = decoder.Read();
@@ -92,6 +93,11 @@ public static async Task Run()
                 sampler.Accept(token);
                 conversation.Prompt(token);
                 remaining--;
+                return remaining > 0;
+            }
+
+            while (remaining > 0 && await ProcessNextAsync())
+            {
             }
 
             Console.WriteLine();

LLama/Batched/BatchedExecutor.cs

@@ -332,17 +332,12 @@ public Task<DecodeResult> DecodeAsync(LLamaContext ctx, CancellationToken token)
             try
             {
                 var nPast = _conversation.GetMtmdPast();
-                for (var i = 0; i < _sequence.Chunks.Count; i++)
+                var status = _clipModel.EvaluateChunks(_sequence.Chunks, ctx.NativeHandle, ref nPast,
+                    (int)_conversation.ConversationId, checked((int)ctx.BatchSize), logitsLast: true);
+                if (status != 0)
                 {
-                    var chunk = _sequence.Chunks[i];
-                    var logitsLast = i == _sequence.Chunks.Count - 1;
-                    var status = _clipModel.EvaluateChunk(chunk.NativePtr, ctx.NativeHandle, ref nPast,
-                        (int)_conversation.ConversationId, checked((int)ctx.BatchSize), logitsLast);
-                    if (status != 0)
-                    {
-                        _conversation.OnMtmdEvaluationFailed(status);
-                        return Task.FromResult(DecodeResult.DecodeFailed);
-                    }
+                    _conversation.OnMtmdEvaluationFailed(status);
+                    return Task.FromResult(DecodeResult.DecodeFailed);
                 }
 
                 _conversation.OnMtmdEvaluationCompleted(nPast, _sequence);

LLama/Batched/Conversation.cs

@@ -75,12 +75,12 @@ internal Conversation(BatchedExecutor batch, LLamaSeqId id)
 
     internal sealed class MtmdChunkSequence : IDisposable
     {
-        public List<SafeMtmdInputChunk> Chunks { get; }
+        public SafeMtmdInputChunks Chunks { get; }
         public List<LLamaToken> TextTokens { get; }
         public int TotalPositions { get; }
         public int TotalTokens => TextTokens.Count;
 
-        private MtmdChunkSequence(List<SafeMtmdInputChunk> chunks, List<LLamaToken> textTokens, int totalPositions)
+        private MtmdChunkSequence(SafeMtmdInputChunks chunks, List<LLamaToken> textTokens, int totalPositions)
         {
             Chunks = chunks;
             TextTokens = textTokens;
@@ -89,38 +89,27 @@ private MtmdChunkSequence(List<SafeMtmdInputChunk> chunks, List<LLamaToken> text
 
         public static MtmdChunkSequence Create(SafeMtmdInputChunks chunks, SafeMtmdWeights clipModel)
         {
-            var copies = new List<SafeMtmdInputChunk>();
             var textTokens = new List<LLamaToken>();
 
             foreach (var chunk in chunks.Enumerate())
             {
-                var copy = chunk.Copy();
-                if (copy == null)
+                using (chunk)
                 {
-                    chunk.Dispose();
-                    continue;
-                }
-
-                copies.Add(copy);
+                    if (chunk.Type != SafeMtmdInputChunk.SafeMtmdInputChunkType.Text)
+                        continue;
 
-                if (copy.Type == SafeMtmdInputChunk.SafeMtmdInputChunkType.Text)
-                {
-                    foreach (var token in copy.GetTextTokensSpan())
+                    foreach (var token in chunk.GetTextTokensSpan())
                         textTokens.Add((LLamaToken)unchecked((int)token));
                 }
-
-                chunk.Dispose();
             }
 
             var totalPositions = (int)clipModel.CountPositions(chunks);
-            return new MtmdChunkSequence(copies, textTokens, totalPositions);
+            return new MtmdChunkSequence(chunks, textTokens, totalPositions);
         }
 
         public void Dispose()
         {
-            foreach (var chunk in Chunks)
-                chunk.Dispose();
-            Chunks.Clear();
+            Chunks.Dispose();
         }
     }