Extend Mtmd test

Author: SignalRT

LLama.Unittest/MtmdWeightsTests.cs

@@ -44,56 +44,97 @@ public void Dispose()
             _llamaWeights.Dispose();
         }
 
-        [Fact,Trait("Category", "NoCI")]
-        public void EmbedImageAsFileName()
+        private SafeMtmdInputChunks TokenizeWithEmbed(Func<SafeMtmdEmbed> loadEmbed)
         {
             _safeMtmdWeights.ClearMedia();
 
-            using var image = _safeMtmdWeights.LoadMedia(Constants.MtmdImage);
-            Assert.NotNull(image);
-            Assert.True(image.Nx > 0);
-
-            var prompt = _mediaMarker;
-            SafeMtmdInputChunks? chunks;
-            var status = _safeMtmdWeights.Tokenize(prompt, addSpecial: true, parseSpecial: true, out chunks);
-            Assert.Equal(0, status);
-            Assert.NotNull(chunks);
+            var embed = loadEmbed();
+            Assert.NotNull(embed);
 
-            var ownedChunks = chunks!;
-            using (ownedChunks)
+            using (embed)
             {
-                long nPast = 0;
-                var eval = _safeMtmdWeights.EvaluateChunks(ownedChunks, _context.NativeHandle, ref nPast, seqId: 0, nBatch: checked((int)_context.BatchSize), logitsLast: true);
-                Assert.Equal(0, eval);
-                Assert.True(nPast > 0);
+                Assert.True(embed.Nx > 0);
+                Assert.True(embed.Ny > 0);
+                Assert.False(embed.IsAudio);
+                Assert.True(embed.GetDataSpan().Length > 0);
+
+                var status = _safeMtmdWeights.Tokenize(_mediaMarker, addSpecial: true, parseSpecial: true, out var chunks);
+                Assert.Equal(0, status);
+                Assert.NotNull(chunks);
+
+                return chunks!;
             }
-        }        
-        
+        }
+
+        private void AssertChunksEvaluate(SafeMtmdInputChunks chunks)
+        {
+            long nPast = 0;
+            var eval = _safeMtmdWeights.EvaluateChunks(chunks, _context.NativeHandle, ref nPast, seqId: 0, nBatch: checked((int)_context.BatchSize), logitsLast: true);
+            Assert.Equal(0, eval);
+            Assert.True(nPast > 0);
+        }
+
+        [Fact,Trait("Category", "NoCI")]
+        public void EmbedImageAsFileName()
+        {
+            using var chunks = TokenizeWithEmbed(() => _safeMtmdWeights.LoadMedia(Constants.MtmdImage));
+            AssertChunksEvaluate(chunks);
+        }
+
         [Fact,Trait("Category", "NoCI")]
         public void EmbedImageAsBinary()
         {
-            _safeMtmdWeights.ClearMedia();
+            var imageBytes = File.ReadAllBytes(Constants.MtmdImage);
+            using var chunks = TokenizeWithEmbed(() => _safeMtmdWeights.LoadMedia(imageBytes));
+            AssertChunksEvaluate(chunks);
+        }
+
+        [Fact,Trait("Category", "NoCI")]
+        public void TokenizeProvidesChunkMetadata()
+        {
+            using var chunks = TokenizeWithEmbed(() => _safeMtmdWeights.LoadMedia(Constants.MtmdImage));
 
-            byte[] imageBytes = File.ReadAllBytes(Constants.MtmdImage);
-            using var image = _safeMtmdWeights.LoadMedia(imageBytes);
-            Assert.NotNull(image);
-            Assert.True(image.Nx > 0);
+            Assert.True(chunks.Size > 0);
 
-            var prompt = _mediaMarker;
-            SafeMtmdInputChunks? chunks;
-            var status = _safeMtmdWeights.Tokenize(prompt, addSpecial: true, parseSpecial: true, out chunks);
-            Assert.Equal(0, status);
-            Assert.NotNull(chunks);
+            ulong totalTokens = 0;
+            long totalPositions = 0;
+            var imageChunks = 0;
 
-            var ownedChunks = chunks!;
-            using (ownedChunks)
+            foreach (var chunk in chunks.Enumerate())
             {
-                long nPast = 0;
-                var eval = _safeMtmdWeights.EvaluateChunks(ownedChunks, _context.NativeHandle, ref nPast, seqId: 0, nBatch: checked((int)_context.BatchSize), logitsLast: true);
-                Assert.Equal(0, eval);
-                Assert.True(nPast > 0);
+                totalTokens += chunk.NTokens;
+                totalPositions += chunk.NPos;
+
+                if (chunk.Type == SafeMtmdInputChunk.SafeMtmdInputChunkType.Image)
+                {
+                    imageChunks++;
+
+                    var copy = chunk.Copy();
+                    try
+                    {
+                        Assert.NotNull(copy);
+                        if (copy != null)
+                        {
+                            Assert.Equal(chunk.NTokens, copy.NTokens);
+                            Assert.Equal(chunk.NPos, copy.NPos);
+                        }
+                    }
+                    finally
+                    {
+                        copy?.Dispose();
+                    }
+                }
             }
-        }      
 
+            Assert.True(imageChunks > 0);
+            Assert.True(totalTokens > 0);
+            Assert.Equal(totalTokens, _safeMtmdWeights.CountTokens(chunks));
+            Assert.Equal(totalPositions, _safeMtmdWeights.CountPositions(chunks));
+            Assert.True(_safeMtmdWeights.SupportsVision);
+            Assert.False(_safeMtmdWeights.SupportsAudio);
+
+            var audioBitrate = _safeMtmdWeights.AudioBitrate;
+            Assert.True(audioBitrate <= 0);
+        }
     }
 }