WIP

Author: SignalRT

LLama/LLamaExecutorBase.cs

@@ -32,11 +32,11 @@ public abstract class StatefulExecutorBase : ILLamaExecutor
         /// </summary>
         protected int _consumedTokensCount; // n_consume
         /// <summary>
-        /// 
+        /// Number of tokens consumed from the session cache during the current run.
         /// </summary>
         protected int _n_session_consumed;
         /// <summary>
-        /// 
+        /// Number of prompt tokens that match the loaded session cache prefix.
         /// </summary>
         protected int _n_matching_session_tokens;
         /// <summary>
@@ -52,7 +52,7 @@ public abstract class StatefulExecutorBase : ILLamaExecutor
         /// </summary>
         protected List<LLamaToken> _embed_inps = new();
         /// <summary>
-        /// 
+        /// Tokens recovered from the session file and reused to warm up the KV cache.
         /// </summary>
         protected List<LLamaToken> _session_tokens = new();
         /// <summary>
@@ -84,10 +84,10 @@ public bool IsMultiModal
         private readonly StreamingTokenDecoder _decoder;
 
         /// <summary>
-        /// 
+        /// Initialize a stateful executor bound to a specific context.
         /// </summary>
-        /// <param name="context"></param>
-        /// <param name="logger"></param>
+        /// <param name="context">LLama context used for all native interactions.</param>
+        /// <param name="logger">Optional logger for diagnostic output.</param>
         protected StatefulExecutorBase(LLamaContext context, ILogger? logger = null)
         {
             Embeds = new List<SafeMtmdEmbed>();
@@ -101,22 +101,22 @@ protected StatefulExecutorBase(LLamaContext context, ILogger? logger = null)
         }
 
         /// <summary>
-        /// 
+        /// Initialize a multimodal executor with the supplied MTMD weights.
         /// </summary>
-        /// <param name="context"></param>
-        /// <param name="safeMtmdWeights"></param>
-        /// <param name="logger"></param>
+        /// <param name="context">LLama context used for all native interactions.</param>
+        /// <param name="safeMtmdWeights">Multimodal weights to associate with this executor.</param>
+        /// <param name="logger">Optional logger for diagnostic output.</param>
         public StatefulExecutorBase(LLamaContext context, SafeMtmdWeights safeMtmdWeights, ILogger? logger = null) : 
                         this( context, logger )
         {
             ClipModel = safeMtmdWeights;
         }
 
         /// <summary>
-        /// This API is currently not verified.
+        /// Attach a session cache file so the executor can reuse previous KV state if compatible.
         /// </summary>
-        /// <param name="filename"></param>
-        /// <returns></returns>
+        /// <param name="filename">Path to the llama.cpp session file.</param>
+        /// <returns>The current executor instance for fluent configuration.</returns>
         /// <exception cref="ArgumentNullException"></exception>
         /// <exception cref="RuntimeError"></exception>
         public StatefulExecutorBase WithSessionFile(string filename)
@@ -173,9 +173,9 @@ public StatefulExecutorBase WithSessionFile(string filename)
         }
 
         /// <summary>
-        /// This API has not been verified currently.
+        /// Persist the current session cache to disk.
         /// </summary>
-        /// <param name="filename"></param>
+        /// <param name="filename">Destination path for the llama.cpp session file.</param>
         public void SaveSessionFile(string filename)
         {
             var session_token_array = _session_tokens.ToArray();
@@ -203,7 +203,7 @@ protected virtual void HandleRunOutOfContext(int tokensToKeep)
         }
 
         /// <summary>
-        /// Try to reuse the matching prefix from the session file.
+        /// Try to reuse the matching prompt prefix from the loaded session cache before evaluating new tokens.
         /// </summary>
         protected virtual void TryReuseMatchingPrefix()
         {
@@ -236,66 +236,66 @@ protected virtual void TryReuseMatchingPrefix()
         }
 
         /// <summary>
-        /// Decide whether to continue the loop.
+        /// Determine whether the inference loop should continue processing tokens.
         /// </summary>
-        /// <param name="args"></param>
-        /// <returns></returns>
+        /// <param name="args">Mutable state associated with the current inference.</param>
+        /// <returns><c>true</c> to continue generating; otherwise <c>false</c>.</returns>
         protected abstract Task<bool> GetLoopCondition(InferStateArgs args);
 
         /// <summary>
-        /// Preprocess the inputs before the inference.
+        /// Prepare the executor for inference by tokenizing input and updating cached state.
         /// </summary>
-        /// <param name="text"></param>
-        /// <param name="args"></param>
+        /// <param name="text">Prompt text to process.</param>
+        /// <param name="args">Mutable state associated with the current inference.</param>
         protected abstract Task PreprocessInputs(string? text, InferStateArgs args);
 
         /// <summary>
-        /// Do some post processing after the inference.
+        /// Perform any post-processing on the generated tokens.
         /// </summary>
-        /// <param name="inferenceParams"></param>
-        /// <param name="args"></param>
-        /// <returns></returns>
+        /// <param name="inferenceParams">Parameters controlling sampling.</param>
+        /// <param name="args">Mutable state associated with the current inference.</param>
+        /// <returns>A tuple indicating whether generation should stop and any extra outputs to emit.</returns>
         protected abstract Task<(bool, IReadOnlyList<string>)> PostProcess(IInferenceParams inferenceParams, InferStateArgs args);
 
         /// <summary>
-        /// The core inference logic.
+        /// Core inference loop that advances the model by one step.
         /// </summary>
-        /// <param name="inferenceParams"></param>
-        /// <param name="args"></param>
+        /// <param name="inferenceParams">Parameters controlling sampling.</param>
+        /// <param name="args">Mutable state associated with the current inference.</param>
         protected abstract Task InferInternal(IInferenceParams inferenceParams, InferStateArgs args);
 
         /// <summary>
-        /// Save the current state to a file.
+        /// Save the executor state to a serialized snapshot file.
         /// </summary>
-        /// <param name="filename"></param>
+        /// <param name="filename">Destination file for the serialized state.</param>
         public abstract Task SaveState(string filename);
 
         /// <summary>
-        /// Get the current state data.
+        /// Capture the executor state in a serializable object.
         /// </summary>
-        /// <returns></returns>
+        /// <returns>State snapshot suitable for persistence.</returns>
         public abstract ExecutorBaseState GetStateData();
 
         /// <summary>
-        /// Load the state from data.
+        /// Restore executor state from a previously captured snapshot.
         /// </summary>
-        /// <param name="data"></param>
+        /// <param name="data">State snapshot created by <see cref="GetStateData"/>.</param>
         public abstract Task LoadState(ExecutorBaseState data);
 
         /// <summary>
-        /// Load the state from a file.
+        /// Restore executor state from a serialized snapshot file.
         /// </summary>
-        /// <param name="filename"></param>
+        /// <param name="filename">Path to the snapshot produced by <see cref="SaveState"/>.</param>
         public abstract Task LoadState(string filename);
 
 
         /// <summary>
-        /// Execute the inference.
+        /// Execute an asynchronous inference session.
         /// </summary>
-        /// <param name="text">The prompt. If null, generation will continue where it left off previously.</param>
-        /// <param name="inferenceParams"></param>
-        /// <param name="cancellationToken"></param>
-        /// <returns></returns>
+        /// <param name="text">Optional prompt; when null generation resumes from prior state.</param>
+        /// <param name="inferenceParams">Sampling parameters to apply; defaults are used when null.</param>
+        /// <param name="cancellationToken">Cancellation token for cooperative cancellation.</param>
+        /// <returns>Stream of decoded text segments as they become available.</returns>
         public virtual async IAsyncEnumerable<string> InferAsync(string? text, IInferenceParams? inferenceParams = null, [EnumeratorCancellation] CancellationToken cancellationToken = default)
         {
             cancellationToken.ThrowIfCancellationRequested();
@@ -370,33 +370,36 @@ public virtual async Task PrefillPromptAsync(string prompt)
         }   
 
         /// <summary>
-        /// State arguments that are used in single inference
+        /// Mutable state passed between inference callbacks during a single generation pass.
         /// </summary>
         protected class InferStateArgs
         {
             /// <summary>
-            /// 
+            /// Anti-prompts that terminate generation when encountered.
             /// </summary>
             public IList<string>? Antiprompts { get; set; }
             /// <summary>
             /// Tokens count remained to be used. (n_remain)
             /// </summary>
             public int RemainedTokens { get; set; }
             /// <summary>
-            /// 
+            /// Indicates whether generated tokens should be returned to the caller.
             /// </summary>
             public bool ReturnValue { get; set; }
             /// <summary>
-            /// 
+            /// Signals that the executor should pause and wait for additional user input.
             /// </summary>
             public bool WaitForInput { get; set; }
             /// <summary>
-            /// 
+            /// Indicates whether the session cache should be persisted after inference completes.
             /// </summary>
             public bool NeedToSaveSession { get; set; }
         }
 
 #pragma warning disable CS1591, CS8618 // Missing XML and irrelevant nullable warnings
+        /// <summary>
+        /// Serializable snapshot of executor state used for persistence and restart.
+        /// </summary>
         [JsonConverter(typeof(PolymorphicJSONConverter<ExecutorBaseState>))]
         public class ExecutorBaseState
         {

LLama/LLamaInteractExecutor.cs

@@ -21,28 +21,29 @@ namespace LLama
     /// </summary>
     public class InteractiveExecutor : StatefulExecutorBase
     {
+        // Indicates whether the executor is currently evaluating the initial prompt or a follow-up turn.
         private bool _is_prompt_run = true;
 
         // MTMD multimodal state
-        private SafeMtmdInputChunks? _mtmdChunks;
-        private string? _mtmdMarker;
+        private SafeMtmdInputChunks? _mtmdChunks;  // Pending chunk collection produced by the multimodal tokenizer.
+        private string? _mtmdMarker;  // Cached multimodal marker returned by the native helper.
 
         /// <summary>
-        /// 
+        /// Create an interactive executor for text-only inference.
         /// </summary>
-        /// <param name="context"></param>
-        /// <param name="logger"></param>
+        /// <param name="context">LLama context to operate against.</param>
+        /// <param name="logger">Optional logger for diagnostic output.</param>
         public InteractiveExecutor(LLamaContext context, ILogger? logger = null)
             : base(context, logger)
         {
         }
 
         /// <summary>
-        /// 
+        /// Create an interactive multimodal executor that can process text alongside media inputs.
         /// </summary>
-        /// <param name="context"></param>
-        /// <param name="clipModel"></param>
-        /// <param name="logger"></param>
+        /// <param name="context">LLama context to operate against.</param>
+        /// <param name="clipModel">Multimodal weights (MTMD) to attach to the executor.</param>
+        /// <param name="logger">Optional logger for diagnostic output.</param>
         public InteractiveExecutor(LLamaContext context, SafeMtmdWeights clipModel, ILogger? logger = null)
             : base(context, clipModel, logger)
         {
@@ -109,15 +110,20 @@ public override async Task LoadState(string filename)
         }
 
         /// <summary>
-        /// Define whether to continue the loop to generate responses.
+        /// Decide whether generation should continue for the current iteration.
         /// </summary>
-        /// <returns></returns>
+        /// <param name="args">Mutable inference state.</param>
+        /// <returns><c>true</c> to keep generating; otherwise <c>false</c>.</returns>
         protected override Task<bool> GetLoopCondition(InferStateArgs args)
         {
             return Task.FromResult(args.RemainedTokens != 0 && !args.WaitForInput || _is_prompt_run);
         }
 
-        /// <inheritdoc />
+        /// <summary>
+        /// Preprocess the incoming prompt or continuation text before inference.
+        /// </summary>
+        /// <param name="text">Prompt text or continuation provided by the caller.</param>
+        /// <param name="args">Mutable inference state.</param>
         protected override Task PreprocessInputs(string? text, InferStateArgs args)
         {
             if (_is_prompt_run)
@@ -159,12 +165,18 @@ protected override Task PreprocessInputs(string? text, InferStateArgs args)
             return Task.CompletedTask;
         }
 
+        /// <summary>
+        /// Release any queued multimodal chunks and reset state.
+        /// </summary>
         private void DisposeMtmdChunks()
         {
             _mtmdChunks?.Dispose();
             _mtmdChunks = null;
         }
 
+        /// <summary>
+        /// Dispose and clear any pending multimodal embeddings queued for evaluation.
+        /// </summary>
         private void DisposeEmbeds()
         {
             if (Embeds.Count == 0)
@@ -180,6 +192,9 @@ private void DisposeEmbeds()
             Embeds.Clear();
         }
 
+        /// <summary>
+        /// Retrieve the marker token used to signal media segments to the tokenizer.
+        /// </summary>
         private string GetMtmdMarker()
         {
             if (_mtmdMarker is not null)
@@ -191,7 +206,12 @@ private string GetMtmdMarker()
             return _mtmdMarker;
         }
 
-        /// <inheritdoc />
+        /// <summary>
+        /// Preprocess multimodal prompts by aligning media markers and tokenizing via MTMD helpers.
+        /// </summary>
+        /// <param name="text">Prompt text containing optional media markers.</param>
+        /// <param name="args">Mutable inference state.</param>
+        /// <param name="addBos">Whether to treat the prompt as a fresh run and add the BOS token.</param>
         private Task PreprocessMtmd(string text, InferStateArgs args, bool addBos = true)
         {
             if (ClipModel is null)
@@ -213,7 +233,7 @@ private Task PreprocessMtmd(string text, InferStateArgs args, bool addBos = true
 
                 if (!prompt.Contains(marker))
                 {
-                    var suffix = string.Concat(Enumerable.Repeat(marker, Embeds.Count));
+                    var suffix = string.Concat(Enumerable.Repeat(marker, Embeds.Count));  // Ensure tokenizer sees one marker per embed.
                     prompt = string.Concat(prompt, suffix);
                 }
             }
@@ -228,7 +248,7 @@ private Task PreprocessMtmd(string text, InferStateArgs args, bool addBos = true
                     throw new RuntimeError($"Failed to tokenize multimodal prompt. Status: {status}.");
                 }
 
-                _mtmdChunks = chunks;
+                _mtmdChunks = chunks;  // Own the chunk collection until evaluation completes.
 
                 var tokens = new List<LLamaToken>();
                 foreach (var chunk in chunks.Enumerate())
@@ -263,18 +283,18 @@ private Task PreprocessMtmd(string text, InferStateArgs args, bool addBos = true
             }
             finally
             {
-                DisposeEmbeds();
+                DisposeEmbeds();  // Flush any embeds decoded in prior step; MTMD replays them via chunk eval.
             }
 
             return Task.CompletedTask;
         }
 
         /// <summary>
-        /// Return whether to break the generation.
+        /// Decide whether generation should stop based on antiprompts, token limits, or end-of-generation markers.
         /// </summary>
-        /// <param name="inferenceParams"></param>
-        /// <param name="args"></param>
-        /// <returns></returns>
+        /// <param name="inferenceParams">Sampling parameters controlling generation.</param>
+        /// <param name="args">Mutable inference state.</param>
+        /// <returns>Tuple describing whether to stop and any additional outputs to emit.</returns>
         protected override async Task<(bool, IReadOnlyList<string>)> PostProcess(IInferenceParams inferenceParams, InferStateArgs args)
         {
             if (_embed_inps.Count <= _consumedTokensCount)
@@ -429,7 +449,7 @@ protected override async Task InferInternal(IInferenceParams inferenceParams, In
         }
 
         /// <summary>
-        /// The descriptor of the state of the interactive executor.
+        /// Serializable state specific to the interactive executor.
         /// </summary>
         public class InteractiveExecutorState
             : StatefulExecutorBase.ExecutorBaseState