VertexAI - Add more functionality around options in, and feedback out (#1192)

* VertexAI - Add logic for text input/output

* Remove debug logs

* Update GenerativeModel.cs

* Implement more of the VertexAI setting structs

* Minor updates

* Fix SafetyRatings missing in PromptFeedback

* Comments for Citation

* Fix merge issues

* Comment cleanup

* More comment cleanup

Author: a-maurice

vertexai/src/Candidate.cs

@@ -17,19 +17,54 @@
 using System;
 using System.Collections.Generic;
 using System.Collections.ObjectModel;
+using System.Linq;
+using Firebase.VertexAI.Internal;
 
 namespace Firebase.VertexAI {
 
+/// <summary>
+/// Represents the reason why the model stopped generating content.
+/// </summary>
 public enum FinishReason {
-  Unknown,
+  /// <summary>
+  /// A new and not yet supported value.
+  /// </summary>
+  Unknown = 0,
+  /// <summary>
+  /// Natural stop point of the model or provided stop sequence.
+  /// </summary>
   Stop,
+  /// <summary>
+  /// The maximum number of tokens as specified in the request was reached.
+  /// </summary>
   MaxTokens,
+  /// <summary>
+  /// The token generation was stopped because the response was flagged for safety reasons.
+  /// </summary>
   Safety,
+  /// <summary>
+  /// The token generation was stopped because the response was flagged for unauthorized citations.
+  /// </summary>
   Recitation,
+  /// <summary>
+  /// All other reasons that stopped token generation.
+  /// </summary>
   Other,
+  /// <summary>
+  /// Token generation was stopped because the response contained forbidden terms.
+  /// </summary>
   Blocklist,
+  /// <summary>
+  /// Token generation was stopped because the response contained potentially prohibited content.
+  /// </summary>
   ProhibitedContent,
+  /// <summary>
+  /// Token generation was stopped because of Sensitive Personally Identifiable Information (SPII).
+  /// </summary>
   SPII,
+  /// <summary>
+  /// Token generation was stopped because the function call generated by the model was invalid.
+  /// </summary>
   MalformedFunctionCall,
 }
 
@@ -62,27 +97,37 @@ public readonly struct Candidate {
   /// </summary>
   public CitationMetadata? CitationMetadata { get; }
 
-  // Hidden constructor, users don't need to make this, though they still technically can.
-  internal Candidate(ModelContent content, List<SafetyRating> safetyRatings,
+  // Hidden constructor, users don't need to make this.
+  private Candidate(ModelContent content, List<SafetyRating> safetyRatings,
       FinishReason? finishReason, CitationMetadata? citationMetadata) {
     Content = content;
     _safetyRatings = new ReadOnlyCollection<SafetyRating>(safetyRatings ?? new List<SafetyRating>());
     FinishReason = finishReason;
     CitationMetadata = citationMetadata;
   }
 
-  internal static Candidate FromJson(Dictionary<string, object> jsonDict) {
-    ModelContent content = new();
-    if (jsonDict.TryGetValue("content", out object contentObj)) {
-      if (contentObj is not Dictionary<string, object> contentDict) {
-        throw new VertexAISerializationException("Invalid JSON format: 'content' is not a dictionary.");
-      }
-      // We expect this to be another dictionary to convert
-      content = ModelContent.FromJson(contentDict);
-    }
+  private static FinishReason ParseFinishReason(string str) {
+    return str switch {
+      "STOP" => Firebase.VertexAI.FinishReason.Stop,
+      "MAX_TOKENS" => Firebase.VertexAI.FinishReason.MaxTokens,
+      "SAFETY" => Firebase.VertexAI.FinishReason.Safety,
+      "RECITATION" => Firebase.VertexAI.FinishReason.Recitation,
+      "OTHER" => Firebase.VertexAI.FinishReason.Other,
+      "BLOCKLIST" => Firebase.VertexAI.FinishReason.Blocklist,
+      "PROHIBITED_CONTENT" => Firebase.VertexAI.FinishReason.ProhibitedContent,
+      "SPII" => Firebase.VertexAI.FinishReason.SPII,
+      "MALFORMED_FUNCTION_CALL" => Firebase.VertexAI.FinishReason.MalformedFunctionCall,
+      _ => Firebase.VertexAI.FinishReason.Unknown,
+    };
+  }
 
-    // TODO: Parse SafetyRatings, FinishReason, and CitationMetadata
-    return new Candidate(content, null, null, null);
+  internal static Candidate FromJson(Dictionary<string, object> jsonDict) {
+    return new Candidate(
+      jsonDict.ParseObject("content", ModelContent.FromJson, defaultValue: new ModelContent("model")),
+      jsonDict.ParseObjectList("safetyRatings", SafetyRating.FromJson),
+      jsonDict.ParseNullableEnum("finishReason", ParseFinishReason),
+      jsonDict.ParseNullableObject("citationMetadata",
+          Firebase.VertexAI.CitationMetadata.FromJson));
   }
 }
 

vertexai/src/Citation.cs

@@ -14,25 +14,101 @@
  * limitations under the License.
  */
 
+using System;
 using System.Collections.Generic;
+using System.Collections.ObjectModel;
+using Firebase.VertexAI.Internal;
 
 namespace Firebase.VertexAI {
 
+/// <summary>
+/// A collection of source attributions for a piece of content.
+/// </summary>
 public readonly struct CitationMetadata {
-  public IEnumerable<Citation> Citations { get; }
+  private readonly ReadOnlyCollection<Citation> _citations;
 
-  // Hidden constructor, users don't need to make this
+  /// <summary>
+  /// A list of individual cited sources and the parts of the content to which they apply.
+  /// </summary>
+  public IEnumerable<Citation> Citations =>
+      _citations ?? new ReadOnlyCollection<Citation>(new List<Citation>());
+
+  // Hidden constructor, users don't need to make this.
+  private CitationMetadata(List<Citation> citations) {
+    _citations = new ReadOnlyCollection<Citation>(citations ?? new List<Citation>());
+  }
+
+  internal static CitationMetadata FromJson(Dictionary<string, object> jsonDict) {
+    return new CitationMetadata(
+      jsonDict.ParseObjectList("citations", Citation.FromJson));
+  }
 }
 
+/// <summary>
+/// A struct describing a source attribution.
+/// </summary>
 public readonly struct Citation {
+  /// <summary>
+  /// The inclusive beginning of a sequence in a model response that derives from a cited source.
+  /// </summary>
   public int StartIndex { get; }
+  /// <summary>
+  /// The exclusive end of a sequence in a model response that derives from a cited source.
+  /// </summary>
   public int EndIndex { get; }
+  /// <summary>
+  /// A link to the cited source, if available.
+  /// </summary>
   public System.Uri Uri { get; }
+  /// <summary>
+  /// The title of the cited source, if available.
+  /// </summary>
   public string Title { get; }
+  /// <summary>
+  /// The license the cited source work is distributed under, if specified.
+  /// </summary>
   public string License { get; }
+  /// <summary>
+  /// The publication date of the cited source, if available.
+  /// </summary>
   public System.DateTime? PublicationDate { get; }
 
-  // Hidden constructor, users don't need to make this
+  // Hidden constructor, users don't need to make this.
+  private Citation(int startIndex, int endIndex, Uri uri, string title,
+      string license, DateTime? publicationDate) {
+    StartIndex = startIndex;
+    EndIndex = endIndex;
+    Uri = uri;
+    Title = title;
+    License = license;
+    PublicationDate = publicationDate;
+  }
+
+  internal static Citation FromJson(Dictionary<string, object> jsonDict) {
+    // If there is a Uri, need to convert it.
+    Uri uri = null;
+    if (jsonDict.TryParseValue("uri", out string uriString)) {
+      uri = new Uri(uriString);
+    }
+
+    // If there is a publication date, we need to convert it.
+    DateTime? pubDate = null;
+    if (jsonDict.TryParseValue("publicationDate", out Dictionary<string, object> dateDict)) {
+      // Make sure that if any key is missing, it has a default value that will work with DateTime.
+      pubDate = new DateTime(
+        dateDict.ParseValue<int>("year", defaultValue: 1),
+        dateDict.ParseValue<int>("month", defaultValue: 1),
+        dateDict.ParseValue<int>("day", defaultValue: 1));
+    }
+
+    return new Citation(
+      jsonDict.ParseValue<int>("startIndex"),
+      jsonDict.ParseValue<int>("endIndex"),
+      uri,
+      jsonDict.ParseValue<string>("title"),
+      jsonDict.ParseValue<string>("license"),
+      pubDate);
+  }
 }
 
 }

vertexai/src/GenerateContentResponse.cs

@@ -18,6 +18,7 @@
 using System.Collections.ObjectModel;
 using System.Linq;
 using Google.MiniJSON;
+using Firebase.VertexAI.Internal;
 
 namespace Firebase.VertexAI {
 
@@ -65,8 +66,8 @@ public IEnumerable<ModelContent.FunctionCallPart> FunctionCalls {
     }
   }
 
-  // Hidden constructor, users don't need to make this, though they still technically can.
-  internal GenerateContentResponse(List<Candidate> candidates, PromptFeedback? promptFeedback,
+  // Hidden constructor, users don't need to make this.
+  private GenerateContentResponse(List<Candidate> candidates, PromptFeedback? promptFeedback,
       UsageMetadata? usageMetadata) {
     _candidates = new ReadOnlyCollection<Candidate>(candidates ?? new List<Candidate>());
     PromptFeedback = promptFeedback;
@@ -78,48 +79,119 @@ internal static GenerateContentResponse FromJson(string jsonString) {
   }
 
   internal static GenerateContentResponse FromJson(Dictionary<string, object> jsonDict) {
-    // Parse the Candidates
-    List<Candidate> candidates = new();
-    if (jsonDict.TryGetValue("candidates", out object candidatesObject)) {
-      if (candidatesObject is not List<object> listOfCandidateObjects) {
-        throw new VertexAISerializationException("Invalid JSON format: 'candidates' is not a list.");
-      }
-
-      candidates = listOfCandidateObjects
-          .Select(o => o as Dictionary<string, object>)
-          .Where(dict => dict != null)
-          .Select(Candidate.FromJson)
-          .ToList();
-    }
-
-    // TODO: Parse PromptFeedback and UsageMetadata
-
-    return new GenerateContentResponse(candidates, null, null);
+    return new GenerateContentResponse(
+      jsonDict.ParseObjectList("candidates", Candidate.FromJson),
+      jsonDict.ParseNullableObject("promptFeedback",
+          Firebase.VertexAI.PromptFeedback.FromJson),
+      jsonDict.ParseNullableObject("usageMetadata",
+          Firebase.VertexAI.UsageMetadata.FromJson));
   }
 }
 
+/// <summary>
+/// A type describing possible reasons to block a prompt.
+/// </summary>
 public enum BlockReason {
-  Unknown,
+  /// <summary>
+  /// A new and not yet supported value.
+  /// </summary>
+  Unknown = 0,
+  /// <summary>
+  /// The prompt was blocked because it was deemed unsafe.
+  /// </summary>
   Safety,
+  /// <summary>
+  /// All other block reasons.
+  /// </summary>
   Other,
+  /// <summary>
+  /// The prompt was blocked because it contained terms from the terminology blocklist.
+  /// </summary>
   Blocklist,
+  /// <summary>
+  /// The prompt was blocked due to prohibited content.
+  /// </summary>
   ProhibitedContent,
 }
 
+/// <summary>
+/// A metadata struct containing any feedback the model had on the prompt it was provided.
+/// </summary>
 public readonly struct PromptFeedback {
+  private readonly ReadOnlyCollection<SafetyRating> _safetyRatings;
+
+  /// <summary>
+  /// The reason a prompt was blocked, if it was blocked.
+  /// </summary>
   public BlockReason? BlockReason { get; }
+  /// <summary>
+  /// A human-readable description of the `BlockReason`.
+  /// </summary>
   public string BlockReasonMessage { get; }
-  public IEnumerable<SafetyRating> SafetyRatings { get; }
+  /// <summary>
+  /// The safety ratings of the prompt.
+  /// </summary>
+  public IEnumerable<SafetyRating> SafetyRatings =>
+      _safetyRatings ?? new ReadOnlyCollection<SafetyRating>(new List<SafetyRating>());
+
+  // Hidden constructor, users don't need to make this.
+  private PromptFeedback(BlockReason? blockReason, string blockReasonMessage,
+                         List<SafetyRating> safetyRatings) {
+    BlockReason = blockReason;
+    BlockReasonMessage = blockReasonMessage;
+    _safetyRatings = new ReadOnlyCollection<SafetyRating>(safetyRatings ?? new List<SafetyRating>());
+  }
 
-  // Hidden constructor, users don't need to make this
+  private static BlockReason ParseBlockReason(string str) {
+    return str switch {
+      "SAFETY" => Firebase.VertexAI.BlockReason.Safety,
+      "OTHER" => Firebase.VertexAI.BlockReason.Other,
+      "BLOCKLIST" => Firebase.VertexAI.BlockReason.Blocklist,
+      "PROHIBITED_CONTENT" => Firebase.VertexAI.BlockReason.ProhibitedContent,
+      _ => Firebase.VertexAI.BlockReason.Unknown,
+    };
+  }
+
+  internal static PromptFeedback FromJson(Dictionary<string, object> jsonDict) {
+    return new PromptFeedback(
+      jsonDict.ParseNullableEnum("blockReason", ParseBlockReason),
+      jsonDict.ParseValue<string>("blockReasonMessage"),
+      jsonDict.ParseObjectList("safetyRatings", SafetyRating.FromJson));
+  }
 }
 
+/// <summary>
+/// Token usage metadata for processing the generate content request.
+/// </summary>
 public readonly struct UsageMetadata {
+  /// <summary>
+  /// The number of tokens in the request prompt.
+  /// </summary>
   public int PromptTokenCount { get; }
+  /// <summary>
+  /// The total number of tokens across the generated response candidates.
+  /// </summary>
   public int CandidatesTokenCount { get; }
+  /// <summary>
+  /// The total number of tokens in both the request and response.
+  /// </summary>
   public int TotalTokenCount { get; }
 
-  // Hidden constructor, users don't need to make this
+  // TODO: New fields about ModalityTokenCount
+
+  // Hidden constructor, users don't need to make this.
+  private UsageMetadata(int promptTC, int candidatesTC, int totalTC) {
+    PromptTokenCount = promptTC;
+    CandidatesTokenCount = candidatesTC;
+    TotalTokenCount = totalTC;
+  }
+
+  internal static UsageMetadata FromJson(Dictionary<string, object> jsonDict) {
+    return new UsageMetadata(
+      jsonDict.ParseValue<int>("promptTokenCount"),
+      jsonDict.ParseValue<int>("candidatesTokenCount"),
+      jsonDict.ParseValue<int>("totalTokenCount"));
+  }
 }
 
 }