[VertexAI] Add response modality to GenerateContent (#1237)

* [VertexAI] Add multi-modalities to GenerationConfig

* Add integration test

Author: a-maurice

vertexai/src/GenerationConfig.cs

@@ -16,6 +16,8 @@
 
 using System;
 using System.Collections.Generic;
+using System.Linq;
+using Firebase.VertexAI.Internal;
 
 namespace Firebase.VertexAI {
 
@@ -34,6 +36,7 @@ public readonly struct GenerationConfig {
   private readonly string[] _stopSequences;
   private readonly string _responseMimeType;
   private readonly Schema _responseSchema;
+  private readonly List<ResponseModality> _responseModalities;
 
   /// <summary>
   /// Creates a new `GenerationConfig` value.
@@ -146,7 +149,8 @@ public GenerationConfig(
       float? frequencyPenalty = null,
       string[] stopSequences = null,
       string responseMimeType = null,
-      Schema responseSchema = null) {
+      Schema responseSchema = null,
+      IEnumerable<ResponseModality> responseModalities = null) {
     _temperature = temperature;
     _topP = topP;
     _topK = topK;
@@ -157,6 +161,8 @@ public GenerationConfig(
     _stopSequences = stopSequences;
     _responseMimeType = responseMimeType;
     _responseSchema = responseSchema;
+    _responseModalities = responseModalities != null ?
+        new List<ResponseModality>(responseModalities) : null;
   }
 
   /// <summary>
@@ -175,6 +181,10 @@ internal Dictionary<string, object> ToJson() {
     if (_stopSequences != null && _stopSequences.Length > 0) jsonDict["stopSequences"] = _stopSequences;
     if (!string.IsNullOrWhiteSpace(_responseMimeType)) jsonDict["responseMimeType"] = _responseMimeType;
     if (_responseSchema != null) jsonDict["responseSchema"] = _responseSchema.ToJson();
+    if (_responseModalities != null && _responseModalities.Count > 0) {
+      jsonDict["responseModalities"] =
+          _responseModalities.Select(EnumConverters.ResponseModalityToString).ToList();
+    }
 
     return jsonDict;
   }

vertexai/testapp/Assets/Firebase/Sample/VertexAI/UIHandlerAutomated.cs

@@ -65,6 +65,7 @@ protected override void Start() {
         TestChatBasicTextStream,
         TestCountTokens,
         TestYoutubeLink,
+        TestGenerateImage,
       };
       // Set of tests that only run the single time.
       Func<Task>[] singleTests = {
@@ -632,6 +633,35 @@ async Task TestYoutubeLink(Backend backend) {
           response.Text.Contains("Firebase", StringComparison.OrdinalIgnoreCase));
     }
 
+    // Test being able to generate an image with GenerateContent.
+    async Task TestGenerateImage(Backend backend) {
+      var model = GetFirebaseAI(backend).GetGenerativeModel("gemini-2.0-flash-exp",
+        generationConfig: new GenerationConfig(
+          responseModalities: new [] { ResponseModality.Text, ResponseModality.Image })
+      );
+
+      GenerateContentResponse response = await model.GenerateContentAsync(
+        ModelContent.Text("Can you give me a picture of a cartoon dog, and a couple of sentences about him?")
+      );
+
+      Assert("Response missing candidates.", response.Candidates.Any());
+
+      // We don't care much about the response, just that there is an image, and text.
+      bool foundText = false;
+      bool foundImage = false;
+      var candidate = response.Candidates.First();
+      foreach (var part in candidate.Content.Parts) {
+        if (part is ModelContent.TextPart) {
+          foundText = true;
+        } else if (part is ModelContent.InlineDataPart dataPart) {
+          if (dataPart.MimeType.Contains("image")) {
+            foundImage = true;
+          }
+        }
+      }
+      Assert($"Missing expected modalities. Text: {foundText}, Image: {foundImage}", foundText && foundImage);
+    }
+
     // Test providing a file from a GCS bucket (Firebase Storage) to the model.
     async Task TestReadFile() {
       // GCS is currently only supported with VertexAI.