[Firebase AI] Add support for Code Execution (#1326)

* [Firebase AI] Add support for Code Execution

* Cleanup pass

Author: a-maurice

docs/readme.md

@@ -109,6 +109,10 @@ Support
 
 Release Notes
 -------------
+### Upcoming
+-   Changes
+    - Firebase AI: Add support for enabling the model to use Code Execution.
+
 ### 13.2.0
 -   Changes
     - General: Update to Firebase C++ SDK version 13.1.0.

firebaseai/src/FunctionCalling.cs

@@ -82,6 +82,14 @@ internal Dictionary<string, object> ToJson() {
 /// </summary>
 public readonly struct GoogleSearch {}
 
+/// <summary>
+/// A tool that allows the model to execute code.
+///
+/// This tool can be used to solve complex problems, for example, by generating and executing Python
+/// code to solve a math problem.
+/// </summary>
+public readonly struct CodeExecution {}
+
 /// <summary>
 /// A helper tool that the model may use when generating responses.
 ///
@@ -93,6 +101,7 @@ public readonly struct Tool {
 
   private List<FunctionDeclaration> FunctionDeclarations { get; }
   private GoogleSearch? GoogleSearch { get; }
+  private CodeExecution? CodeExecution { get; }
 
   /// <summary>
   /// Creates a tool that allows the model to perform function calling.
@@ -102,6 +111,7 @@ public readonly struct Tool {
   public Tool(params FunctionDeclaration[] functionDeclarations) {
     FunctionDeclarations = new List<FunctionDeclaration>(functionDeclarations);
     GoogleSearch = null;
+    CodeExecution = null;
   }
   /// <summary>
   /// Creates a tool that allows the model to perform function calling.
@@ -111,6 +121,7 @@ public Tool(params FunctionDeclaration[] functionDeclarations) {
   public Tool(IEnumerable<FunctionDeclaration> functionDeclarations) {
     FunctionDeclarations = new List<FunctionDeclaration>(functionDeclarations);
     GoogleSearch = null;
+    CodeExecution = null;
   }
 
   /// <summary>
@@ -121,6 +132,18 @@ public Tool(IEnumerable<FunctionDeclaration> functionDeclarations) {
   public Tool(GoogleSearch googleSearch) {
     FunctionDeclarations = null;
     GoogleSearch = googleSearch;
+    CodeExecution = null;
+  }
+  
+  /// <summary>
+  /// Creates a tool that allows the model to use Code Execution.
+  /// </summary>
+  /// <param name="codeExecution">An empty `CodeExecution` object. The presence of this object
+  ///     in the list of tools enables the model to use Code Execution.</param>
+  public Tool(CodeExecution codeExecution) {
+    FunctionDeclarations = null;
+    GoogleSearch = null;
+    CodeExecution = codeExecution;
   }
 
   /// <summary>
@@ -135,6 +158,9 @@ internal Dictionary<string, object> ToJson() {
     if (GoogleSearch.HasValue) {
       json["googleSearch"] = new Dictionary<string, object>();
     }
+    if (CodeExecution.HasValue) {
+      json["codeExecution"] = new Dictionary<string, object>();
+    }
     return json;
   }
 }

firebaseai/src/ModelContent.cs

@@ -379,6 +379,153 @@ Dictionary<string, object> Part.ToJson() {
       };
     }
   }
+  
+  /// <summary>
+  /// A part containing code that was executed by the model.
+  /// </summary>
+  public readonly struct ExecutableCodePart : Part {
+    public enum CodeLanguage {
+      Unspecified = 0,
+      Python
+    }
+
+    /// <summary>
+    /// The language
+    /// </summary>
+    public CodeLanguage Language { get; }
+    /// <summary>
+    /// The code that was executed.
+    /// </summary>
+    public string Code { get; }
+
+    private readonly bool? _isThought;
+    public bool IsThought { get { return _isThought ?? false; } }
+
+    private readonly string _thoughtSignature;
+
+    private static CodeLanguage ParseLanguage(string str) {
+      return str switch {
+        "PYTHON" => CodeLanguage.Python,
+        _ => CodeLanguage.Unspecified,
+      };
+    }
+
+    private string LanguageAsString {
+      get {
+        return Language switch {
+          CodeLanguage.Python => "PYTHON",
+          _ => "LANGUAGE_UNSPECIFIED"
+        };
+      }
+    }
+
+    /// <summary>
+    /// Intended for internal use only.
+    /// </summary>
+    internal ExecutableCodePart(string language, string code,
+        bool? isThought, string thoughtSignature) {
+      Language = ParseLanguage(language);
+      Code = code;
+      _isThought = isThought;
+      _thoughtSignature = thoughtSignature;
+    }
+
+    Dictionary<string, object> Part.ToJson() {
+      var jsonDict = new Dictionary<string, object>() {
+        { "executableCode", new Dictionary<string, object>() {
+            { "language", LanguageAsString },
+            { "code", Code }
+          }
+        }
+      };
+      jsonDict.AddIfHasValue("thought", _isThought);
+      jsonDict.AddIfHasValue("thoughtSignature", _thoughtSignature);
+      return jsonDict;
+    }
+  }
+  
+  /// <summary>
+  /// A part containing the result of executing code.
+  /// </summary>
+  public readonly struct CodeExecutionResultPart : Part {
+    /// <summary>
+    /// The outcome of a code execution.
+    /// </summary>
+    public enum ExecutionOutcome {
+      Unspecified = 0,
+      /// <summary>
+      /// The code executed without errors.
+      /// </summary>
+      Ok,
+      /// <summary>
+      /// The code failed to execute.
+      /// </summary>
+      Failed,
+      /// <summary>
+      /// The code took too long to execute.
+      /// </summary>
+      DeadlineExceeded
+    }
+    
+    /// <summary>
+    /// The outcome of the code execution.
+    /// </summary>
+    public ExecutionOutcome Outcome { get; }
+    /// <summary>
+    /// The output of the code execution.
+    /// </summary>
+    public string Output { get; }
+    
+    private readonly bool? _isThought;
+    public bool IsThought { get { return _isThought ?? false; } }
+
+    private readonly string _thoughtSignature;
+    
+    private static ExecutionOutcome ParseOutcome(string str) {
+      return str switch {
+        "OUTCOME_UNSPECIFIED" => ExecutionOutcome.Unspecified,
+        "OUTCOME_OK" => ExecutionOutcome.Ok,
+        "OUTCOME_FAILED" => ExecutionOutcome.Failed,
+        "OUTCOME_DEADLINE_EXCEEDED" => ExecutionOutcome.DeadlineExceeded,
+        _ => ExecutionOutcome.Unspecified,
+      };
+    }
+    
+    private string OutcomeAsString {
+      get {
+        return Outcome switch {
+          ExecutionOutcome.Ok => "OUTCOME_OK",
+          ExecutionOutcome.Failed => "OUTCOME_FAILED",
+          ExecutionOutcome.DeadlineExceeded => "OUTCOME_DEADLINE_EXCEEDED",
+          _ => "OUTCOME_UNSPECIFIED"
+        };
+      }
+    }
+
+    /// <summary>
+    /// Intended for internal use only.
+    /// </summary>
+    internal CodeExecutionResultPart(string outcome, string output,
+        bool? isThought, string thoughtSignature) {
+        Outcome = ParseOutcome(outcome);
+        Output = output;
+        _isThought = isThought;
+        _thoughtSignature = thoughtSignature;
+      }
+    
+    Dictionary<string, object> Part.ToJson() {
+      var jsonDict = new Dictionary<string, object>() {
+        { "codeExecutionResult", new Dictionary<string, object>() {
+            { "outcome", OutcomeAsString },
+            { "output", Output }
+          }
+        }
+      };
+      jsonDict.AddIfHasValue("thought", _isThought);
+      jsonDict.AddIfHasValue("thoughtSignature", _thoughtSignature);
+      return jsonDict;
+    }
+  }
 
 #endregion
 
@@ -413,6 +560,24 @@ private static InlineDataPart InlineDataPartFromJson(Dictionary<string, object>
       isThought,
       thoughtSignature);
   }
+  
+  private static ExecutableCodePart ExecutableCodePartFromJson(Dictionary<string, object> jsonDict,
+      bool? isThought, string thoughtSignature) {
+    return new ExecutableCodePart(
+      jsonDict.ParseValue<string>("language", JsonParseOptions.ThrowEverything),
+      jsonDict.ParseValue<string>("code", JsonParseOptions.ThrowEverything),
+      isThought,
+      thoughtSignature);
+  }
+  
+  private static CodeExecutionResultPart CodeExecutionResultPartFromJson(Dictionary<string, object> jsonDict,
+      bool? isThought, string thoughtSignature) {
+    return new CodeExecutionResultPart(
+      jsonDict.ParseValue<string>("outcome", JsonParseOptions.ThrowEverything),
+      jsonDict.ParseValue<string>("output", JsonParseOptions.ThrowEverything),
+      isThought,
+      thoughtSignature);
+  }
 
   private static Part PartFromJson(Dictionary<string, object> jsonDict) {
     bool? isThought = jsonDict.ParseNullableValue<bool>("thought");
@@ -427,6 +592,14 @@ private static Part PartFromJson(Dictionary<string, object> jsonDict) {
         innerDict => InlineDataPartFromJson(innerDict, isThought, thoughtSignature),
         out var inlineDataPart)) {
       return inlineDataPart;
+    } else if (jsonDict.TryParseObject("executableCode",
+        innerDict => ExecutableCodePartFromJson(innerDict, isThought, thoughtSignature),
+        out var executableCodePart)) {
+      return executableCodePart;
+    } else if (jsonDict.TryParseObject("codeExecutionResult",
+        innerDict => CodeExecutionResultPartFromJson(innerDict, isThought, thoughtSignature),
+        out var codeExecutionResultPart)) {
+      return codeExecutionResultPart;
     } else {
 #if FIREBASEAI_DEBUG_LOGGING
       UnityEngine.Debug.LogWarning($"Received unknown part, with keys: {string.Join(',', jsonDict.Keys)}");

firebaseai/testapp/Assets/Firebase/Sample/FirebaseAI/UIHandlerAutomated.cs

@@ -75,6 +75,7 @@ protected override void Start() {
         TestImagenGenerateImageOptions,
         TestThinkingBudget,
         TestIncludeThoughts,
+        TestCodeExecution,
       };
       // Set of tests that only run the single time.
       Func<Task>[] singleTests = {
@@ -100,6 +101,7 @@ protected override void Start() {
         InternalTestGenerateImagesAllFiltered,
         InternalTestGenerateImagesBase64SomeFiltered,
         InternalTestThoughtSummary,
+        InternalTestCodeExecution,
       };
 
       // Create the set of tests, combining the above lists.
@@ -819,11 +821,6 @@ async Task TestThinkingBudget(Backend backend) {
     // Test requesting thought summaries.
     async Task TestIncludeThoughts(Backend backend) {
       // Thinking Budget requires at least the 2.5 model.
-      var tool = new Tool(new FunctionDeclaration(
-        "GetKeyword", "Call to retrieve a special keyword.",
-        new Dictionary<string, Schema>() {
-          { "input", Schema.String("Input string") },
-        }));
       var model = GetFirebaseAI(backend).GetGenerativeModel(
         modelName: "gemini-2.5-flash",
         generationConfig: new GenerationConfig(
@@ -844,6 +841,26 @@ async Task TestIncludeThoughts(Backend backend) {
       Assert("ThoughtSummary was missing", !string.IsNullOrWhiteSpace(response.ThoughtSummary));
     }
 
+    async Task TestCodeExecution(Backend backend) {
+      var model = GetFirebaseAI(backend).GetGenerativeModel(
+        modelName: ModelName,
+        tools: new Tool[] { new Tool(new CodeExecution()) }
+      );
+
+      var prompt = "What is the sum of the first 50 prime numbers? Generate and run code for the calculation.";
+      var chat = model.StartChat();
+      var response = await chat.SendMessageAsync(prompt);
+
+      string result = response.Text;
+      Assert("Response text was missing", !string.IsNullOrWhiteSpace(result));
+
+      var executableCodeParts = response.Candidates.First().Content.Parts.OfType<ModelContent.ExecutableCodePart>();
+      Assert("Missing ExecutableCodeParts", executableCodeParts.Any());
+
+      var codeExecutionResultParts = response.Candidates.First().Content.Parts.OfType<ModelContent.CodeExecutionResultPart>();
+      Assert("Missing CodeExecutionResultParts", codeExecutionResultParts.Any());
+    }
+
     // Test providing a file from a GCS bucket (Firebase Storage) to the model.
     async Task TestReadFile() {
       // GCS is currently only supported with VertexAI.
@@ -1519,5 +1536,23 @@ async Task InternalTestThoughtSummary() {
 
       ValidateUsageMetadata(response.UsageMetadata, 13, 2, 39, 54);
     }
+
+    async Task InternalTestCodeExecution() {
+      Dictionary<string, object> json = await GetVertexJsonTestData("unary-success-code-execution.json");
+      GenerateContentResponse response = GenerateContentResponse.FromJson(json, FirebaseAI.Backend.InternalProvider.VertexAI);
+
+      AssertEq("Candidate count", response.Candidates.Count(), 1);
+      var candidate = response.Candidates.First();
+
+      var executableCodeParts = candidate.Content.Parts.OfType<ModelContent.ExecutableCodePart>().ToList();
+      AssertEq("ExecutableCodePart count", executableCodeParts.Count(), 1);
+      AssertEq("ExecutableCodePart language", executableCodeParts[0].Language, ModelContent.ExecutableCodePart.CodeLanguage.Python);
+      AssertEq("ExecutableCodePart code", executableCodeParts[0].Code, "prime_numbers = [2, 3, 5, 7, 11]\nsum_of_primes = sum(prime_numbers)\nprint(f'The sum of the first 5 prime numbers is: {sum_of_primes}')\n");
+
+      var codeExecutionResultParts = candidate.Content.Parts.OfType<ModelContent.CodeExecutionResultPart>().ToList();
+      AssertEq("CodeExecutionResultPart count", codeExecutionResultParts.Count(), 1);
+      AssertEq("CodeExecutionResultPart outcome", codeExecutionResultParts[0].Outcome, ModelContent.CodeExecutionResultPart.ExecutionOutcome.Ok);
+      AssertEq("CodeExecutionResultPart output", codeExecutionResultParts[0].Output, "The sum of the first 5 prime numbers is: 28\n");
+    }
   }
 }