get tool calls from run steps

Author: CJCrafter

examples/src/main/java/chat/ChatCompletion.java

@@ -1,5 +1,6 @@
 package chat;
 
+import com.cjcrafter.openai.Models;
 import com.cjcrafter.openai.OpenAI;
 import com.cjcrafter.openai.chat.ChatMessage;
 import com.cjcrafter.openai.chat.ChatRequest;
@@ -29,7 +30,7 @@ public static void main(String[] args) {
 
         // Here you can change the model's settings, add tools, and more.
         ChatRequest request = ChatRequest.builder()
-                .model("gpt-3.5-turbo")
+                .model(Models.Chat.GPT_3_5_TURBO)
                 .messages(messages)
                 .build();
 

examples/src/main/kotlin/assistant/Thread.kt

@@ -16,26 +16,36 @@ fun main() {
     val choice = readln().toInt()
     val assistant = assistants.data[choice]
 
+    // We have to create a new thread. We'll save this thread, so we can add
+    // user messages and get responses later.
     val thread = openai.threads.create()
-    openai.threads.messages(thread).create {
-        role(ThreadUser.USER)
-        content("Hi! I am using the threads API right now through Java!")
-    }
-
-    var run = openai.threads.runs(thread).create {
-        assistant(assistant)
-        //instructions("You can override instructions, model, etc.")
-    }
 
-    // This is a known limitation in OpenAI, and they are working to address
-    // this so that we can easily stream a response without nonsense like this.
-    while (!run.status.isTerminal) {
-        Thread.sleep(2500)
-        run = openai.threads.runs(thread).retrieve(run)
+    while (true) {
+
+        // Handle user input
+        println("Type your input below: ")
+        val input = readln()
+        openai.threads.messages(thread).create {
+            role(ThreadUser.USER)
+            content(input)
+
+            // You can also add files and metadata to the message
+            //addFile(file)
+            //addMetadata("key", "value")
+        }
+
+        // After adding a message to the thread, we have to "run" the thread
+        var run = openai.threads.runs(thread).create {
+            assistant(assistant)
+            //instructions("You can override instructions, model, etc.")
+        }
+
+        // This is a known limitation in OpenAI, and they are working to address
+        // this so that we can easily stream a response without nonsense like this.
+        while (!run.status.isTerminal) {
+            Thread.sleep(2500)
+            run = openai.threads.runs(thread).retrieve(run)
+        }
     }
 
-    println(run)
-
-    // Cleanup... Optional
-    //openai.threads.delete(thread)
 }

src/main/kotlin/com/cjcrafter/openai/chat/tool/CodeInterpreter.kt

@@ -0,0 +1,84 @@
+package com.cjcrafter.openai.chat.tool
+
+import com.fasterxml.jackson.annotation.JsonProperty
+import com.fasterxml.jackson.annotation.JsonSubTypes
+import com.fasterxml.jackson.annotation.JsonTypeInfo
+
+/**
+ * Represents the data for a [Tool.Type.CODE_INTERPRETER] tool call. This
+ * contains the code that was run by the code interpreter, and the outputs
+ * of the code interpreter.
+ *
+ * @property input The code that was run by the code interpreter
+ * @property outputs The data output by the code interpreter
+ */
+data class CodeInterpreter(
+    @JsonProperty(required = true) val input: String,
+    @JsonProperty(required = true) val outputs: List<Output>
+) {
+
+    /**
+     * A sealed class that represents the output of 1 [CodeInterpreter] step.
+     */
+    @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "type")
+    @JsonSubTypes(
+        JsonSubTypes.Type(LogOutput::class, name = "logs"),
+        JsonSubTypes.Type(ImageOutput::class, name = "image"),
+    )
+    sealed class Output {
+
+        /**
+         * The type of the output.
+         */
+        abstract val type: Type
+
+        enum class Type {
+
+            /**
+             * When the code interpreter outputs text.
+             */
+            @JsonProperty("logs")
+            LOGS,
+
+            /**
+             * When the code interpreter outputs an image file.
+             */
+            @JsonProperty("image")
+            IMAGE,
+        }
+    }
+
+    /**
+     * Represents an [Output.Type.LOGS] output. This only holds the text that
+     * was outputted by the code interpreter.
+     *
+     * @property text The text output
+     */
+    data class LogOutput(
+        val text: String,
+    ) : Output() {
+        override val type: Type = Type.LOGS
+    }
+
+    /**
+     * Represents an [Output.Type.IMAGE] output. This only holds the ID of the
+     * generated image file. You can use this ID to retrieve the image file.
+     *
+     * @property image The image output
+     * @see com.cjcrafter.openai.files.FileHandler.retrieve
+     */
+    data class ImageOutput(
+        val image: Image,
+    ) : Output() {
+        override val type: Type = Type.IMAGE
+
+        /**
+         * Holds data about the image output by the code interpreter.
+         *
+         * @property fileId The unique ID of the output file, which can be used to retrieve the file
+         */
+        data class Image(
+            @JsonProperty("file_id", required = true) val fileId: String,
+        )
+    }
+}

src/main/kotlin/com/cjcrafter/openai/chat/tool/CodeInterpreterToolCall.kt

@@ -0,0 +1,18 @@
+package com.cjcrafter.openai.chat.tool
+
+import com.fasterxml.jackson.annotation.JsonProperty
+
+/**
+ * Represents a tool call by an [com.cjcrafter.openai.assistants.Assistant]
+ * to a code interpreter.
+ *
+ * @property id The unique id of this tool call
+ * @property codeInterpreter The details about the input and output
+ */
+data class CodeInterpreterToolCall(
+    @JsonProperty(required = true) override val id: String,
+    @JsonProperty("code_interpreter", required = true) val codeInterpreter: CodeInterpreter,
+) : ToolCall() {
+    override val type = Tool.Type.CODE_INTERPRETER
+
+}

src/main/kotlin/com/cjcrafter/openai/chat/tool/FunctionCall.kt

@@ -1,28 +1,50 @@
 package com.cjcrafter.openai.chat.tool
 
+import com.cjcrafter.openai.OpenAI
+import com.cjcrafter.openai.assistants.Assistant
 import com.cjcrafter.openai.exception.HallucinationException
 import com.fasterxml.jackson.core.JsonProcessingException
 import com.fasterxml.jackson.databind.JsonNode
 import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
 import com.fasterxml.jackson.module.kotlin.readValue
-import org.jetbrains.annotations.ApiStatus
 
 /**
- * Represents a function call by ChatGPT. When ChatGPT calls a function, you
- * should be parsing the arguments, calling some method (getting current weather
- * at a location, getting a stock price, modifying a database, etc.), and then
- * replying to ChatGPT with the result.
+ * Represents a function call by either a chat completion, or an [Assistant].
  *
- * ChatGPT may *hallucinate*, or make up, function calls. To handle hallucinations,
+ * When a function call is made, you MUST respond with the result of the function.
+ * This means that you should parse the arguments, call some function (an API call,
+ * getting current weather, getting a stock price, modifying a database, etc.), and
+ * sending the result of that function back.
+ *
+ * For chat completions ([OpenAI.createChatCompletion]), you should send the result
+ * of the function as a [com.cjcrafter.openai.chat.ChatMessage] with the
+ * corresponding [ToolCall.id] as the function id.
+ *
+ * For [Assistant]s, you should use [com.cjcrafter.openai.threads.runs.RunHandler.submitToolOutputs]
+ * with the corresponding [ToolCall.id] as the tool call id. For [Assistant]s,
+ * it is important to submit tool outputs _within a timely manner_, usually
+ * within 10 minutes of starting a [com.cjcrafter.openai.threads.runs.Run].
+ * Otherwise, the [com.cjcrafter.openai.threads.runs.Run] will expire, and you
+ * will not be able to submit your tool call.
+ *
+ * ChatGPT may _hallucinate_, or make up, function calls. To handle hallucinations,
  * we have provided [tryParseArguments].
  *
  * @property name The name of the function which was called
  * @property arguments The raw json representation of the arguments
+ * @property output The result of the function call if it has been set, only used for [Assistant]s. You should not set this.
  */
 data class FunctionCall(
     var name: String,
     var arguments: String,
+    var output: String? = null,
 ) {
+
+    /**
+     * Used internally to update the function call. This is used when the chat
+     * completion is streamed via [OpenAI.streamChatCompletion]. This is not
+     * used by [Assistant]s.
+     */
     internal fun update(delta: FunctionCallDelta) {
         // The only field that updates is arguments
         arguments += delta.arguments
@@ -44,7 +66,6 @@ data class FunctionCall(
      * @param tools The list of tools that ChatGPT has access to, or null to skip advanced checking.
      * @return The parsed arguments.
      */
-
     @JvmOverloads
     @Throws(HallucinationException::class)
     fun tryParseArguments(tools: List<Tool>? = null): Map<String, JsonNode> {

src/main/kotlin/com/cjcrafter/openai/chat/tool/FunctionToolCall.kt

@@ -0,0 +1,21 @@
+package com.cjcrafter.openai.chat.tool
+
+import com.fasterxml.jackson.annotation.JsonProperty
+
+/**
+ * Represents a tool call for [Tool.Type.FUNCTION].
+ *
+ * @property id The unique id of the tool call. You should use this id in your reply.
+ * @property function The details about which function was called, parameters, etc.
+ */
+data class FunctionToolCall(
+    @JsonProperty(required = true) override val id: String,
+    @JsonProperty(required = true) val function: FunctionCall,
+) : ToolCall() {
+    override val type = Tool.Type.FUNCTION
+
+    override fun update(delta: ToolCallDelta) {
+        if (delta.function != null)
+            function.update(delta.function)
+    }
+}

src/main/kotlin/com/cjcrafter/openai/chat/tool/RetrievalToolCall.kt

@@ -0,0 +1,23 @@
+package com.cjcrafter.openai.chat.tool
+
+import com.fasterxml.jackson.annotation.JsonProperty
+import org.jetbrains.annotations.ApiStatus
+
+/**
+ * Represents a tool call for [Tool.Type.RETRIEVAL].
+ *
+ * Currently, this tool call serves no functionality other than to inform you
+ * that some retrieval operation has occurred. Currently, the OpenAI API does
+ * not return any information about the retrieval ([retrieval] will always be
+ * an empty map. [retrieval]'s data type may be changed in the future, I
+ * recommend you __DO NOT__ use it).
+ *
+ * @property id The unique id of this tool call
+ * @property retrieval An empty map, do not use this!
+ */
+data class RetrievalToolCall(
+    @JsonProperty(required = true) override val id: String,
+    @JsonProperty(required = true) @ApiStatus.Experimental val retrieval: Map<String, String>,
+): ToolCall() {
+    override val type = Tool.Type.RETRIEVAL
+}

src/main/kotlin/com/cjcrafter/openai/chat/tool/Tool.kt

@@ -11,8 +11,9 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo
  */
 @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "type")
 @JsonSubTypes(
-    JsonSubTypes.Type(TextAnnotation.FileCitation::class, name = "file_citation"),
-    JsonSubTypes.Type(TextAnnotation.FilePath::class, name = "file_path"),
+    JsonSubTypes.Type(Tool.FunctionTool::class, name = "function"),
+    JsonSubTypes.Type(Tool.RetrievalTool::class, name = "retrieval"),
+    JsonSubTypes.Type(Tool.CodeInterpreterTool::class, name = "code_interpreter"),
 )
 sealed class Tool {
 
@@ -22,15 +23,25 @@ sealed class Tool {
      * @see Function.builder
      */
     data class FunctionTool(
-        var function: Function,
+        @JsonProperty(required = true) var function: Function,
     ): Tool() {
         override val type = Type.FUNCTION
     }
 
+    /**
+     * Represents a tool that retrieves data from uploaded files.
+     *
+     * Note that retrieval tools are only supported by [Assistant]s
+     */
     data object RetrievalTool: Tool() {
         override val type = Type.RETRIEVAL
     }
 
+    /**
+     * Represents a tool that runs Python code on the OpenAI server.
+     *
+     * Note that code interpreter tools are only supported by [Assistant]s
+     */
     data object CodeInterpreterTool: Tool() {
         override val type = Type.CODE_INTERPRETER
     }
@@ -58,6 +69,8 @@ sealed class Tool {
 
         /**
          * A tool that runs Python code on the OpenAI server.
+         *
+         * Note that code interpreter tools are only supported by [Assistant]s
          */
         @JsonProperty("code_interpreter")
         CODE_INTERPRETER,

src/main/kotlin/com/cjcrafter/openai/chat/tool/ToolCall.kt

@@ -1,5 +1,8 @@
 package com.cjcrafter.openai.chat.tool
 
+import com.fasterxml.jackson.annotation.JsonSubTypes
+import com.fasterxml.jackson.annotation.JsonTypeInfo
+
 /**
  * Wraps a tool call by ChatGPT. You should check the [type] of the tool call,
  * and handle the request. For example, if the type is [Tool.Type.FUNCTION], you
@@ -10,17 +13,20 @@ package com.cjcrafter.openai.chat.tool
  * call.
  *
  * @property id The id of this call. You should use this to construct a [com.cjcrafter.openai.chat.ChatUser.TOOL] message.
- * @property type The type of tool call. Currently, the only type is [Tool.Type.FUNCTION].
- * @property function The function call containing the function name and arguments.
+ * @property type The type of tool call.
  */
-data class ToolCall(
-    var id: String,
-    var type: Tool.Type,
-    var function: FunctionCall,
-) {
-    internal fun update(delta: ToolCallDelta) {
-        // The only field that updates is function
-        if (delta.function != null)
-            function.update(delta.function)
+@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "type")
+@JsonSubTypes(
+    JsonSubTypes.Type(FunctionToolCall::class, name = "function"),
+    JsonSubTypes.Type(RetrievalToolCall::class, name = "retrieval"),
+    JsonSubTypes.Type(CodeInterpreterToolCall::class, name = "code_interpreter"),
+)
+sealed class ToolCall {
+
+    abstract val id: String
+    abstract val type: Tool.Type
+
+    internal open fun update(delta: ToolCallDelta) {
+        // Nothing to update
     }
 }

src/main/kotlin/com/cjcrafter/openai/chat/tool/ToolCallDelta.kt

@@ -15,9 +15,11 @@ data class ToolCallDelta(
     val type: Tool.Type? = null,
     val function: FunctionCallDelta? = null,
 ) {
-    internal fun toToolCall() = ToolCall(
-        id = id ?: throw IllegalStateException("id must be set"),
-        type = type ?: throw IllegalStateException("type must be set"),
-        function = function?.toFunctionCall() ?: throw IllegalStateException("function must be set"),
-    )
+    internal fun toToolCall(): ToolCall {
+        // This is for chat, which is always a function tool call
+        return FunctionToolCall(
+            id = id ?: throw IllegalStateException("id must be set"),
+            function = function?.toFunctionCall() ?: throw IllegalStateException("function must be set"),
+        )
+    }
 }