Merge pull request #59 from yml-org/feature/CM-1314/audio-transcriptions

feature: create audio transcriptions entrypoint

Author: osugikoji

README.md

@@ -91,6 +91,7 @@ try {
 - [ChatCompletions](guides/Features.md#chatcompletions)
 - [ImageGenerations](guides/Features.md#imagegenerations)
 - [Edits](guides/Features.md#edits)
+- [AudioTranscriptions](guides/Features.md#audioTranscriptions)
 
 ## ℹ️ Sample apps
 

guides/Features.md

@@ -6,6 +6,7 @@
 - [ChatCompletions](#chatcompletions)
 - [ImageGenerations](#imagegenerations)
 - [Edits](#edits)
+- [AudioTranscriptions](#audioTranscriptions)
 
 ## ListModels
 
@@ -86,9 +87,12 @@ var yChat: YChat {
 
 do {
   let result = try await yChat.completion()
+                    .setModel(input: "text-davinci-003")
                     .setInput(input: "Say this is a test.")
                     .setMaxTokens(tokens: 1024)
-                    .set... // you can set more parameters
+                    .setTemperature(temperature: 1.0)
+                    .setTopP(topP: 1.0)
+                    .saveHistory(isSaveHistory: false)
                     .execute()
 } catch {
   // catch any error that may occurs on api call.  
@@ -104,9 +108,12 @@ val yChat by lazy {
 
 try {
     val result = yChat.completion()
+        .setModel("text-davinci-003")
         .setInput("Say this is a test.")
         .setMaxTokens(1024)
-        .set... // you can set more parameters
+        .setTemperature(1.0)
+        .setTopP(1.0)
+        .saveHistory(false)
         .execute()
 } catch (e: exception) {
     // catch any error that may occurs on api call.  
@@ -126,12 +133,15 @@ var yChat: YChat {
 
 do {
   let result = try await yChat.chatCompletions()
+                    .setModel(model: "gpt-3.5-turbo")
                     .setMaxTokens(tokens: 1024)
+                    .setMaxResults(results: 1)
+                    .setTemperature(temperature: 1.0)
+                    .setTopP(topP: 1.0)
                     .addMessage(
                         role: "assistant",
                         content: "You are a helpful assistant that only answers questions related to fitness"
                     )
-                    .set... // you can set more parameters
                     .execute(content: "What is the best exercise for building muscle?")
 } catch {
   // catch any error that may occurs on api call.  
@@ -147,12 +157,15 @@ val yChat by lazy {
 
 try {
     val result = yChat.chatCompletions()
+        .setModel("gpt-3.5-turbo")
         .setMaxTokens(1024)
+        .setMaxResults(1)
+        .setTemperature(1.0)
+        .setTopP(1.0)
         .addMessage(
             role = "assistant",
             content = "You are a helpful assistant that only answers questions related to fitness"
         )
-        .set... // you can set more parameters
         .execute("What is the best exercise for building muscle?")
 } catch (e: exception) {
     // catch any error that may occurs on api call.  
@@ -172,9 +185,9 @@ var yChat: YChat {
 
 do {
   let result = try await yChat.imageGenerations()
-                    .setResults(results: 2)
+                    .setResults(results: 1)
                     .setSize(size: "1024x1024")
-                    .set... // you can set more parameters
+                    .setResponseFormat(responseFormat: "url")
                     .execute(prompt: "ocean")
 } catch {
   // catch any error that may occurs on api call.  
@@ -190,9 +203,9 @@ val yChat by lazy {
 
 try {
     val result = yChat.imageGenerations()
-        .setResults(2)
+        .setResults(1)
         .setSize("1024x1024")
-        .set... // you can set more parameters
+        .setResponseFormat("url")
         .execute("ocean")
 } catch (e: exception) {
     // catch any error that may occurs on api call.  
@@ -214,7 +227,9 @@ do {
   let result = try await yChat.edits()
                     .setInput(input: "What day of the wek is it?")
                     .setResults(result: 1)
-                    .set... // you can set more parameters
+                    .setModel(model: "text-davinci-edit-001")
+                    .setTemperature(temperature: 1.0)
+                    .setTopP(topP: 1.0)
                     .execute(instruction: "Fix the spelling mistakes")
 } catch {
   // catch any error that may occurs on api call.  
@@ -232,9 +247,65 @@ try {
     val result = yChat.edits()
         .setInput("What day of the wek is it?")
         .setResults(1)
-        .set... // you can set more parameters
+        .setModel("text-davinci-edit-001")
+        .setTemperature(1.0)
+        .setTopP(1.0)
         .execute("Fix the spelling mistakes")
 } catch (e: exception) {
     // catch any error that may occurs on api call.  
 }
+```
+
+## AudioTranscriptions
+
+The audioTranscriptions api is used to transcribes audio into the input language.
+
+### Swift
+
+```swift
+var yChat: YChat {
+    YChatCompanion.shared.create(apiKey: "your-api-key") 
+}
+
+guard let audioFileUrl = Bundle.main.url(forResource: "audio", withExtension: "m4a") else {
+    print("Unable to find the audio file.")
+    return
+}
+
+let audioData = try! Data(contentsOf: audioFileUrl)
+
+do {
+  let result = try await yChat.audioTranscriptions()
+                    .setModel(model: "whisper-1")
+                    .setPrompt(prompt: "")
+                    .setResponseFormat(format: "json")
+                    .setTemperature(temperature: 0.4)
+                    .setLanguage(language: "en")
+                    .execute(filename: "audio.m4a", audioFile: audioData)
+} catch {
+  // catch any error that may occurs on api call.  
+}
+```
+
+### Kotlin
+
+```kotlin
+val yChat by lazy {
+    YChat.create("your-api-key")
+}
+
+val inputStream = application.resources.openRawResource(R.raw.audio)
+val byteArray = inputStream.readBytes()
+
+try {
+    val result = yChat.audioTranscriptions()
+        .setModel("whisper-1")
+        .setPrompt("")
+        .setResponseFormat("json")
+        .setTemperature(0.4)
+        .setLanguage("en")
+        .execute("audio.m4a", byteArray)
+} catch (e: exception) {
+    // catch any error that may occurs on api call.  
+}
 ```

sample/jvm/README.md

@@ -91,4 +91,19 @@ This endpoint retrieve the artificial intelligence model based on the given ID.
 
 ##### Example:
 
-`GET http://localhost:8080/api/ychat/models/babbage`
+`GET http://localhost:8080/api/ychat/models/babbage`
+
+### Audio Transcriptions Endpoint
+
+This endpoint transcribes audio into the input language.
+
+##### Endpoint: http://localhost:[port_number]/api/ychat/audio/transcriptions
+
+##### Example:
+
+```
+curl -X POST \
+  -H "Content-Type: multipart/form-data" \
+  -F "file=@/path/to/audio/file" \
+  "http://localhost:8080/api/ychat/audio/transcriptions"
+```

sample/jvm/src/main/java/co/yml/ychat/jvm/controller/YChatController.java

@@ -7,9 +7,11 @@
 import org.springframework.http.ResponseEntity;
 import org.springframework.web.bind.annotation.GetMapping;
 import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.PostMapping;
 import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RequestParam;
 import org.springframework.web.bind.annotation.RestController;
+import org.springframework.web.multipart.MultipartFile;
 
 @RestController
 @RequestMapping("api/ychat")
@@ -64,6 +66,14 @@ public ResponseEntity<AIModel> model(@PathVariable String id) throws Exception {
         return ResponseEntity.ok(result);
     }
 
+    @PostMapping("audio/transcriptions")
+    public ResponseEntity<String> audioTranscriptions(
+            @RequestParam("file") MultipartFile multipartFile
+    ) throws Exception {
+        String result = YChatService.getAudioTranscription(multipartFile);
+        return ResponseEntity.ok(result);
+    }
+
     private static class Defaults {
         static final String COMPLETION_INPUT = "Say this is a test.";
         static final String CHAT_COMPLETION_INPUT = "Tell me one strength exercise";

sample/jvm/src/main/java/co/yml/ychat/jvm/services/YChatService.java

@@ -4,10 +4,12 @@
 import co.yml.ychat.domain.model.AIModel;
 import co.yml.ychat.domain.model.ChatMessage;
 import java.util.List;
+import java.util.Optional;
 import java.util.concurrent.CompletableFuture;
 import org.jetbrains.annotations.NotNull;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
+import org.springframework.web.multipart.MultipartFile;
 
 @Service
 public class YChatService {
@@ -65,6 +67,15 @@ public AIModel getModel(String id) throws Exception {
         return future.get();
     }
 
+    public String getAudioTranscription(MultipartFile multipartFile) throws Exception {
+        final CompletableFuture<String> future = new CompletableFuture<>();
+        String filename = Optional.ofNullable(multipartFile.getOriginalFilename())
+                .orElseThrow(() -> new IllegalStateException("Filename not found"));
+        byte[] bytes = multipartFile.getBytes();
+        ychat.audioTranscriptions().execute(filename, bytes, new CompletionCallbackResult<>(future));
+        return future.get();
+    }
+
     private static class CompletionCallbackResult<T> implements YChat.Callback<T> {
 
         private final CompletableFuture<T> future;

ychat/src/androidMain/kotlin/co/yml/ychat/domain/model/FileBytes.kt

@@ -0,0 +1,7 @@
+package co.yml.ychat.domain.model
+
+actual typealias FileBytes = ByteArray
+
+actual fun FileBytes.toByteArray(): ByteArray {
+    return this
+}

ychat/src/commonMain/kotlin/co/yml/ychat/YChat.kt

@@ -1,5 +1,6 @@
 package co.yml.ychat
 
+import co.yml.ychat.entrypoint.features.AudioTranscriptions
 import co.yml.ychat.entrypoint.features.ChatCompletions
 import co.yml.ychat.entrypoint.features.Completion
 import co.yml.ychat.entrypoint.features.Edits
@@ -111,7 +112,7 @@ interface YChat {
      * The image generations api is used to generate images based on a prompt. You input some text as a
      * prompt, and the model will generate one or more images.
      *
-     * You can configure the parameters of the completion before executing it. Example:
+     * You can configure the parameters before executing it. Example:
      * ```
      * val result = YChat.create(apiKey).imageGenerations()
      *      .setResults(2)
@@ -137,6 +138,20 @@ interface YChat {
      */
     fun edits(): Edits
 
+    /**
+     * The audioTranscriptions api is used to transcribes audio into the input language.
+     *
+     * You can configure the parameters before executing it. Example:
+     * ```
+     * val result = YChat.create(apiKey).audioTranscriptions()
+     *      .setTemperature(0.4)
+     *      .setResponseFormat("json")
+     *      .set...
+     *      .execute("file.mp4", byteArrayFile)
+     * ```
+     */
+    fun audioTranscriptions(): AudioTranscriptions
+
     /**
      * Callback is an interface used for handling the results of an operation.
      * It provides two methods, `onSuccess` and `onError`, for handling the success

ychat/src/commonMain/kotlin/co/yml/ychat/data/api/ChatGptApi.kt

@@ -1,5 +1,7 @@
 package co.yml.ychat.data.api
 
+import co.yml.ychat.data.dto.AudioParamsDto
+import co.yml.ychat.data.dto.AudioResultDto
 import co.yml.ychat.data.dto.ChatCompletionParamsDto
 import co.yml.ychat.data.dto.ChatCompletionsDto
 import co.yml.ychat.data.dto.CompletionDto
@@ -25,4 +27,6 @@ internal interface ChatGptApi {
     suspend fun models(): ApiResult<ModelListDto>
 
     suspend fun model(id: String): ApiResult<ModelDto>
+
+    suspend fun audioTranscriptions(audioParamsDto: AudioParamsDto): ApiResult<AudioResultDto>
 }