Handle text case in computer-use tool (#543)

## Summary
Change the computer-use tool to handle text content in addition to image
content. Previously, the tool only accepted single image blocks, but now
it can process text blocks, mixed content (text + images), and multiple
text blocks.

- Relaxed validation to allow multiple content blocks instead of
requiring exactly one
- Added support for text-based tool results with proper concatenation
- Enhanced error messages to be more descriptive about content
requirements

## How was it tested?
Added unit tests. 

## Community Contribution License

All community contributions in this pull request are licensed to the
project maintainers under the terms of the [Apache 2
License](https://www.apache.org/licenses/LICENSE-2.0).

By creating this pull request I represent that I have the right to
license the contributions to the project maintainers under the Apache 2
License as stated in the [Community Contribution
License](https://github.com/jetify-com/opensource/blob/main/CONTRIBUTING.md#community-contribution-license).

Author: loreto

aisdk/ai/provider/openai/internal/codec/encode_prompt.go

@@ -4,6 +4,7 @@ import (
 	"encoding/base64"
 	"encoding/json"
 	"fmt"
+	"strings"
 
 	"github.com/openai/openai-go"
 	"github.com/openai/openai-go/responses"
@@ -409,52 +410,104 @@ func EncodeToolMessage(message *api.ToolMessage) ([]responses.ResponseInputItemU
 
 // encodeComputerToolResult handles encoding the result of a computer use tool call
 func encodeComputerToolResult(result *api.ToolResultBlock) (responses.ResponseInputItemUnionParam, error) {
-	if len(result.Content) != 1 {
-		return responses.ResponseInputItemUnionParam{}, fmt.Errorf("expected 1 content block for computer use tool result, got %d", len(result.Content))
-	}
+	// No content at all - this is an error
+	if len(result.Content) == 0 {
+		return responses.ResponseInputItemUnionParam{}, fmt.Errorf("expected at least 1 content block for computer use tool result, got 0")
+	}
+
+	// Single content block - try to handle as screenshot
+	if len(result.Content) == 1 {
+		content := result.Content[0]
+		var imageBlock *api.ImageBlock
+		switch b := content.(type) {
+		case *api.ImageBlock:
+			imageBlock = b
+		case api.ImageBlock:
+			imageBlock = &b
+		default:
+			// Single non-image block - check if it's text
+			if _, ok := content.(*api.TextBlock); ok {
+				return encodeTextToolResult(result)
+			}
+			// Single block that's neither image nor text - this is invalid
+			return responses.ResponseInputItemUnionParam{}, fmt.Errorf("computer use tool result has 1 content block of type %s, expected image or text", content.Type())
+		}
 
-	content := result.Content[0]
-	var imageBlock *api.ImageBlock
-	switch b := content.(type) {
-	case *api.ImageBlock:
-		imageBlock = b
-	case api.ImageBlock:
-		imageBlock = &b
-	default:
-		return responses.ResponseInputItemUnionParam{}, fmt.Errorf("expected image block for computer use tool result, got %T", content)
+		// Create data URL from image data
+		// TODO: Add helper methods to the image and file blocks to make this easier
+		dataURL := "data:" + imageBlock.MediaType + ";base64," + base64.StdEncoding.EncodeToString(imageBlock.Data)
+
+		screenshot := responses.ResponseComputerToolCallOutputScreenshotParam{
+			Type:     "computer_screenshot",
+			ImageURL: openai.String(dataURL),
+		}
+
+		// Extract safety checks from provider metadata if available
+		var acknowledgedSafetyChecks []responses.ResponseInputItemComputerCallOutputAcknowledgedSafetyCheckParam
+		if metadata := GetMetadata(result); metadata != nil {
+			for _, check := range metadata.ComputerSafetyChecks {
+				acknowledgedSafetyChecks = append(acknowledgedSafetyChecks, responses.ResponseInputItemComputerCallOutputAcknowledgedSafetyCheckParam{
+					ID:      check.ID,
+					Code:    openai.String(check.Code),
+					Message: openai.String(check.Message),
+				})
+			}
+		}
+
+		// Create the computer call output parameter
+		output := responses.ResponseInputItemComputerCallOutputParam{
+			CallID:                   result.ToolCallID,
+			Output:                   screenshot,
+			AcknowledgedSafetyChecks: acknowledgedSafetyChecks,
+		}
+
+		return responses.ResponseInputItemUnionParam{
+			OfComputerCallOutput: &output,
+		}, nil
 	}
 
-	// Create data URL from image data
-	// TODO: Add helper methods to the image and file blocks to make this easier
-	dataURL := "data:" + imageBlock.MediaType + ";base64," + base64.StdEncoding.EncodeToString(imageBlock.Data)
+	// Multiple blocks - check if any are text
+	hasText := false
+	for _, content := range result.Content {
+		if _, ok := content.(*api.TextBlock); ok {
+			hasText = true
+			break
+		}
+	}
 
-	screenshot := responses.ResponseComputerToolCallOutputScreenshotParam{
-		Type:     "computer_screenshot",
-		ImageURL: openai.String(dataURL),
+	if !hasText {
+		// Multiple blocks but none are text - this is ambiguous
+		return responses.ResponseInputItemUnionParam{}, fmt.Errorf("computer use tool result has %d content blocks but no text content", len(result.Content))
 	}
 
-	// Extract safety checks from provider metadata if available
-	var acknowledgedSafetyChecks []responses.ResponseInputItemComputerCallOutputAcknowledgedSafetyCheckParam
-	if metadata := GetMetadata(result); metadata != nil {
-		for _, check := range metadata.ComputerSafetyChecks {
-			acknowledgedSafetyChecks = append(acknowledgedSafetyChecks, responses.ResponseInputItemComputerCallOutputAcknowledgedSafetyCheckParam{
-				ID:      check.ID,
-				Code:    openai.String(check.Code),
-				Message: openai.String(check.Message),
-			})
+	// Has text content - use fallback
+	return encodeTextToolResult(result)
+}
+
+// encodeTextToolResult handles encoding text-based tool results, with Content[] taking precedence over Result
+func encodeTextToolResult(result *api.ToolResultBlock) (responses.ResponseInputItemUnionParam, error) {
+	output := ""
+
+	// Check Content[] first - more expressive when available
+	if len(result.Content) > 0 {
+		for _, content := range result.Content {
+			if textBlock, ok := content.(*api.TextBlock); ok {
+				output += textBlock.Text + "\n"
+			}
 		}
+		output = strings.TrimSuffix(output, "\n")
 	}
 
-	// Create the computer call output parameter
-	output := responses.ResponseInputItemComputerCallOutputParam{
-		CallID:                   result.ToolCallID,
-		Output:                   screenshot,
-		AcknowledgedSafetyChecks: acknowledgedSafetyChecks,
+	// If no text content found, use Result field
+	if output == "" && result.Result != nil {
+		resultJSON, err := json.Marshal(result.Result)
+		if err != nil {
+			return responses.ResponseInputItemUnionParam{}, fmt.Errorf("failed to marshal tool result: %v", err)
+		}
+		output = string(resultJSON)
 	}
 
-	return responses.ResponseInputItemUnionParam{
-		OfComputerCallOutput: &output,
-	}, nil
+	return responses.ResponseInputItemParamOfFunctionCallOutput(result.ToolCallID, output), nil
 }
 
 func EncodeToolResultBlock(result *api.ToolResultBlock) (responses.ResponseInputItemUnionParam, error) {

aisdk/ai/provider/openai/internal/codec/encode_prompt_test.go

@@ -1023,10 +1023,10 @@ var toolMessageTests = []testCase{
 				},
 			},
 		},
-		expectedError: "expected 1 content block for computer use tool result, got 0",
+		expectedError: "expected at least 1 content block for computer use tool result, got 0",
 	},
 	{
-		name: "computer tool result with multiple content blocks",
+		name: "computer tool result with multiple content blocks but no text",
 		input: []api.Message{
 			&api.ToolMessage{
 				Content: []api.ToolResultBlock{
@@ -1046,25 +1046,120 @@ var toolMessageTests = []testCase{
 				},
 			},
 		},
-		expectedError: "expected 1 content block for computer use tool result, got 2",
+		expectedError: "computer use tool result has 2 content blocks but no text content",
 	},
 	{
-		name: "computer tool result with wrong content type",
+		name: "computer tool result with text content",
 		input: []api.Message{
 			&api.ToolMessage{
 				Content: []api.ToolResultBlock{
 					{
 						ToolCallID: "openai.computer_use_preview",
 						Content: []api.ContentBlock{
 							&api.TextBlock{
-								Text: "not an image",
+								Text: "Action completed successfully",
 							},
 						},
 					},
 				},
 			},
 		},
-		expectedError: "expected image block for computer use tool result",
+		expectedMessages: []string{
+			`{
+				"type": "function_call_output",
+				"call_id": "openai.computer_use_preview",
+				"output": "Action completed successfully"
+			}`,
+		},
+	},
+	{
+		name: "computer tool result with mixed content (text and image)",
+		input: []api.Message{
+			&api.ToolMessage{
+				Content: []api.ToolResultBlock{
+					{
+						ToolCallID: "openai.computer_use_preview",
+						Content: []api.ContentBlock{
+							&api.TextBlock{
+								Text: "Screenshot taken",
+							},
+							&api.ImageBlock{
+								Data:      []byte("test-image-data"),
+								MediaType: "image/png",
+							},
+						},
+					},
+				},
+			},
+		},
+		expectedMessages: []string{
+			`{
+				"type": "function_call_output",
+				"call_id": "openai.computer_use_preview",
+				"output": "Screenshot taken"
+			}`,
+		},
+	},
+	{
+		name: "computer tool result with multiple text blocks",
+		input: []api.Message{
+			&api.ToolMessage{
+				Content: []api.ToolResultBlock{
+					{
+						ToolCallID: "openai.computer_use_preview",
+						Content: []api.ContentBlock{
+							&api.TextBlock{
+								Text: "First line",
+							},
+							&api.TextBlock{
+								Text: "Second line",
+							},
+						},
+					},
+				},
+			},
+		},
+		expectedMessages: []string{
+			`{
+				"type": "function_call_output",
+				"call_id": "openai.computer_use_preview",
+				"output": "First line\nSecond line"
+			}`,
+		},
+	},
+	{
+		name: "computer tool result falls back to Result field",
+		input: []api.Message{
+			&api.ToolMessage{
+				Content: []api.ToolResultBlock{
+					{
+						ToolCallID: "openai.computer_use_preview",
+						Content:    []api.ContentBlock{},
+						Result:     json.RawMessage(`{"status":"error","message":"Action failed"}`),
+					},
+				},
+			},
+		},
+		expectedError: "expected at least 1 content block for computer use tool result, got 0",
+	},
+	{
+		name: "computer tool result with single reasoning block",
+		input: []api.Message{
+			&api.ToolMessage{
+				Content: []api.ToolResultBlock{
+					{
+						ToolCallID: "openai.computer_use_preview",
+						Content: []api.ContentBlock{
+							&api.ReasoningBlock{
+								Text:      "Processing the request...",
+								Signature: "signature_123",
+							},
+						},
+					},
+				},
+			},
+		},
+		expectedError: "computer use tool result has 1 content block of type reasoning, expected image or text",
 	},
 	{
 		name: "tool message value type (not pointer)",