diff --git a/core/llm/llms/Anthropic.ts b/core/llm/llms/Anthropic.ts index ff3f85e51eb..a1fdfbe3598 100644 --- a/core/llm/llms/Anthropic.ts +++ b/core/llm/llms/Anthropic.ts @@ -28,6 +28,7 @@ import { } from "../../index.js"; import { safeParseToolCallArgs } from "../../tools/parseArgs.js"; import { renderChatMessage, stripImages } from "../../util/messageContent.js"; +import { extractBase64FromDataUrl } from "../../util/url.js"; import { DEFAULT_REASONING_TOKENS } from "../constants.js"; import { BaseLLM } from "../index.js"; @@ -105,14 +106,22 @@ class Anthropic extends BaseLLM { }); } } else { - parts.push({ - type: "image", - source: { - type: "base64", - media_type: getAnthropicMediaTypeFromDataUrl(part.imageUrl.url), - data: part.imageUrl.url.split(",")[1], - }, - }); + const base64Data = extractBase64FromDataUrl(part.imageUrl.url); + if (base64Data) { + parts.push({ + type: "image", + source: { + type: "base64", + media_type: getAnthropicMediaTypeFromDataUrl(part.imageUrl.url), + data: base64Data, + }, + }); + } else { + console.warn( + "Anthropic: skipping image with invalid data URL format", + part.imageUrl.url, + ); + } } } } diff --git a/core/llm/llms/Bedrock.ts b/core/llm/llms/Bedrock.ts index ea47b63ae1f..7b338e9df59 100644 --- a/core/llm/llms/Bedrock.ts +++ b/core/llm/llms/Bedrock.ts @@ -21,6 +21,7 @@ import type { CompletionOptions } from "../../index.js"; import { ChatMessage, Chunk, LLMOptions, MessageContent } from "../../index.js"; import { safeParseToolCallArgs } from "../../tools/parseArgs.js"; import { renderChatMessage, stripImages } from "../../util/messageContent.js"; +import { parseDataUrl } from "../../util/url.js"; import { BaseLLM } from "../index.js"; import { PROVIDER_TOOL_SUPPORT } from "../toolSupport.js"; import { getSecureID } from "../utils/getSecureID.js"; @@ -545,8 +546,9 @@ class Bedrock extends BaseLLM { if (part.type === "text") { blocks.push({ text: part.text }); } else if (part.type === "imageUrl" && part.imageUrl) { - try { - const [mimeType, base64Data] = part.imageUrl.url.split(","); + const parsed = parseDataUrl(part.imageUrl.url); + if (parsed) { + const { mimeType, base64Data } = parsed; const format = mimeType.split("/")[1]?.split(";")[0] || "jpeg"; if ( format === ImageFormat.JPEG || @@ -568,8 +570,8 @@ class Bedrock extends BaseLLM { part, ); } - } catch (error) { - console.warn("Bedrock: failed to process image part", error, part); + } else { + console.warn("Bedrock: failed to process image part", part); } } } diff --git a/core/llm/llms/Gemini.ts b/core/llm/llms/Gemini.ts index 7caf2db7d75..0e0b9b243fe 100644 --- a/core/llm/llms/Gemini.ts +++ b/core/llm/llms/Gemini.ts @@ -11,6 +11,7 @@ import { } from "../../index.js"; import { safeParseToolCallArgs } from "../../tools/parseArgs.js"; import { renderChatMessage, stripImages } from "../../util/messageContent.js"; +import { extractBase64FromDataUrl } from "../../util/url.js"; import { BaseLLM } from "../index.js"; import { GeminiChatContent, @@ -184,16 +185,31 @@ class Gemini extends BaseLLM { } continuePartToGeminiPart(part: MessagePart): GeminiChatContentPart { - return part.type === "text" - ? { - text: part.text, - } - : { - inlineData: { - mimeType: "image/jpeg", - data: part.imageUrl?.url.split(",")[1], - }, - }; + if (part.type === "text") { + return { + text: part.text, + }; + } + + let data = ""; + if (part.imageUrl?.url) { + const extracted = extractBase64FromDataUrl(part.imageUrl.url); + if (extracted) { + data = extracted; + } else { + console.warn( + "Gemini: skipping image with invalid data URL format", + part.imageUrl.url, + ); + } + } + + return { + inlineData: { + mimeType: "image/jpeg", + data, + }, + }; } public prepareBody( diff --git a/core/llm/llms/Ollama.ts b/core/llm/llms/Ollama.ts index 0a239fce909..7e36ac83647 100644 --- a/core/llm/llms/Ollama.ts +++ b/core/llm/llms/Ollama.ts @@ -13,6 +13,7 @@ import { } from "../../index.js"; import { renderChatMessage } from "../../util/messageContent.js"; import { getRemoteModelInfo } from "../../util/ollamaHelper.js"; +import { extractBase64FromDataUrl } from "../../util/url.js"; import { BaseLLM } from "../index.js"; type OllamaChatMessage = { @@ -303,9 +304,16 @@ class Ollama extends BaseLLM implements ModelInstaller { const images: string[] = []; message.content.forEach((part) => { if (part.type === "imageUrl" && part.imageUrl) { - const image = part.imageUrl?.url.split(",").at(-1); + const image = part.imageUrl?.url + ? extractBase64FromDataUrl(part.imageUrl.url) + : undefined; if (image) { images.push(image); + } else if (part.imageUrl?.url) { + console.warn( + "Ollama: skipping image with invalid data URL format", + part.imageUrl.url, + ); } } }); diff --git a/core/util/url.ts b/core/util/url.ts index 83e0edcba14..3bdc0fc6907 100644 --- a/core/util/url.ts +++ b/core/util/url.ts @@ -1,3 +1,8 @@ +import { + extractBase64FromDataUrl as extractBase64FromDataUrlFromAdapter, + parseDataUrl as parseDataUrlFromAdapter, +} from "@continuedev/openai-adapters"; + export function canParseUrl(url: string): boolean { if ((URL as any)?.canParse) { return (URL as any).canParse(url); @@ -9,3 +14,6 @@ export function canParseUrl(url: string): boolean { return false; } } + +export const parseDataUrl = parseDataUrlFromAdapter; +export const extractBase64FromDataUrl = extractBase64FromDataUrlFromAdapter; diff --git a/packages/openai-adapters/src/apis/Anthropic.ts b/packages/openai-adapters/src/apis/Anthropic.ts index 0a7bc573cc5..66656957a13 100644 --- a/packages/openai-adapters/src/apis/Anthropic.ts +++ b/packages/openai-adapters/src/apis/Anthropic.ts @@ -34,6 +34,7 @@ import { } from "../util.js"; import { EMPTY_CHAT_COMPLETION } from "../util/emptyChatCompletion.js"; import { safeParseArgs } from "../util/parseArgs.js"; +import { extractBase64FromDataUrl } from "../util/url.js"; import { CACHING_STRATEGIES, CachingStrategyName, @@ -194,14 +195,22 @@ export class AnthropicApi implements BaseLlmApi { if (part.type === "image_url") { const dataUrl = part.image_url.url; if (dataUrl?.startsWith("data:")) { - blocks.push({ - type: "image", - source: { - type: "base64", - media_type: getAnthropicMediaTypeFromDataUrl(dataUrl), - data: dataUrl.split(",")[1], - }, - }); + const base64Data = extractBase64FromDataUrl(dataUrl); + if (base64Data) { + blocks.push({ + type: "image", + source: { + type: "base64", + media_type: getAnthropicMediaTypeFromDataUrl(dataUrl), + data: base64Data, + }, + }); + } else { + console.warn( + "Anthropic: skipping image with invalid data URL format", + dataUrl, + ); + } } } else { const text = part.type === "text" ? part.text : part.refusal; diff --git a/packages/openai-adapters/src/apis/Bedrock.ts b/packages/openai-adapters/src/apis/Bedrock.ts index cf5588d686d..d98ad25d2f5 100644 --- a/packages/openai-adapters/src/apis/Bedrock.ts +++ b/packages/openai-adapters/src/apis/Bedrock.ts @@ -34,6 +34,7 @@ import { fromStatic } from "@aws-sdk/token-providers"; import { BedrockConfig } from "../types.js"; import { chatChunk, chatChunkFromDelta, embedding, rerank } from "../util.js"; import { safeParseArgs } from "../util/parseArgs.js"; +import { parseDataUrl } from "../util/url.js"; import { BaseLlmApi, CreateRerankResponse, @@ -134,35 +135,35 @@ export class BedrockApi implements BaseLlmApi { throw new Error("Unsupported part type: input_audio"); case "image_url": default: - try { - const [mimeType, base64Data] = ( - part as ChatCompletionContentPartImage - ).image_url.url.split(","); - const format = mimeType.split("/")[1]?.split(";")[0] || "jpeg"; - if ( - format === ImageFormat.JPEG || - format === ImageFormat.PNG || - format === ImageFormat.WEBP || - format === ImageFormat.GIF - ) { - return { - image: { - format, - source: { - bytes: Uint8Array.from(Buffer.from(base64Data, "base64")), - }, - }, - }; - } else { - console.warn( - `Bedrock: skipping unsupported image part format: ${format}`, - ); - return { text: "[Unsupported image format]" }; - } - } catch (error) { - console.warn("Bedrock: failed to process image part", error); + const parsed = parseDataUrl( + (part as ChatCompletionContentPartImage).image_url.url, + ); + if (!parsed) { + console.warn("Bedrock: failed to process image part - invalid URL"); return { text: "[Failed to process image]" }; } + const { mimeType, base64Data } = parsed; + const format = mimeType.split("/")[1]?.split(";")[0] || "jpeg"; + if ( + format === ImageFormat.JPEG || + format === ImageFormat.PNG || + format === ImageFormat.WEBP || + format === ImageFormat.GIF + ) { + return { + image: { + format, + source: { + bytes: Uint8Array.from(Buffer.from(base64Data, "base64")), + }, + }, + }; + } else { + console.warn( + `Bedrock: skipping unsupported image part format: ${format}`, + ); + return { text: "[Unsupported image format]" }; + } } } diff --git a/packages/openai-adapters/src/index.ts b/packages/openai-adapters/src/index.ts index a7ee579f9f6..09b3cbacd44 100644 --- a/packages/openai-adapters/src/index.ts +++ b/packages/openai-adapters/src/index.ts @@ -190,3 +190,4 @@ export { } from "./apis/AnthropicUtils.js"; export { isResponsesModel } from "./apis/openaiResponses.js"; +export { parseDataUrl, extractBase64FromDataUrl } from "./util/url.js"; diff --git a/packages/openai-adapters/src/util/url.ts b/packages/openai-adapters/src/util/url.ts new file mode 100644 index 00000000000..be013bb2e85 --- /dev/null +++ b/packages/openai-adapters/src/util/url.ts @@ -0,0 +1,21 @@ +export function parseDataUrl(dataUrl: string): + | { + mimeType: string; + base64Data: string; + } + | undefined { + const urlParts = dataUrl.split(","); + + if (urlParts.length < 2) { + return undefined; + } + + const [mimeType, ...base64Parts] = urlParts; + const base64Data = base64Parts.join(","); + + return { mimeType, base64Data }; +} + +export function extractBase64FromDataUrl(dataUrl: string): string | undefined { + return parseDataUrl(dataUrl)?.base64Data; +}