Skip to content

Commit b4526bf

Browse files
dcs-sonidcs-soni
committed
fix: move url utils from core to openai adapters
1 parent 978a79d commit b4526bf

File tree

9 files changed

+133
-59
lines changed

9 files changed

+133
-59
lines changed

‎core/llm/llms/Anthropic.ts‎

Lines changed: 17 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ import { safeParseToolCallArgs } from "../../tools/parseArgs.js";
3030
import { renderChatMessage, stripImages } from "../../util/messageContent.js";
3131
import { DEFAULT_REASONING_TOKENS } from "../constants.js";
3232
import { BaseLLM } from "../index.js";
33+
import { extractBase64FromDataUrl } from "../../util/url.js";
3334

3435
class Anthropic extends BaseLLM {
3536
static providerName = "anthropic";
@@ -105,14 +106,22 @@ class Anthropic extends BaseLLM {
105106
});
106107
}
107108
} else {
108-
parts.push({
109-
type: "image",
110-
source: {
111-
type: "base64",
112-
media_type: getAnthropicMediaTypeFromDataUrl(part.imageUrl.url),
113-
data: part.imageUrl.url.split(",")[1],
114-
},
115-
});
109+
const base64Data = extractBase64FromDataUrl(part.imageUrl.url);
110+
if (base64Data) {
111+
parts.push({
112+
type: "image",
113+
source: {
114+
type: "base64",
115+
media_type: getAnthropicMediaTypeFromDataUrl(part.imageUrl.url),
116+
data: base64Data,
117+
},
118+
});
119+
} else {
120+
console.warn(
121+
"Anthropic: skipping image with invalid data URL format",
122+
part.imageUrl.url,
123+
);
124+
}
116125
}
117126
}
118127
}

‎core/llm/llms/Bedrock.ts‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ import { BaseLLM } from "../index.js";
2525
import { PROVIDER_TOOL_SUPPORT } from "../toolSupport.js";
2626
import { getSecureID } from "../utils/getSecureID.js";
2727
import { withLLMRetry } from "../utils/retry.js";
28+
import { parseDataUrl } from "../../util/url.js";
2829

2930
interface ModelConfig {
3031
formatPayload: (text: string) => any;
@@ -545,8 +546,9 @@ class Bedrock extends BaseLLM {
545546
if (part.type === "text") {
546547
blocks.push({ text: part.text });
547548
} else if (part.type === "imageUrl" && part.imageUrl) {
548-
try {
549-
const [mimeType, base64Data] = part.imageUrl.url.split(",");
549+
const parsed = parseDataUrl(part.imageUrl.url);
550+
if (parsed) {
551+
const { mimeType, base64Data } = parsed;
550552
const format = mimeType.split("/")[1]?.split(";")[0] || "jpeg";
551553
if (
552554
format === ImageFormat.JPEG ||
@@ -568,8 +570,8 @@ class Bedrock extends BaseLLM {
568570
part,
569571
);
570572
}
571-
} catch (error) {
572-
console.warn("Bedrock: failed to process image part", error, part);
573+
} else {
574+
console.warn("Bedrock: failed to process image part", part);
573575
}
574576
}
575577
}

‎core/llm/llms/Gemini.ts‎

Lines changed: 26 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ import {
2121
GeminiToolFunctionDeclaration,
2222
convertContinueToolToGeminiFunction,
2323
} from "./gemini-types";
24+
import { extractBase64FromDataUrl } from "../../util/url.js";
2425

2526
class Gemini extends BaseLLM {
2627
static providerName = "gemini";
@@ -184,16 +185,31 @@ class Gemini extends BaseLLM {
184185
}
185186

186187
continuePartToGeminiPart(part: MessagePart): GeminiChatContentPart {
187-
return part.type === "text"
188-
? {
189-
text: part.text,
190-
}
191-
: {
192-
inlineData: {
193-
mimeType: "image/jpeg",
194-
data: part.imageUrl?.url.split(",")[1],
195-
},
196-
};
188+
if (part.type === "text") {
189+
return {
190+
text: part.text,
191+
};
192+
}
193+
194+
let data = "";
195+
if (part.imageUrl?.url) {
196+
const extracted = extractBase64FromDataUrl(part.imageUrl.url);
197+
if (extracted) {
198+
data = extracted;
199+
} else {
200+
console.warn(
201+
"Gemini: skipping image with invalid data URL format",
202+
part.imageUrl.url,
203+
);
204+
}
205+
}
206+
207+
return {
208+
inlineData: {
209+
mimeType: "image/jpeg",
210+
data,
211+
},
212+
};
197213
}
198214

199215
public prepareBody(

‎core/llm/llms/Ollama.ts‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ import {
1414
import { renderChatMessage } from "../../util/messageContent.js";
1515
import { getRemoteModelInfo } from "../../util/ollamaHelper.js";
1616
import { BaseLLM } from "../index.js";
17+
import { extractBase64FromDataUrl } from "../../util/url.js";
1718

1819
type OllamaChatMessage = {
1920
role: ChatMessageRole;
@@ -303,10 +304,16 @@ class Ollama extends BaseLLM implements ModelInstaller {
303304
const images: string[] = [];
304305
message.content.forEach((part) => {
305306
if (part.type === "imageUrl" && part.imageUrl) {
306-
const image = part.imageUrl?.url.split(",").at(-1);
307+
const image = part.imageUrl?.url
308+
? extractBase64FromDataUrl(part.imageUrl.url)
309+
: undefined;
307310
if (image) {
308311
images.push(image);
309-
}
312+
} else if (part.imageUrl?.url) {
313+
console.warn(
314+
"Ollama: skipping image with invalid data URL format",
315+
part.imageUrl.url,
316+
);
310317
}
311318
});
312319
if (images.length > 0) {

‎core/util/url.ts‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,8 @@
1+
import {
2+
parseDataUrl as parseDataUrlFromAdapter,
3+
extractBase64FromDataUrl as extractBase64FromDataUrlFromAdapter,
4+
} from "@continuedev/openai-adapters";
5+
16
export function canParseUrl(url: string): boolean {
27
if ((URL as any)?.canParse) {
38
return (URL as any).canParse(url);
@@ -9,3 +14,6 @@ export function canParseUrl(url: string): boolean {
914
return false;
1015
}
1116
}
17+
18+
export const parseDataUrl = parseDataUrlFromAdapter;
19+
export const extractBase64FromDataUrl = extractBase64FromDataUrlFromAdapter;

‎packages/openai-adapters/src/apis/Anthropic.ts‎

Lines changed: 17 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ import {
2525
CompletionUsage,
2626
} from "openai/resources/index";
2727
import { ChatCompletionCreateParams } from "openai/resources/index.js";
28+
import { extractBase64FromDataUrl } from "../util/url.js";
2829
import { AnthropicConfig } from "../types.js";
2930
import {
3031
chatChunk,
@@ -194,14 +195,22 @@ export class AnthropicApi implements BaseLlmApi {
194195
if (part.type === "image_url") {
195196
const dataUrl = part.image_url.url;
196197
if (dataUrl?.startsWith("data:")) {
197-
blocks.push({
198-
type: "image",
199-
source: {
200-
type: "base64",
201-
media_type: getAnthropicMediaTypeFromDataUrl(dataUrl),
202-
data: dataUrl.split(",")[1],
203-
},
204-
});
198+
const base64Data = extractBase64FromDataUrl(dataUrl);
199+
if (base64Data) {
200+
blocks.push({
201+
type: "image",
202+
source: {
203+
type: "base64",
204+
media_type: getAnthropicMediaTypeFromDataUrl(dataUrl),
205+
data: base64Data,
206+
},
207+
});
208+
} else {
209+
console.warn(
210+
"Anthropic: skipping image with invalid data URL format",
211+
dataUrl,
212+
);
213+
}
205214
}
206215
} else {
207216
const text = part.type === "text" ? part.text : part.refusal;

‎packages/openai-adapters/src/apis/Bedrock.ts‎

Lines changed: 28 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ import {
3131

3232
import { fromNodeProviderChain } from "@aws-sdk/credential-providers";
3333
import { fromStatic } from "@aws-sdk/token-providers";
34+
import { parseDataUrl } from "../util/url.js";
3435
import { BedrockConfig } from "../types.js";
3536
import { chatChunk, chatChunkFromDelta, embedding, rerank } from "../util.js";
3637
import { safeParseArgs } from "../util/parseArgs.js";
@@ -134,35 +135,35 @@ export class BedrockApi implements BaseLlmApi {
134135
throw new Error("Unsupported part type: input_audio");
135136
case "image_url":
136137
default:
137-
try {
138-
const [mimeType, base64Data] = (
139-
part as ChatCompletionContentPartImage
140-
).image_url.url.split(",");
141-
const format = mimeType.split("/")[1]?.split(";")[0] || "jpeg";
142-
if (
143-
format === ImageFormat.JPEG ||
144-
format === ImageFormat.PNG ||
145-
format === ImageFormat.WEBP ||
146-
format === ImageFormat.GIF
147-
) {
148-
return {
149-
image: {
150-
format,
151-
source: {
152-
bytes: Uint8Array.from(Buffer.from(base64Data, "base64")),
153-
},
154-
},
155-
};
156-
} else {
157-
console.warn(
158-
`Bedrock: skipping unsupported image part format: ${format}`,
159-
);
160-
return { text: "[Unsupported image format]" };
161-
}
162-
} catch (error) {
163-
console.warn("Bedrock: failed to process image part", error);
138+
const parsed = parseDataUrl(
139+
(part as ChatCompletionContentPartImage).image_url.url,
140+
);
141+
if (!parsed) {
142+
console.warn("Bedrock: failed to process image part - invalid URL");
164143
return { text: "[Failed to process image]" };
165144
}
145+
const { mimeType, base64Data } = parsed;
146+
const format = mimeType.split("/")[1]?.split(";")[0] || "jpeg";
147+
if (
148+
format === ImageFormat.JPEG ||
149+
format === ImageFormat.PNG ||
150+
format === ImageFormat.WEBP ||
151+
format === ImageFormat.GIF
152+
) {
153+
return {
154+
image: {
155+
format,
156+
source: {
157+
bytes: Uint8Array.from(Buffer.from(base64Data, "base64")),
158+
},
159+
},
160+
};
161+
} else {
162+
console.warn(
163+
`Bedrock: skipping unsupported image part format: ${format}`,
164+
);
165+
return { text: "[Unsupported image format]" };
166+
}
166167
}
167168
}
168169

‎packages/openai-adapters/src/index.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -190,3 +190,4 @@ export {
190190
} from "./apis/AnthropicUtils.js";
191191

192192
export { isResponsesModel } from "./apis/openaiResponses.js";
193+
export { parseDataUrl, extractBase64FromDataUrl } from "./util/url.js";

‎packages/openai-adapters/src/util/url.ts‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
export function parseDataUrl(dataUrl: string):
2+
| {
3+
mimeType: string;
4+
base64Data: string;
5+
}
6+
| undefined {
7+
const urlParts = dataUrl.split(",");
8+
9+
if (urlParts.length < 2) {
10+
return undefined;
11+
}
12+
13+
const [mimeType, ...base64Parts] = urlParts;
14+
const base64Data = base64Parts.join(",");
15+
16+
return { mimeType, base64Data };
17+
}
18+
19+
export function extractBase64FromDataUrl(dataUrl: string): string | undefined {
20+
return parseDataUrl(dataUrl)?.base64Data;
21+
}

0 commit comments

Comments
 (0)