Skip to content

Commit b4f9fff

Browse files
connor4312connor4312
authored
mcp: support resource_links and structured output (microsoft#257161)
Closes microsoft#248418
1 parent 4ea0d4b commit b4f9fff

File tree

7 files changed

+81
-36
lines changed

7 files changed

+81
-36
lines changed

‎src/vs/workbench/contrib/chat/browser/chatContentParts/chatToolInputOutputContentPart.ts‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@
55

66
import * as dom from '../../../../../base/browser/dom.js';
77
import { ButtonWithIcon } from '../../../../../base/browser/ui/button/button.js';
8-
import { VSBuffer } from '../../../../../base/common/buffer.js';
98
import { Codicon } from '../../../../../base/common/codicons.js';
109
import { Emitter } from '../../../../../base/common/event.js';
1110
import { IMarkdownString } from '../../../../../base/common/htmlContent.js';
@@ -52,7 +51,7 @@ export interface IChatCollapsibleIOCodePart {
5251

5352
export interface IChatCollapsibleIODataPart {
5453
kind: 'data';
55-
value: Uint8Array;
54+
value?: Uint8Array;
5655
mimeType: string | undefined;
5756
uri: URI;
5857
}
@@ -313,7 +312,7 @@ class SaveResourcesAction extends Action2 {
313312
const target = isFolder ? joinPath(uri, basename(part.uri)) : uri;
314313
try {
315314
if (part.kind === 'data') {
316-
await fileService.writeFile(target, VSBuffer.wrap(part.value));
315+
await fileService.copy(part.uri, target, true);
317316
} else {
318317
// MCP doesn't support streaming data, so no sense trying
319318
const contents = await fileService.readFile(part.uri);

‎src/vs/workbench/contrib/chat/browser/chatContentParts/toolInvocationParts/chatInputOutputMarkdownProgressPart.ts‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,7 @@ export class ChatInputOutputMarkdownProgressPart extends BaseChatToolInvocationS
8888

8989
let processedOutput = output;
9090
if (typeof output === 'string') { // back compat with older stored versions
91-
processedOutput = [{ value: output, isText: true }];
91+
processedOutput = [{ type: 'embed', value: output, isText: true }];
9292
}
9393

9494
const requestId = isResponseVM(context.element) ? context.element.requestId : context.element.id;
@@ -101,15 +101,16 @@ export class ChatInputOutputMarkdownProgressPart extends BaseChatToolInvocationS
101101
toCodePart(input),
102102
processedOutput && {
103103
parts: processedOutput.map((o, i): ChatCollapsibleIOPart => {
104-
const permalinkBasename = o.uri
105-
? basename(o.uri)
104+
const permalinkBasename = o.type === 'ref' || o.uri
105+
? basename(o.uri!)
106106
: o.mimeType && getExtensionForMimeType(o.mimeType)
107107
? `file${getExtensionForMimeType(o.mimeType)}`
108108
: 'file' + (o.isText ? '.txt' : '.bin');
109109

110-
const permalinkUri = ChatResponseResource.createUri(context.element.sessionId, requestId, toolInvocation.toolCallId, i, permalinkBasename);
111110

112-
if (o.isText && !o.asResource) {
111+
if (o.type === 'ref') {
112+
return { kind: 'data', uri: o.uri, mimeType: o.mimeType };
113+
} else if (o.isText && !o.asResource) {
113114
return toCodePart(o.value);
114115
} else {
115116
let decoded: Uint8Array | undefined;
@@ -122,6 +123,7 @@ export class ChatInputOutputMarkdownProgressPart extends BaseChatToolInvocationS
122123
}
123124

124125
// Fall back to text if it's not valid base64
126+
const permalinkUri = ChatResponseResource.createUri(context.element.sessionId, requestId, toolInvocation.toolCallId, i, permalinkBasename);
125127
return { kind: 'data', value: decoded || new TextEncoder().encode(o.value), mimeType: o.mimeType, uri: permalinkUri };
126128
}
127129
}),

‎src/vs/workbench/contrib/chat/browser/languageModelToolsService.ts‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -339,7 +339,7 @@ export class LanguageModelToolsService extends Disposable implements ILanguageMo
339339
toolResult ??= { content: [] };
340340
toolResult.toolResultError = err instanceof Error ? err.message : String(err);
341341
if (tool.data.alwaysDisplayInputOutput) {
342-
toolResult.toolResultDetails = { input: this.formatToolInput(dto), output: [{ isText: true, value: String(err) }], isError: true };
342+
toolResult.toolResultDetails = { input: this.formatToolInput(dto), output: [{ type: 'embed', isText: true, value: String(err) }], isError: true };
343343
}
344344

345345
throw err;
@@ -410,11 +410,11 @@ export class LanguageModelToolsService extends Disposable implements ILanguageMo
410410
private toolResultToIO(toolResult: IToolResult): IToolResultInputOutputDetails['output'] {
411411
return toolResult.content.map(part => {
412412
if (part.kind === 'text') {
413-
return { isText: true, value: part.value };
413+
return { type: 'embed', isText: true, value: part.value };
414414
} else if (part.kind === 'promptTsx') {
415-
return { isText: true, value: stringifyPromptTsxPart(part) };
415+
return { type: 'embed', isText: true, value: stringifyPromptTsxPart(part) };
416416
} else if (part.kind === 'data') {
417-
return { value: encodeBase64(part.value.data), mimeType: part.value.mimeType };
417+
return { type: 'embed', value: encodeBase64(part.value.data), mimeType: part.value.mimeType };
418418
} else {
419419
assertNever(part);
420420
}

‎src/vs/workbench/contrib/chat/common/chatResponseResourceFileSystemProvider.ts‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -46,18 +46,18 @@ export class ChatResponseResourceFileSystemProvider extends Disposable implement
4646

4747
readFileStream(resource: URI): ReadableStreamEvents<Uint8Array> {
4848
const stream = newWriteableStream<Uint8Array>(data => VSBuffer.concat(data.map(data => VSBuffer.wrap(data))).buffer);
49-
stream.end(this.lookupURI(resource));
49+
Promise.resolve(this.lookupURI(resource)).then(v => stream.end(v));
5050
return stream;
5151
}
5252

53-
stat(resource: URI): Promise<IStat> {
54-
const r = this.lookupURI(resource);
55-
return Promise.resolve({
53+
async stat(resource: URI): Promise<IStat> {
54+
const r = await this.lookupURI(resource);
55+
return {
5656
type: FileType.File,
5757
ctime: 0,
5858
mtime: 0,
5959
size: r.length,
60-
});
60+
};
6161
}
6262

6363
delete(): Promise<void> {
@@ -84,7 +84,7 @@ export class ChatResponseResourceFileSystemProvider extends Disposable implement
8484
throw createFileSystemProviderError('fs is readonly', FileSystemProviderErrorCode.NoPermissions);
8585
}
8686

87-
private lookupURI(uri: URI): Uint8Array {
87+
private lookupURI(uri: URI): Uint8Array | Promise<Uint8Array> {
8888
const parsed = ChatResponseResource.parseUri(uri);
8989
if (!parsed) {
9090
throw createFileSystemProviderError(`File not found`, FileSystemProviderErrorCode.FileNotFound);
@@ -109,6 +109,10 @@ export class ChatResponseResourceFileSystemProvider extends Disposable implement
109109
throw createFileSystemProviderError(`Tool does not have part`, FileSystemProviderErrorCode.FileNotFound);
110110
}
111111

112+
if (part.type === 'ref') {
113+
return this._fileService.readFile(part.uri).then(r => r.value.buffer);
114+
}
115+
112116
return part.isText ? new TextEncoder().encode(part.value) : decodeBase64(part.value).buffer;
113117
}
114118
}

‎src/vs/workbench/contrib/chat/common/languageModelToolsService.ts‎

Lines changed: 19 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -132,19 +132,27 @@ export interface IToolInvocationPreparationContext {
132132
chatInteractionId?: string;
133133
}
134134

135+
export type ToolInputOutputBase = {
136+
/** Mimetype of the value, optional */
137+
mimeType?: string;
138+
/** URI of the resource on the MCP server. */
139+
uri?: URI;
140+
/** If true, this part came in as a resource reference rather than direct data. */
141+
asResource?: boolean;
142+
};
143+
144+
export type ToolInputOutputEmbedded = ToolInputOutputBase & {
145+
type: 'embed';
146+
value: string;
147+
/** If true, value is text. If false or not given, value is base64 */
148+
isText?: boolean;
149+
};
150+
151+
export type ToolInputOutputReference = ToolInputOutputBase & { type: 'ref'; uri: URI };
152+
135153
export interface IToolResultInputOutputDetails {
136154
readonly input: string;
137-
readonly output: ({
138-
value: string;
139-
/** If true, value is text. If false or not given, value is base64 */
140-
isText?: boolean;
141-
/** Mimetype of the value, optional */
142-
mimeType?: string;
143-
/** URI of the resource on the MCP server. */
144-
uri?: URI;
145-
/** If true, this part came in as a resource reference rather than direct data. */
146-
asResource?: boolean;
147-
})[];
155+
readonly output: (ToolInputOutputEmbedded | ToolInputOutputReference)[];
148156
readonly isError?: boolean;
149157
}
150158

‎src/vs/workbench/contrib/extensions/common/searchExtensionsTool.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -143,7 +143,7 @@ export class SearchExtensionsTool implements IToolImpl {
143143
}],
144144
toolResultDetails: {
145145
input: JSON.stringify(params),
146-
output: [{ isText: true, value: JSON.stringify(result.map(extension => extension.id)) }]
146+
output: [{ type: 'embed', isText: true, value: JSON.stringify(result.map(extension => extension.id)) }]
147147
}
148148
};
149149
}

‎src/vs/workbench/contrib/mcp/common/mcpService.ts‎

Lines changed: 38 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
*--------------------------------------------------------------------------------------------*/
55

66
import { RunOnceScheduler } from '../../../../base/common/async.js';
7-
import { decodeBase64 } from '../../../../base/common/buffer.js';
7+
import { decodeBase64, VSBuffer } from '../../../../base/common/buffer.js';
88
import { CancellationToken } from '../../../../base/common/cancellation.js';
99
import { Codicon } from '../../../../base/common/codicons.js';
1010
import { markdownCommandLink, MarkdownString } from '../../../../base/common/htmlContent.js';
@@ -15,6 +15,7 @@ import { autorun, IObservable, observableValue, transaction } from '../../../../
1515
import { basename } from '../../../../base/common/resources.js';
1616
import { URI } from '../../../../base/common/uri.js';
1717
import { localize } from '../../../../nls.js';
18+
import { IFileService } from '../../../../platform/files/common/files.js';
1819
import { IInstantiationService } from '../../../../platform/instantiation/common/instantiation.js';
1920
import { ILogService } from '../../../../platform/log/common/log.js';
2021
import { IProductService } from '../../../../platform/product/common/productService.js';
@@ -257,6 +258,7 @@ class McpToolImplementation implements IToolImpl {
257258
private readonly _tool: IMcpTool,
258259
private readonly _server: IMcpServer,
259260
@IProductService private readonly _productService: IProductService,
261+
@IFileService private readonly _fileService: IFileService,
260262
) { }
261263

262264
async prepareToolInvocation(context: IToolInvocationPreparationContext): Promise<IPreparedToolInvocation> {
@@ -318,7 +320,7 @@ class McpToolImplementation implements IToolImpl {
318320

319321
// Rewrite image rsources to images so they are inlined nicely
320322
const addAsInlineData = (mimeType: string, value: string, uri?: URI) => {
321-
details.output.push({ mimeType, value, uri });
323+
details.output.push({ type: 'embed', mimeType, value, uri });
322324
if (isForModel) {
323325
result.content.push({
324326
kind: 'data',
@@ -329,8 +331,10 @@ class McpToolImplementation implements IToolImpl {
329331

330332
const isForModel = audience.includes('assistant');
331333
if (item.type === 'text') {
332-
details.output.push({ isText: true, value: item.text });
333-
if (isForModel) {
334+
details.output.push({ type: 'embed', isText: true, value: item.text });
335+
// structured content 'represents the result of the tool call', so take
336+
// that in place of any textual description when present.
337+
if (isForModel && !callResult.structuredContent) {
334338
result.content.push({
335339
kind: 'text',
336340
value: item.text
@@ -340,13 +344,36 @@ class McpToolImplementation implements IToolImpl {
340344
// default to some image type if not given to hint
341345
addAsInlineData(item.mimeType || 'image/png', item.data);
342346
} else if (item.type === 'resource_link') {
343-
// todo@connor4312 look at what we did before #250329 and use that here
347+
const uri = McpResourceURI.fromServer(this._server.definition, item.uri);
348+
details.output.push({
349+
type: 'ref',
350+
uri,
351+
mimeType: item.mimeType,
352+
});
353+
354+
if (isForModel) {
355+
if (item.mimeType && getAttachableImageExtension(item.mimeType)) {
356+
result.content.push({
357+
kind: 'data',
358+
value: {
359+
mimeType: item.mimeType,
360+
data: await this._fileService.readFile(uri).then(f => f.value).catch(() => VSBuffer.alloc(0)),
361+
}
362+
});
363+
} else {
364+
result.content.push({
365+
kind: 'text',
366+
value: `The tool returns a resource which can be read from the URI ${uri}\n`,
367+
});
368+
}
369+
}
344370
} else if (item.type === 'resource') {
345371
const uri = McpResourceURI.fromServer(this._server.definition, item.resource.uri);
346372
if (item.resource.mimeType && getAttachableImageExtension(item.resource.mimeType) && 'blob' in item.resource) {
347373
addAsInlineData(item.resource.mimeType, item.resource.blob, uri);
348374
} else {
349375
details.output.push({
376+
type: 'embed',
350377
uri,
351378
isText: 'text' in item.resource,
352379
mimeType: item.resource.mimeType,
@@ -359,13 +386,18 @@ class McpToolImplementation implements IToolImpl {
359386

360387
result.content.push({
361388
kind: 'text',
362-
value: 'text' in item.resource ? item.resource.text : `The tool returns a resource which can be read from the URI ${permalink || uri}`,
389+
value: 'text' in item.resource ? item.resource.text : `The tool returns a resource which can be read from the URI ${permalink || uri}\n`,
363390
});
364391
}
365392
}
366393
}
367394
}
368395

396+
if (callResult.structuredContent) {
397+
details.output.push({ type: 'embed', isText: true, value: JSON.stringify(callResult.structuredContent, null, 2) });
398+
result.content.push({ kind: 'text', value: JSON.stringify(callResult.structuredContent) });
399+
}
400+
369401
result.toolResultDetails = details;
370402
return result;
371403
}

0 commit comments

Comments
 (0)