Merge pull request microsoft#255000 from mjbvz/quiet-leopon

Add basic API for extensible rendering inside of chat output

Author: mjbvz

src/vs/platform/extensions/common/extensionsApiProposals.ts

@@ -37,6 +37,9 @@ const _allApiProposals = {
 	chatEditing: {
 		proposal: 'https://raw.githubusercontent.com/microsoft/vscode/main/src/vscode-dts/vscode.proposed.chatEditing.d.ts',
 	},
+	chatOutputRenderer: {
+		proposal: 'https://raw.githubusercontent.com/microsoft/vscode/main/src/vscode-dts/vscode.proposed.chatOutputRenderer.d.ts',
+	},
 	chatParticipantAdditions: {
 		proposal: 'https://raw.githubusercontent.com/microsoft/vscode/main/src/vscode-dts/vscode.proposed.chatParticipantAdditions.d.ts',
 	},

src/vs/workbench/api/browser/extensionHost.contribution.ts

@@ -91,6 +91,7 @@ import './mainThreadAiEmbeddingVector.js';
 import './mainThreadAiSettingsSearch.js';
 import './mainThreadMcp.js';
 import './mainThreadChatStatus.js';
+import './mainThreadChatOutputRenderer.js';
 import './mainThreadChatSessions.js';
 import './mainThreadDataChannels.js';
 

src/vs/workbench/api/browser/mainThreadChatOutputRenderer.ts

@@ -0,0 +1,58 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { VSBuffer } from '../../../base/common/buffer.js';
+import { Disposable, IDisposable } from '../../../base/common/lifecycle.js';
+import { URI, UriComponents } from '../../../base/common/uri.js';
+import { ExtensionIdentifier } from '../../../platform/extensions/common/extensions.js';
+import { IChatOutputRendererService } from '../../contrib/chat/browser/chatOutputItemRenderer.js';
+import { IExtHostContext } from '../../services/extensions/common/extHostCustomers.js';
+import { ExtHostChatOutputRendererShape, ExtHostContext, MainThreadChatOutputRendererShape } from '../common/extHost.protocol.js';
+import { MainThreadWebviews } from './mainThreadWebviews.js';
+
+export class MainThreadChatOutputRenderer extends Disposable implements MainThreadChatOutputRendererShape {
+
+	private readonly _proxy: ExtHostChatOutputRendererShape;
+
+	private _webviewHandlePool = 0;
+
+	private readonly registeredRenderers = new Map<string, IDisposable>();
+
+	constructor(
+		extHostContext: IExtHostContext,
+		private readonly _mainThreadWebview: MainThreadWebviews,
+		@IChatOutputRendererService private readonly _rendererService: IChatOutputRendererService,
+	) {
+		super();
+		this._proxy = extHostContext.getProxy(ExtHostContext.ExtHostChatOutputRenderer);
+	}
+
+	override dispose(): void {
+		super.dispose();
+
+		this.registeredRenderers.forEach(disposable => disposable.dispose());
+		this.registeredRenderers.clear();
+	}
+
+	$registerChatOutputRenderer(mime: string, extensionId: ExtensionIdentifier, extensionLocation: UriComponents): void {
+		this._rendererService.registerRenderer(mime, {
+			renderOutputPart: async (mime, data, webview, token) => {
+				const webviewHandle = `chat-output-${++this._webviewHandlePool}`;
+
+				this._mainThreadWebview.addWebview(webviewHandle, webview, {
+					serializeBuffersForPostMessage: true,
+				});
+
+				this._proxy.$renderChatOutput(mime, VSBuffer.wrap(data), webviewHandle, token);
+			},
+		}, {
+			extension: { id: extensionId, location: URI.revive(extensionLocation) }
+		});
+	}
+
+	$unregisterChatOutputRenderer(mime: string): void {
+		this.registeredRenderers.get(mime)?.dispose();
+	}
+}

src/vs/workbench/api/browser/mainThreadWebviewManager.ts

@@ -11,6 +11,7 @@ import { MainThreadWebviews } from './mainThreadWebviews.js';
 import { MainThreadWebviewsViews } from './mainThreadWebviewViews.js';
 import * as extHostProtocol from '../common/extHost.protocol.js';
 import { extHostCustomer, IExtHostContext } from '../../services/extensions/common/extHostCustomers.js';
+import { MainThreadChatOutputRenderer } from './mainThreadChatOutputRenderer.js';
 
 @extHostCustomer
 export class MainThreadWebviewManager extends Disposable {
@@ -31,5 +32,8 @@ export class MainThreadWebviewManager extends Disposable {
 
 		const webviewViews = this._register(instantiationService.createInstance(MainThreadWebviewsViews, context, webviews));
 		context.set(extHostProtocol.MainContext.MainThreadWebviewViews, webviewViews);
+
+		const chatOutputRenderers = this._register(instantiationService.createInstance(MainThreadChatOutputRenderer, context, webviews));
+		context.set(extHostProtocol.MainContext.MainThreadChatOutputRenderer, chatOutputRenderers);
 	}
 }

src/vs/workbench/api/browser/mainThreadWebviews.ts

@@ -13,11 +13,11 @@ import { localize } from '../../../nls.js';
 import { ExtensionIdentifier } from '../../../platform/extensions/common/extensions.js';
 import { IOpenerService } from '../../../platform/opener/common/opener.js';
 import { IProductService } from '../../../platform/product/common/productService.js';
-import * as extHostProtocol from '../common/extHost.protocol.js';
-import { deserializeWebviewMessage, serializeWebviewMessage } from '../common/extHostWebviewMessaging.js';
-import { IOverlayWebview, IWebview, WebviewContentOptions, WebviewExtensionDescription } from '../../contrib/webview/browser/webview.js';
+import { IWebview, WebviewContentOptions, WebviewExtensionDescription } from '../../contrib/webview/browser/webview.js';
 import { IExtHostContext } from '../../services/extensions/common/extHostCustomers.js';
 import { SerializableObjectWithBuffers } from '../../services/extensions/common/proxyIdentifier.js';
+import * as extHostProtocol from '../common/extHost.protocol.js';
+import { deserializeWebviewMessage, serializeWebviewMessage } from '../common/extHostWebviewMessaging.js';
 
 export class MainThreadWebviews extends Disposable implements extHostProtocol.MainThreadWebviewsShape {
 
@@ -43,7 +43,7 @@ export class MainThreadWebviews extends Disposable implements extHostProtocol.Ma
 		this._proxy = context.getProxy(extHostProtocol.ExtHostContext.ExtHostWebviews);
 	}
 
-	public addWebview(handle: extHostProtocol.WebviewHandle, webview: IOverlayWebview, options: { serializeBuffersForPostMessage: boolean }): void {
+	public addWebview(handle: extHostProtocol.WebviewHandle, webview: IWebview, options: { serializeBuffersForPostMessage: boolean }): void {
 		if (this._webviews.has(handle)) {
 			throw new Error('Webview already registered');
 		}
@@ -72,7 +72,7 @@ export class MainThreadWebviews extends Disposable implements extHostProtocol.Ma
 		return webview.postMessage(message, arrayBuffers);
 	}
 
-	private hookupWebviewEventDelegate(handle: extHostProtocol.WebviewHandle, webview: IOverlayWebview, options: { serializeBuffersForPostMessage: boolean }) {
+	private hookupWebviewEventDelegate(handle: extHostProtocol.WebviewHandle, webview: IWebview, options: { serializeBuffersForPostMessage: boolean }) {
 		const disposables = new DisposableStore();
 
 		disposables.add(webview.onDidClickLink((uri) => this.onDidClickLink(handle, uri)));

src/vs/workbench/api/common/extHost.api.impl.ts

@@ -113,6 +113,7 @@ import { IExtHostWindow } from './extHostWindow.js';
 import { IExtHostWorkspace } from './extHostWorkspace.js';
 import { ExtHostAiSettingsSearch } from './extHostAiSettingsSearch.js';
 import { ExtHostChatSessions } from './extHostChatSessions.js';
+import { ExtHostChatOutputRenderer } from './extHostChatOutputRenderer.js';
 
 export interface IExtensionRegistries {
 	mine: ExtensionDescriptionRegistry;
@@ -220,6 +221,7 @@ export function createApiFactoryAndRegisterActors(accessor: ServicesAccessor): I
 	const extHostTesting = rpcProtocol.set(ExtHostContext.ExtHostTesting, accessor.get(IExtHostTesting));
 	const extHostUriOpeners = rpcProtocol.set(ExtHostContext.ExtHostUriOpeners, new ExtHostUriOpeners(rpcProtocol));
 	const extHostProfileContentHandlers = rpcProtocol.set(ExtHostContext.ExtHostProfileContentHandlers, new ExtHostProfileContentHandlers(rpcProtocol));
+	const extHostChatOutputRenderer = rpcProtocol.set(ExtHostContext.ExtHostChatOutputRenderer, new ExtHostChatOutputRenderer(rpcProtocol, extHostWebviews));
 	rpcProtocol.set(ExtHostContext.ExtHostInteractive, new ExtHostInteractive(rpcProtocol, extHostNotebook, extHostDocumentsAndEditors, extHostCommands, extHostLogService));
 	const extHostLanguageModelTools = rpcProtocol.set(ExtHostContext.ExtHostLanguageModelTools, new ExtHostLanguageModelTools(rpcProtocol, extHostLanguageModels));
 	const extHostChatAgents2 = rpcProtocol.set(ExtHostContext.ExtHostChatAgents2, new ExtHostChatAgents2(rpcProtocol, extHostLogService, extHostCommands, extHostDocuments, extHostLanguageModels, extHostDiagnostics, extHostLanguageModelTools));
@@ -1505,10 +1507,14 @@ export function createApiFactoryAndRegisterActors(accessor: ServicesAccessor): I
 				checkProposedApiEnabled(extension, 'chatParticipantPrivate');
 				return _asExtensionEvent(extHostChatAgents2.onDidDisposeChatSession)(listeners, thisArgs, disposables);
 			},
-			registerChatSessionItemProvider(chatSessionType: string, provider: vscode.ChatSessionItemProvider) {
+			registerChatSessionItemProvider: (chatSessionType: string, provider: vscode.ChatSessionItemProvider) => {
 				checkProposedApiEnabled(extension, 'chatSessionsProvider');
 				return extHostChatSessions.registerChatSessionItemProvider(chatSessionType, provider);
 			},
+			registerChatOutputRenderer: (mime: string, renderer: vscode.ChatOutputRenderer) => {
+				checkProposedApiEnabled(extension, 'chatOutputRenderer');
+				return extHostChatOutputRenderer.registerChatOutputRenderer(extension, mime, renderer);
+			},
 		};
 
 		// namespace: lm

src/vs/workbench/api/common/extHost.protocol.ts

@@ -1452,6 +1452,15 @@ export interface ExtHostUriOpenersShape {
 	$openUri(id: string, context: { resolvedUri: UriComponents; sourceUri: UriComponents }, token: CancellationToken): Promise<void>;
 }
 
+export interface MainThreadChatOutputRendererShape extends IDisposable {
+	$registerChatOutputRenderer(mime: string, extensionId: ExtensionIdentifier, extensionLocation: UriComponents): void;
+	$unregisterChatOutputRenderer(mime: string): void;
+}
+
+export interface ExtHostChatOutputRendererShape {
+	$renderChatOutput(mime: string, valueData: VSBuffer, webviewHandle: string, token: CancellationToken): Promise<void>;
+}
+
 export interface MainThreadProfileContentHandlersShape {
 	$registerProfileContentHandler(id: string, name: string, description: string | undefined, extensionId: string): Promise<void>;
 	$unregisterProfileContentHandler(id: string): Promise<void>;
@@ -3188,6 +3197,7 @@ export const MainContext = {
 	MainThreadAiSettingsSearch: createProxyIdentifier<MainThreadAiSettingsSearchShape>('MainThreadAiSettingsSearch'),
 	MainThreadDataChannels: createProxyIdentifier<MainThreadDataChannelsShape>('MainThreadDataChannels'),
 	MainThreadChatSessions: createProxyIdentifier<MainThreadChatSessionsShape>('MainThreadChatSessions'),
+	MainThreadChatOutputRenderer: createProxyIdentifier<MainThreadChatOutputRendererShape>('MainThreadChatOutputRenderer'),
 };
 
 export const ExtHostContext = {
@@ -3233,6 +3243,7 @@ export const ExtHostContext = {
 	ExtHostStorage: createProxyIdentifier<ExtHostStorageShape>('ExtHostStorage'),
 	ExtHostUrls: createProxyIdentifier<ExtHostUrlsShape>('ExtHostUrls'),
 	ExtHostUriOpeners: createProxyIdentifier<ExtHostUriOpenersShape>('ExtHostUriOpeners'),
+	ExtHostChatOutputRenderer: createProxyIdentifier<ExtHostChatOutputRendererShape>('ExtHostChatOutputRenderer'),
 	ExtHostProfileContentHandlers: createProxyIdentifier<ExtHostProfileContentHandlersShape>('ExtHostProfileContentHandlers'),
 	ExtHostOutputService: createProxyIdentifier<ExtHostOutputServiceShape>('ExtHostOutputService'),
 	ExtHostLabelService: createProxyIdentifier<ExtHostLabelServiceShape>('ExtHostLabelService'),

src/vs/workbench/api/common/extHostChatOutputRenderer.ts

@@ -0,0 +1,53 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import type * as vscode from 'vscode';
+import { CancellationToken } from '../../../base/common/cancellation.js';
+import { ExtHostChatOutputRendererShape, IMainContext, MainContext, MainThreadChatOutputRendererShape } from './extHost.protocol.js';
+import { Disposable } from './extHostTypes.js';
+import { ExtHostWebviews } from './extHostWebview.js';
+import { IExtensionDescription } from '../../../platform/extensions/common/extensions.js';
+import { VSBuffer } from '../../../base/common/buffer.js';
+
+export class ExtHostChatOutputRenderer implements ExtHostChatOutputRendererShape {
+
+	private readonly _proxy: MainThreadChatOutputRendererShape;
+
+	private readonly _renderers = new Map</*mime*/ string, {
+		readonly renderer: vscode.ChatOutputRenderer;
+		readonly extension: IExtensionDescription;
+	}>();
+
+	constructor(
+		mainContext: IMainContext,
+		private readonly webviews: ExtHostWebviews,
+	) {
+		this._proxy = mainContext.getProxy(MainContext.MainThreadChatOutputRenderer);
+	}
+
+	registerChatOutputRenderer(extension: IExtensionDescription, mime: string, renderer: vscode.ChatOutputRenderer): vscode.Disposable {
+		if (this._renderers.has(mime)) {
+			throw new Error(`Chat output renderer already registered for mime type: ${mime}`);
+		}
+
+		this._renderers.set(mime, { extension, renderer });
+		this._proxy.$registerChatOutputRenderer(mime, extension.identifier, extension.extensionLocation);
+
+		return new Disposable(() => {
+			this._renderers.delete(mime);
+			this._proxy.$unregisterChatOutputRenderer(mime);
+		});
+	}
+
+	async $renderChatOutput(mime: string, valueData: VSBuffer, webviewHandle: string, token: CancellationToken): Promise<void> {
+		const entry = this._renderers.get(mime);
+		if (!entry) {
+			throw new Error(`No chat output renderer registered for mime type: ${mime}`);
+		}
+
+		const webview = this.webviews.createNewWebview(webviewHandle, {}, entry.extension);
+		return entry.renderer.renderChatOutput(valueData.buffer, webview, {}, token);
+	}
+}

src/vs/workbench/api/common/extHostTypeConverters.ts

@@ -42,7 +42,7 @@ import { IChatAgentRequest, IChatAgentResult } from '../../contrib/chat/common/c
 import { IChatRequestDraft } from '../../contrib/chat/common/chatEditingService.js';
 import { IChatRequestVariableEntry, isImageVariableEntry } from '../../contrib/chat/common/chatVariableEntries.js';
 import { IChatAgentMarkdownContentWithVulnerability, IChatCodeCitation, IChatCommandButton, IChatConfirmation, IChatContentInlineReference, IChatContentReference, IChatExtensionsContent, IChatFollowup, IChatMarkdownContent, IChatMoveMessage, IChatPrepareToolInvocationPart, IChatProgressMessage, IChatResponseCodeblockUriPart, IChatTaskDto, IChatTaskResult, IChatTextEdit, IChatTreeData, IChatUserActionEvent, IChatWarningMessage } from '../../contrib/chat/common/chatService.js';
-import { IToolResult, ToolDataSource } from '../../contrib/chat/common/languageModelToolsService.js';
+import { IToolResult, IToolResultInputOutputDetails, IToolResultOutputDetails, ToolDataSource } from '../../contrib/chat/common/languageModelToolsService.js';
 import * as chatProvider from '../../contrib/chat/common/languageModels.js';
 import { IChatMessageDataPart, IChatResponseDataPart, IChatResponsePromptTsxPart, IChatResponseTextPart } from '../../contrib/chat/common/languageModels.js';
 import { DebugTreeItemCollapsibleState, IDebugVisualizationTreeItem } from '../../contrib/debug/common/debug.js';
@@ -3346,12 +3346,30 @@ export namespace LanguageModelToolResult2 {
 		}));
 	}
 
-	export function from(result: vscode.ExtendedLanguageModelToolResult, extension: IExtensionDescription): Dto<IToolResult> | SerializableObjectWithBuffers<Dto<IToolResult>> {
+	export function from(result: vscode.ExtendedLanguageModelToolResult2, extension: IExtensionDescription): Dto<IToolResult> | SerializableObjectWithBuffers<Dto<IToolResult>> {
 		if (result.toolResultMessage) {
 			checkProposedApiEnabled(extension, 'chatParticipantPrivate');
 		}
 
 		let hasBuffers = false;
+		let detailsDto: Dto<Array<URI | types.Location> | IToolResultInputOutputDetails | IToolResultOutputDetails | undefined> = undefined;
+		if (Array.isArray(result.toolResultDetails)) {
+			detailsDto = result.toolResultDetails?.map(detail => {
+				return URI.isUri(detail) ? detail : Location.from(detail as vscode.Location);
+			});
+		} else {
+			if (result.toolResultDetails2) {
+				detailsDto = {
+					output: {
+						type: 'data',
+						mimeType: (result.toolResultDetails2 as vscode.ToolResultDataOutput).mime,
+						value: VSBuffer.wrap((result.toolResultDetails2 as vscode.ToolResultDataOutput).value),
+					}
+				} satisfies IToolResultOutputDetails;
+				hasBuffers = true;
+			}
+		}
+
 		const dto: Dto<IToolResult> = {
 			content: result.content.map(item => {
 				if (item instanceof types.LanguageModelTextPart) {
@@ -3378,7 +3396,7 @@ export namespace LanguageModelToolResult2 {
 				}
 			}),
 			toolResultMessage: MarkdownString.fromStrict(result.toolResultMessage),
-			toolResultDetails: result.toolResultDetails?.map(detail => URI.isUri(detail) ? detail : Location.from(detail as vscode.Location)),
+			toolResultDetails: detailsDto,
 		};
 
 		return hasBuffers ? new SerializableObjectWithBuffers(dto) : dto;

src/vs/workbench/contrib/chat/browser/chat.contribution.ts

@@ -115,6 +115,7 @@ import { ChatAttachmentResolveService, IChatAttachmentResolveService } from './c
 import { registerLanguageModelActions } from './actions/chatLanguageModelActions.js';
 import { PromptUrlHandler } from './promptSyntax/promptUrlHandler.js';
 import { ChatTaskServiceImpl, IChatTasksService } from '../common/chatTasksService.js';
+import { ChatOutputRendererService, IChatOutputRendererService } from './chatOutputItemRenderer.js';
 
 // Register configuration
 const configurationRegistry = Registry.as<IConfigurationRegistry>(ConfigurationExtensions.Configuration);
@@ -803,6 +804,7 @@ registerSingleton(IChatContextPickService, ChatContextPickService, Instantiation
 registerSingleton(IChatModeService, ChatModeService, InstantiationType.Delayed);
 registerSingleton(IChatAttachmentResolveService, ChatAttachmentResolveService, InstantiationType.Delayed);
 registerSingleton(IChatTasksService, ChatTaskServiceImpl, InstantiationType.Delayed);
+registerSingleton(IChatOutputRendererService, ChatOutputRendererService, InstantiationType.Delayed);
 
 registerWorkbenchContribution2(ChatEditingNotebookFileSystemProviderContrib.ID, ChatEditingNotebookFileSystemProviderContrib, WorkbenchPhase.BlockStartup);