feat: include tools and resources in MCP server info

Author: amishne

report-app/src/app/pages/report-viewer/report-viewer.html

@@ -218,7 +218,7 @@ <h4>Servers</h4>
         <ul>
           @for (server of details.mcp.servers; track server) {
             <li>
-              <span>
+              <div>
                 <strong>{{ server.name }}</strong
                 >:
                 <code
@@ -227,7 +227,27 @@ <h4>Servers</h4>
                     {{ arg }}
                   }
                 </code>
-              </span>
+              </div>
+              @if (server.tools && server.tools.length > 0) {
+                <div>
+                  Tools
+                  <ul>
+                    @for (tool of server.tools; track tool) {
+                      <li>{{ tool }}</li>
+                    }
+                  </ul>
+                </div>
+              }
+              @if (server.resources && server.resources.length > 0) {
+                <div>
+                  Resources
+                  <ul>
+                    @for (tool of server.resources; track tool) {
+                      <li>{{ tool }}</li>
+                    }
+                  </ul>
+                </div>
+              }
             </li>
           }
         </ul>

runner/codegen/genkit/genkit-runner.ts

@@ -1,4 +1,11 @@
-import {DynamicResourceAction, GenerateResponse, genkit, ModelReference, ToolAction} from 'genkit';
+import {
+  Action,
+  DynamicResourceAction,
+  GenerateResponse,
+  genkit,
+  ModelReference,
+  ToolAction,
+} from 'genkit';
 import {GenkitMcpHost, McpServerConfig, createMcpHost} from '@genkit-ai/mcp';
 import {GenkitPlugin, GenkitPluginV2} from 'genkit/plugin';
 import {z} from 'zod';
@@ -11,6 +18,7 @@ import {
   LocalLlmGenerateTextResponse,
   LocalLlmGenerateTextRequestOptions,
   LocalLlmGenerateFilesRequestOptions,
+  McpServerDetails,
 } from '../llm-runner.js';
 import {setTimeout} from 'node:timers/promises';
 import {callWithTimeout} from '../../utils/timeout.js';
@@ -21,10 +29,18 @@ import {UserFacingError} from '../../utils/errors.js';
 import {GenkitModelProvider, PromptDataForCounting} from './model-provider.js';
 import {ToolLogEntry} from '../../shared-interfaces.js';
 import {combineAbortSignals} from '../../utils/abort-signal.js';
+import {toToolDefinition} from 'genkit/tool';
 
 const globalLogger = new GenkitLogger();
 logger.init(globalLogger);
 
+/**
+ * Gets the name of a Genkit action.
+ */
+function getActionName(action: Action<any, any>): string {
+  return toToolDefinition(action).name;
+}
+
 /** Runner that uses the Genkit API under the hood. */
 export class GenkitRunner implements LlmRunner {
   readonly id = 'genkit';
@@ -199,7 +215,10 @@ export class GenkitRunner implements LlmRunner {
     }
   }
 
-  startMcpServerHost(hostName: string, servers: McpServerOptions[]): void {
+  async startMcpServerHost(
+    hostName: string,
+    servers: McpServerOptions[],
+  ): Promise<McpServerDetails> {
     if (this.mcpHost !== null) {
       throw new Error('MCP host is already started');
     }
@@ -216,6 +235,12 @@ export class GenkitRunner implements LlmRunner {
 
     globalLogger.startCapturingLogs();
     this.mcpHost = createMcpHost({name: hostName, mcpServers});
+    const tools = await this.mcpHost.getActiveTools(this.genkitInstance);
+    const resources = await this.mcpHost.getActiveResources(this.genkitInstance);
+    return {
+      tools: tools.map(getActionName),
+      resources: resources.map(getActionName),
+    };
   }
 
   flushMcpServerLogs(): string[] {

runner/codegen/llm-runner.ts

@@ -58,8 +58,9 @@ export interface LlmRunner {
    * Optional since not all runners may support MCP.
    * @param hostName Name for the MCP host.
    * @param servers Configured servers that should be started.
+   * @returns Details about the created server.
    */
-  startMcpServerHost?(hostName: string, servers: McpServerOptions[]): void;
+  startMcpServerHost?(hostName: string, servers: McpServerOptions[]): Promise<McpServerDetails>;
 
   /** Stops tracking MCP server logs and returns the current ones. */
   flushMcpServerLogs?(): string[];
@@ -171,6 +172,12 @@ export const mcpServerOptionsSchema = z.object({
 /** Options used to start an MCP server. */
 export type McpServerOptions = z.infer<typeof mcpServerOptionsSchema>;
 
+/** Details about an MCP server. */
+export interface McpServerDetails {
+  tools: string[];
+  resources: string[];
+}
+
 /**
  * Type for a prompt message may be passed to LLM runner in the eval tool.
  *

runner/orchestration/executors/local-executor.ts

@@ -1,7 +1,7 @@
 import {ChildProcess, fork} from 'node:child_process';
 import path, {join} from 'node:path';
 import PQueue from 'p-queue';
-import {LlmRunner} from '../../codegen/llm-runner.js';
+import {LlmRunner, McpServerDetails} from '../../codegen/llm-runner.js';
 import {getRunnerByName, RunnerName} from '../../codegen/runner-creation.js';
 import {ProgressLogger} from '../../progress/progress-logger.js';
 import {
@@ -225,16 +225,16 @@ export class LocalExecutor implements Executor {
     };
   }
 
-  async startMcpServerHost(hostName: string) {
+  async startMcpServerHost(hostName: string): Promise<McpServerDetails | undefined> {
     const llm = await this.llm;
     if (llm.startMcpServerHost === undefined) {
-      return;
+      return undefined;
     }
 
-    llm.startMcpServerHost(hostName, this.config.mcpServers ?? []);
+    return llm.startMcpServerHost(hostName, this.config.mcpServers ?? []);
   }
 
-  async collectMcpServerLogs() {
+  async collectMcpServerLogs(mcpServerDetails: McpServerDetails | undefined) {
     const llm = await this.llm;
     if (llm.flushMcpServerLogs === undefined) {
       return;
@@ -245,6 +245,8 @@ export class LocalExecutor implements Executor {
         name: m.name,
         command: m.command,
         args: m.args,
+        tools: mcpServerDetails?.tools ?? [],
+        resources: mcpServerDetails?.resources ?? [],
       })),
       logs: llm.flushMcpServerLogs().join('\n'),
     };

runner/orchestration/generate.ts

@@ -85,9 +85,10 @@ export async function generateCodeAndAssess(options: AssessmentConfig): Promise<
     // We need Chrome to collect runtime information.
     await installChrome();
 
-    if (options.startMcp && env.executor instanceof LocalExecutor) {
-      env.executor.startMcpServerHost(`mcp-${env.clientSideFramework.id}`);
-    }
+    const mcpServerDetails =
+      env.executor instanceof LocalExecutor && options.startMcp && env.executor.startMcpServerHost
+        ? await env.executor.startMcpServerHost(`mcp-${env.clientSideFramework.id}`)
+        : undefined;
 
     progress.initialize(promptsToProcess.length);
 
@@ -163,7 +164,7 @@ export async function generateCodeAndAssess(options: AssessmentConfig): Promise<
 
     const mcp =
       env.executor instanceof LocalExecutor && options.startMcp
-        ? await env.executor.collectMcpServerLogs()
+        ? await env.executor.collectMcpServerLogs(mcpServerDetails)
         : undefined;
 
     const timestamp = new Date();

runner/shared-interfaces.ts

@@ -437,7 +437,15 @@ export interface RunDetails {
   /** Information about configured MCP servers, if any. */
   mcp?: {
     /** MCP servers that were configured. */
-    servers: {name: string; command: string; args: string[]}[];
+    servers: {
+      name: string;
+      command: string;
+      args: string[];
+      /** Tools reported for this server. */
+      tools?: string[];
+      /** Resources reported for this server. */
+      resources?: string[];
+    }[];
 
     /** Logs produced by all of the servers. */
     logs: string;