feat: Ask v1

Author: kgilpin

packages/cli/src/cli.ts

@@ -36,6 +36,7 @@ const CompareCommand = require('./cmds/compare/compare');
 const CompareReportCommand = require('./cmds/compare-report/compareReport');
 const InventoryCommand = require('./cmds/inventory/inventory');
 const InventoryReportCommand = require('./cmds/inventory-report/inventoryReport');
+const Ask = require('./cmds/ask/ask');
 import UploadCommand from './cmds/upload';
 import { default as sqlErrorLog } from './lib/sqlErrorLog';
 
@@ -192,6 +193,7 @@ yargs(process.argv.slice(2))
   .command(CompareReportCommand)
   .command(InventoryCommand)
   .command(InventoryReportCommand)
+  .command(Ask)
   .option('verbose', {
     alias: 'v',
     type: 'boolean',

packages/cli/src/cmds/ask/ask.ts

@@ -0,0 +1,307 @@
+import { warn } from 'console';
+import { handleWorkingDirectory } from '../../lib/handleWorkingDirectory';
+import { locateAppMapDir } from '../../lib/locateAppMapDir';
+import { exists, verbose } from '../../utils';
+import OpenAI from 'openai';
+import { ChatCompletionMessageParam } from 'openai/resources';
+import Context from '../../inspect/context';
+import { EventMatch } from '../../search/types';
+import { readFile, writeFile } from 'fs/promises';
+import { join } from 'path';
+import { AppMapFilter, Metadata, buildAppMap } from '@appland/models';
+import { loadSequenceDiagram } from '../compare/loadSequenceDiagram';
+import {
+  Action,
+  Diagram,
+  FormatType,
+  Specification,
+  buildDiagram,
+  format,
+  nodeName,
+} from '@appland/sequence-diagram';
+import { match } from 'assert';
+
+export const command = 'ask <question>';
+export const describe =
+  'Ask a plain text question and get a filtered and configured AppMap as a response';
+
+export const builder = (args) => {
+  args.positional('question', {
+    describe: 'plain text question about the code base',
+  });
+  args.option('directory', {
+    describe: 'program working directory',
+    type: 'string',
+    alias: 'd',
+  });
+  return args.strict();
+};
+
+function buildOpenAI(): OpenAI {
+  const OPENAI_API_KEY = process.env.OPENAI_API_KEY;
+  if (!OPENAI_API_KEY) {
+    throw new Error('OPENAI_API_KEY environment variable must be set');
+  }
+  return new OpenAI({ apiKey: OPENAI_API_KEY });
+}
+
+type SearchDiagramsParam = {
+  keywords: string[];
+};
+
+type DiagramEvent = {
+  name: string;
+  id: number;
+};
+
+type SearchDiagramResult = {
+  diagramId: string;
+};
+
+type DiagramDetailsParam = {
+  diagramIds: string[];
+};
+
+type DiagramDetailsResult = {
+  metadata: Metadata;
+  summary: string;
+};
+
+class Ask {
+  constructor(public appmapDir: string) {}
+
+  async searchDiagrams(keywords: string[]): Promise<SearchDiagramResult[] | undefined> {
+    const regexpOfGlob = (glob: string) =>
+      glob.replace(/\./g, '\\.').replace(/\*/g, '.*').replace(/\?/g, '.');
+
+    // TODO: Use proper keyword search
+    const searchTerms = keywords.map((keyword) => keyword.split(' ')).flat();
+    const matchesForKeyword = new Array<Set<string>>();
+    for (const keyword of searchTerms) {
+      matchesForKeyword.push(new Set<string>());
+      const queryStr = `function:%r{.*${regexpOfGlob(keyword)}.*}`;
+      const context = new Context(this.appmapDir, queryStr);
+      await context.findCodeObjects();
+      if (!context.codeObjectMatches) continue;
+
+      const stats = await context.buildStats();
+      /*
+      const eventName = (eventMatch: EventMatch): string => {
+        if (eventMatch.event.sqlQuery) return eventMatch.event.sqlQuery;
+        else if (eventMatch.event.httpServerRequest)
+          return eventMatch.event.httpServerRequest.normalized_path;
+
+        return [eventMatch.event.definedClass, eventMatch.event.methodId].join('#');
+      };
+      */
+
+      stats.eventMatches.forEach((eventMatch) => {
+        const diagramId = eventMatch.appmap;
+        matchesForKeyword[matchesForKeyword.length - 1].add(diagramId);
+        /*
+        const event: DiagramEvent = {
+          name: eventName(eventMatch),
+          id: eventMatch.event.id,
+        };
+        const eventMatches = eventMatchesByDiagramId.get(diagramId);
+        if (eventMatches) {
+          eventMatches.push(event);
+        } else {
+          eventMatchesByDiagramId.set(diagramId, [event]);
+        }
+        */
+      });
+    }
+
+    // Collect all diagrams that match all keywords
+    const matchingDiagrams = new Set<string>();
+    for (const diagramId of matchesForKeyword[0]) {
+      let isMatch = true;
+      for (const matches of matchesForKeyword) {
+        if (!matches.has(diagramId)) {
+          isMatch = false;
+          break;
+        }
+      }
+      if (isMatch) matchingDiagrams.add(diagramId);
+    }
+
+    return Array.from(matchingDiagrams).map((diagramId) => ({ diagramId }));
+    // return Array.from(eventMatchesByDiagramId.entries()).map(([diagramId, matchingEvents]) => ({
+    //   diagramId,
+    //   matchingEvents,
+    // }));
+  }
+}
+
+export const handler = async (argv: any) => {
+  verbose(argv.verbose);
+  handleWorkingDirectory(argv.directory);
+  const appmapDir = await locateAppMapDir(argv.appmapDir);
+
+  const ask = new Ask(appmapDir);
+
+  function showPlan(paramStr: string) {
+    const params = JSON.parse(paramStr) as { plan: string };
+    warn(`Plan:\n${params.plan}`);
+  }
+
+  async function searchDiagrams(paramStr: string): Promise<SearchDiagramResult[]> {
+    const params = JSON.parse(paramStr) as SearchDiagramsParam;
+    let { keywords } = params;
+    warn(`Searching for diagrams matching "${keywords}"`);
+    const eventMatches = await ask.searchDiagrams(keywords);
+    if (!eventMatches) return [];
+
+    return eventMatches;
+  }
+
+  async function getDiagramDetails(paramStr: string): Promise<DiagramDetailsResult[]> {
+    const params = JSON.parse(paramStr) as DiagramDetailsParam;
+    const { diagramIds } = params;
+    warn(`Getting details for diagram ${diagramIds}`);
+    const result = new Array<DiagramDetailsResult>();
+    for (const diagramId of diagramIds) {
+      const metadata = JSON.parse(
+        await readFile(join(diagramId, 'metadata.json'), 'utf-8')
+      ) as Metadata;
+      delete metadata['git'];
+
+      warn(`Building sequence diagram for AppMap ${diagramId}`);
+      const appmapFile = [diagramId, 'appmap.json'].join('.');
+      const prunedAppMap = buildAppMap()
+        .source(await readFile(appmapFile, 'utf-8'))
+        .prune(10 * 1000 * 1000)
+        .build();
+      const filter = new AppMapFilter();
+      if (metadata.language?.name !== 'java') filter.declutter.hideExternalPaths.on = true;
+
+      filter.declutter.limitRootEvents.on = true;
+      const filteredAppMap = filter.filter(prunedAppMap, []);
+      const specification = Specification.build(filteredAppMap, { loops: true });
+      const diagram = buildDiagram(appmapFile, filteredAppMap, specification);
+
+      const diagramText: string[] = [];
+      const maxDepth = 2;
+      const collectAction = (action: Action, depth = 0) => {
+        if (depth > maxDepth) return;
+
+        const indent = '  '.repeat(depth);
+        diagramText.push(`${indent}${nodeName(action)}(events:${action.eventIds.join(', ')}})`);
+        if (action.children) {
+          action.children.forEach((child) => collectAction(child, depth + 1));
+        }
+      };
+      diagram.rootActions.forEach((action) => collectAction(action));
+      result.push({ metadata, summary: diagramText.join('\n') });
+    }
+    return result;
+  }
+
+  const question = argv.question;
+
+  const systemMessages: ChatCompletionMessageParam[] = [
+    'You are an assistant that answers questions about the design and architecture of code.',
+    'You answer these questions by accessing a knowledge base of sequence diagrams.',
+    'Each sequence diagram conists of a series of events, such as function calls, HTTP server requests, SQL queries, etc.',
+    'You are given a question about the code, and you have to (a) Locate the sequence diagram that best answers the question (b) Focus the sequence diagram on the sub-section of the diagram that best answers the question.',
+    'You use the function calling interface to choose the best diagram and configure the diagram view to reveal the answer.',
+    'Always try multiple keywords, and collate the results together',
+    'If a diagram keyword search returns no diagrams, try other keywords.',
+    'The first function you should call is the "showPlan" function, to which you will provide a markdown document describing your general strategy for answering the question.',
+    'In your response, include links to the diagram files, with anchor tags to events that are particular interest',
+    'In your response, include links to source code files, with anchor tags to the lines of code that are of particular interest',
+  ].map((msg) => ({
+    content: msg,
+    role: 'system',
+  }));
+
+  const userMessage: ChatCompletionMessageParam = {
+    content: question,
+    role: 'user',
+  };
+
+  const messages = [...systemMessages, userMessage];
+
+  const openai = buildOpenAI();
+  const runFunctions = openai.beta.chat.completions.runFunctions({
+    model: 'gpt-4',
+    messages,
+    function_call: 'auto',
+    functions: [
+      {
+        function: showPlan,
+        description: 'Print the plan for answering the question',
+        parameters: {
+          type: 'object',
+          properties: {
+            plan: {
+              type: 'string',
+              description: 'The plan in Markdown format',
+            },
+          },
+        },
+      },
+      {
+        function: searchDiagrams,
+        description: `List sequence diagrams that match a keyword. The response is an array of search matches. Each match includes a diagram id, plus information about the events (function calls, HTTP server requests, SQL queries, etc) within that diagram that match the search term.`,
+        parameters: {
+          type: 'object',
+          properties: {
+            keywords: {
+              type: 'array',
+              description: 'Array of keywords to find in the diagrams',
+              items: {
+                type: 'string',
+              },
+            },
+          },
+        },
+      },
+      {
+        function: getDiagramDetails,
+        description: `Get details about diagrams, including their name, code language, frameworks, source location, exceptions raised.`,
+        parameters: {
+          type: 'object',
+          properties: {
+            diagramIds: {
+              type: 'array',
+              description: 'Array of diagram ids',
+              items: {
+                type: 'string',
+              },
+            },
+          },
+        },
+      },
+    ],
+  });
+
+  runFunctions.on('functionCall', (data) => {
+    warn(JSON.stringify(data));
+  });
+  runFunctions.on('finalFunctionCall', (data) => {
+    warn(JSON.stringify(data));
+  });
+  runFunctions.on('functionCallResult', (data) => {
+    warn(JSON.stringify(data));
+  });
+  runFunctions.on('finalFunctionCallResult', (data) => {
+    warn(JSON.stringify(data));
+  });
+
+  const response = await runFunctions.finalContent();
+  if (!response) {
+    warn(`No response from OpenAI`);
+    return;
+  }
+  console.log(response);
+
+  // const context = new Context(appmapDir, codeObjectId);
+  // await context.findCodeObjects();
+  // if (context.codeObjectMatches?.length === 0) {
+  //   return yargs.exit(1, new Error(`Code object '${context.codeObjectId}' not found`));
+  // }
+  // await context.buildStats();
+  // console.log(JSON.stringify(context.stats, null, 2));
+};

packages/cli/src/functionStats.js

@@ -41,6 +41,7 @@ class FunctionStats {
     const trigram = (/** @type {Trigram} */ t) =>
       [t.callerId, t.codeObjectId, t.calleeId].join(' ->\n');
     return {
+      appmaps: this.appMapNames,
       returnValues: this.returnValues,
       httpServerRequests: this.httpServerRequests,
       sqlQueries: this.sqlQueries,
@@ -58,6 +59,10 @@ class FunctionStats {
     return [...new Set(this.eventMatches.map((e) => e.appmap))].sort();
   }
 
+  get appmaps() {
+    return this.appMapNames;
+  }
+
   get returnValues() {
     return [...new Set(this.eventMatches.map((e) => e.event.returnValue).map(formatValue))].sort();
   }

packages/cli/src/inspect/context.ts

@@ -3,7 +3,7 @@ import EventEmitter from 'events';
 import { default as FunctionStatsImpl } from '../functionStats';
 import FindCodeObjects from '../search/findCodeObjects';
 import FindEvents from '../search/findEvents';
-import { FunctionStats, Filter, CodeObjectMatch } from '../search/types';
+import { FunctionStats, Filter, CodeObjectMatch, EventMatch } from '../search/types';
 
 export default class Context extends EventEmitter {
   public filters: Filter[] = [];
@@ -38,12 +38,12 @@ export default class Context extends EventEmitter {
     this.emit('stop');
   }
 
-  async buildStats() {
+  async buildStats(): Promise<FunctionStats> {
     assert(this.codeObjectMatches, `codeObjectMatches is not yet computed`);
 
     this.emit('start', this.codeObjectMatches.length);
 
-    const result: any[] = [];
+    const result: EventMatch[] = [];
     await Promise.all(
       this.codeObjectMatches.map(async (codeObjectMatch) => {
         const findEvents = new FindEvents(codeObjectMatch.appmap, codeObjectMatch.codeObject);
@@ -57,6 +57,7 @@ export default class Context extends EventEmitter {
 
     this.emit('collate');
     this.stats = new FunctionStatsImpl(result);
+    return this.stats;
   }
 
   save() {

packages/cli/src/search/types.d.ts

@@ -95,6 +95,7 @@ export interface Filter {
 }
 
 export interface FunctionStats {
+  appmaps: string[];
   eventMatches: EventMatch[];
   returnValues: string[];
   httpServerRequests: string[];