Skip to content

Commit f3a22ec

Browse files
kgilpinkgilpin
committed
feat (wip): Search for snippets
1 parent 2ce79e9 commit f3a22ec

File tree

17 files changed

+302
-186
lines changed

17 files changed

+302
-186
lines changed

‎packages/cli/src/cmds/search/searchSingleAppMap.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ export default async function searchSingleAppMap(
1212
query: string,
1313
options: SearchOptions = {}
1414
): Promise<SearchResponse> {
15+
// eslint-disable-next-line no-param-reassign
1516
if (appmap.endsWith('.appmap.json')) appmap = appmap.slice(0, -'.appmap.json'.length);
1617

1718
const findEvents = new FindEvents(appmap);

‎packages/cli/src/fulltext/appmap-index.ts‎

Lines changed: 27 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ import {
1616
} from './appmap-match';
1717
import loadAppMapConfig from '../lib/loadAppMapConfig';
1818

19-
type ClassMapEntry = {
19+
export type ClassMapEntry = {
2020
name: string;
2121
type: string;
2222
labels: string[];
@@ -48,43 +48,46 @@ export async function listAppMaps(directory: string): Promise<string[]> {
4848
return appmapFiles.map(relativeToPath);
4949
}
5050

51+
export async function readIndexFile<T>(
52+
appmapName: string,
53+
indexName: string
54+
): Promise<T | undefined> {
55+
const indexFile = join(appmapName, [indexName, '.json'].join(''));
56+
let indexStr: string;
57+
try {
58+
indexStr = await readFile(indexFile, 'utf-8');
59+
} catch (e) {
60+
if (isNativeError(e) && !isNodeError(e, 'ENOENT')) {
61+
warn(`Error reading metadata file ${indexFile}: ${e.message}`);
62+
}
63+
return undefined;
64+
}
65+
66+
try {
67+
return JSON.parse(indexStr) as T;
68+
} catch (e) {
69+
const errorMessage = isNativeError(e) ? e.message : String(e);
70+
warn(`Error parsing metadata file ${indexFile}: ${errorMessage}`);
71+
}
72+
}
73+
5174
/**
5275
* Read all content for an AppMap. For efficiency, utilizes the AppMap index files, rather
5376
* than reading the entire AppMap file directly.
5477
*/
5578
export async function readAppMapContent(appmapFile: string): Promise<string> {
5679
const appmapName = appmapFile.replace(/\.appmap\.json$/, '');
5780

58-
async function readIndexFile<T>(name: string): Promise<T | undefined> {
59-
const indexFile = join(appmapName, [name, '.json'].join(''));
60-
let indexStr: string;
61-
try {
62-
indexStr = await readFile(indexFile, 'utf-8');
63-
} catch (e) {
64-
if (isNativeError(e) && !isNodeError(e, 'ENOENT')) {
65-
warn(`Error reading metadata file ${indexFile}: ${e.message}`);
66-
}
67-
return undefined;
68-
}
69-
70-
try {
71-
return JSON.parse(indexStr) as T;
72-
} catch (e) {
73-
const errorMessage = isNativeError(e) ? e.message : String(e);
74-
warn(`Error parsing metadata file ${indexFile}: ${errorMessage}`);
75-
}
76-
}
77-
7881
const appmapWords = new Array<string>();
7982

80-
const metadata = await readIndexFile<Metadata>('metadata');
83+
const metadata = await readIndexFile<Metadata>(appmapName, 'metadata');
8184
if (metadata) {
8285
appmapWords.push(metadata.name);
8386
if (metadata.labels) appmapWords.push(...metadata.labels);
8487
if (metadata.exception) appmapWords.push(metadata.exception.message);
8588
}
8689

87-
const classMap = (await readIndexFile<ClassMapEntry[]>('classMap')) ?? [];
90+
const classMap = (await readIndexFile<ClassMapEntry[]>(appmapName, 'classMap')) ?? [];
8891

8992
const queries = new Array<string>();
9093
const codeObjects = new Array<string>();
@@ -119,7 +122,7 @@ export async function readAppMapContent(appmapFile: string): Promise<string> {
119122
classMap.forEach((co) => collectClassMapEntry(co));
120123
appmapWords.push(...queries, ...codeObjects, ...routes, ...externalRoutes);
121124

122-
const parameters = (await readIndexFile<string[]>('canonical.parameters')) ?? [];
125+
const parameters = (await readIndexFile<string[]>(appmapName, 'canonical.parameters')) ?? [];
123126
appmapWords.push(...parameters);
124127
appmapWords.push(...types);
125128

‎packages/cli/src/rpc/explain/EventCollector.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ export default class EventCollector {
5959
return { results, context, contextSize };
6060
}
6161

62-
async appmapIndex(appmap: string): Promise<FindEvents> {
62+
protected async appmapIndex(appmap: string): Promise<FindEvents> {
6363
let index = this.appmapIndexes.get(appmap);
6464
if (!index) {
6565
index = new FindEvents(appmap);
@@ -69,7 +69,7 @@ export default class EventCollector {
6969
return index;
7070
}
7171

72-
async findEvents(appmap: string, options: SearchOptions): Promise<EventSearchResponse> {
72+
protected async findEvents(appmap: string, options: SearchOptions): Promise<EventSearchResponse> {
7373
if (appmap.endsWith('.appmap.json')) appmap = appmap.slice(0, -'.appmap.json'.length);
7474

7575
const index = await this.appmapIndex(appmap);

‎packages/cli/src/rpc/explain/SearchContextCollector.ts‎

Lines changed: 76 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,23 @@
1-
import { log } from 'console';
1+
import { log, warn } from 'console';
22
import sqlite3 from 'better-sqlite3';
33

44
import { ContextV2, applyContext } from '@appland/navie';
55
import { SearchRpc } from '@appland/rpc';
6-
import { FileIndex, FileSearchResult } from '@appland/search';
6+
import { FileIndex, FileSearchResult, SnippetSearchResult } from '@appland/search';
77

88
import { SearchResponse as AppMapSearchResponse } from '../../fulltext/appmap-match';
99
import { DEFAULT_MAX_DIAGRAMS } from '../search/search';
10-
import EventCollector from './EventCollector';
1110
import indexFiles from './index-files';
1211
import indexSnippets from './index-snippets';
13-
import collectSnippets from './collect-snippets';
1412
import buildIndex from './buildIndex';
1513
import { buildAppMapIndex, search } from '../../fulltext/appmap-index';
14+
import indexEvents from './index-events';
15+
16+
type ContextCandidate = {
17+
results: SearchRpc.SearchResult[];
18+
context: ContextV2.ContextResponse;
19+
contextSize: number;
20+
};
1621

1722
export default class SearchContextCollector {
1823
public excludePatterns: RegExp[] | undefined;
@@ -98,41 +103,80 @@ export default class SearchContextCollector {
98103

99104
const snippetIndex = await buildIndex('snippets', async (indexFile) => {
100105
const db = new sqlite3(indexFile);
101-
return await indexSnippets(db, fileSearchResults);
106+
const snippetIndex = await indexSnippets(db, fileSearchResults);
107+
await indexEvents(snippetIndex, appmapSearchResponse.results);
108+
return snippetIndex;
102109
});
103110

104-
let contextCandidate: {
105-
results: SearchRpc.SearchResult[];
106-
context: ContextV2.ContextResponse;
107-
contextSize: number;
108-
};
111+
let contextCandidate: ContextCandidate;
109112
try {
110-
const eventsCollector = new EventCollector(this.vectorTerms.join(' '), appmapSearchResponse);
111-
112113
let charCount = 0;
113-
let maxEventsPerDiagram = 5;
114+
let maxSnippets = 50;
114115
log(`[search-context] Requested char limit: ${this.charLimit}`);
115116
for (;;) {
116-
log(`[search-context] Collecting context with ${maxEventsPerDiagram} events per diagram.`);
117-
118-
contextCandidate = await eventsCollector.collectEvents(
119-
maxEventsPerDiagram,
120-
this.excludePatterns,
121-
this.includePatterns,
122-
this.includeTypes
117+
log(`[search-context] Collecting context with ${maxSnippets} events per diagram.`);
118+
119+
// Collect all code objects from AppMaps and use them to build the sequence diagram
120+
// const codeSnippets = new Array<SnippetSearchResult>();
121+
// TODO: Apply this.includeTypes
122+
123+
const snippetContextItem = (
124+
snippet: SnippetSearchResult
125+
): ContextV2.ContextItem | ContextV2.FileContextItem | undefined => {
126+
const { snippetId, directory, score, content } = snippet;
127+
128+
const { type: snippetIdType, id: snippetIdValue } = snippetId;
129+
130+
let location: string | undefined;
131+
if (snippetIdType === 'code-snippet') location = snippetIdValue;
132+
133+
switch (snippetId.type) {
134+
case 'query':
135+
case 'route':
136+
case 'external-route':
137+
return {
138+
type: ContextV2.ContextItemType.DataRequest,
139+
content,
140+
directory,
141+
score,
142+
};
143+
case 'code-snippet':
144+
return {
145+
type: ContextV2.ContextItemType.CodeSnippet,
146+
content,
147+
directory,
148+
score,
149+
location,
150+
};
151+
default:
152+
warn(`[search-context] Unknown snippet type: ${snippetId.type}`);
153+
154+
// TODO: Collect all matching events, then build a sequence diagram
155+
// case 'event':
156+
// return await buildSequenceDiagram(snippet);
157+
// default:
158+
// codeSnippets.push(snippet);
159+
}
160+
};
161+
162+
const snippetSearchResults = snippetIndex.index.searchSnippets(
163+
this.vectorTerms.join(' OR '),
164+
maxSnippets
123165
);
166+
const context: ContextV2.ContextItem[] = [];
167+
for (const result of snippetSearchResults) {
168+
const contextItem = snippetContextItem(result);
169+
if (contextItem) context.push(contextItem);
170+
}
124171

125-
const codeSnippetCount = contextCandidate.context.filter(
126-
(item) => item.type === ContextV2.ContextItemType.CodeSnippet
127-
).length;
172+
// TODO: Build sequence diagrams
128173

129-
const charLimit = codeSnippetCount === 0 ? this.charLimit : this.charLimit / 4;
130-
const sourceContext = collectSnippets(
131-
snippetIndex.index,
132-
this.vectorTerms.join(' OR '),
133-
charLimit
134-
);
135-
contextCandidate.context = contextCandidate.context.concat(sourceContext);
174+
contextCandidate = {
175+
// TODO: Fixme remove hard coded cast
176+
results: appmapSearchResponse.results as SearchRpc.SearchResult[],
177+
context,
178+
contextSize: snippetSearchResults.reduce((acc, result) => acc + result.content.length, 0),
179+
};
136180

137181
const appliedContext = applyContext(contextCandidate.context, this.charLimit);
138182
const appliedContextSize = appliedContext.reduce(
@@ -147,8 +191,8 @@ export default class SearchContextCollector {
147191
break;
148192
}
149193
charCount = appliedContextSize;
150-
maxEventsPerDiagram = Math.ceil(maxEventsPerDiagram * 1.5);
151-
log(`[search-context] Increasing max events per diagram to ${maxEventsPerDiagram}.`);
194+
maxSnippets = Math.ceil(maxSnippets * 1.5);
195+
log(`[search-context] Increasing max events per diagram to ${maxSnippets}.`);
152196
}
153197
} finally {
154198
snippetIndex.close();

‎packages/cli/src/rpc/explain/appmap-location.ts‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
import { SearchRpc } from "@appland/rpc";
2+
3+
export default function appmapLocation(appmap: string, event?: SearchRpc.EventMatch): string {
4+
const appmapFile = [appmap, 'appmap.json'].join('.');
5+
const tokens = [appmapFile];
6+
if (event?.eventIds.length) tokens.push(String(event.eventIds[0]));
7+
return tokens.join(':');
8+
}

‎packages/cli/src/rpc/explain/build-sequence-diagram.ts‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
import { AppMapFilter, serializeFilter } from '@appland/models';
2+
import { SearchRpc } from '@appland/rpc';
3+
import assert from 'assert';
4+
5+
import { handler as sequenceDiagramHandler } from '../appmap/sequenceDiagram';
6+
import { ContextV2 } from '@appland/navie';
7+
import appmapLocation from './appmap-location';
8+
9+
export default async function buildSequenceDiagram(
10+
result: SearchRpc.SearchResult
11+
): Promise<ContextV2.FileContextItem> {
12+
const codeObjects = result.events.map((event) => event.fqid);
13+
const appmapFilter = new AppMapFilter();
14+
appmapFilter.declutter.context.on = true;
15+
appmapFilter.declutter.context.names = codeObjects;
16+
const filterState = serializeFilter(appmapFilter);
17+
18+
const plantUML = await sequenceDiagramHandler(result.appmap, {
19+
filter: filterState,
20+
format: 'plantuml',
21+
formatOptions: { disableMarkup: true },
22+
});
23+
assert(typeof plantUML === 'string');
24+
return {
25+
directory: result.directory,
26+
location: appmapLocation(result.appmap),
27+
type: ContextV2.ContextItemType.SequenceDiagram,
28+
content: plantUML,
29+
score: result.score,
30+
};
31+
}

‎packages/cli/src/rpc/explain/buildContext.ts‎

Lines changed: 7 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,11 @@
11
import { SearchRpc } from '@appland/rpc';
2-
import { AppMapFilter, serializeFilter } from '@appland/models';
3-
import assert from 'assert';
4-
5-
import { handler as sequenceDiagramHandler } from '../appmap/sequenceDiagram';
6-
import lookupSourceCode from './lookupSourceCode';
72
import { warn } from 'console';
83
import { ContextV2 } from '@appland/navie';
94

5+
import lookupSourceCode from './lookupSourceCode';
6+
import buildSequenceDiagram from './build-sequence-diagram';
7+
import appmapLocation from './appmap-location';
8+
109
/**
1110
* Processes search results to build sequence diagrams, code snippets, and code object sets. This is the format
1211
* expected by the Navie AI.
@@ -33,39 +32,10 @@ export default async function buildContext(
3332
const codeSnippetLocations = new Set<string>();
3433
const dataRequestContent = new Set<string>();
3534

36-
const appmapLocation = (appmap: string, event?: SearchRpc.EventMatch) => {
37-
const appmapFile = [appmap, 'appmap.json'].join('.');
38-
const tokens = [appmapFile];
39-
if (event?.eventIds.length) tokens.push(String(event.eventIds[0]));
40-
return tokens.join(':');
41-
};
42-
43-
const buildSequenceDiagram = async (result: SearchRpc.SearchResult) => {
44-
const codeObjects = result.events.map((event) => event.fqid);
45-
const appmapFilter = new AppMapFilter();
46-
appmapFilter.declutter.context.on = true;
47-
appmapFilter.declutter.context.names = codeObjects;
48-
const filterState = serializeFilter(appmapFilter);
49-
50-
const plantUML = await sequenceDiagramHandler(result.appmap, {
51-
filter: filterState,
52-
format: 'plantuml',
53-
formatOptions: { disableMarkup: true },
54-
});
55-
assert(typeof plantUML === 'string');
56-
sequenceDiagrams.push({
57-
directory: result.directory,
58-
location: appmapLocation(result.appmap),
59-
type: ContextV2.ContextItemType.SequenceDiagram,
60-
content: plantUML,
61-
score: result.score,
62-
});
63-
};
64-
6535
const examinedLocations = new Set<string>();
6636
for (const result of searchResults) {
6737
try {
68-
await buildSequenceDiagram(result);
38+
sequenceDiagrams.push(await buildSequenceDiagram(result));
6939
} catch (e) {
7040
warn(`Failed to build sequence diagram for ${result.appmap}`);
7141
warn(e);
@@ -93,6 +63,8 @@ export default async function buildContext(
9363

9464
codeSnippetLocations.add(event.location);
9565

66+
// TODO: Snippets from appmap events will no longer be needed, because the snippets come
67+
// from the search results in the index (boosted by AppMap references).
9668
const snippets = await lookupSourceCode(result.directory, event.location);
9769
if (snippets) {
9870
codeSnippets.push({

‎packages/cli/src/rpc/explain/collect-snippets.ts‎

Lines changed: 0 additions & 22 deletions
This file was deleted.

‎packages/cli/src/rpc/explain/collectContext.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -109,7 +109,7 @@ export default async function collectContext(
109109
}> {
110110
const keywords = searchTerms.map((term) => queryKeywords(term)).flat();
111111

112-
// recent?: boolean;
112+
// recent?: boolean;
113113
// locations?: string[];
114114
// itemTypes?: ContextItemType[];
115115
// labels?: ContextLabel[];

0 commit comments

Comments
 (0)