refactor: Relocate code from fulltext to explain/index

Author: kgilpin

packages/cli/src/cmds/search/search.ts

@@ -9,13 +9,13 @@ import { FileIndex } from '@appland/search';
 import { handleWorkingDirectory } from '../../lib/handleWorkingDirectory';
 import { verbose } from '../../utils';
 import searchSingleAppMap, { SearchOptions as SingleSearchOptions } from './searchSingleAppMap';
-import { SearchResponse as DiagramsSearchResponse } from '../../fulltext/appmap-match';
+import { SearchResponse as DiagramsSearchResponse } from '../../rpc/explain/index/appmap-match';
 import {
   SearchResult as EventSearchResult,
   SearchResponse as EventSearchResponse,
 } from '../../fulltext/FindEvents';
 import { openInBrowser } from '../open/openers';
-import { buildAppMapIndex, search } from '../../fulltext/appmap-index';
+import { buildAppMapIndex, search } from '../../rpc/explain/index/appmap-index';
 import buildIndexInTempDir from '../../rpc/explain/index/build-index-in-temp-dir';
 
 export const command = 'search <query>';

packages/cli/src/fulltext/FindEvents.ts

@@ -7,7 +7,7 @@ import assert from 'assert';
 
 import { verbose } from '../utils';
 import { collectParameters } from './collectParameters';
-import { fileNameMatchesFilterPatterns } from './fileNameMatchesFilterPatterns';
+import { fileNameMatchesFilterPatterns } from '../rpc/explain/index/filter-patterns';
 
 type IndexItem = {
   fqid: string;

packages/cli/src/fulltext/ref.ts

@@ -4,7 +4,7 @@ import { ContextV2, applyContext } from '@appland/navie';
 import { SearchRpc } from '@appland/rpc';
 
 import { DEFAULT_MAX_DIAGRAMS } from '../search/search';
-import { SearchResponse as AppMapSearchResponse } from '../../fulltext/appmap-match';
+import { SearchResponse as AppMapSearchResponse } from './index/appmap-match';
 import { searchAppMapFiles } from './index/appmap-file-index';
 import { searchProjectFiles } from './index/project-file-index';
 import {

packages/cli/src/rpc/explain/collect-search-context.ts

@@ -0,0 +1,29 @@
+import { FilterFn, isBinaryFile, isDataFile, isLargeFile } from '@appland/search';
+import makeDebug from 'debug';
+import { fileNameMatchesFilterPatterns } from './index/filter-patterns';
+
+const debug = makeDebug('appmap:rpc:explain:file-filter');
+
+export default function fileFilter(
+  includePatterns: RegExp[] | undefined,
+  excludePatterns: RegExp[] | undefined
+): FilterFn {
+  return async (path: string) => {
+    debug('Filtering: %s', path);
+    if (isBinaryFile(path)) {
+      debug('Skipping binary file: %s', path);
+      return false;
+    }
+
+    const includeFile = fileNameMatchesFilterPatterns(path, includePatterns, excludePatterns);
+    if (!includeFile) return false;
+
+    const isData = isDataFile(path);
+    if (isData && (await isLargeFile(path))) {
+      debug('Skipping large data file: %s', path);
+      return false;
+    }
+
+    return true;
+  };
+}

packages/cli/src/rpc/explain/fileFilter.ts

@@ -0,0 +1,55 @@
+import sqlite3 from 'better-sqlite3';
+import makeDebug from 'debug';
+
+import {
+  buildFileIndex,
+  FileIndex,
+  fileTokens,
+  FilterFn,
+  isBinaryFile,
+  isDataFile,
+  isLargeFile,
+  listProjectFiles,
+  readFileSafe,
+} from '@appland/search';
+import { fileNameMatchesFilterPatterns } from './index/filter-patterns';
+
+const debug = makeDebug('appmap:rpc:explain:index-files');
+
+function fileFilter(
+  includePatterns: RegExp[] | undefined,
+  excludePatterns: RegExp[] | undefined
+): FilterFn {
+  return async (path: string) => {
+    debug('Filtering: %s', path);
+    if (isBinaryFile(path)) {
+      debug('Skipping binary file: %s', path);
+      return false;
+    }
+
+    const includeFile = fileNameMatchesFilterPatterns(path, includePatterns, excludePatterns);
+    if (!includeFile) return false;
+
+    const isData = isDataFile(path);
+    if (isData && (await isLargeFile(path))) {
+      debug('Skipping large data file: %s', path);
+      return false;
+    }
+
+    return true;
+  };
+}
+
+export default async function indexFiles(
+  db: sqlite3.Database,
+  directories: string[],
+  includePatterns: RegExp[] | undefined,
+  excludePatterns: RegExp[] | undefined
+): Promise<FileIndex> {
+  const fileIndex = new FileIndex(db);
+
+  const filter = fileFilter(includePatterns, excludePatterns);
+  await buildFileIndex(fileIndex, directories, listProjectFiles, filter, readFileSafe, fileTokens);
+
+  return fileIndex;
+}

packages/cli/src/rpc/explain/index-files.ts

@@ -3,8 +3,8 @@ import sqlite3 from 'better-sqlite3';
 import { FileIndex } from '@appland/search';
 
 import buildIndexInTempDir, { CloseableIndex } from './build-index-in-temp-dir';
-import { buildAppMapIndex, search } from '../../../fulltext/appmap-index';
-import { SearchResponse } from '../../../fulltext/appmap-match';
+import { buildAppMapIndex, search } from './appmap-index';
+import { SearchResponse } from './appmap-match';
 
 export async function buildAppMapFileIndex(
   appmapDirectories: string[]

packages/cli/src/rpc/explain/index/appmap-file-index.ts

@@ -5,7 +5,7 @@ import { readFile } from 'fs/promises';
 import { Metadata } from '@appland/models';
 import { buildFileIndex, FileIndex, fileTokens } from '@appland/search';
 
-import { findFiles, isNodeError, verbose } from '../utils';
+import { findFiles, isNodeError, verbose } from '../../../utils';
 import {
   downscoreOutOfDateMatches,
   Match,
@@ -14,7 +14,7 @@ import {
   scoreMatches,
   SearchResponse,
 } from './appmap-match';
-import loadAppMapConfig from '../lib/loadAppMapConfig';
+import loadAppMapConfig from '../../../lib/loadAppMapConfig';
 
 export type ClassMapEntry = {
   name: string;

packages/cli/src/fulltext/appmap-index.ts renamed to packages/cli/src/rpc/explain/index/appmap-index.ts

@@ -1,5 +1,5 @@
-import UpToDate from '../lib/UpToDate';
-import { exists } from '../utils';
+import UpToDate from '../../../lib/UpToDate';
+import { exists } from '../../../utils';
 
 import makeDebug from 'debug';