Merge pull request #2045 from getappmap/feat/ignore-large-data-files

feat: Ignore large data files

Author: kgilpin

packages/cli/src/fulltext/FileIndex.ts

@@ -173,73 +173,85 @@ export async function buildFileIndex(
 }
 
 const BINARY_FILE_EXTENSIONS: string[] = [
-  'png',
-  'jpg',
-  'jpeg',
-  'gif',
+  '7z',
+  'aac',
+  'avi',
   'bmp',
-  'ico',
-  'tiff',
-  'webp',
-  'svg',
-  'mp3',
-  'wav',
-  'ogg',
+  'bz2',
+  'class',
+  'dll',
+  'doc',
+  'docx',
+  'dylib',
+  'ear',
+  'exe',
+  'eot',
   'flac',
-  'aac',
-  'mp4',
-  'webm',
+  'flv',
+  'gif',
+  'gz',
+  'ico',
+  'jar',
+  'jpeg',
+  'jpg',
+  'min.js',
+  'min.css',
+  'mjs',
   'mkv',
-  'avi',
+  'mo',
   'mov',
-  'wmv',
+  'mp3',
+  'mp4',
   'mpg',
-  'flv',
-  'zip',
-  'tar',
-  'gz',
-  'bz2',
-  'xz',
-  '7z',
-  'rar',
+  'odt',
+  'odp',
+  'ods',
+  'ogg',
+  'otf',
   'pdf',
-  'doc',
-  'docx',
-  'xls',
-  'xlsx',
+  'po',
+  'png',
   'ppt',
   'pptx',
-  'odt',
-  'ods',
-  'odp',
+  'pyc',
+  'rar',
   'rtf',
+  'so',
+  'svg',
+  'tar',
+  'tiff',
+  'ttf',
+  'wav',
+  'webm',
+  'webp',
   'woff',
   'woff2',
-  'eot',
-  'ttf',
-  'otf',
-  'mo',
-  'po',
-  'pyc',
-  'ico',
-  'flv',
-  'avi',
-  'mov',
   'wmv',
-  'mpg',
-  'jar',
-  'war',
-  'ear',
-  'class',
-  'so',
-  'dll',
-  'dylib',
-  'o',
-  'exe',
-  'min.js',
-  'min.css',
+  'xls',
+  'xlsx',
+  'xz',
+  'zip',
 ].map((ext) => '.' + ext);
 
+const DATA_FILE_EXTENSIONS: string[] = [
+  'csv',
+  'dat',
+  'log',
+  'json',
+  'tsv',
+  'yaml',
+  'yml',
+  'xml',
+].map((ext) => '.' + ext);
+
+const isBinaryFile = (fileName: string) => {
+  return BINARY_FILE_EXTENSIONS.some((ext) => fileName.endsWith(ext));
+};
+
+const isDataFile = (fileName: string) => {
+  return DATA_FILE_EXTENSIONS.some((ext) => fileName.endsWith(ext));
+};
+
 export async function filterFiles(
   directory: string,
   fileNames: string[],
@@ -248,23 +260,31 @@ export async function filterFiles(
 ): Promise<string[]> {
   const result: string[] = [];
   for (const fileName of fileNames) {
-    const fileExtension = path.extname(fileName).toLowerCase();
-    if (BINARY_FILE_EXTENSIONS.some((ext) => ext === fileExtension)) continue;
+    if (isBinaryFile(fileName)) continue;
 
     const includeFile = fileNameMatchesFilterPatterns(fileName, includePatterns, excludePatterns);
     if (!includeFile) continue;
 
+    let appendFile = false;
     try {
       const stats = await stat(join(directory, fileName));
       if (stats.isFile()) {
-        if (stats.size > 50_000) debug(`WARNING Large file ${fileName} with size ${stats.size}`);
-
-        result.push(fileName);
+        appendFile = true;
+        if (stats.size > 50_000) {
+          if (isDataFile(fileName)) {
+            debug(`Skipping large data file ${fileName} with size ${stats.size}`);
+            appendFile = false;
+          } else {
+            debug(`WARNING Large file ${fileName} with size ${stats.size}`);
+          }
+        }
       }
     } catch (error) {
       console.warn(`Error checking file ${fileName}`);
       console.warn(error);
     }
+
+    if (appendFile) result.push(fileName);
   }
   return result;
 }

packages/cli/tests/unit/fulltext/FileIndex.spec.ts

@@ -155,16 +155,31 @@ describe(filterFiles, () => {
     jest.resetAllMocks();
   });
 
-  it('filters out files with binary extensions and non-files', async () => {
+  it('filters out binary files, non-files, and large data files', async () => {
     const dir = tmp.dirSync({ unsafeCleanup: true }).name;
     writeFileSync(join(dir, 'file.txt'), 'hello');
     writeFileSync(join(dir, 'file.zip'), 'hello');
     writeFileSync(join(dir, 'file.json'), 'hello');
     writeFileSync(join(dir, 'large.txt'), Buffer.alloc(100_000));
+    writeFileSync(join(dir, 'large.js'), Buffer.alloc(100_000));
+    writeFileSync(join(dir, 'large.ts'), Buffer.alloc(100_000));
+    writeFileSync(join(dir, 'large.haml'), Buffer.alloc(100_000));
+    writeFileSync(join(dir, 'large.java'), Buffer.alloc(100_000));
+    writeFileSync(join(dir, 'large.mjs'), Buffer.alloc(100_000));
+    writeFileSync(join(dir, 'large.pyc'), Buffer.alloc(100_000));
+    writeFileSync(join(dir, 'large.json'), Buffer.alloc(100_000));
     mkdirSync(join(dir, 'dir'));
 
     const fileList = readdirSync(dir);
     const filtered = await filterFiles(dir, fileList);
-    expect(filtered).toEqual(['file.json', 'file.txt', 'large.txt']);
+    expect(filtered).toEqual([
+      'file.json',
+      'file.txt',
+      'large.haml',
+      'large.java',
+      'large.js',
+      'large.ts',
+      'large.txt',
+    ]);
   });
 });

packages/cli/tests/unit/rpc/explain/pattern.spec.ts

@@ -17,6 +17,7 @@ describe('Regex patterns', () => {
     { path: '_.navie', pattern: EXCLUDE_DOT_NAVIE_DIR, shouldMatch: false },
     { path: '._navie', pattern: EXCLUDE_DOT_NAVIE_DIR, shouldMatch: false },
     { path: '/navie/.navie/', pattern: EXCLUDE_DOT_NAVIE_DIR, shouldMatch: true },
+    { path: '/non-matching/test/file', pattern: /some_other_regex/, shouldMatch: false },
   ];
 
   testCases.forEach(({ path, pattern, shouldMatch }) => {