fix: Migrate from better-sqlite3 to node-sqlite3-wasm for improved compatibility

dividedmind · dividedmind · commit a0038c5c12c3 · 2025-02-17T19:09:23.000+01:00
diff --git a/packages/cli/package.json b/packages/cli/package.json
@@ -38,7 +38,6 @@
     "@eslint/js": "~8.57.0",
     "@octokit/types": "^11.1.0",
     "@swc/core": "^1.3.76",
-    "@types/better-sqlite3": "^7.6.11",
     "@types/cors": "^2.8.17",
     "@types/fs-extra": "^9.0.13",
     "@types/gitconfiglocal": "^2.0.1",
@@ -103,7 +102,6 @@
     "applicationinsights": "^2.1.4",
     "async": "^3.2.4",
     "axios": "^0.27.2",
-    "better-sqlite3": "^11.5.0",
     "body-parser": "^1.20.2",
     "boxen": "^5.0.1",
     "chalk": "^4.1.2",
@@ -131,6 +129,7 @@
     "lunr": "^2.3.9",
     "minimatch": "^5.1.2",
     "moo": "^0.5.1",
+    "node-sqlite3-wasm": "^0.8.34",
     "open": "^8.2.1",
     "openapi-diff": "^0.23.6",
     "openapi-types": "^12.1.3",
@@ -167,9 +166,7 @@
       "resources",
       "built/appmap.html",
       "built/sequenceDiagram.html",
-      "built/docs",
-      "../../node_modules/better-sqlite3/build/Release/better_sqlite3.node",
-      "node_modules/better-sqlite3/build/Release/better_sqlite3.node"
+      "built/docs"
     ],
     "outputPath": "dist"
   },
diff --git a/packages/cli/src/cmds/search/search.ts b/packages/cli/src/cmds/search/search.ts
@@ -1,5 +1,5 @@
+import sqlite3 from 'node-sqlite3-wasm';
 import yargs from 'yargs';
-import sqlite3 from 'better-sqlite3';
 import assert from 'assert';
 import { readFileSync } from 'fs';
 import { writeFile } from 'fs/promises';
@@ -186,7 +186,7 @@ export const handler = async (argv: ArgumentTypes) => {
     };
 
     const index = await buildIndexInTempDir('appmaps', async (indexFile) => {
-      const db = new sqlite3(indexFile);
+      const db = new sqlite3.Database(indexFile);
       const fileIndex = new FileIndex(db);
       await buildAppMapIndex(fileIndex, [process.cwd()]);
       return fileIndex;
diff --git a/packages/cli/src/rpc/explain/index-files.ts b/packages/cli/src/rpc/explain/index-files.ts
@@ -1,4 +1,4 @@
-import sqlite3 from 'better-sqlite3';
+import type sqlite3 from 'node-sqlite3-wasm';
 
 import {
   buildFileIndex,
diff --git a/packages/cli/src/rpc/explain/index-snippets.ts b/packages/cli/src/rpc/explain/index-snippets.ts
@@ -1,3 +1,5 @@
+import sqlite3 from 'node-sqlite3-wasm';
+
 import {
   buildSnippetIndex,
   FileSearchResult,
@@ -6,7 +8,6 @@ import {
   readFileSafe,
   SnippetIndex,
 } from '@appland/search';
-import sqlite3 from 'better-sqlite3';
 
 export default async function indexSnippets(
   db: sqlite3.Database,
diff --git a/packages/cli/src/rpc/explain/index/appmap-file-index.ts b/packages/cli/src/rpc/explain/index/appmap-file-index.ts
@@ -1,4 +1,4 @@
-import sqlite3 from 'better-sqlite3';
+import sqlite3 from 'node-sqlite3-wasm';
 
 import { FileIndex, SessionId } from '@appland/search';
 
@@ -10,7 +10,7 @@ export async function buildAppMapFileIndex(
   appmapDirectories: string[]
 ): Promise<CloseableIndex<FileIndex>> {
   return await buildIndexInTempDir<FileIndex>('appmaps', async (indexFile) => {
-    const db = new sqlite3(indexFile);
+    const db = new sqlite3.Database(indexFile);
     const fileIndex = new FileIndex(db);
     await buildAppMapIndex(fileIndex, appmapDirectories);
     return fileIndex;
diff --git a/packages/cli/src/rpc/explain/index/project-file-index.ts b/packages/cli/src/rpc/explain/index/project-file-index.ts
@@ -1,5 +1,5 @@
-import sqlite3 from 'better-sqlite3';
 import makeDebug from 'debug';
+import sqlite3 from 'node-sqlite3-wasm';
 
 import {
   buildFileIndex,
@@ -71,7 +71,7 @@ export async function buildProjectFileIndex(
   excludePatterns: RegExp[] | undefined
 ): Promise<CloseableIndex<FileIndex>> {
   return await buildIndexInTempDir('files', async (indexFile) => {
-    const db = new sqlite3(indexFile);
+    const db = new sqlite3.Database(indexFile);
     return await indexFiles(db, sourceDirectories, includePatterns, excludePatterns);
   });
 }
diff --git a/packages/cli/src/rpc/explain/index/project-file-snippet-index.ts b/packages/cli/src/rpc/explain/index/project-file-snippet-index.ts
@@ -1,4 +1,5 @@
-import sqlite3 from 'better-sqlite3';
+import sqlite3 from 'node-sqlite3-wasm';
+
 import { FileSearchResult, SessionId, SnippetIndex } from '@appland/search';
 
 import buildIndexInTempDir, { CloseableIndex } from './build-index-in-temp-dir';
@@ -76,7 +77,7 @@ export async function buildProjectFileSnippetIndex(
   fileSearchResults: FileSearchResult[]
 ): Promise<ProjectFileSnippetIndex> {
   const snippetIndex = await buildIndexInTempDir('snippets', async (indexFile) => {
-    const db = new sqlite3(indexFile);
+    const db = new sqlite3.Database(indexFile);
     return await indexSnippets(db, fileSearchResults);
   });
 
diff --git a/packages/cli/src/rpc/search/search.ts b/packages/cli/src/rpc/search/search.ts
@@ -1,5 +1,7 @@
 import { isAbsolute, join } from 'path';
-import sqlite3 from 'better-sqlite3';
+
+import sqlite3 from 'node-sqlite3-wasm';
+
 import { FileIndex, generateSessionId } from '@appland/search';
 import { SearchRpc } from '@appland/rpc';
 
@@ -62,7 +64,7 @@ export async function handler(
     // Search across all AppMaps, creating a map from AppMap id to AppMapSearchResult
     const maxResults = options.maxDiagrams || options.maxResults || DEFAULT_MAX_DIAGRAMS;
     const index = await buildIndexInTempDir('appmaps', async (indexFile) => {
-      const db = new sqlite3(indexFile);
+      const db = new sqlite3.Database(indexFile);
       const fileIndex = new FileIndex(db);
       await buildAppMapIndex(
         fileIndex,
diff --git a/packages/search/package.json b/packages/search/package.json
@@ -21,7 +21,6 @@
   "author": "AppLand, Inc",
   "license": "Commons Clause + MIT",
   "devDependencies": {
-    "@types/better-sqlite3": "^7.6.11",
     "@types/jest": "^29.5.4",
     "@types/node": "^16",
     "eslint": "^9",
@@ -39,8 +38,8 @@
     "typescript-eslint": "^8.11.0"
   },
   "dependencies": {
-    "better-sqlite3": "^11.5.0",
     "isbinaryfile": "^5.0.4",
+    "node-sqlite3-wasm": "^0.8.34",
     "yargs": "^17.7.2"
   }
 }
diff --git a/packages/search/src/cli.ts b/packages/search/src/cli.ts
@@ -1,7 +1,7 @@
 import yargs from 'yargs';
 import { hideBin } from 'yargs/helpers';
-import sqlite3 from 'better-sqlite3';
 import makeDebug from 'debug';
+import sqlite3 from 'node-sqlite3-wasm';
 
 import { fileTokens } from './tokenize';
 import FileIndex from './file-index';
@@ -62,7 +62,7 @@ const cli = yargs(hideBin(process.argv))
         return !filterRE.test(path);
       };
 
-      const db = new sqlite3(':memory:');
+      const db = new sqlite3.Database(':memory:');
       const fileIndex = new FileIndex(db);
       const sessionId = generateSessionId();
 
diff --git a/packages/search/src/file-index.ts b/packages/search/src/file-index.ts
@@ -1,4 +1,5 @@
-import sqlite3 from 'better-sqlite3';
+import type sqlite3 from 'node-sqlite3-wasm';
+
 import { SessionId } from './session-id';
 
 const CREATE_TABLE_SQL = `CREATE VIRTUAL TABLE file_content USING fts5(
@@ -74,22 +75,22 @@ export type FileSearchResult = {
 export default class FileIndex {
   #insert: sqlite3.Statement;
   #updateBoost: sqlite3.Statement;
-  #deleteSession: sqlite3.Statement<string>;
-  #search: sqlite3.Statement<[string, string, number]>;
+  #deleteSession: sqlite3.Statement;
+  #search: sqlite3.Statement;
 
   constructor(public database: sqlite3.Database) {
     this.database.exec(CREATE_TABLE_SQL);
     this.database.exec(CREATE_BOOST_TABLE_SQL);
-    this.database.pragma('journal_mode = OFF');
-    this.database.pragma('synchronous = OFF');
+    this.database.exec('PRAGMA journal_mode = OFF');
+    this.database.exec('PRAGMA synchronous = OFF');
     this.#insert = this.database.prepare(INSERT_SQL);
     this.#updateBoost = this.database.prepare(UPDATE_BOOST_SQL);
     this.#deleteSession = this.database.prepare(DELETE_SESSION_SQL);
     this.#search = this.database.prepare(SEARCH_SQL);
   }
 
   indexFile(directory: string, filePath: string, symbols: string, words: string): void {
-    this.#insert.run(directory, filePath, symbols, words);
+    this.#insert.run([directory, filePath, symbols, words]);
   }
 
   /**
@@ -99,7 +100,7 @@ export default class FileIndex {
    * @param boostFactor - The factor by which to boost the file's relevance.
    */
   boostFile(sessionId: SessionId, filePath: string, boostFactor: number): void {
-    this.#updateBoost.run(sessionId, filePath, boostFactor);
+    this.#updateBoost.run([sessionId, filePath, boostFactor]);
   }
 
   /**
@@ -118,7 +119,7 @@ export default class FileIndex {
    * @returns An array of search results with directory, file path, and score.
    */
   search(sessionId: SessionId, query: string, limit = 10): FileSearchResult[] {
-    const rows = this.#search.all(sessionId, query, limit) as FileIndexRow[];
+    const rows = this.#search.all([sessionId, query, limit]) as FileIndexRow[];
     return rows.map((row) => ({
       directory: row.directory,
       filePath: row.file_path,
diff --git a/packages/search/src/snippet-index.ts b/packages/search/src/snippet-index.ts
@@ -1,5 +1,7 @@
 import assert from 'assert';
-import sqlite3 from 'better-sqlite3';
+
+import type sqlite3 from 'node-sqlite3-wasm';
+
 import { SessionId } from './session-id';
 
 const CREATE_SNIPPET_CONTENT_TABLE_SQL = `CREATE VIRTUAL TABLE snippet_content USING fts5(
@@ -115,14 +117,14 @@ type SnippetSearchRow = {
 export default class SnippetIndex {
   #insertSnippet: sqlite3.Statement;
   #updateSnippetBoost: sqlite3.Statement;
-  #deleteSession: sqlite3.Statement<[string]>;
-  #searchSnippet: sqlite3.Statement<[string, string, number]>;
+  #deleteSession: sqlite3.Statement;
+  #searchSnippet: sqlite3.Statement;
 
   constructor(public database: sqlite3.Database) {
     this.database.exec(CREATE_SNIPPET_CONTENT_TABLE_SQL);
     this.database.exec(CREATE_SNIPPET_BOOST_TABLE_SQL);
-    this.database.pragma('journal_mode = OFF');
-    this.database.pragma('synchronous = OFF');
+    this.database.exec('PRAGMA journal_mode = OFF');
+    this.database.exec('PRAGMA synchronous = OFF');
     this.#insertSnippet = this.database.prepare(INSERT_SNIPPET_SQL);
     this.#deleteSession = this.database.prepare(DELETE_SESSION_SQL);
     this.#updateSnippetBoost = this.database.prepare(UPDATE_SNIPPET_BOOST_SQL);
@@ -152,7 +154,7 @@ export default class SnippetIndex {
     words: string,
     content: string
   ): void {
-    this.#insertSnippet.run(encodeSnippetId(snippetId), directory, symbols, words, content);
+    this.#insertSnippet.run([encodeSnippetId(snippetId), directory, symbols, words, content]);
   }
 
   /**
@@ -162,11 +164,11 @@ export default class SnippetIndex {
    * @param boostFactor - The factor by which to boost the snippet's relevance.
    */
   boostSnippet(sessionId: SessionId, snippetId: SnippetId, boostFactor: number): void {
-    this.#updateSnippetBoost.run(sessionId, encodeSnippetId(snippetId), boostFactor);
+    this.#updateSnippetBoost.run([sessionId, encodeSnippetId(snippetId), boostFactor]);
   }
 
   searchSnippets(sessionId: SessionId, query: string, limit = 10): SnippetSearchResult[] {
-    const rows = this.#searchSnippet.all(sessionId, query, limit) as SnippetSearchRow[];
+    const rows = this.#searchSnippet.all([sessionId, query, limit]) as SnippetSearchRow[];
     return rows.map((row) => ({
       directory: row.directory,
       snippetId: parseSnippetId(row.snippet_id),
diff --git a/packages/search/test/file-index.spec.ts b/packages/search/test/file-index.spec.ts
@@ -1,5 +1,7 @@
 import { strict as assert } from 'assert';
-import sqlite3 from 'better-sqlite3';
+
+import sqlite3 from 'node-sqlite3-wasm';
+
 import FileIndex, { FileSearchResult } from '../src/file-index';
 import { generateSessionId, SessionId } from '../src/session-id';
 
@@ -16,7 +18,7 @@ describe('FileIndex', () => {
   const directory = 'src';
 
   beforeEach(() => {
-    db = new sqlite3(':memory:');
+    db = new sqlite3.Database(':memory:');
     index = new FileIndex(db);
     sessionId = generateSessionId();
   });
diff --git a/packages/search/test/snippet-index.spec.ts b/packages/search/test/snippet-index.spec.ts
@@ -1,5 +1,6 @@
 import { strict as assert } from 'assert';
-import sqlite3 from 'better-sqlite3';
+
+import sqlite3 from 'node-sqlite3-wasm';
 
 import SnippetIndex, {
   fileChunkSnippetId,
@@ -26,7 +27,7 @@ describe('SnippetIndex', () => {
   const snippet4: SnippetId = { type: 'code-snippet', id: 'test4.txt:31' };
 
   beforeEach(() => {
-    db = new sqlite3(':memory:');
+    db = new sqlite3.Database(':memory:');
     index = new SnippetIndex(db);
     sessionId = generateSessionId();
   });
diff --git a/yarn.lock b/yarn.lock

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-import sqlite3 from 'better-sqlite3';`
	`1`	`+import type sqlite3 from 'node-sqlite3-wasm';`
`2`	`2`
`3`	`3`	`import {`
`4`	`4`	`buildFileIndex,`