feat: parallel log scanning

Author: esbena

extensions/ql-vscode/src/compare-performance/compare-performance-view.ts

@@ -1,3 +1,4 @@
+import { statSync } from "fs";
 import { ViewColumn } from "vscode";
 
 import type { App } from "../common/app";
@@ -11,6 +12,7 @@ import { showAndLogExceptionWithTelemetry } from "../common/logging";
 import { extLogger } from "../common/logging/vscode";
 import type { WebviewPanelConfig } from "../common/vscode/abstract-webview";
 import { AbstractWebview } from "../common/vscode/abstract-webview";
+import { withProgress } from "../common/vscode/progress";
 import { telemetryListener } from "../common/vscode/telemetry";
 import type { HistoryItemLabelProvider } from "../query-history/history-item-label-provider";
 import { PerformanceOverviewScanner } from "../log-insights/performance-comparison";
@@ -41,12 +43,22 @@ export class ComparePerformanceView extends AbstractWebview<
 
     await this.waitForPanelLoaded();
 
-    // TODO: try processing in (async) parallel once readJsonl is streaming
-    const fromPerf = await scanLog(
-      fromJsonLog,
-      new PerformanceOverviewScanner(),
-    );
-    const toPerf = await scanLog(toJsonLog, new PerformanceOverviewScanner());
+    function scanLogWithProgress(log: string, logDescription: string) {
+      const bytes = statSync(log).size;
+      return withProgress(
+        async (progress) =>
+          scanLog(log, new PerformanceOverviewScanner(), progress),
+
+        {
+          title: `Scanning evaluator log ${logDescription} (${(bytes / 1024 / 1024).toFixed(1)} MB)`,
+        },
+      );
+    }
+
+    const [fromPerf, toPerf] = await Promise.all([
+      scanLogWithProgress(fromJsonLog, "1/2"),
+      scanLogWithProgress(toJsonLog, "2/2"),
+    ]);
 
     await this.postMessage({
       t: "setPerformanceComparison",

extensions/ql-vscode/src/log-insights/log-scanner.ts

@@ -1,6 +1,7 @@
-import type { SummaryEvent } from "./log-summary";
-import { readJsonlFile } from "../common/jsonl-reader";
 import type { Disposable } from "../common/disposable-object";
+import { readJsonlFile } from "../common/jsonl-reader";
+import type { ProgressCallback } from "../common/vscode/progress";
+import type { SummaryEvent } from "./log-summary";
 
 /**
  * Callback interface used to report diagnostics from a log scanner.
@@ -114,15 +115,22 @@ export class EvaluationLogScannerSet {
 }
 
 /**
- * Scan the evaluator summary log using the given scanner. For conveience, returns the scanner.
+ * Scan the evaluator summary log using the given scanner. For convenience, returns the scanner.
  *
  * @param jsonSummaryLocation The file path of the JSON summary log.
  * @param scanner The scanner to process events from the log
  */
 export async function scanLog<T extends EvaluationLogScanner>(
   jsonSummaryLocation: string,
   scanner: T,
+  progress?: ProgressCallback,
 ): Promise<T> {
+  progress?.({
+    // XXX all scans have step 1 - the backing progress tracker allows increments instead of steps - but for now we are happy with a tiny UI that says what is happening
+    message: `Scanning ...`,
+    step: 1,
+    maxStep: 2,
+  });
   await readJsonlFile<SummaryEvent>(jsonSummaryLocation, async (obj) => {
     scanner.onEvent(obj);
   });