Merge pull request #3843 from asgerf/asgerf/compare-perf-view

Add 'compare performance' view

Author: asgerf

extensions/ql-vscode/package.json

@@ -942,6 +942,10 @@
         "command": "codeQLQueryHistory.compareWith",
         "title": "Compare Results"
       },
+      {
+        "command": "codeQLQueryHistory.comparePerformanceWith",
+        "title": "Compare Performance"
+      },
       {
         "command": "codeQLQueryHistory.openOnGithub",
         "title": "View Logs"
@@ -1213,6 +1217,11 @@
           "group": "3_queryHistory@0",
           "when": "viewItem == rawResultsItem || viewItem == interpretedResultsItem"
         },
+        {
+          "command": "codeQLQueryHistory.comparePerformanceWith",
+          "group": "3_queryHistory@1",
+          "when": "viewItem == rawResultsItem && config.codeQL.canary || viewItem == interpretedResultsItem && config.codeQL.canary"
+        },
         {
           "command": "codeQLQueryHistory.showQueryLog",
           "group": "4_queryHistory@4",
@@ -1716,6 +1725,10 @@
           "command": "codeQLQueryHistory.compareWith",
           "when": "false"
         },
+        {
+          "command": "codeQLQueryHistory.comparePerformanceWith",
+          "when": "false"
+        },
         {
           "command": "codeQLQueryHistory.sortByName",
           "when": "false"

extensions/ql-vscode/src/common/commands.ts

@@ -180,6 +180,7 @@ export type QueryHistoryCommands = {
   "codeQLQueryHistory.removeHistoryItemContextInline": TreeViewContextMultiSelectionCommandFunction<QueryHistoryInfo>;
   "codeQLQueryHistory.renameItem": TreeViewContextMultiSelectionCommandFunction<QueryHistoryInfo>;
   "codeQLQueryHistory.compareWith": TreeViewContextMultiSelectionCommandFunction<QueryHistoryInfo>;
+  "codeQLQueryHistory.comparePerformanceWith": TreeViewContextMultiSelectionCommandFunction<QueryHistoryInfo>;
   "codeQLQueryHistory.showEvalLog": TreeViewContextMultiSelectionCommandFunction<QueryHistoryInfo>;
   "codeQLQueryHistory.showEvalLogSummary": TreeViewContextMultiSelectionCommandFunction<QueryHistoryInfo>;
   "codeQLQueryHistory.showEvalLogViewer": TreeViewContextMultiSelectionCommandFunction<QueryHistoryInfo>;

extensions/ql-vscode/src/common/interface-types.ts

@@ -27,6 +27,7 @@ import type {
 } from "./raw-result-types";
 import type { AccessPathSuggestionOptions } from "../model-editor/suggestions";
 import type { ModelEvaluationRunState } from "../model-editor/shared/model-evaluation-run-state";
+import type { PerformanceComparisonDataFromLog } from "../log-insights/performance-comparison";
 
 /**
  * This module contains types and code that are shared between
@@ -396,6 +397,17 @@ export interface SetComparisonsMessage {
   readonly message: string | undefined;
 }
 
+export type ToComparePerformanceViewMessage = SetPerformanceComparisonQueries;
+
+export interface SetPerformanceComparisonQueries {
+  readonly t: "setPerformanceComparison";
+  readonly from: PerformanceComparisonDataFromLog;
+  readonly to: PerformanceComparisonDataFromLog;
+  readonly comparison: boolean;
+}
+
+export type FromComparePerformanceViewMessage = CommonFromViewMessages;
+
 export type QueryCompareResult =
   | RawQueryCompareResult
   | InterpretedQueryCompareResult;

extensions/ql-vscode/src/common/vscode/abstract-webview.ts

@@ -41,6 +41,13 @@ export abstract class AbstractWebview<
 
   constructor(protected readonly app: App) {}
 
+  public hidePanel() {
+    if (this.panel !== undefined) {
+      this.panel.dispose();
+      this.panel = undefined;
+    }
+  }
+
   public async restoreView(panel: WebviewPanel): Promise<void> {
     this.panel = panel;
     const config = await this.getPanelConfig();

extensions/ql-vscode/src/common/vscode/webview-html.ts

@@ -7,6 +7,7 @@ import type { App } from "../app";
 export type WebviewKind =
   | "results"
   | "compare"
+  | "compare-performance"
   | "variant-analysis"
   | "data-flow-paths"
   | "model-editor"

extensions/ql-vscode/src/compare-performance/compare-performance-view.ts

@@ -0,0 +1,108 @@
+import { statSync } from "fs";
+import { ViewColumn } from "vscode";
+
+import type { App } from "../common/app";
+import { redactableError } from "../common/errors";
+import type {
+  FromComparePerformanceViewMessage,
+  ToComparePerformanceViewMessage,
+} from "../common/interface-types";
+import type { Logger } from "../common/logging";
+import { showAndLogExceptionWithTelemetry } from "../common/logging";
+import { extLogger } from "../common/logging/vscode";
+import type { WebviewPanelConfig } from "../common/vscode/abstract-webview";
+import { AbstractWebview } from "../common/vscode/abstract-webview";
+import { withProgress } from "../common/vscode/progress";
+import { telemetryListener } from "../common/vscode/telemetry";
+import type { HistoryItemLabelProvider } from "../query-history/history-item-label-provider";
+import { PerformanceOverviewScanner } from "../log-insights/performance-comparison";
+import { scanLog } from "../log-insights/log-scanner";
+import type { ResultsView } from "../local-queries";
+
+export class ComparePerformanceView extends AbstractWebview<
+  ToComparePerformanceViewMessage,
+  FromComparePerformanceViewMessage
+> {
+  constructor(
+    app: App,
+    public logger: Logger,
+    public labelProvider: HistoryItemLabelProvider,
+    private resultsView: ResultsView,
+  ) {
+    super(app);
+  }
+
+  async showResults(fromJsonLog: string, toJsonLog: string) {
+    const panel = await this.getPanel();
+    panel.reveal(undefined, false);
+
+    // Close the results viewer as it will have opened when the user clicked the query in the history view
+    // (which they must do as part of the UI interaction for opening the performance view).
+    // The performance view generally needs a lot of width so it's annoying to have the result viewer open.
+    this.resultsView.hidePanel();
+
+    await this.waitForPanelLoaded();
+
+    function scanLogWithProgress(log: string, logDescription: string) {
+      const bytes = statSync(log).size;
+      return withProgress(
+        async (progress) =>
+          scanLog(log, new PerformanceOverviewScanner(), progress),
+
+        {
+          title: `Scanning evaluator log ${logDescription} (${(bytes / 1024 / 1024).toFixed(1)} MB)`,
+        },
+      );
+    }
+
+    const [fromPerf, toPerf] = await Promise.all([
+      fromJsonLog === ""
+        ? new PerformanceOverviewScanner()
+        : scanLogWithProgress(fromJsonLog, "1/2"),
+      scanLogWithProgress(toJsonLog, fromJsonLog === "" ? "1/1" : "2/2"),
+    ]);
+
+    await this.postMessage({
+      t: "setPerformanceComparison",
+      from: fromPerf.getData(),
+      to: toPerf.getData(),
+      comparison: fromJsonLog !== "",
+    });
+  }
+
+  protected getPanelConfig(): WebviewPanelConfig {
+    return {
+      viewId: "comparePerformanceView",
+      title: "Compare CodeQL Performance",
+      viewColumn: ViewColumn.Active,
+      preserveFocus: true,
+      view: "compare-performance",
+    };
+  }
+
+  protected onPanelDispose(): void {}
+
+  protected async onMessage(
+    msg: FromComparePerformanceViewMessage,
+  ): Promise<void> {
+    switch (msg.t) {
+      case "viewLoaded":
+        this.onWebViewLoaded();
+        break;
+
+      case "telemetry":
+        telemetryListener?.sendUIInteraction(msg.action);
+        break;
+
+      case "unhandledError":
+        void showAndLogExceptionWithTelemetry(
+          extLogger,
+          telemetryListener,
+          redactableError(
+            msg.error,
+          )`Unhandled error in performance comparison view: ${msg.error.message}`,
+        );
+        break;
+    }
+  }
+}

extensions/ql-vscode/src/extension.ts

@@ -135,6 +135,7 @@ import { LanguageContextStore } from "./language-context-store";
 import { LanguageSelectionPanel } from "./language-selection-panel/language-selection-panel";
 import { GitHubDatabasesModule } from "./databases/github-databases";
 import { DatabaseFetcher } from "./databases/database-fetcher";
+import { ComparePerformanceView } from "./compare-performance/compare-performance-view";
 
 /**
  * extension.ts
@@ -928,6 +929,11 @@ async function activateWithInstalledDistribution(
       from: CompletedLocalQueryInfo,
       to: CompletedLocalQueryInfo,
     ): Promise<void> => showResultsForComparison(compareView, from, to),
+    async (
+      from: CompletedLocalQueryInfo,
+      to: CompletedLocalQueryInfo | undefined,
+    ): Promise<void> =>
+      showPerformanceComparison(comparePerformanceView, from, to),
   );
 
   ctx.subscriptions.push(qhm);
@@ -953,6 +959,15 @@ async function activateWithInstalledDistribution(
   );
   ctx.subscriptions.push(compareView);
 
+  void extLogger.log("Initializing performance comparison view.");
+  const comparePerformanceView = new ComparePerformanceView(
+    app,
+    queryServerLogger,
+    labelProvider,
+    localQueryResultsView,
+  );
+  ctx.subscriptions.push(comparePerformanceView);
+
   void extLogger.log("Initializing source archive filesystem provider.");
   archiveFilesystemProvider_activate(ctx, dbm);
 
@@ -1191,6 +1206,30 @@ async function showResultsForComparison(
   }
 }
 
+async function showPerformanceComparison(
+  view: ComparePerformanceView,
+  from: CompletedLocalQueryInfo,
+  to: CompletedLocalQueryInfo | undefined,
+): Promise<void> {
+  let fromLog = from.evaluatorLogPaths?.jsonSummary;
+  let toLog = to?.evaluatorLogPaths?.jsonSummary;
+
+  if (to === undefined) {
+    toLog = fromLog;
+    fromLog = "";
+  }
+  if (fromLog === undefined || toLog === undefined) {
+    return extLogger.showWarningMessage(
+      `Cannot compare performance as the structured logs are missing. Did they queries complete normally?`,
+    );
+  }
+  await extLogger.log(
+    `Comparing performance of ${from.getQueryName()} and ${to?.getQueryName() ?? "baseline"}`,
+  );
+
+  await view.showResults(fromLog, toLog);
+}
+
 function addUnhandledRejectionListener() {
   const handler = (error: unknown) => {
     // This listener will be triggered for errors from other extensions as

extensions/ql-vscode/src/log-insights/log-scanner.ts

@@ -1,6 +1,7 @@
-import type { SummaryEvent } from "./log-summary";
-import { readJsonlFile } from "../common/jsonl-reader";
 import type { Disposable } from "../common/disposable-object";
+import { readJsonlFile } from "../common/jsonl-reader";
+import type { ProgressCallback } from "../common/vscode/progress";
+import type { SummaryEvent } from "./log-summary";
 
 /**
  * Callback interface used to report diagnostics from a log scanner.
@@ -112,3 +113,27 @@ export class EvaluationLogScannerSet {
     scanners.forEach((scanner) => scanner.onDone());
   }
 }
+
+/**
+ * Scan the evaluator summary log using the given scanner. For convenience, returns the scanner.
+ *
+ * @param jsonSummaryLocation The file path of the JSON summary log.
+ * @param scanner The scanner to process events from the log
+ */
+export async function scanLog<T extends EvaluationLogScanner>(
+  jsonSummaryLocation: string,
+  scanner: T,
+  progress?: ProgressCallback,
+): Promise<T> {
+  progress?.({
+    // all scans have step 1 - the backing progress tracker allows increments instead of steps - but for now we are happy with a tiny UI that says what is happening
+    message: `Scanning ...`,
+    step: 1,
+    maxStep: 2,
+  });
+  await readJsonlFile<SummaryEvent>(jsonSummaryLocation, async (obj) => {
+    scanner.onEvent(obj);
+  });
+  scanner.onDone();
+  return scanner;
+}

extensions/ql-vscode/src/log-insights/log-summary.ts

@@ -33,6 +33,7 @@ interface ResultEventBase extends SummaryEventBase {
 export interface ComputeSimple extends ResultEventBase {
   evaluationStrategy: "COMPUTE_SIMPLE";
   ra: Ra;
+  millis: number;
   pipelineRuns?: [PipelineRun];
   queryCausingWork?: string;
   dependencies: { [key: string]: string };
@@ -42,6 +43,7 @@ export interface ComputeRecursive extends ResultEventBase {
   evaluationStrategy: "COMPUTE_RECURSIVE";
   deltaSizes: number[];
   ra: Ra;
+  millis: number;
   pipelineRuns: PipelineRun[];
   queryCausingWork?: string;
   dependencies: { [key: string]: string };