feat: Cache parsed SQL when building sequence diagram

Author: ahtrotta

packages/models/package.json

@@ -38,6 +38,7 @@
     "eslint-plugin-import": "^2.22.1",
     "jest": "^27.4.7",
     "lint-staged": "^10.5.4",
+    "lru-cache": "6.0.0",
     "prettier": "^2.7.1",
     "ts-jest": "^27.1.4",
     "tsup": "^6.5.0",

packages/models/src/event.js

@@ -1,3 +1,4 @@
+/* eslint-disable */
 import { addHiddenProperty, hasProp, transformToJSON } from './util';
 import analyzeSQL, { abstractSqlAstJSON } from './sql/analyze';
 import normalizeSQL from './sql/normalize';
@@ -362,14 +363,14 @@ export default class Event {
 
   get identityHash() {
     if (!this.$hidden.identityHash) {
-      this.$hidden.identityHash = this.buildIdentityHash(this).digest();
+      this.$hidden.identityHash = this.buildIdentityHash().digest();
     }
     return this.$hidden.identityHash;
   }
 
   get hash() {
     if (!this.$hidden.hash) {
-      this.$hidden.hash = this.buildStableHash(this).digest();
+      this.$hidden.hash = this.buildStableHash().digest();
     }
     return this.$hidden.hash;
   }
@@ -381,6 +382,13 @@ export default class Event {
     return this.$hidden.stableProperties;
   }
 
+  getHashWithSqlCache(parsedSqlCache) {
+    if (!this.$hidden.hash) {
+      this.$hidden.hash = this.buildStableHash(parsedSqlCache).digest();
+    }
+    return this.$hidden.hash;
+  }
+
   callStack() {
     const stack = this.ancestors().reverse();
     stack.push(this.callEvent);
@@ -501,7 +509,7 @@ export default class Event {
 
   // Collects properties of an event which are not dependent on the specifics
   // of invocation.
-  gatherStableProperties() {
+  gatherStableProperties(parsedSqlCache) {
     const { sqlQuery } = this;
 
     // Convert null and undefined values to empty strings
@@ -526,10 +534,17 @@ export default class Event {
 
     let properties;
     if (sqlQuery) {
-      const sqlNormalized = abstractSqlAstJSON(sqlQuery, this.sql.database_type)
-        // Collapse repeated variable literals and parameter tokens (e.g. '?, ?' in an IN clause)
-        .split(/{"type":"variable"}(?:,{"type":"variable"})*/g)
-        .join(`{"type":"variable"}`);
+      let sqlNormalized;
+      const cacheKey = `${this.sql.database_type}:${sqlQuery}`;
+      if (parsedSqlCache) sqlNormalized = parsedSqlCache.get(cacheKey);
+      if (!sqlNormalized) {
+        sqlNormalized = abstractSqlAstJSON(sqlQuery, this.sql.database_type)
+          // Collapse repeated variable literals and parameter tokens (e.g. '?, ?' in an IN clause)
+          .split(/{"type":"variable"}(?:,{"type":"variable"})*/g)
+          .join(`{"type":"variable"}`);
+        parsedSqlCache.set(cacheKey, sqlNormalized);
+      }
+
       properties = {
         event_type: 'sql',
         sql_normalized: sqlNormalized,
@@ -554,7 +569,10 @@ export default class Event {
     return HashBuilder.buildHash('event-identity-v2', this.gatherIdentityProperties());
   }
 
-  buildStableHash() {
-    return HashBuilder.buildHash('event-stable-properties-v2', this.gatherStableProperties());
+  buildStableHash(parsedSqlCache) {
+    return HashBuilder.buildHash(
+      'event-stable-properties-v2',
+      this.gatherStableProperties(parsedSqlCache)
+    );
   }
 }

packages/models/types/index.d.ts

@@ -1,4 +1,5 @@
 import type { SqliteParser } from './sqlite-parser';
+import type { LRUCache } from 'lru-cache';
 
 declare module '@appland/models' {
   export type CodeObjectType =
@@ -214,6 +215,7 @@ declare module '@appland/models' {
     dataObjects(): Array<ParameterObject | ReturnValueObject>;
     toString(): string;
     toJSON(): any;
+    getHashWithSqlCache(parsedSqlCache: LRUCache<string, any>): string;
   }
 
   export class EventNavigator {

packages/sequence-diagram/src/buildDiagram.ts

@@ -1,6 +1,7 @@
 import { AppMap, Event } from '@appland/models';
 import { classNameToOpenAPIType } from '@appland/openapi';
 import sha256 from 'crypto-js/sha256.js';
+import LRUCache from 'lru-cache';
 import { merge } from './mergeWindow';
 import { selectEvents } from './selectEvents';
 import Specification from './specification';
@@ -19,6 +20,7 @@ import {
 } from './types';
 
 const MAX_WINDOW_SIZE = 5;
+const parsedSqlCache = new LRUCache<string, any>({ max: 1000 });
 
 class ActorManager {
   private _actorsByCodeObjectId = new Map<string, Actor>();
@@ -68,7 +70,7 @@ export default function buildDiagram(
         callee: actorManager.findOrCreateActor(callee),
         route: callee.route,
         status: response.status || response.status_code,
-        digest: callee.hash,
+        digest: callee.getHashWithSqlCache(parsedSqlCache),
         subtreeDigest: 'undefined',
         children: [],
         elapsed: callee.elapsedTime,
@@ -83,7 +85,7 @@ export default function buildDiagram(
         callee: actorManager.findOrCreateActor(callee),
         route: callee.route,
         status: response.status || response.status_code,
-        digest: callee.hash,
+        digest: callee.getHashWithSqlCache(parsedSqlCache),
         subtreeDigest: 'undefined',
         children: [],
         elapsed: callee.elapsedTime,
@@ -96,7 +98,7 @@ export default function buildDiagram(
         caller: caller ? actorManager.findOrCreateActor(caller) : undefined,
         callee: actorManager.findOrCreateActor(callee),
         query: callee.sqlQuery,
-        digest: truncatedQuery ? 'truncatedQuery' : callee.hash,
+        digest: truncatedQuery ? 'truncatedQuery' : callee.getHashWithSqlCache(parsedSqlCache),
         subtreeDigest: 'undefined',
         children: [],
         elapsed: callee.elapsedTime,
@@ -109,7 +111,7 @@ export default function buildDiagram(
         callee: actorManager.findOrCreateActor(callee),
         name: callee.codeObject.name,
         static: callee.codeObject.static,
-        digest: callee.hash,
+        digest: callee.getHashWithSqlCache(parsedSqlCache),
         subtreeDigest: 'undefined',
         stableProperties: { ...callee.stableProperties },
         returnValue: buildReturnValue(callee),

yarn.lock

@@ -380,6 +380,7 @@ __metadata:
     eslint-plugin-import: ^2.22.1
     jest: ^27.4.7
     lint-staged: ^10.5.4
+    lru-cache: 6.0.0
     prettier: ^2.7.1
     ts-jest: ^27.1.4
     tsup: ^6.5.0
@@ -24788,6 +24789,15 @@ __metadata:
   languageName: node
   linkType: hard
 
+"lru-cache@npm:6.0.0, lru-cache@npm:^6.0.0":
+  version: 6.0.0
+  resolution: "lru-cache@npm:6.0.0"
+  dependencies:
+    yallist: ^4.0.0
+  checksum: f97f499f898f23e4585742138a22f22526254fdba6d75d41a1c2526b3b6cc5747ef59c5612ba7375f42aca4f8461950e925ba08c991ead0651b4918b7c978297
+  languageName: node
+  linkType: hard
+
 "lru-cache@npm:^4.0.1, lru-cache@npm:^4.1.2, lru-cache@npm:^4.1.5":
   version: 4.1.5
   resolution: "lru-cache@npm:4.1.5"
@@ -24807,15 +24817,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"lru-cache@npm:^6.0.0":
-  version: 6.0.0
-  resolution: "lru-cache@npm:6.0.0"
-  dependencies:
-    yallist: ^4.0.0
-  checksum: f97f499f898f23e4585742138a22f22526254fdba6d75d41a1c2526b3b6cc5747ef59c5612ba7375f42aca4f8461950e925ba08c991ead0651b4918b7c978297
-  languageName: node
-  linkType: hard
-
 "lru-cache@npm:^7.14.1":
   version: 7.18.3
   resolution: "lru-cache@npm:7.18.3"