refactor(@angular-devkit/build-angular): cache downlevel bundles

Author: clydin

package.json

@@ -92,6 +92,7 @@
     "@types/clean-css": "^4.2.1",
     "@types/copy-webpack-plugin": "^4.4.1",
     "@types/express": "^4.16.0",
+    "@types/find-cache-dir": "^2.0.0",
     "@types/glob": "^7.0.0",
     "@types/inquirer": "^0.0.44",
     "@types/jasmine": "~3.4.0",

packages/angular_devkit/build_angular/package.json

@@ -17,12 +17,14 @@
     "ajv": "6.10.2",
     "autoprefixer": "9.6.1",
     "browserslist": "4.6.6",
+    "cacache": "12.0.2",
     "caniuse-lite": "1.0.30000989",
     "circular-dependency-plugin": "5.2.0",
     "clean-css": "4.2.1",
     "copy-webpack-plugin": "5.0.4",
     "core-js": "3.2.1",
     "file-loader": "4.2.0",
+    "find-cache-dir": "3.0.0",
     "glob": "7.1.4",
     "istanbul-instrumenter-loader": "3.0.1",
     "karma-source-map-support": "1.4.0",

packages/angular_devkit/build_angular/src/browser/index.ts

@@ -24,7 +24,10 @@ import {
   virtualFs,
 } from '@angular-devkit/core';
 import { NodeJsSyncHost } from '@angular-devkit/core/node';
+import { createHash } from 'crypto';
+import * as findCacheDirectory from 'find-cache-dir';
 import * as fs from 'fs';
+import * as os from 'os';
 import * as path from 'path';
 import { from, of } from 'rxjs';
 import { bufferCount, catchError, concatMap, map, mergeScan, switchMap } from 'rxjs/operators';
@@ -62,6 +65,7 @@ import {
   normalizeOptimization,
   normalizeSourceMaps,
 } from '../utils';
+import { CacheKey, ProcessBundleOptions } from '../utils/process-bundle';
 import { assertCompatibleAngularVersion } from '../utils/version';
 import {
   generateBrowserWebpackConfigFromContext,
@@ -70,6 +74,10 @@ import {
 } from '../utils/webpack-browser-config';
 import { Schema as BrowserBuilderSchema } from './schema';
 
+const cacache = require('cacache');
+const cacheDownlevelPath = findCacheDirectory({ name: 'angular-build-dl' });
+const packageVersion = require('../../package.json').version;
+
 export type BrowserBuilderOutput = json.JsonObject &
   BuilderOutput & {
     outputPath: string;
@@ -240,6 +248,7 @@ export function buildWebpackBrowser(
           1,
         ),
         bufferCount(configs.length),
+        // tslint:disable-next-line: no-big-function
         switchMap(async buildEvents => {
           configs.length = 0;
           const success = buildEvents.every(r => r.success);
@@ -274,9 +283,10 @@ export function buildWebpackBrowser(
                 optimize: normalizeOptimization(options.optimization).scripts,
                 sourceMaps: sourceMapOptions.scripts,
                 hiddenSourceMaps: sourceMapOptions.hidden,
+                vendorSourceMaps: sourceMapOptions.vendor,
               };
 
-              const actions: {}[] = [];
+              const actions: ProcessBundleOptions[] = [];
               const seen = new Set<string>();
               for (const file of emittedFiles) {
                 // Scripts and non-javascript files are not processed
@@ -348,6 +358,7 @@ export function buildWebpackBrowser(
                   code,
                   map,
                   runtime: file.file.startsWith('runtime'),
+                  ignoreOriginal: es5Polyfills,
                 });
 
                 // Add the newly created ES5 bundles to the index as nomodule scripts
@@ -359,30 +370,133 @@ export function buildWebpackBrowser(
 
               // Execute the bundle processing actions
               context.logger.info('Generating ES5 bundles for differential loading...');
-              await new Promise<void>((resolve, reject) => {
-                const workerFile = require.resolve('../utils/process-bundle');
-                const workers = workerFarm(
-                  {
-                    maxRetries: 1,
-                  },
-                  path.extname(workerFile) !== '.ts'
-                    ? workerFile
-                    : require.resolve('../utils/process-bundle-bootstrap'),
-                  ['process'],
-                );
-                let completed = 0;
-                const workCallback = (error: Error | null) => {
-                  if (error) {
-                    workerFarm.end(workers);
-                    reject(error);
-                  } else if (++completed === actions.length) {
-                    workerFarm.end(workers);
-                    resolve();
+
+              const processActions: typeof actions = [];
+              const cacheActions: { src: string; dest: string }[] = [];
+              for (const action of actions) {
+                // Create base cache key with elements:
+                // * package version - different build-angular versions cause different final outputs
+                // * code length/hash - ensure cached version matches the same input code
+                const codeHash = createHash('sha1')
+                  .update(action.code)
+                  .digest('hex');
+                const baseCacheKey = `${packageVersion}|${action.code.length}|${codeHash}`;
+
+                // Postfix added to sourcemap cache keys when vendor sourcemaps are present
+                // Allows non-destructive caching of both variants
+                const SourceMapVendorPostfix =
+                  !!action.sourceMaps && action.vendorSourceMaps ? '|vendor' : '';
+
+                // Determine cache entries required based on build settings
+                const cacheKeys = [];
+
+                // If optimizing and the original is not ignored, add original as required
+                if ((action.optimize || action.optimizeOnly) && !action.ignoreOriginal) {
+                  cacheKeys[CacheKey.OriginalCode] = baseCacheKey + '|orig';
+
+                  // If sourcemaps are enabled, add original sourcemap as required
+                  if (action.sourceMaps) {
+                    cacheKeys[CacheKey.OriginalMap] =
+                      baseCacheKey + SourceMapVendorPostfix + '|orig-map';
+                  }
+                }
+                // If not only optimizing, add downlevel as required
+                if (!action.optimizeOnly) {
+                  cacheKeys[CacheKey.DownlevelCode] = baseCacheKey + '|dl';
+
+                  // If sourcemaps are enabled, add downlevel sourcemap as required
+                  if (action.sourceMaps) {
+                    cacheKeys[CacheKey.DownlevelMap] =
+                      baseCacheKey + SourceMapVendorPostfix + '|dl-map';
+                  }
+                }
+
+                // Attempt to get required cache entries
+                const cacheEntries = [];
+                for (const key of cacheKeys) {
+                  if (key) {
+                    cacheEntries.push(await cacache.get.info(cacheDownlevelPath, key));
+                  } else {
+                    cacheEntries.push(null);
+                  }
+                }
+
+                // Check if required cache entries are present
+                let cached = cacheKeys.length > 0;
+                for (let i = 0; i < cacheKeys.length; ++i) {
+                  if (cacheKeys[i] && !cacheEntries[i]) {
+                    cached = false;
+                    break;
+                  }
+                }
+
+                // If all required cached entries are present, use the cached entries
+                // Otherwise process the files
+                if (cached) {
+                  if (cacheEntries[CacheKey.OriginalCode]) {
+                    cacheActions.push({
+                      src: cacheEntries[CacheKey.OriginalCode].path,
+                      dest: action.filename,
+                    });
                   }
-                };
+                  if (cacheEntries[CacheKey.OriginalMap]) {
+                    cacheActions.push({
+                      src: cacheEntries[CacheKey.OriginalMap].path,
+                      dest: action.filename + '.map',
+                    });
+                  }
+                  if (cacheEntries[CacheKey.DownlevelCode]) {
+                    cacheActions.push({
+                      src: cacheEntries[CacheKey.DownlevelCode].path,
+                      dest: action.filename.replace('es2015', 'es5'),
+                    });
+                  }
+                  if (cacheEntries[CacheKey.DownlevelMap]) {
+                    cacheActions.push({
+                      src: cacheEntries[CacheKey.DownlevelMap].path,
+                      dest: action.filename.replace('es2015', 'es5') + '.map',
+                    });
+                  }
+                } else {
+                  processActions.push({
+                    ...action,
+                    cacheKeys,
+                    cachePath: cacheDownlevelPath || undefined,
+                  });
+                }
+              }
+
+              for (const action of cacheActions) {
+                fs.copyFileSync(action.src, action.dest, fs.constants.COPYFILE_FICLONE);
+              }
+
+              if (processActions.length > 0) {
+                await new Promise<void>((resolve, reject) => {
+                  const workerFile = require.resolve('../utils/process-bundle');
+                  const workers = workerFarm(
+                    {
+                      maxRetries: 1,
+                    },
+                    path.extname(workerFile) !== '.ts'
+                      ? workerFile
+                      : require.resolve('../utils/process-bundle-bootstrap'),
+                    ['process'],
+                  );
+                  let completed = 0;
+                  const workCallback = (error: Error | null) => {
+                    if (error) {
+                      workerFarm.end(workers);
+                      reject(error);
+                    } else if (++completed === processActions.length) {
+                      workerFarm.end(workers);
+                      resolve();
+                    }
+                  };
+
+                  processActions.forEach(action => workers['process'](action, workCallback));
+                });
+              }
 
-                actions.forEach(action => workers['process'](action, workCallback));
-              });
               context.logger.info('ES5 bundle generation complete.');
             } else {
               const { emittedFiles = [] } = firstBuild;

packages/angular_devkit/build_angular/src/utils/process-bundle.ts

@@ -11,16 +11,28 @@ import { SourceMapConsumer, SourceMapGenerator } from 'source-map';
 import { minify } from 'terser';
 
 const { transformAsync } = require('@babel/core');
+const cacache = require('cacache');
 
-interface ProcessBundleOptions {
+export interface ProcessBundleOptions {
   filename: string;
   code: string;
   map?: string;
-  sourceMaps: boolean;
-  hiddenSourceMaps: boolean;
-  runtime: boolean;
+  sourceMaps?: boolean;
+  hiddenSourceMaps?: boolean;
+  vendorSourceMaps?: boolean;
+  runtime?: boolean;
   optimize: boolean;
   optimizeOnly?: boolean;
+  ignoreOriginal?: boolean;
+  cacheKeys?: (string | null)[];
+  cachePath?: string;
+}
+
+export const enum CacheKey {
+  OriginalCode = 0,
+  OriginalMap = 1,
+  DownlevelCode = 2,
+  DownlevelMap = 3,
 }
 
 export function process(
@@ -31,6 +43,10 @@ export function process(
 }
 
 async function processWorker(options: ProcessBundleOptions): Promise<void> {
+  if (!options.cacheKeys) {
+    options.cacheKeys = [];
+  }
+
   // If no downlevelling required than just mangle code and return
   if (options.optimizeOnly) {
     return mangleOriginal(options);
@@ -139,7 +155,9 @@ async function processWorker(options: ProcessBundleOptions): Promise<void> {
     map = result.map;
 
     // Mangle original code
-    mangleOriginal(options);
+    if (!options.ignoreOriginal) {
+      await mangleOriginal(options);
+    }
   } else if (map) {
     map = JSON.stringify(map);
   }
@@ -149,13 +167,20 @@ async function processWorker(options: ProcessBundleOptions): Promise<void> {
       code += `\n//# sourceMappingURL=${path.basename(newFilePath)}.map`;
     }
 
+    if (options.cachePath && options.cacheKeys[CacheKey.DownlevelMap]) {
+      await cacache.put(options.cachePath, options.cacheKeys[CacheKey.DownlevelMap], map);
+    }
+
     fs.writeFileSync(newFilePath + '.map', map);
   }
 
+  if (options.cachePath && options.cacheKeys[CacheKey.DownlevelCode]) {
+    await cacache.put(options.cachePath, options.cacheKeys[CacheKey.DownlevelCode], code);
+  }
   fs.writeFileSync(newFilePath, code);
 }
 
-function mangleOriginal(options: ProcessBundleOptions): void {
+async function mangleOriginal(options: ProcessBundleOptions): Promise<void> {
   const resultOriginal = minify(options.code, {
     compress: false,
     ecma: 6,
@@ -176,8 +201,25 @@ function mangleOriginal(options: ProcessBundleOptions): void {
     throw resultOriginal.error;
   }
 
+  if (options.cachePath && options.cacheKeys && options.cacheKeys[CacheKey.OriginalCode]) {
+    await cacache.put(
+      options.cachePath,
+      options.cacheKeys[CacheKey.OriginalCode],
+      resultOriginal.code,
+    );
+  }
+
   fs.writeFileSync(options.filename, resultOriginal.code);
+
   if (resultOriginal.map) {
+    if (options.cachePath && options.cacheKeys && options.cacheKeys[CacheKey.OriginalMap]) {
+      await cacache.put(
+        options.cachePath,
+        options.cacheKeys[CacheKey.OriginalMap],
+        resultOriginal.map,
+      );
+    }
+
     fs.writeFileSync(options.filename + '.map', resultOriginal.map);
   }
 }