refactor(@angular-devkit/build-angular): improve performance of parallel bundle processing

clydin · vikerman · commit bdedff809069 · 2019-09-30T10:17:08.000-07:00
diff --git a/packages/angular_devkit/build_angular/package.json b/packages/angular_devkit/build_angular/package.json
@@ -27,6 +27,7 @@
     "find-cache-dir": "3.0.0",
     "glob": "7.1.4",
     "istanbul-instrumenter-loader": "3.0.1",
+    "jest-worker": "24.9.0",
     "karma-source-map-support": "1.4.0",
     "less": "3.9.0",
     "less-loader": "5.0.0",
@@ -61,7 +62,6 @@
     "webpack-merge": "4.2.1",
     "webpack-sources": "1.4.3",
     "webpack-subresource-integrity": "1.1.0-rc.6",
-    "worker-farm": "1.7.0",
     "worker-plugin": "3.2.0"
   },
   "devDependencies": {
diff --git a/packages/angular_devkit/build_angular/src/browser/action-executor.ts b/packages/angular_devkit/build_angular/src/browser/action-executor.ts
@@ -0,0 +1,53 @@
+/**
+ * @license
+ * Copyright Google Inc. All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+import JestWorker from 'jest-worker';
+import * as os from 'os';
+
+export class ActionExecutor<Input extends { size: number }, Output> {
+  private largeWorker: JestWorker;
+  private smallWorker: JestWorker;
+
+  private smallThreshold = 32 * 1024;
+
+  constructor(actionFile: string, private readonly actionName: string) {
+    // larger files are processed in a separate process to limit memory usage in the main process
+    this.largeWorker = new JestWorker(actionFile, {
+      exposedMethods: [actionName],
+    });
+
+    // small files are processed in a limited number of threads to improve speed
+    // The limited number also prevents a large increase in memory usage for an otherwise short operation
+    this.smallWorker = new JestWorker(actionFile, {
+      exposedMethods: [actionName],
+      numWorkers: os.cpus().length < 2 ? 1 : 2,
+      // Will automatically fallback to processes if not supported
+      enableWorkerThreads: true,
+    });
+  }
+
+  execute(options: Input): Promise<Output> {
+    if (options.size > this.smallThreshold) {
+      return ((this.largeWorker as unknown) as Record<string, (options: Input) => Promise<Output>>)[
+        this.actionName
+      ](options);
+    } else {
+      return ((this.smallWorker as unknown) as Record<string, (options: Input) => Promise<Output>>)[
+        this.actionName
+      ](options);
+    }
+  }
+
+  executeAll(options: Input[]): Promise<Output[]> {
+    return Promise.all(options.map(o => this.execute(o)));
+  }
+
+  stop() {
+    this.largeWorker.end();
+    this.smallWorker.end();
+  }
+}
diff --git a/packages/angular_devkit/build_angular/src/browser/index.ts b/packages/angular_devkit/build_angular/src/browser/index.ts
@@ -33,7 +33,6 @@ import { from, of } from 'rxjs';
 import { bufferCount, catchError, concatMap, map, mergeScan, switchMap } from 'rxjs/operators';
 import { ScriptTarget } from 'typescript';
 import * as webpack from 'webpack';
-import * as workerFarm from 'worker-farm';
 import { NgBuildAnalyticsPlugin } from '../../plugins/webpack/analytics';
 import { WebpackConfigOptions } from '../angular-cli-files/models/build-options';
 import {
@@ -80,6 +79,7 @@ import {
   getIndexInputFile,
   getIndexOutputFile,
 } from '../utils/webpack-browser-config';
+import { ActionExecutor } from './action-executor';
 import { Schema as BrowserBuilderSchema } from './schema';
 
 const cacache = require('cacache');
@@ -598,35 +598,25 @@ export function buildWebpackBrowser(
               }
 
               if (processActions.length > 0) {
-                await new Promise<void>((resolve, reject) => {
-                  const workerFile = require.resolve('../utils/process-bundle');
-                  const workers = workerFarm(
-                    {
-                      maxRetries: 1,
-                    },
-                    path.extname(workerFile) !== '.ts'
-                      ? workerFile
-                      : require.resolve('../utils/process-bundle-bootstrap'),
-                    ['process'],
-                  );
-                  let completed = 0;
-                  const workCallback = (error: Error | null, result: ProcessBundleResult) => {
-                    if (error) {
-                      workerFarm.end(workers);
-                      reject(error);
-
-                      return;
-                    }
-
-                    processResults.push(result);
-                    if (++completed === processActions.length) {
-                      workerFarm.end(workers);
-                      resolve();
-                    }
-                  };
+                const workerFile = require.resolve('../utils/process-bundle');
+                const executor = new ActionExecutor<
+                  ProcessBundleOptions & { size: number },
+                  ProcessBundleResult
+                >(
+                  path.extname(workerFile) !== '.ts'
+                    ? workerFile
+                    : require.resolve('../utils/process-bundle-bootstrap'),
+                  'process',
+                );
 
-                  processActions.forEach(action => workers['process'](action, workCallback));
-                });
+                try {
+                  const results = await executor.executeAll(
+                    processActions.map(a => ({ ...a, size: a.code.length })),
+                  );
+                  results.forEach(result => processResults.push(result));
+                } finally {
+                  executor.stop();
+                }
               }
 
               // Runtime must be processed after all other files
@@ -636,7 +626,7 @@ export function buildWebpackBrowser(
                   runtimeData: processResults,
                 };
                 processResults.push(
-                  await import('../utils/process-bundle').then(m => m.processAsync(runtimeOptions)),
+                  await import('../utils/process-bundle').then(m => m.process(runtimeOptions)),
                 );
               }
 
diff --git a/packages/angular_devkit/build_angular/src/utils/process-bundle.ts b/packages/angular_devkit/build_angular/src/utils/process-bundle.ts
@@ -57,14 +57,7 @@ export const enum CacheKey {
   DownlevelMap = 3,
 }
 
-export function process(
-  options: ProcessBundleOptions,
-  callback: (error: Error | null, result?: ProcessBundleResult) => void,
-): void {
-  processAsync(options).then(result => callback(null, result), error => callback(error));
-}
-
-export async function processAsync(options: ProcessBundleOptions): Promise<ProcessBundleResult> {
+export async function process(options: ProcessBundleOptions): Promise<ProcessBundleResult> {
   if (!options.cacheKeys) {
     options.cacheKeys = [];
   }
diff --git a/yarn.lock b/yarn.lock
@@ -6531,6 +6531,14 @@ jasminewd2@^2.1.0:
   resolved "https://registry.yarnpkg.com/jasminewd2/-/jasminewd2-2.2.0.tgz#e37cf0b17f199cce23bea71b2039395246b4ec4e"
   integrity sha1-43zwsX8ZnM4jvqcbIDk5Uka07E4=
 
+jest-worker@24.9.0:
+  version "24.9.0"
+  resolved "https://registry.yarnpkg.com/jest-worker/-/jest-worker-24.9.0.tgz#5dbfdb5b2d322e98567898238a9697bcce67b3e5"
+  integrity sha512-51PE4haMSXcHohnSMdM42anbvZANYTqMrr52tVKPqqsPJMzoP6FYYDVqahX/HrAoKEKz3uUPzSvKs9A3qR4iVw==
+  dependencies:
+    merge-stream "^2.0.0"
+    supports-color "^6.1.0"
+
 jquery@^3.3.1:
   version "3.3.1"
   resolved "https://registry.yarnpkg.com/jquery/-/jquery-3.3.1.tgz#958ce29e81c9790f31be7792df5d4d95fc57fbca"
@@ -7515,6 +7523,11 @@ merge-source-map@1.0.4:
   dependencies:
     source-map "^0.5.6"
 
+merge-stream@^2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/merge-stream/-/merge-stream-2.0.0.tgz#52823629a14dd00c9770fb6ad47dc6310f2c1f60"
+  integrity sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==
+
 methods@~1.1.2:
   version "1.1.2"
   resolved "https://registry.yarnpkg.com/methods/-/methods-1.1.2.tgz#5529a4d67654134edcc5266656835b0f851afcee"
@@ -12177,7 +12190,7 @@ wordwrap@~0.0.2:
   resolved "https://registry.yarnpkg.com/wordwrap/-/wordwrap-0.0.3.tgz#a3d5da6cd5c0bc0008d37234bbaf1bed63059107"
   integrity sha1-o9XabNXAvAAI03I0u68b7WMFkQc=
 
-worker-farm@1.7.0, worker-farm@^1.7.0:
+worker-farm@^1.7.0:
   version "1.7.0"
   resolved "https://registry.yarnpkg.com/worker-farm/-/worker-farm-1.7.0.tgz#26a94c5391bbca926152002f69b84a4bf772e5a8"
   integrity sha512-rvw3QTZc8lAxyVrqcSGVm5yP/IJ2UcB3U0graE3LCFoZ0Yn2x4EoVSqJKdB/T5M+FLcRPjz4TDacRf3OCfNUzw==
