refactor(@angular-devkit/build-angular): use localize babel plugins directly

With recent improvements in the performance of babel parsing and AST traversal, the localize babel plugins can now be leveraged directly.

Author: clydin

packages/angular_devkit/build_angular/package.json

@@ -12,8 +12,8 @@
     "@angular-devkit/build-webpack": "0.0.0",
     "@angular-devkit/core": "0.0.0",
     "@babel/core": "7.8.4",
-    "@babel/generator": "7.8.4",
     "@babel/preset-env": "7.8.4",
+    "@babel/template": "7.8.3",
     "@ngtools/webpack": "0.0.0",
     "ajv": "6.11.0",
     "autoprefixer": "9.7.4",
@@ -35,7 +35,6 @@
     "less-loader": "5.0.0",
     "license-webpack-plugin": "2.1.3",
     "loader-utils": "1.2.3",
-    "magic-string": "0.25.6",
     "mini-css-extract-plugin": "0.9.0",
     "minimatch": "3.0.4",
     "parse5": "4.0.0",

packages/angular_devkit/build_angular/src/utils/process-bundle.ts

@@ -11,9 +11,10 @@ import {
   PluginObj,
   parseSync,
   transformAsync,
-  traverse,
+  transformFromAstSync,
   types,
 } from '@babel/core';
+import templateBuilder from '@babel/template';
 import { createHash } from 'crypto';
 import * as fs from 'fs';
 import * as path from 'path';
@@ -490,6 +491,59 @@ function createReplacePlugin(replacements: [string, string][]): PluginObj {
   };
 }
 
+async function createI18nPlugins(
+  locale: string,
+  translation: unknown | undefined,
+  missingTranslation: 'error' | 'warning' | 'ignore',
+  localeDataContent: string | undefined,
+) {
+  const plugins = [];
+  // tslint:disable-next-line: no-implicit-dependencies
+  const localizeDiag = await import('@angular/localize/src/tools/src/diagnostics');
+
+  const diagnostics = new localizeDiag.Diagnostics();
+
+  const es2015 = await import(
+    // tslint:disable-next-line: trailing-comma no-implicit-dependencies
+    '@angular/localize/src/tools/src/translate/source_files/es2015_translate_plugin'
+  );
+  plugins.push(
+    // tslint:disable-next-line: no-any
+    es2015.makeEs2015TranslatePlugin(diagnostics, (translation || {}) as any, {
+      missingTranslation: translation === undefined ? 'ignore' : missingTranslation,
+    }),
+  );
+
+  const es5 = await import(
+    // tslint:disable-next-line: trailing-comma no-implicit-dependencies
+    '@angular/localize/src/tools/src/translate/source_files/es5_translate_plugin'
+  );
+  plugins.push(
+    // tslint:disable-next-line: no-any
+    es5.makeEs5TranslatePlugin(diagnostics, (translation || {}) as any, {
+      missingTranslation: translation === undefined ? 'ignore' : missingTranslation,
+    }),
+  );
+
+  const inlineLocale = await import(
+    // tslint:disable-next-line: trailing-comma no-implicit-dependencies
+    '@angular/localize/src/tools/src/translate/source_files/locale_plugin'
+  );
+  plugins.push(inlineLocale.makeLocalePlugin(locale));
+
+  if (localeDataContent) {
+    plugins.push({
+      visitor: {
+        Program(path: NodePath<types.Program>) {
+          path.unshiftContainer('body', templateBuilder.ast(localeDataContent));
+        },
+      },
+    });
+  }
+
+  return { diagnostics, plugins };
+}
+
 export interface InlineOptions {
   filename: string;
   code: string;
@@ -500,13 +554,6 @@ export interface InlineOptions {
   setLocale?: boolean;
 }
 
-interface LocalizePosition {
-  start: number;
-  end: number;
-  messageParts: TemplateStringsArray;
-  expressions: types.Expression[];
-}
-
 const localizeName = '$localize';
 
 export async function inlineLocales(options: InlineOptions) {
@@ -522,88 +569,91 @@ export async function inlineLocales(options: InlineOptions) {
     return inlineCopyOnly(options);
   }
 
-  const { default: MagicString } = await import('magic-string');
-  const { default: generate } = await import('@babel/generator');
-  const utils = await import(
-    // tslint:disable-next-line: trailing-comma no-implicit-dependencies
-    '@angular/localize/src/tools/src/translate/source_files/source_file_utils'
-  );
-  // tslint:disable-next-line: no-implicit-dependencies
-  const localizeDiag = await import('@angular/localize/src/tools/src/diagnostics');
-
-  const diagnostics = new localizeDiag.Diagnostics();
+  let ast: ParseResult | undefined | null;
+  try {
+    ast = parseSync(options.code, {
+      babelrc: false,
+      configFile: false,
+      sourceType: 'script',
+      filename: options.filename,
+    });
+  } catch (error) {
+    if (error.message) {
+      // Make the error more readable.
+      // Same errors will contain the full content of the file as the error message
+      // Which makes it hard to find the actual error message.
+      const index = error.message.indexOf(')\n');
+      const msg = index !== -1 ? error.message.substr(0, index + 1) : error.message;
+      throw new Error(`${msg}\nAn error occurred inlining file "${options.filename}"`);
+    }
+  }
 
-  const positions = findLocalizePositions(options, utils);
-  if (positions.length === 0 && !options.setLocale) {
-    return inlineCopyOnly(options);
+  if (!ast) {
+    throw new Error(`Unknown error occurred inlining file "${options.filename}"`);
   }
 
-  // tslint:disable-next-line: no-any
-  let content = new MagicString(options.code, { filename: options.filename } as any);
+  const diagnostics = [];
   const inputMap = options.map && (JSON.parse(options.map) as RawSourceMap);
-  let contentClone;
   for (const locale of i18n.inlineLocales) {
     const isSourceLocale = locale === i18n.sourceLocale;
     // tslint:disable-next-line: no-any
     const translations: any = isSourceLocale ? {} : i18n.locales[locale].translation || {};
-    for (const position of positions) {
-      const translated = utils.translate(
-        diagnostics,
-        translations,
-        position.messageParts,
-        position.expressions,
-        isSourceLocale ? 'ignore' : options.missingTranslation || 'warning',
-      );
-
-      const expression = utils.buildLocalizeReplacement(translated[0], translated[1]);
-      const { code } = generate(expression);
-
-      content.overwrite(position.start, position.end, code);
-    }
-
+    let localeDataContent;
     if (options.setLocale) {
-      const setLocaleText = `var $localize=Object.assign(void 0===$localize?{}:$localize,{locale:"${locale}"});`;
-      contentClone = content.clone();
-      content.prepend(setLocaleText);
-
       // If locale data is provided, load it and prepend to file
-      const localeDataPath = i18n.locales[locale] && i18n.locales[locale].dataPath;
+      const localeDataPath = i18n.locales[locale]?.dataPath;
       if (localeDataPath) {
-        const localDataContent = await loadLocaleData(localeDataPath, true);
-        // The semicolon ensures that there is no syntax error between statements
-        content.prepend(localDataContent + ';');
+        localeDataContent = await loadLocaleData(localeDataPath, true);
       }
     }
 
-    const output = content.toString();
+    const { diagnostics: localeDiagnostics, plugins } = await createI18nPlugins(
+      locale,
+      translations,
+      isSourceLocale ? 'ignore' : options.missingTranslation || 'warning',
+      localeDataContent,
+    );
+    const transformResult = await transformFromAstSync(ast, options.code, {
+      filename: options.filename,
+      // using false ensures that babel will NOT search and process sourcemap comments (large memory usage)
+      // The types do not include the false option even though it is valid
+      // tslint:disable-next-line: no-any
+      inputSourceMap: false as any,
+      babelrc: false,
+      configFile: false,
+      plugins,
+      compact: !shouldBeautify,
+      sourceMaps: !!inputMap,
+    });
+
+    diagnostics.push(...localeDiagnostics.messages);
+
+    if (!transformResult || !transformResult.code) {
+      throw new Error(`Unknown error occurred processing bundle for "${options.filename}".`);
+    }
+
     const outputPath = path.join(
       options.outputPath,
       i18n.flatOutput ? '' : locale,
       options.filename,
     );
-    fs.writeFileSync(outputPath, output);
+    fs.writeFileSync(outputPath, transformResult.code);
 
-    if (inputMap) {
-      const contentMap = content.generateMap();
+    if (inputMap && transformResult.map) {
       const outputMap = mergeSourceMaps(
         options.code,
         inputMap,
-        output,
-        contentMap,
+        transformResult.code,
+        transformResult.map,
         options.filename,
         options.code.length > FAST_SOURCEMAP_THRESHOLD,
       );
 
       fs.writeFileSync(outputPath + '.map', JSON.stringify(outputMap));
     }
-
-    if (contentClone) {
-      content = contentClone;
-      contentClone = undefined;
-    }
   }
 
-  return { file: options.filename, diagnostics: diagnostics.messages, count: positions.length };
+  return { file: options.filename, diagnostics };
 }
 
 function inlineCopyOnly(options: InlineOptions) {
@@ -626,85 +676,6 @@ function inlineCopyOnly(options: InlineOptions) {
   return { file: options.filename, diagnostics: [], count: 0 };
 }
 
-function findLocalizePositions(
-  options: InlineOptions,
-  // tslint:disable-next-line: no-implicit-dependencies
-  utils: typeof import('@angular/localize/src/tools/src/translate/source_files/source_file_utils'),
-): LocalizePosition[] {
-  let ast: ParseResult | undefined | null;
-
-  try {
-    ast = parseSync(options.code, {
-      babelrc: false,
-      configFile: false,
-      sourceType: 'script',
-      filename: options.filename,
-    });
-  } catch (error) {
-    if (error.message) {
-      // Make the error more readable.
-      // Same errors will contain the full content of the file as the error message
-      // Which makes it hard to find the actual error message.
-      const index = error.message.indexOf(')\n');
-      const msg = index !== -1 ? error.message.substr(0, index + 1) : error.message;
-      throw new Error(`${msg}\nAn error occurred inlining file "${options.filename}"`);
-    }
-  }
-
-  if (!ast) {
-    throw new Error(`Unknown error occurred inlining file "${options.filename}"`);
-  }
-
-  const positions: LocalizePosition[] = [];
-  if (options.es5) {
-    traverse(ast, {
-      CallExpression(path: NodePath<types.CallExpression>) {
-        const callee = path.get('callee');
-        if (
-          callee.isIdentifier() &&
-          callee.node.name === localizeName &&
-          utils.isGlobalIdentifier(callee)
-        ) {
-          const messageParts = utils.unwrapMessagePartsFromLocalizeCall(path);
-          const expressions = utils.unwrapSubstitutionsFromLocalizeCall(path.node);
-          positions.push({
-            // tslint:disable-next-line: no-non-null-assertion
-            start: path.node.start!,
-            // tslint:disable-next-line: no-non-null-assertion
-            end: path.node.end!,
-            messageParts,
-            expressions,
-          });
-        }
-      },
-    });
-  } else {
-    const traverseFast = ((types as unknown) as {
-      traverseFast: (node: types.Node, enter: (node: types.Node) => void) => void;
-    }).traverseFast;
-
-    traverseFast(ast, node => {
-      if (
-        node.type === 'TaggedTemplateExpression' &&
-        types.isIdentifier(node.tag) &&
-        node.tag.name === localizeName
-      ) {
-        const messageParts = utils.unwrapMessagePartsFromTemplateLiteral(node.quasi.quasis);
-        positions.push({
-          // tslint:disable-next-line: no-non-null-assertion
-          start: node.start!,
-          // tslint:disable-next-line: no-non-null-assertion
-          end: node.end!,
-          messageParts,
-          expressions: node.quasi.expressions,
-        });
-      }
-    });
-  }
-
-  return positions;
-}
-
 async function loadLocaleData(path: string, optimize: boolean): Promise<string> {
   // The path is validated during option processing before the build starts
   const content = fs.readFileSync(path, 'utf8');

tests/legacy-cli/e2e/tests/i18n/ivy-localize-es2015.ts

@@ -20,7 +20,10 @@ export default async function() {
     await expectFileToMatch(`${outputPath}/main.js`, translation.helloPartial);
     await expectToFail(() => expectFileToMatch(`${outputPath}/main.js`, '$localize`'));
     await expectFileNotToExist(`${outputPath}/main-es5.js`);
-    await expectFileToMatch(`${outputPath}/main.js`, lang);
+
+    // Ensure locale is inlined (@angular/localize plugin inlines `$localize.locale` references)
+    // The only reference in a new application is in @angular/core
+    await expectFileToMatch(`${outputPath}/vendor.js`, lang);
 
     // Verify the HTML lang attribute is present
     await expectFileToMatch(`${outputPath}/index.html`, `lang="${lang}"`);

tests/legacy-cli/e2e/tests/i18n/ivy-localize-es5.ts

@@ -20,7 +20,10 @@ export default async function() {
     await expectFileToMatch(`${outputPath}/main.js`, translation.helloPartial);
     await expectToFail(() => expectFileToMatch(`${outputPath}/main.js`, '$localize`'));
     await expectFileNotToExist(`${outputPath}/main-es2015.js`);
-    await expectFileToMatch(`${outputPath}/main.js`, lang);
+
+    // Ensure locale is inlined (@angular/localize plugin inlines `$localize.locale` references)
+    // The only reference in a new application is in @angular/core
+    await expectFileToMatch(`${outputPath}/vendor.js`, lang);
 
     // Verify the HTML lang attribute is present
     await expectFileToMatch(`${outputPath}/index.html`, `lang="${lang}"`);

yarn.lock

@@ -151,7 +151,7 @@
     semver "^5.4.1"
     source-map "^0.5.0"
 
-"@babel/generator@7.8.4", "@babel/generator@^7.4.0", "@babel/generator@^7.8.3", "@babel/generator@^7.8.4":
+"@babel/generator@^7.4.0", "@babel/generator@^7.8.3", "@babel/generator@^7.8.4":
   version "7.8.4"
   resolved "https://registry.yarnpkg.com/@babel/generator/-/generator-7.8.4.tgz#35bbc74486956fe4251829f9f6c48330e8d0985e"
   integrity sha512-PwhclGdRpNAf3IxZb0YVuITPZmmrXz9zf6fH8lT4XbrmfQKr6ryBzhv593P5C6poJRciFCL/eHGW2NuGrgEyxA==
@@ -794,7 +794,7 @@
   dependencies:
     regenerator-runtime "^0.13.2"
 
-"@babel/template@^7.4.0", "@babel/template@^7.7.4", "@babel/template@^7.8.3":
+"@babel/template@7.8.3", "@babel/template@^7.4.0", "@babel/template@^7.7.4", "@babel/template@^7.8.3":
   version "7.8.3"
   resolved "https://registry.yarnpkg.com/@babel/template/-/template-7.8.3.tgz#e02ad04fe262a657809327f578056ca15fd4d1b8"
   integrity sha512-04m87AcQgAFdvuoyiQ2kgELr2tV8B4fP/xJAVUL3Yb3bkNdMedD3d0rlSQr3PegP0cms3eHjl1F7PWlvWbU8FQ==