build: use bazel to perform release builds

clydin · angular-robot[bot] · commit 7f93735e98e4 · 2023-01-05T03:21:58.000Z
When performing a release via the dev-infra `ng-dev` tooling, the release
builds for the packages that will be published are now performed using bazel.
Prior to this, the release builds were performed using a custom build script
that programmatically invoked TypeScript APIs. The Bazel build and discovery
process for the releasable packages is performed by a script that is based on
the scripts from components and framework repositories. Several small modifications
were performed to match the behavior and structure of the cli repository:
* Use of `packages` as the source root in the bazel query
* Use of `pkg_npm` rule in the bazel query
* Partial transition to native Node.js `fs` APIs instead of `shelljs`
* Directory creation per package when copying output (supports multiple package scopes)
* Copying of archives (tgz) for each package

The snapshot and local build capabilities are not modified as part of this change
but will be merged in a followup as part of a larger transition to use bazel
throughout the package build process.
diff --git a/.circleci/dynamic_config.yml b/.circleci/dynamic_config.yml
@@ -332,6 +332,7 @@ jobs:
 
   publish_artifacts:
     executor: action-executor
+    resource_class: medium
     environment:
     steps:
       - custom_attach_workspace
@@ -342,7 +343,7 @@ jobs:
           name: Copy tarballs to folder
           command: |
             mkdir -p dist/artifacts/
-            cp dist/*.tgz dist/artifacts/
+            cp dist/releases/*.tgz dist/artifacts/
       - store_artifacts:
           path: dist/artifacts/
           destination: angular
diff --git a/.ng-dev/release.mts b/.ng-dev/release.mts
@@ -1,8 +1,5 @@
-import '../lib/bootstrap-local.js';
-
 import { ReleaseConfig } from '@angular/ng-dev';
 import packages from '../lib/packages.js';
-import buildPackages from '../scripts/build.js';
 
 const npmPackages = Object.entries(packages.releasePackages).map(([name, { experimental }]) => ({
   name,
@@ -13,7 +10,12 @@ const npmPackages = Object.entries(packages.releasePackages).map(([name, { exper
 export const release: ReleaseConfig = {
   representativeNpmPackage: '@angular/cli',
   npmPackages,
-  buildPackages: () => buildPackages.default(),
+  buildPackages: async () => {
+    // The `performNpmReleaseBuild` function is loaded at runtime to avoid loading additional
+    // files and dependencies unless a build is required.
+    const { performNpmReleaseBuild } = await import('../scripts/build-packages-dist.mjs');
+    return performNpmReleaseBuild();
+  },
   releaseNotes: {
     groupOrder: [
       '@angular/cli',
diff --git a/package.json b/package.json
@@ -117,6 +117,7 @@
     "@types/progress": "^2.0.3",
     "@types/resolve": "^1.17.1",
     "@types/semver": "^7.3.12",
+    "@types/shelljs": "^0.8.11",
     "@types/tar": "^6.1.2",
     "@types/text-table": "^0.2.1",
     "@types/yargs": "^17.0.8",
diff --git a/packages/angular/cli/BUILD.bazel b/packages/angular/cli/BUILD.bazel
@@ -176,6 +176,7 @@ pkg_npm(
         "//packages/angular_devkit/schematics:package.json",
         "//packages/schematics/angular:package.json",
     ],
+    tags = ["release-package"],
     deps = [
         ":README.md",
         ":angular-cli",
diff --git a/packages/angular/create/BUILD.bazel b/packages/angular/create/BUILD.bazel
@@ -32,6 +32,7 @@ genrule(
 
 pkg_npm(
     name = "npm_package",
+    tags = ["release-package"],
     visibility = ["//visibility:public"],
     deps = [
         ":README.md",
diff --git a/packages/angular/pwa/BUILD.bazel b/packages/angular/pwa/BUILD.bazel
@@ -88,6 +88,7 @@ pkg_npm(
         "//packages/angular_devkit/schematics:package.json",
         "//packages/schematics/angular:package.json",
     ],
+    tags = ["release-package"],
     deps = [
         ":README.md",
         ":license",
diff --git a/packages/angular_devkit/architect/BUILD.bazel b/packages/angular_devkit/architect/BUILD.bazel
@@ -112,6 +112,7 @@ pkg_npm(
     pkg_deps = [
         "//packages/angular_devkit/core:package.json",
     ],
+    tags = ["release-package"],
     deps = [
         ":README.md",
         ":architect",
diff --git a/packages/angular_devkit/architect/node/BUILD.bazel b/packages/angular_devkit/architect/node/BUILD.bazel
@@ -23,7 +23,6 @@ ts_library(
         "//packages/angular_devkit/architect",
         "//packages/angular_devkit/core",
         "//packages/angular_devkit/core/node",
-        "//tests/angular_devkit/architect/node/jobs:jobs_test_lib",
         "@npm//@types/node",
         "@npm//rxjs",
     ],
@@ -40,6 +39,7 @@ ts_library(
     deps = [
         ":node",
         "//packages/angular_devkit/architect",
+        "//tests/angular_devkit/architect/node/jobs:jobs_test_lib",
     ],
 )
 
diff --git a/packages/angular_devkit/architect_cli/BUILD.bazel b/packages/angular_devkit/architect_cli/BUILD.bazel
@@ -40,6 +40,7 @@ pkg_npm(
         "//packages/angular_devkit/architect:package.json",
         "//packages/angular_devkit/core:package.json",
     ],
+    tags = ["release-package"],
     deps = [
         ":README.md",
         ":architect_cli",
diff --git a/packages/angular_devkit/build_angular/BUILD.bazel b/packages/angular_devkit/build_angular/BUILD.bazel
@@ -233,6 +233,7 @@ pkg_npm(
         "//packages/angular_devkit/core:package.json",
         "//packages/ngtools/webpack:package.json",
     ],
+    tags = ["release-package"],
     deps = [
         ":README.md",
         ":build_angular",
diff --git a/packages/angular_devkit/build_webpack/BUILD.bazel b/packages/angular_devkit/build_webpack/BUILD.bazel
@@ -120,6 +120,7 @@ pkg_npm(
     pkg_deps = [
         "//packages/angular_devkit/architect:package.json",
     ],
+    tags = ["release-package"],
     deps = [
         ":README.md",
         ":build_webpack",
diff --git a/packages/angular_devkit/core/BUILD.bazel b/packages/angular_devkit/core/BUILD.bazel
@@ -83,6 +83,7 @@ genrule(
 
 pkg_npm(
     name = "npm_package",
+    tags = ["release-package"],
     deps = [
         ":README.md",
         ":core",
diff --git a/packages/angular_devkit/schematics/BUILD.bazel b/packages/angular_devkit/schematics/BUILD.bazel
@@ -86,6 +86,7 @@ pkg_npm(
     pkg_deps = [
         "//packages/angular_devkit/core:package.json",
     ],
+    tags = ["release-package"],
     deps = [
         ":README.md",
         ":collection-schema.json",
diff --git a/packages/angular_devkit/schematics_cli/BUILD.bazel b/packages/angular_devkit/schematics_cli/BUILD.bazel
@@ -110,6 +110,7 @@ pkg_npm(
         "//packages/angular_devkit/schematics:package.json",
         "//packages/angular_devkit/core:package.json",
     ],
+    tags = ["release-package"],
     deps = [
         ":README.md",
         ":license",
diff --git a/packages/ngtools/webpack/BUILD.bazel b/packages/ngtools/webpack/BUILD.bazel
@@ -85,6 +85,7 @@ genrule(
 
 pkg_npm(
     name = "npm_package",
+    tags = ["release-package"],
     deps = [
         ":README.md",
         ":license",
diff --git a/packages/schematics/angular/BUILD.bazel b/packages/schematics/angular/BUILD.bazel
@@ -166,6 +166,7 @@ pkg_npm(
         "//packages/angular_devkit/schematics:package.json",
         "//packages/angular_devkit/core:package.json",
     ],
+    tags = ["release-package"],
     deps = [
         ":README.md",
         ":angular",
diff --git a/scripts/build-packages-dist.mts b/scripts/build-packages-dist.mts
@@ -0,0 +1,168 @@
+/**
+ * @license
+ * Copyright Google LLC All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+
+/**
+ * Script that builds the release output of all packages which have the "release-package
+ * Bazel tag set. The script builds all those packages and copies the release output to the
+ * distribution folder within the project.
+ */
+
+import { BuiltPackage } from '@angular/ng-dev';
+import { execSync } from 'node:child_process';
+import { chmodSync, copyFileSync, mkdirSync, rmSync } from 'node:fs';
+import { dirname, join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import sh from 'shelljs';
+
+/** Name of the Bazel tag that will be used to find release package targets. */
+const releaseTargetTag = 'release-package';
+
+/** Path to the project directory. */
+const projectDir = join(dirname(fileURLToPath(import.meta.url)), '../');
+
+/** Command that runs Bazel. */
+const bazelCmd = process.env.BAZEL || `yarn -s bazel`;
+
+/** Command that queries Bazel for all release package targets. */
+const queryPackagesCmd =
+  `${bazelCmd} query --output=label "attr('tags', '\\[.*${releaseTargetTag}.*\\]', //packages/...) ` +
+  `intersect kind('pkg_npm', //packages/...)"`;
+
+/** Path for the default distribution output directory. */
+const defaultDistPath = join(projectDir, 'dist/releases');
+
+/** Builds the release packages for NPM. */
+export function performNpmReleaseBuild(): BuiltPackage[] {
+  return buildReleasePackages(defaultDistPath, /* isSnapshotBuild */ false);
+}
+
+/**
+ * Builds the release packages as snapshot build. This means that the current
+ * Git HEAD SHA is included in the version (for easier debugging and back tracing).
+ */
+export function performDefaultSnapshotBuild(): BuiltPackage[] {
+  return buildReleasePackages(defaultDistPath, /* isSnapshotBuild */ true);
+}
+
+/**
+ * Builds the release packages with the given compile mode and copies
+ * the package output into the given directory.
+ */
+function buildReleasePackages(distPath: string, isSnapshotBuild: boolean): BuiltPackage[] {
+  console.log('######################################');
+  console.log('  Building release packages...');
+  console.log('######################################');
+
+  // List of targets to build. e.g. "packages/angular/cli:npm_package"
+  const targets = exec(queryPackagesCmd, true).split(/\r?\n/);
+  const packageNames = getPackageNamesOfTargets(targets);
+  const bazelBinPath = exec(`${bazelCmd} info bazel-bin`, true);
+  const getBazelOutputPath = (pkgName: string) =>
+    join(bazelBinPath, 'packages', pkgName, 'npm_package');
+  const getDistPath = (pkgName: string) => join(distPath, pkgName);
+
+  // Build with "--config=release" or `--config=snapshot` so that Bazel
+  // runs the workspace stamping script. The stamping script ensures that the
+  // version placeholder is populated in the release output.
+  const stampConfigArg = `--config=${isSnapshotBuild ? 'snapshot' : 'release'}`;
+
+  // Walk through each release package and clear previous "npm_package" outputs. This is
+  // a workaround for: https://github.com/bazelbuild/rules_nodejs/issues/1219. We need to
+  // do this to ensure that the version placeholders are properly populated.
+  packageNames.forEach((pkgName) => {
+    // Directory output is created by the npm_package target
+    const directoryOutputPath = getBazelOutputPath(pkgName);
+    // Archive output is created by the npm_package_archive target
+    const archiveOutputPath = directoryOutputPath + '_archive.tgz';
+
+    if (sh.test('-d', directoryOutputPath)) {
+      sh.chmod('-R', 'u+w', directoryOutputPath);
+      sh.rm('-rf', directoryOutputPath);
+    }
+    try {
+      chmodSync(archiveOutputPath, '0755');
+      rmSync(archiveOutputPath, { force: true });
+    } catch {}
+  });
+
+  // Build both the npm_package and npm_package_archive targets for each package
+  // TODO: Consider switching to only using the archive for publishing
+  const buildTargets = targets.flatMap((target) => [target, target + '_archive']);
+  exec(`${bazelCmd} build ${stampConfigArg} ${buildTargets.join(' ')}`);
+
+  // Delete the distribution directory so that the output is guaranteed to be clean. Re-create
+  // the empty directory so that we can copy the release packages into it later.
+  rmSync(distPath, { force: true, recursive: true, maxRetries: 3 });
+  mkdirSync(distPath, { recursive: true });
+
+  // Copy the package output into the specified distribution folder.
+  packageNames.forEach((pkgName) => {
+    // Directory output is created by the npm_package target
+    const directoryOutputPath = getBazelOutputPath(pkgName);
+    // Archive output is created by the npm_package_archive target
+    const archiveOutputPath = directoryOutputPath + '_archive.tgz';
+
+    const targetFolder = getDistPath(pkgName);
+    console.log(`> Copying package output to "${targetFolder}"`);
+
+    // Ensure package scope directory exists prior to copying
+    mkdirSync(dirname(targetFolder), { recursive: true });
+
+    // Copy package contents to target directory
+    sh.cp('-R', directoryOutputPath, targetFolder);
+    sh.chmod('-R', 'u+w', targetFolder);
+
+    // Copy archive of package to target directory
+    const archiveTargetPath = join(distPath, `${pkgName.replace('/', '_')}.tgz`);
+    copyFileSync(archiveOutputPath, archiveTargetPath);
+    chmodSync(archiveTargetPath, '0755');
+  });
+
+  return packageNames.map((pkg) => {
+    return {
+      // Package names on disk do not have the @ scope prefix and use underscores instead of dashes
+      name: `@${pkg.replace(/_/g, '-')}`,
+      outputPath: getDistPath(pkg),
+    };
+  });
+}
+
+/**
+ * Gets the package names of the specified Bazel targets.
+ * e.g. //packages/angular/cli:npm_package = angular/cli
+ */
+function getPackageNamesOfTargets(targets: string[]): string[] {
+  return targets.map((targetName) => {
+    const matches = targetName.match(/\/\/packages\/(.*?):/);
+    if (matches === null) {
+      throw Error(
+        `Found Bazel target with "${releaseTargetTag}" tag, but could not ` +
+          `determine release output name: ${targetName}`,
+      );
+    }
+
+    return matches[1];
+  });
+}
+
+/** Executes the given command in the project directory. */
+function exec(command: string): void;
+/** Executes the given command in the project directory and returns its stdout. */
+function exec(command: string, captureStdout: true): string;
+function exec(command: string, captureStdout?: true) {
+  const stdout = execSync(command, {
+    cwd: projectDir,
+    stdio: ['inherit', captureStdout ? 'pipe' : 'inherit', 'inherit'],
+  });
+
+  if (captureStdout) {
+    process.stdout.write(stdout);
+
+    return stdout.toString().trim();
+  }
+}
diff --git a/tsconfig-build.json b/tsconfig-build.json
@@ -21,6 +21,7 @@
     "tests/**/*",
     "tools/**/*",
     ".ng-dev/**/*",
-    "**/*_spec.ts"
+    "**/*_spec.ts",
+    "scripts/**/*.mts"
   ]
 }
diff --git a/yarn.lock b/yarn.lock
@@ -3154,7 +3154,7 @@
     "@types/qs" "*"
     "@types/serve-static" "*"
 
-"@types/glob@^8.0.0":
+"@types/glob@*", "@types/glob@^8.0.0":
   version "8.0.0"
   resolved "https://registry.yarnpkg.com/@types/glob/-/glob-8.0.0.tgz#321607e9cbaec54f687a0792b2d1d370739455d2"
   integrity sha512-l6NQsDDyQUVeoTynNpC9uRvCUint/gSUXQA2euwmTuWGvPY5LSDUu6tkCtJB2SvGQlJQzLaKqcGZP4//7EDveA==
@@ -3422,6 +3422,14 @@
     "@types/mime" "*"
     "@types/node" "*"
 
+"@types/shelljs@^0.8.11":
+  version "0.8.11"
+  resolved "https://registry.yarnpkg.com/@types/shelljs/-/shelljs-0.8.11.tgz#17a5696c825974e96828e96e89585d685646fcb8"
+  integrity sha512-x9yaMvEh5BEaZKeVQC4vp3l+QoFj3BXcd4aYfuKSzIIyihjdVARAadYy3SMNIz0WCCdS2vB9JL/U6GQk5PaxQw==
+  dependencies:
+    "@types/glob" "*"
+    "@types/node" "*"
+
 "@types/sockjs@^0.3.33":
   version "0.3.33"
   resolved "https://registry.yarnpkg.com/@types/sockjs/-/sockjs-0.3.33.tgz#570d3a0b99ac995360e3136fd6045113b1bd236f"
