ci: add sharding for e2e tests

Author: hansl

.travis.yml

@@ -40,12 +40,20 @@ matrix:
       env: test
     - node_js: "6"
       os: linux
-      script: node tests/run_e2e.js "--glob=tests/build/**"
-      env: build
+      script: node tests/run_e2e.js "--ignore=**/tests/build/**" --nb-shards=4 --shard=0
+      env: e2e-0
+    - node_js: "6"
+      os: linux
+      script: node tests/run_e2e.js "--ignore=**/tests/build/**" --nb-shards=4 --shard=1
+      env: e2e-1
+    - node_js: "6"
+      os: linux
+      script: node tests/run_e2e.js "--ignore=**/tests/build/**" --nb-shards=4 --shard=2
+      env: e2e-2
     - node_js: "6"
       os: linux
-      script: travis_wait 120 node tests/run_e2e.js "--ignore=**/tests/build/**"
-      env: e2e
+      script: node tests/run_e2e.js "--ignore=**/tests/build/**" --nb-shards=4 --shard=3
+      env: e2e-3
     - node_js: "6"
       os: linux
       script: node tests/run_e2e.js --eject "--glob=tests/build/**"

tests/e2e_runner.ts

@@ -36,11 +36,15 @@ Error.stackTraceLimit = Infinity;
  *   --reuse=/path    Use a path instead of create a new project. That project should have been
  *                    created, and npm installed. Ideally you want a project created by a previous
  *                    run of e2e.
+ *   --nb-shards      Total number of shards that this is part of. Default is 2 if --shard is
+ *                    passed in.
+ *   --shard          Index of this processes' shard.
  * If unnamed flags are passed in, the list of tests will be filtered to include only those passed.
  */
 const argv = minimist(process.argv.slice(2), {
   'boolean': ['debug', 'nolink', 'nightly', 'noproject', 'verbose', 'eject', 'appveyor'],
-  'string': ['glob', 'ignore', 'reuse', 'ng-sha', ]
+  'string': ['glob', 'ignore', 'reuse', 'ng-sha', ],
+  'number': ['nb-shards', 'shard']
 });
 
 
@@ -73,7 +77,6 @@ ConsoleLoggerStack.start(new IndentLogger('name'))
 
 const testGlob = argv.glob || 'tests/**/*.ts';
 let currentFileName = null;
-let index = 0;
 
 const e2eRoot = path.join(__dirname, 'e2e');
 const allSetups = glob.sync(path.join(e2eRoot, 'setup/**/*.ts'), { nodir: true })
@@ -83,20 +86,26 @@ const allTests = glob.sync(path.join(e2eRoot, testGlob), { nodir: true, ignore:
   .map(name => path.relative(e2eRoot, name))
   .sort();
 
-const testsToRun = allSetups
-  .concat(allTests
-    .filter(name => {
-      // Check for naming tests on command line.
-      if (argv._.length == 0) {
-        return true;
-      }
+const shardId = ('shard' in argv) ? argv['shard'] : null;
+const nbShards = (shardId === null ? 1 : argv['nb-shards']) || 2;
+const tests = allTests
+  .filter(name => {
+    // Check for naming tests on command line.
+    if (argv._.length == 0) {
+      return true;
+    }
+
+    return argv._.some(argName => {
+      return path.join(process.cwd(), argName) == path.join(__dirname, 'e2e', name)
+        || argName == name
+        || argName == name.replace(/\.ts$/, '');
+    });
+  });
 
-      return argv._.some(argName => {
-        return path.join(process.cwd(), argName) == path.join(__dirname, 'e2e', name)
-          || argName == name
-          || argName == name.replace(/\.ts$/, '');
-      });
-    }));
+// Remove tests that are not part of this shard.
+const shardedTests = tests
+  .filter((name, i) => (shardId === null || (i % nbShards) == shardId));
+const testsToRun = allSetups.concat(shardedTests);
 
 
 /**
@@ -111,7 +120,7 @@ if (testsToRun.length == allTests.length) {
 
 setGlobalVariable('argv', argv);
 
-testsToRun.reduce((previous, relativeName) => {
+testsToRun.reduce((previous, relativeName, testIndex) => {
   // Make sure this is a windows compatible path.
   let absoluteName = path.join(e2eRoot, relativeName);
   if (/^win/.test(process.platform)) {
@@ -131,7 +140,7 @@ testsToRun.reduce((previous, relativeName) => {
     let clean = true;
     let previousDir = null;
     return Promise.resolve()
-      .then(() => printHeader(currentFileName))
+      .then(() => printHeader(currentFileName, testIndex))
       .then(() => previousDir = process.cwd())
       .then(() => ConsoleLoggerStack.push(currentFileName))
       .then(() => fn(() => clean = false))
@@ -196,9 +205,14 @@ function isTravis() {
   return process.env['TRAVIS'];
 }
 
-function printHeader(testName) {
-  const text = `${++index} of ${testsToRun.length}`;
-  console.log(green(`Running "${bold(blue(testName))}" (${bold(white(text))})...`));
+function printHeader(testName: string, testIndex: number) {
+  const text = `${testIndex + 1} of ${testsToRun.length}`;
+  const fullIndex = (testIndex < allSetups.length ? testIndex
+      : (testIndex - allSetups.length) * nbShards + shardId + allSetups.length) + 1;
+  const length = tests.length + allSetups.length;
+  const shard = shardId === null ? ''
+      : yellow(` [${shardId}:${nbShards}]` + bold(` (${fullIndex}/${length})`));
+  console.log(green(`Running "${bold(blue(testName))}" (${bold(white(text))}${shard})...`));
 
   if (isTravis()) {
     console.log(`travis_fold:start:${encode(testName)}`);