Make the integration test fully-automatic. (#81)

This entails two major changes:
- In the Cloud Functions, keep track of which tests pass, and return 200 "PASS" when all tests pass, or 500 "FAIL - [debug-info]" when they don't.
- On the client-side, trigger the HTTP function and interpret the return code as a pass or fail.

Also tries to improve the readability of 'run_tests.sh' by indenting appropriately, and improve the usability by more clearly indicating whether tests have passed or failed.

Author: Robert-Jan Huijsman

integration_test/README.md

@@ -6,8 +6,8 @@ How to Use
 Run the integration test as follows:
 
 ```bash
-firebase use $YOURPROJECTID # add your own project
+firebase init # Don't choose to use any features, select your project.
 ./run_tests.sh
 ```
 
-Follow the instructions output by the script. You'll click on a number of HTTPS function links.  The integration test for HTTPS is that it properly kicks off other integration tests and redirects you to the database console. From there the other integration test suites will write their results back to the database, where you can check the results.
+The tests run fully automatically, and will print the result on standard out. The integration test for HTTPS is that it properly kicks off other integration tests and returns a result. From there the other integration test suites will write their results back to the database, where you can check the detailed results if you'd like.

integration_test/functions/src/index.ts

@@ -7,6 +7,7 @@ import { Request, Response } from 'express';
 export * from './pubsub-tests';
 export * from './database-tests';
 export * from './auth-tests';
+const numTests = Object.keys(exports).length;  // Assumption: every exported function is its own test.
 
 firebase.initializeApp(_.omit(functions.config().firebase, 'credential'));  // Explicitly decline admin privileges.
 admin.initializeApp(functions.config().firebase);
@@ -16,7 +17,7 @@ export const integrationTests: any = functions.https.onRequest((req: Request, re
 
   const testId = firebase.database().ref().push().key;
 
-  Promise.all([
+  return Promise.all([
     // A database write to trigger the Firebase Realtime Database tests.
     // The database write happens without admin privileges, so that the triggered function's "event.data.ref" also
     // doesn't have admin privileges.
@@ -33,8 +34,37 @@ export const integrationTests: any = functions.https.onRequest((req: Request, re
       admin.auth().deleteUser(userRecord.uid);
     }),
   ]).then(() => {
-    // On test completion, redirect the user to the Firebase RTDB dashboard, to
-    // see the results stored there.
-    resp.redirect(`https://${process.env.GCLOUD_PROJECT}.firebaseio.com/testRuns/${testId}`);
+    // On test completion, check that all tests pass and reply "PASS", or provide further details.
+    console.log('Waiting for all tests to report they pass...');
+    let ref = admin.database().ref(`testRuns/${testId}`);
+    return new Promise((resolve, reject) => {
+      let testsExecuted = 0;
+      ref.on('child_added', (snapshot) => {
+        testsExecuted += 1;
+        if (!snapshot.val().passed) {
+          reject(new Error(`test ${snapshot.key} failed; see database for details.`));
+          return;
+        }
+        console.log(`${snapshot.key} passed (${testsExecuted} of ${numTests})`);
+        if (testsExecuted < numTests) {
+          // Not all tests have completed. Wait longer.
+          return;
+        }
+        // All tests have passed!
+        resolve();
+      });
+    }).then(() => {
+      ref.off();  // No more need to listen.
+      return Promise.resolve();
+    }).catch(err => {
+      ref.off();  // No more need to listen.
+      return Promise.reject(err);
+    });
+  }).then(() => {
+    console.log('All tests pass!');
+    resp.status(200).send('PASS');
+  }).catch(err => {
+    console.log(`Some tests failed: ${err}`);
+    resp.status(500).send(`FAIL - details at https://${process.env.GCLOUD_PROJECT}.firebaseio.com/testRuns/${testId}`);
   });
 });

integration_test/functions/src/testing.ts

@@ -39,8 +39,9 @@ export class TestSuite {
       let sum = 0;
       results.forEach((val) => sum = sum + val.passed);
       const summary = `passed ${sum} of ${running.length}`;
+      const passed = sum === running.length;
       console.log(summary);
-      const result = { summary, tests: results };
+      const result = { passed, summary, tests: results };
       return firebase.database().ref(`testRuns/${testId}/${this.name}`).set(result);
     }).then(() => null);
   }

integration_test/run_tests.sh

@@ -1,28 +1,47 @@
 #!/bin/bash
-echo "#### Building SDK..." &&
-(cd ../ && rm firebase-functions-*.tgz && npm run build:pack &&
-mv firebase-functions-*.tgz integration_test/functions/firebase-functions.tgz) &&
-echo "#### Installing dependencies..." &&
-(cd functions && npm install) &&
-echo "##### Deploying empty index.js to project..." &&
-./functions/node_modules/.bin/tsc -p functions/ &&  # Make sure the functions/lib directory actually exists.
-echo "" > functions/lib/index.js &&
-firebase deploy --debug 2> ./deploy-debug.log &&
-echo &&
-echo "##### Project emptied. Deploying functions..." &&
-./functions/node_modules/.bin/tsc -p functions/ &&
-firebase deploy --only functions --debug 2> ./deploy-debug.log &&
-echo &&
-echo "##### Please click the 'integrationTests' link above!" &&
-read -n1 -r -p "##### Press [return] when you've confirmed the tests have passed." &&
-echo "##### Re-deploying the same functions, to make sure updates work..." &&
-firebase deploy --only functions --debug 2> ./deploy-debug.log &&
-echo &&
-echo "##### Please click the 'integrationTests' link above!" &&
-read -n1 -r -p "##### Press [return] when you've confirmed the tests have passed." &&
-echo "##### Removing all functions" &&
-echo "" > functions/lib/index.js &&
-firebase deploy --debug 2> ./deploy-debug.log &&
-rm functions/firebase-functions.tgz
-echo &&
-echo "##### Done! Please verify that your project is empty."
+(
+  echo "##### Building SDK..." &&
+  (
+    cd ../ &&
+    rm -f firebase-functions-*.tgz &&
+    npm run build:pack &&
+    mv firebase-functions-*.tgz integration_test/functions/firebase-functions.tgz
+  ) &&
+  echo "##### Installing dependencies..." &&
+  (
+    cd functions &&
+    npm install
+  ) &&
+  echo "##### Deploying empty index.js to project..." &&
+  ./functions/node_modules/.bin/tsc -p functions/ &&  # Make sure the functions/lib directory actually exists.
+  echo "" > functions/lib/index.js &&
+  firebase deploy --only functions --debug 2> /dev/null &&
+  echo &&
+  echo "##### Project emptied. Deploying functions..." &&
+  ./functions/node_modules/.bin/tsc -p functions/ &&
+  firebase deploy --only functions --debug 2> /dev/null &&
+  echo &&
+  echo "##### Running the integration tests..." &&
+  funcURL=$(tail -n 1 firebase-debug.log | awk '{ print $5 }') &&  # URL is printed on last line of log.
+  wget --content-on-error -qO- $funcURL &&
+  echo &&
+  echo "##### Integration test run passed!" &&
+  echo "##### Re-deploying the same functions, to make sure updates work..." &&
+  firebase deploy --only functions --debug 2> /dev/null &&
+  echo &&
+  echo "##### Running the integration tests..." &&
+  wget --content-on-error -qO- $funcURL &&
+  echo &&
+  echo "##### Integration test run passed!" &&
+  echo "##### Removing all functions" &&
+  echo "" > functions/lib/index.js &&
+  firebase deploy --only functions --debug 2> /dev/null &&
+  rm functions/firebase-functions.tgz &&
+  rm functions/firebase-debug.log &&
+  echo &&
+  echo "##### All tests pass!"
+) || (
+  echo &&
+  echo "XXXXX Something failed XXXXX" &&
+  false  # Finish with an error code.
+)