Add initial benchmarking support.

Author: davidlehn

CHANGELOG.md

@@ -1,5 +1,8 @@
 # jsonld ChangeLog
 
+### Added
+- Initial support for benchmarking.
+
 ## 1.1.0 - 2018-09-05
 
 ### Added

README.md

@@ -363,6 +363,14 @@ To generate earl reports:
     # generate the earl report for the browser
     EARL=earl-firefox.jsonld npm run test-karma -- --browser Firefox
 
+Benchmarks
+----------
+
+Benchmarks can be created from any manifest that the test system supports.
+Use a command line with a test suite and a benchmark flag:
+
+    JSONLD_TESTS=/tmp/benchmark-manifest.jsonld JSONLD_BENCHMARK=1 npm test
+
 [Digital Bazaar]: http://digitalbazaar.com/
 [JSON-LD]: http://json-ld.org/
 [JSON-LD 1.0]: http://www.w3.org/TR/2014/REC-json-ld-20140116/

karma.conf.js

@@ -65,6 +65,8 @@ module.exports = function(config) {
       plugins: [
         new webpack.DefinePlugin({
           'process.env.JSONLD_TESTS': JSON.stringify(process.env.JSONLD_TESTS),
+          'process.env.JSONLD_BENCHMARK':
+            JSON.stringify(process.env.JSONLD_BENCHMARK),
           'process.env.TEST_ROOT_DIR': JSON.stringify(__dirname),
           'process.env.EARL': JSON.stringify(process.env.EARL),
           'process.env.BAIL': JSON.stringify(process.env.BAIL)

package.json

@@ -42,6 +42,7 @@
     "babel-plugin-transform-object-rest-spread": "^6.26.0",
     "babel-preset-env": "^1.6.1",
     "babel-preset-node6-es6": "^11.2.5",
+    "benchmark": "^2.1.4",
     "browserify": "^15.2.0",
     "chai": "^4.1.2",
     "commander": "^2.14.1",

tests/test-common.js

@@ -7,6 +7,7 @@
  * Copyright (c) 2011-2017 Digital Bazaar, Inc. All rights reserved.
  */
 const EarlReport = require('./earl-report');
+const benchmark = require('benchmark');
 const join = require('join-path-js');
 
 module.exports = function(options) {
@@ -269,14 +270,22 @@ function addTest(manifest, test, tests) {
   test.manifest = manifest;
   const description = test_id + ' ' + (test.purpose || test.name);
 
-  tests.push({
-    title: description + ' (promise)',
-    f: makeFn({useCallbacks: false})
-  });
-  tests.push({
-    title: description + ' (callback)',
-    f: makeFn({useCallbacks: true})
-  });
+  if(options.benchmark) {
+    // only promises
+    tests.push({
+      title: description,
+      f: makeFn({useCallbacks: false})
+    });
+  } else {
+    tests.push({
+      title: description + ' (promise)',
+      f: makeFn({useCallbacks: false})
+    });
+    tests.push({
+      title: description + ' (callback)',
+      f: makeFn({useCallbacks: true})
+    });
+  }
 
   function makeFn({useCallbacks}) {
     return function(done) {
@@ -340,8 +349,7 @@ function addTest(manifest, test, tests) {
       });
 
       const fn = testInfo.fn;
-      let params = testInfo.params;
-      params = params.map(function(param) {return param(test);});
+      const params = testInfo.params.map(param => param(test));
       const callback = function(err, result) {
         Promise.resolve().then(() => {
           if(isNegativeTest(test)) {
@@ -353,6 +361,43 @@ function addTest(manifest, test, tests) {
             }
             return testInfo.compare(test, result);
           }
+        }).then(() => {
+          if(options.benchmark) {
+            // pre-load params to avoid doc loader and parser timing
+            const benchParams = testInfo.params.map(param => param(test, {
+              load: true
+            }));
+            return Promise.all(benchParams);
+          }
+        }).then(values => {
+          if(options.benchmark) {
+            return new Promise((resolve, reject) => {
+              const suite = new benchmark.Suite();
+              suite.add({
+                name: test.name,
+                defer: true,
+                fn: deferred => {
+                  jsonld[fn].apply(null, values).then(() => {
+                    deferred.resolve();
+                  });
+                }
+              });
+              suite
+                .on('start', e => {
+                  self.timeout((e.target.maxTime + 2) * 1000);
+                })
+                .on('cycle', e => {
+                  console.log(String(e.target));
+                })
+                .on('error', err => {
+                  reject(new Error(err));
+                })
+                .on('complete', e => {
+                  resolve();
+                })
+                .run({async: true});
+            });
+          }
         }).then(() => {
           if(options.earl.report) {
             options.earl.report.addAssertion(test, true);
@@ -458,10 +503,15 @@ function readManifestEntry(manifest, entry) {
 }
 
 function readTestUrl(property) {
-  return function(test) {
+  return function(test, options) {
     if(!test[property]) {
       return null;
     }
+    if(options && options.load) {
+      // always load
+      return joinPath(test.dirname, test[property])
+        .then(readJson);
+    }
     return test.manifest.baseIri + test[property];
   };
 }

tests/test-karma.js

@@ -9,6 +9,11 @@
  *   EARL=filename
  * Bail with tests fail:
  *   BAIL=true
+ * Benchmark mode:
+ *   Basic:
+ *   JSONLD_BENCHMARK=1
+ *   With options:
+ *   JSONLD_BENCHMARK=key1=value1,key2=value2,...
  *
  * @author Dave Longley
  * @author David I. Lehn
@@ -64,6 +69,17 @@ if(process.env.JSONLD_TESTS) {
   entries.push(webidl);
 }
 
+let benchmark = null;
+if(process.env.JSONLD_BENCHMARK) {
+  benchmark = {};
+  if(!(['1', 'true'].includes(process.env.JSONLD_BENCHMARK))) {
+    process.env.JSONLD_BENCHMARK.split(',').forEach(pair => {
+      const kv = pair.split('=');
+      benchmark[kv[0]] = kv[1];
+    });
+  }
+}
+
 const options = {
   nodejs: false,
   assert: assert,
@@ -82,6 +98,7 @@ const options = {
   },
   bailOnError: process.env.BAIL === 'true',
   entries: entries,
+  benchmark: benchmark,
   readFile: filename => {
     return server.run(filename, function(filename) {
       const fs = serverRequire('fs-extra');

tests/test.js

@@ -9,6 +9,11 @@
  *   EARL=filename
  * Bail with tests fail:
  *   BAIL=true
+ * Benchmark mode:
+ *   Basic:
+ *   JSONLD_BENCHMARK=1
+ *   With options:
+ *   JSONLD_BENCHMARK=key1=value1,key2=value2,...
  *
  * @author Dave Longley
  * @author David I. Lehn
@@ -54,6 +59,17 @@ if(process.env.JSONLD_TESTS) {
   //entries.push(path.resolve(_top, 'tests/node-document-loader-tests.js'));
 }
 
+let benchmark = null;
+if(process.env.JSONLD_BENCHMARK) {
+  benchmark = {};
+  if(!(['1', 'true'].includes(process.env.JSONLD_BENCHMARK))) {
+    process.env.JSONLD_BENCHMARK.split(',').forEach(pair => {
+      const kv = pair.split('=');
+      benchmark[kv[0]] = kv[1];
+    });
+  }
+}
+
 const options = {
   nodejs: {
     path: path
@@ -67,6 +83,7 @@ const options = {
   },
   bailOnError: process.env.BAIL === 'true',
   entries: entries,
+  benchmark: benchmark,
   readFile: filename => {
     return fs.readFile(filename, 'utf8');
   },