Align test and benchmark code with rdf-canonize.

- Align test and benchmark code with rdf-canonize.
  - **NOTE**: This changes various testing and benchmark runner features
    and options.
  - Update env var usage.
  - Use more common code between Node.js and karma tests.
  - Conditionally load test suites.
  - Fix various minor bugs.
  - Add multiple jobs benchmarking support.

Author: davidlehn

CHANGELOG.md

@@ -1,13 +1,21 @@
 # jsonld ChangeLog
 
-## 8.1.2 - 2023-03-xx
+## 8.2.0 - 2023-03-xx
 
 ### Changed
 - Update for latest [rdf-canon][] changes: test suite location, README, links,
   and identifiers.
   - Skip test with 'U' escapes. Will enable when [rdf-canonize][] dependency is
     updated.
 - Test on Node.js 20.x.
+- Align test and benchmark code with [rdf-canonize][].
+  - **NOTE**: This changes various testing and benchmark runner features and
+    options.
+  - Update env var usage.
+  - Use more common code between Node.js and karma tests.
+  - Conditionally load test suites.
+  - Fix various minor bugs.
+  - Add multiple jobs benchmarking support.
 
 ### Fixed
 - Improve safe mode for `@graph` use cases.

README.md

@@ -389,13 +389,13 @@ Node.js tests can be run with a simple command:
     npm test
 
 If you installed the test suites elsewhere, or wish to run other tests, use
-the `JSONLD_TESTS` environment var:
+the `TESTS` environment var:
 
-    JSONLD_TESTS="/tmp/org/test-suites /tmp/norm/tests" npm test
+    TESTS="/tmp/org/test-suites /tmp/norm/tests" npm test
 
 This feature can be used to run the older json-ld.org test suite:
 
-    JSONLD_TESTS=/tmp/json-ld.org/test-suite npm test
+    TESTS=/tmp/json-ld.org/test-suite npm test
 
 Browser testing can be done with Karma:
 
@@ -419,7 +419,7 @@ Remote context tests are also available:
     # run the context server in the background or another terminal
     node tests/remote-context-server.js
 
-    JSONLD_TESTS=`pwd`/tests npm test
+    TESTS=`pwd`/tests npm test
 
 To generate EARL reports:
 
@@ -432,7 +432,7 @@ To generate EARL reports:
 To generate an EARL report with the `json-ld-api` and `json-ld-framing` tests
 as used on the official [JSON-LD Processor Conformance][] page
 
-    JSONLD_TESTS="`pwd`/../json-ld-api/tests `pwd`/../json-ld-framing/tests" EARL="jsonld-js-earl.jsonld" npm test
+    TESTS="`pwd`/../json-ld-api/tests `pwd`/../json-ld-framing/tests" EARL="jsonld-js-earl.jsonld" npm test
 
 The EARL `.jsonld` output can be converted to `.ttl` using the [rdf][] tool:
 
@@ -449,14 +449,14 @@ Benchmarks
 Benchmarks can be created from any manifest that the test system supports.
 Use a command line with a test suite and a benchmark flag:
 
-    JSONLD_TESTS=/tmp/benchmark-manifest.jsonld JSONLD_BENCHMARK=1 npm test
+    TESTS=/tmp/benchmark-manifest.jsonld BENCHMARK=1 npm test
 
 EARL reports with benchmark data can be generated with an optional environment
 details:
 
-    JSONLD_TESTS=`pwd`/../json-ld.org/benchmarks/b001-manifiest.jsonld JSONLD_BENCHMARK=1 EARL=earl-test.jsonld TEST_ENV=1 npm test
+    TESTS=`pwd`/../json-ld.org/benchmarks/b001-manifiest.jsonld BENCHMARK=1 EARL=earl-test.jsonld TEST_ENV=1 npm test
 
-See `tests/test.js` for more `TEST_ENV` control and options.
+See `tests/test.js` for more `TEST_ENV` and `BENCHMARK` control and options.
 
 These reports can be compared with the `benchmarks/compare/` tool and at the
 [JSON-LD Benchmarks][] site.

karma.conf.js

@@ -1,17 +1,12 @@
 /**
- * Karam configuration for jsonld.js.
+ * Karma configuration for jsonld.js.
  *
- * Set dirs, manifests, or js to run:
- *   JSONLD_TESTS="f1 f2 ..."
- * Output an EARL report:
- *   EARL=filename
- * Bail with tests fail:
- *   BAIL=true
+ * See ./test/test.js for env options.
  *
  * @author Dave Longley
  * @author David I. Lehn
  *
- * Copyright (c) 2011-2017 Digital Bazaar, Inc. All rights reserved.
+ * Copyright (c) 2011-2023 Digital Bazaar, Inc. All rights reserved.
  */
 const os = require('os');
 const webpack = require('webpack');
@@ -67,11 +62,10 @@ module.exports = function(config) {
       plugins: [
         new webpack.DefinePlugin({
           'process.env.BAIL': JSON.stringify(process.env.BAIL),
+          'process.env.BENCHMARK': JSON.stringify(process.env.BENCHMARK),
           'process.env.EARL': JSON.stringify(process.env.EARL),
+          'process.env.TESTS': JSON.stringify(process.env.TESTS),
           'process.env.TEST_ENV': JSON.stringify(process.env.TEST_ENV),
-          'process.env.JSONLD_BENCHMARK':
-            JSON.stringify(process.env.JSONLD_BENCHMARK),
-          'process.env.JSONLD_TESTS': JSON.stringify(process.env.JSONLD_TESTS),
           'process.env.TEST_ROOT_DIR': JSON.stringify(__dirname),
           'process.env.VERBOSE_SKIP': JSON.stringify(process.env.VERBOSE_SKIP),
           // for 'auto' test env
@@ -149,10 +143,10 @@ module.exports = function(config) {
         [
           'envify', {
             BAIL: process.env.BAIL,
+            BENCHMARK: process.env.BENCHMARK,
             EARL: process.env.EARL,
+            TESTS: process.env.TESTS,
             TEST_ENV: process.env.TEST_ENV,
-            JSONLD_BENCHMARK: process.env.JSONLD_BENCHMARK,
-            JSONLD_TESTS: process.env.JSONLD_TESTS,
             TEST_ROOT_DIR: __dirname,
             VERBOSE_SKIP: process.env.VERBOSE_SKIP,
             // for 'auto' test env

tests/test-karma.js

@@ -1,82 +1,66 @@
 /**
  * Karma test runner for jsonld.js.
  *
- * Use environment vars to control, set via karma.conf.js/webpack:
- *
- * Set dirs, manifests, or js to run:
- *   JSONLD_TESTS="r1 r2 ..."
- * Output an EARL report:
- *   EARL=filename
- * Test environment details for EARL report:
- *   This is useful for benchmark comparison.
- *   By default no details are added for privacy reasons.
- *   Automatic details can be added for all fields with '1', 'true', or 'auto':
- *   TEST_ENV=1
- *   To include only certain fields, set them, or use 'auto':
- *   TEST_ENV=cpu='Intel i7-4790K @ 4.00GHz',runtime='Node.js',...
- *   TEST_ENV=cpu=auto # only cpu
- *   TEST_ENV=cpu,runtime # only cpu and runtime
- *   TEST_ENV=auto,comment='special test' # all auto with override
- *   Available fields:
- *   - arch - ex: 'x64'
- *   - cpu - ex: 'Intel(R) Core(TM) i7-4790K CPU @ 4.00GHz'
- *   - cpuCount - ex: 8
- *   - platform - ex: 'linux'
- *   - runtime - ex: 'Node.js'
- *   - runtimeVersion - ex: 'v14.19.0'
- *   - comment: any text
- *   - version: jsonld.js version
- * Bail with tests fail:
- *   BAIL=true
- * Verbose skip reasons:
- *   VERBOSE_SKIP=true
- * Benchmark mode:
- *   Basic:
- *   JSONLD_BENCHMARK=1
- *   With options:
- *   JSONLD_BENCHMARK=key1=value1,key2=value2,...
+ * See ./test.js for environment vars options.
  *
  * @author Dave Longley
  * @author David I. Lehn
  *
- * Copyright (c) 2011-2022 Digital Bazaar, Inc. All rights reserved.
+ * Copyright (c) 2011-2023 Digital Bazaar, Inc. All rights reserved.
  */
 /* global serverRequire */
 // FIXME: hack to ensure delay is set first
 mocha.setup({delay: true, ui: 'bdd'});
 
 const assert = require('chai').assert;
-const common = require('./test');
-const jsonld = require('..');
+const benchmark = require('benchmark');
+const common = require('./test.js');
 const server = require('karma-server-side');
 const webidl = require('./test-webidl');
 const join = require('join-path-js');
 
 // special benchmark setup
 const _ = require('lodash');
-//const _process = require('process');
-const benchmark = require('benchmark');
-//const Benchmark = benchmark.runInContext({_, _process});
 const Benchmark = benchmark.runInContext({_});
 window.Benchmark = Benchmark;
 
 const entries = [];
 
-if(process.env.JSONLD_TESTS) {
-  entries.push(...process.env.JSONLD_TESTS.split(' '));
+if(process.env.TESTS) {
+  entries.push(...process.env.TESTS.split(' '));
 } else {
   const _top = process.env.TEST_ROOT_DIR;
   // TODO: support just adding certain entries in EARL mode?
 
   // json-ld-api main test suite
-  // FIXME: add path detection
-  entries.push(join(_top, 'test-suites/json-ld-api/tests'));
-  entries.push(join(_top, '../json-ld-api/tests'));
+  entries.push((async () => {
+    const testPath = join(_top, 'test-suites/json-ld-api/tests');
+    const siblingPath = join(_top, '../json-ld-api/tests');
+    return server.run(testPath, siblingPath, function(testPath, siblingPath) {
+      const fs = serverRequire('fs-extra');
+      // use local tests if setup
+      if(fs.existsSync(testPath)) {
+        return testPath;
+      }
+      // default to sibling dir
+      return siblingPath;
+    });
+  })());
 
   // json-ld-framing main test suite
-  // FIXME: add path detection
-  entries.push(join(_top, 'test-suites/json-ld-framing/tests'));
-  entries.push(join(_top, '../json-ld-framing/tests'));
+  entries.push((async () => {
+    const testPath = join(_top, 'test-suites/json-ld-framing/tests');
+    const siblingPath = join(_top, '../json-ld-framing/tests');
+    return server.run(testPath, siblingPath, function(testPath, siblingPath) {
+      const fs = serverRequire('fs-extra');
+      // use local tests if setup
+      if(fs.existsSync(testPath)) {
+        return testPath;
+      }
+      // default to sibling dir
+      return siblingPath;
+    });
+  })());
 
   /*
   // TODO: use json-ld-framing once tests are moved
@@ -89,9 +73,19 @@ if(process.env.JSONLD_TESTS) {
   */
 
   // W3C RDF Dataset Canonicalization "rdf-canon" test suite
-  // FIXME: add path detection
-  entries.push(join(_top, 'test-suites/rdf-canon/tests'));
-  entries.push(join(_top, '../rdf-canon/tests'));
+  entries.push((async () => {
+    const testPath = join(_top, 'test-suites/rdf-canon/tests');
+    const siblingPath = join(_top, '../rdf-canon/tests');
+    return server.run(testPath, siblingPath, function(testPath, siblingPath) {
+      const fs = serverRequire('fs-extra');
+      // use local tests if setup
+      if(fs.existsSync(testPath)) {
+        return testPath;
+      }
+      // default to sibling dir
+      return siblingPath;
+    });
+  })());
 
   // other tests
   entries.push(join(_top, 'tests/misc.js'));
@@ -102,79 +96,31 @@ if(process.env.JSONLD_TESTS) {
   entries.push(webidl);
 }
 
-// test environment
-let testEnv = null;
-if(process.env.TEST_ENV) {
-  let _test_env = process.env.TEST_ENV;
-  if(!(['0', 'false'].includes(_test_env))) {
-    testEnv = {};
-    if(['1', 'true', 'auto'].includes(_test_env)) {
-      _test_env = 'auto';
-    }
-    _test_env.split(',').forEach(pair => {
-      if(pair === 'auto') {
-        testEnv.arch = 'auto';
-        testEnv.cpu = 'auto';
-        testEnv.cpuCount = 'auto';
-        testEnv.platform = 'auto';
-        testEnv.runtime = 'auto';
-        testEnv.runtimeVersion = 'auto';
-        testEnv.comment = 'auto';
-        testEnv.version = 'auto';
-      } else {
-        const kv = pair.split('=');
-        if(kv.length === 1) {
-          testEnv[kv[0]] = 'auto';
-        } else {
-          testEnv[kv[0]] = kv.slice(1).join('=');
-        }
-      }
-    });
-    if(testEnv.arch === 'auto') {
-      testEnv.arch = process.env._TEST_ENV_ARCH;
-    }
-    if(testEnv.cpu === 'auto') {
-      testEnv.cpu = process.env._TEST_ENV_CPU;
-    }
-    if(testEnv.cpuCount === 'auto') {
-      testEnv.cpuCount = process.env._TEST_ENV_CPU_COUNT;
-    }
-    if(testEnv.platform === 'auto') {
-      testEnv.platform = process.env._TEST_ENV_PLATFORM;
-    }
-    if(testEnv.runtime === 'auto') {
-      testEnv.runtime = 'browser';
-    }
-    if(testEnv.runtimeVersion === 'auto') {
-      testEnv.runtimeVersion = '(unknown)';
-    }
-    if(testEnv.comment === 'auto') {
-      testEnv.comment = '';
-    }
-    if(testEnv.version === 'auto') {
-      testEnv.version = require('../package.json').version;
-    }
-  }
-}
+// test environment defaults
+const testEnvDefaults = {
+  label: '',
+  arch: process.env._TEST_ENV_ARCH,
+  cpu: process.env._TEST_ENV_CPU,
+  cpuCount: process.env._TEST_ENV_CPU_COUNT,
+  platform: process.env._TEST_ENV_PLATFORM,
+  runtime: 'browser',
+  runtimeVersion: '(unknown)',
+  comment: '',
+  version: require('../package.json').version
+};
 
-let benchmarkOptions = null;
-if(process.env.JSONLD_BENCHMARK) {
-  if(!(['0', 'false'].includes(process.env.JSONLD_BENCHMARK))) {
-    benchmarkOptions = {};
-    if(!(['1', 'true'].includes(process.env.JSONLD_BENCHMARK))) {
-      process.env.JSONLD_BENCHMARK.split(',').forEach(pair => {
-        const kv = pair.split('=');
-        benchmarkOptions[kv[0]] = kv[1];
-      });
-    }
-  }
-}
+const env = {
+  BAIL: process.env.BAIL,
+  BENCHMARK: process.env.BENCHMARK,
+  TEST_ENV: process.env.TEST_ENV,
+  VERBOSE_SKIP: process.env.VERBOSE_SKIP
+};
 
 const options = {
+  env,
   nodejs: false,
   assert,
   benchmark,
-  jsonld,
   /* eslint-disable-next-line no-unused-vars */
   exit: code => {
     console.error('exit not implemented');
@@ -183,11 +129,8 @@ const options = {
   earl: {
     filename: process.env.EARL
   },
-  verboseSkip: process.env.VERBOSE_SKIP === 'true',
-  bailOnError: process.env.BAIL === 'true',
   entries,
-  testEnv,
-  benchmarkOptions,
+  testEnvDefaults,
   readFile: filename => {
     return server.run(filename, function(filename) {
       const fs = serverRequire('fs-extra');