Skip to content

Commit d4d9ba3

Browse files
davidlehndavidlehn
committed
Benchmarking updates.
- Support test environment in EARL output. - Support benchmark output in EARL output. - Add extra test files to karma setup.
1 parent 110523d commit d4d9ba3

File tree

7 files changed

+303
-47
lines changed

7 files changed

+303
-47
lines changed

‎CHANGELOG.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@
22

33
### Added
44
- Support benchmarks in Karma tests.
5+
- Support test environment in EARL output.
6+
- Support benchmark output in EARL output.
57

68
### Changed
79
- Change EARL Assertor to Digital Bazaar, Inc.

‎README.md‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -456,11 +456,14 @@ Use a command line with a test suite and a benchmark flag:
456456
JSONLD_TESTS=/tmp/benchmark-manifest.jsonld JSONLD_BENCHMARK=1 npm test
457457

458458
EARL reports with benchmark data can be generated with an optional environment
459-
tag:
459+
details:
460460

461-
JSONLD_TESTS=`pwd`/../json-ld.org/benchmarks/b001-manifiest.jsonld JSONLD_BENCHMARK=1 EARL=earl-test.jsonld EARL_ENV='CPU=Intel-i7-4790K@4.00GHz,Node.js=v10.16.3,jsonld.js=v1.7.0' npm test
461+
JSONLD_TESTS=`pwd`/../json-ld.org/benchmarks/b001-manifiest.jsonld JSONLD_BENCHMARK=1 EARL=earl-test.jsonld TEST_ENV=1 npm test
462462

463-
These reports can be compared at the [JSON-LD Benchmarks][] site.
463+
See `tests/test.js` for more `TEST_ENV` control and options.
464+
465+
These reports can be compared with the `tests/benchmark-compare` tool and at
466+
the [JSON-LD Benchmarks][] site.
464467

465468
[Digital Bazaar]: https://digitalbazaar.com/
466469

‎karma.conf.js‎

Lines changed: 18 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
*
1414
* Copyright (c) 2011-2017 Digital Bazaar, Inc. All rights reserved.
1515
*/
16+
const os = require('os');
1617
const webpack = require('webpack');
1718

1819
module.exports = function(config) {
@@ -67,12 +68,19 @@ module.exports = function(config) {
6768
new webpack.DefinePlugin({
6869
'process.env.BAIL': JSON.stringify(process.env.BAIL),
6970
'process.env.EARL': JSON.stringify(process.env.EARL),
70-
'process.env.EARL_ENV': JSON.stringify(process.env.EARL_ENV),
71+
'process.env.TEST_ENV': JSON.stringify(process.env.TEST_ENV),
7172
'process.env.JSONLD_BENCHMARK':
7273
JSON.stringify(process.env.JSONLD_BENCHMARK),
7374
'process.env.JSONLD_TESTS': JSON.stringify(process.env.JSONLD_TESTS),
7475
'process.env.TEST_ROOT_DIR': JSON.stringify(__dirname),
75-
'process.env.VERBOSE_SKIP': JSON.stringify(process.env.VERBOSE_SKIP)
76+
'process.env.VERBOSE_SKIP': JSON.stringify(process.env.VERBOSE_SKIP),
77+
// for 'auto' test env
78+
'process.env._TEST_ENV_ARCH': JSON.stringify(process.arch),
79+
'process.env._TEST_ENV_CPU': JSON.stringify(os.cpus()[0].model),
80+
'process.env._TEST_ENV_CPU_COUNT': JSON.stringify(os.cpus().length),
81+
'process.env._TEST_ENV_PLATFORM': JSON.stringify(process.platform),
82+
'process.env._TEST_VERSION':
83+
JSON.stringify(require('./package.json').version)
7684
})
7785
],
7886
module: {
@@ -142,10 +150,17 @@ module.exports = function(config) {
142150
'envify', {
143151
BAIL: process.env.BAIL,
144152
EARL: process.env.EARL,
153+
TEST_ENV: process.env.TEST_ENV,
145154
JSONLD_BENCHMARK: process.env.JSONLD_BENCHMARK,
146155
JSONLD_TESTS: process.env.JSONLD_TESTS,
147156
TEST_ROOT_DIR: __dirname,
148-
VERBOSE_SKIP: process.env.VERBOSE_SKIP
157+
VERBOSE_SKIP: process.env.VERBOSE_SKIP,
158+
// for 'auto' test env
159+
_TEST_ENV_ARCH: process.arch,
160+
_TEST_ENV_CPU: os.cpus()[0].model,
161+
_TEST_ENV_CPU_COUNT: os.cpus().length,
162+
_TEST_ENV_PLATFORM: process.platform,
163+
_TEST_VERSION: require('./package.json').version
149164
}
150165
]
151166
],

‎tests/earl-report.js‎

Lines changed: 96 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111
*
1212
* @param options {Object} reporter options
1313
* id: {String} report id
14-
* env: {String} environment description
14+
* env: {Object} environment description
1515
*/
1616
function EarlReport(options) {
1717
let today = new Date();
@@ -24,6 +24,8 @@ function EarlReport(options) {
2424
this.now.setMilliseconds(0);
2525
this.id = options.id;
2626
this.env = options.env;
27+
// test environment
28+
this._environment = null;
2729
/* eslint-disable quote-props */
2830
this._report = {
2931
'@context': {
@@ -69,35 +71,40 @@ function EarlReport(options) {
6971
'foaf:homepage': 'https://digitalbazaar.com/'
7072
},
7173
'doap:release': {
72-
'doap:name': '',
7374
'doap:revision': '',
7475
'doap:created': today
7576
},
7677
'subjectOf': []
7778
};
7879
/* eslint-enable quote-props */
79-
this._report['@id'] +=
80-
'#' + this.id;
81-
this._report['doap:name'] +=
82-
' ' + this.id + (this.env ? ' ' + this.env : '');
83-
this._report['dc:title'] +=
84-
' ' + this.id + (this.env ? ' ' + this.env : '');
80+
if(this.env && this.env.version) {
81+
this._report['doap:release']['doap:name'] = this.env.version;
82+
}
8583
}
8684

8785
EarlReport.prototype.addAssertion = function(test, pass, options) {
8886
options = options || {};
89-
this._report.subjectOf.push({
87+
const assertion = {
9088
'@type': 'earl:Assertion',
9189
'earl:assertedBy': this._report['doap:developer']['@id'],
9290
'earl:mode': 'earl:automatic',
9391
'earl:test': test['@id'],
9492
'earl:result': {
9593
'@type': 'earl:TestResult',
9694
'dc:date': this.now.toISOString(),
97-
'earl:outcome': pass ? 'earl:passed' : 'earl:failed',
98-
...options.extra
95+
'earl:outcome': pass ? 'earl:passed' : 'earl:failed'
9996
}
100-
});
97+
};
98+
if(options.benchmarkResult) {
99+
const result = {
100+
...options.benchmarkResult
101+
};
102+
if(this._environment) {
103+
result['jldb:environment'] = this._environment['@id'];
104+
}
105+
assertion['jldb:result'] = result;
106+
}
107+
this._report.subjectOf.push(assertion);
101108
return this;
102109
};
103110

@@ -109,4 +116,81 @@ EarlReport.prototype.reportJson = function() {
109116
return JSON.stringify(this._report, null, 2);
110117
};
111118

119+
/* eslint-disable quote-props */
120+
const _benchmarkContext = {
121+
'jldb': 'http://json-ld.org/benchmarks/vocab#',
122+
'rdfs': 'http://www.w3.org/2000/01/rdf-schema#',
123+
'xsd': 'http://www.w3.org/2001/XMLSchema#',
124+
125+
// environment description
126+
'jldb:Environment': {'@type': '@id'},
127+
128+
// per environment
129+
// architecture type
130+
// ex: x86
131+
'jldb:arch': {'@type': 'xsd:string'},
132+
// cpu model description (may show multiple cpus)
133+
// ex: 'Intel(R) Core(TM) i7-4790K CPU @ 4.00GHz'
134+
'jldb:cpu': {'@type': 'xsd:string'},
135+
// count of cpus, may not be uniform, just informative
136+
'jldb:cpuCount': {'@type': 'xsd:integer'},
137+
// platform name
138+
// ex: linux
139+
'jldb:platform': {'@type': 'xsd:string'},
140+
// runtime name
141+
// ex: Node.js, Chromium, Ruby
142+
'jldb:runtime': {'@type': 'xsd:string'},
143+
// runtime version
144+
// ex: v14.19.0
145+
'jldb:runtimeVersion': {'@type': 'xsd:string'},
146+
// arbitrary comment
147+
'jldb:comment': 'rdfs:comment',
148+
149+
// benchmark result
150+
'jldb:BenchmarkResult': {'@type': '@id'},
151+
152+
// use in earl:Assertion, type jldb:BenchmarkResult
153+
'jldb:result': {'@type': '@id'},
154+
155+
// per BenchmarkResult
156+
'jldb:environment': {'@type': '@id'},
157+
'jldb:hz': {'@type': 'xsd:float'},
158+
'jldb:rme': {'@type': 'xsd:float'}
159+
};
160+
/* eslint-enable quote-props */
161+
162+
// setup @context and environment to handle benchmark data
163+
EarlReport.prototype.setupForBenchmarks = function(options) {
164+
// add context if needed
165+
if(!Array.isArray(this._report['@context'])) {
166+
this._report['@context'] = [this._report['@context']];
167+
}
168+
if(!this._report['@context'].some(c => c === _benchmarkContext)) {
169+
this._report['@context'].push(_benchmarkContext);
170+
}
171+
if(options.testEnv) {
172+
// add report environment
173+
const fields = [
174+
['arch', 'jldb:arch'],
175+
['cpu', 'jldb:cpu'],
176+
['cpuCount', 'jldb:cpuCount'],
177+
['platform', 'jldb:platform'],
178+
['runtime', 'jldb:runtime'],
179+
['runtimeVersion', 'jldb:runtimeVersion'],
180+
['comment', 'jldb:comment']
181+
];
182+
const _env = {
183+
'@id': '_:environment:0'
184+
};
185+
for(const [field, property] of fields) {
186+
if(options.testEnv[field]) {
187+
_env[property] = options.testEnv[field];
188+
}
189+
}
190+
this._environment = _env;
191+
this._report['@included'] = this._report['@included'] || [];
192+
this._report['@included'].push(_env);
193+
}
194+
};
195+
112196
module.exports = EarlReport;

‎tests/test-common.js‎

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -266,8 +266,11 @@ const SKIP_TESTS = [];
266266
if(options.earl && options.earl.filename) {
267267
options.earl.report = new EarlReport({
268268
id: options.earl.id,
269-
env: options.earl.env
269+
env: options.testEnv
270270
});
271+
if(options.benchmarkOptions) {
272+
options.earl.report.setupForBenchmarks({testEnv: options.testEnv});
273+
}
271274
}
272275

273276
return new Promise(resolve => {
@@ -573,7 +576,7 @@ function addTest(manifest, test, tests) {
573576
throw Error('Unknown test type: ' + test.type);
574577
}
575578

576-
let benchResult = null;
579+
let benchmarkResult = null;
577580
if(options.benchmarkOptions) {
578581
const result = await runBenchmark({
579582
test,
@@ -584,16 +587,16 @@ function addTest(manifest, test, tests) {
584587
})),
585588
mochaTest: self
586589
});
587-
benchResult = {
588-
'jsonld:benchmarkHz': result.target.hz
590+
benchmarkResult = {
591+
'@type': 'jldb:BenchmarkResult',
592+
'jldb:hz': result.target.hz,
593+
'jldb:rme': result.target.stats.rme
589594
};
590595
}
591596

592597
if(options.earl.report) {
593598
options.earl.report.addAssertion(test, true, {
594-
extra: {
595-
...benchResult
596-
}
599+
benchmarkResult
597600
});
598601
}
599602
} catch(err) {

0 commit comments

Comments
 (0)