Skip to content

Commit 3c1adeb

Browse files
etpinardetpinard
authored
Merge pull request #2933 from plotly/shard-gl-tests
Shard gl jasmine tests
2 parents a8c6217 + 006d5d1 commit 3c1adeb

21 files changed

+437
-250
lines changed

‎.circleci/test.sh‎

Lines changed: 21 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -4,23 +4,31 @@
44
set +e
55
set +o pipefail
66

7+
ROOT=$(dirname $0)/..
78
EXIT_STATE=0
89
MAX_AUTO_RETRY=5
910

11+
log () {
12+
echo -e "\n$1"
13+
}
14+
1015
# inspired by https://unix.stackexchange.com/a/82602
1116
retry () {
12-
local n=0
17+
local n=1
1318

1419
until [ $n -ge $MAX_AUTO_RETRY ]; do
15-
"$@" && break
20+
"$@" --failFast && break
21+
log "run $n of $MAX_AUTO_RETRY failed, trying again ..."
1622
n=$[$n+1]
17-
echo ''
18-
echo run $n of $MAX_AUTO_RETRY failed, trying again ...
19-
echo ''
20-
sleep 15
2123
done
2224

2325
if [ $n -eq $MAX_AUTO_RETRY ]; then
26+
log "one last time, w/o failing fast"
27+
"$@" && n=0
28+
fi
29+
30+
if [ $n -eq $MAX_AUTO_RETRY ]; then
31+
log "all $n runs failed, moving on."
2432
EXIT_STATE=1
2533
fi
2634
}
@@ -29,13 +37,18 @@ case $1 in
2937

3038
jasmine)
3139
npm run test-jasmine -- --skip-tags=gl,noCI,flaky || EXIT_STATE=$?
40+
retry npm run test-jasmine -- --tags=flaky --skip-tags=noCI
3241
npm run test-bundle || EXIT_STATE=$?
3342
exit $EXIT_STATE
3443
;;
3544

3645
jasmine2)
37-
retry npm run test-jasmine -- --tags=gl --skip-tags=noCI,flaky
38-
retry npm run test-jasmine -- --tags=flaky --skip-tags=noCI
46+
SHARDS=($(node $ROOT/tasks/shard_jasmine_tests.js --tag=gl))
47+
48+
for s in ${SHARDS[@]}; do
49+
retry npm run test-jasmine -- "$s" --tags=gl --skip-tags=noCI
50+
done
51+
3952
exit $EXIT_STATE
4053
;;
4154

‎package-lock.json‎

Lines changed: 6 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -136,6 +136,7 @@
136136
"karma": "^3.0.0",
137137
"karma-browserify": "^5.3.0",
138138
"karma-chrome-launcher": "^2.0.0",
139+
"karma-fail-fast-reporter": "^1.0.5",
139140
"karma-firefox-launcher": "^1.0.1",
140141
"karma-jasmine": "^1.1.2",
141142
"karma-jasmine-spec-tags": "^1.0.1",

‎tasks/shard_jasmine_tests.js‎

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
var fs = require('fs');
2+
var path = require('path');
3+
4+
var falafel = require('falafel');
5+
var glob = require('glob');
6+
var minimist = require('minimist');
7+
8+
var pathToJasmineTests = require('./util/constants').pathToJasmineTests;
9+
var isJasmineTestIt = require('./util/common').isJasmineTestIt;
10+
11+
var argv = minimist(process.argv.slice(2), {
12+
string: ['tag', 'limit'],
13+
alias: {
14+
tag: ['t'],
15+
limit: ['l'],
16+
},
17+
default: {
18+
limit: 20
19+
}
20+
});
21+
22+
var tag = argv.tag;
23+
var limit = argv.limit;
24+
25+
glob(path.join(pathToJasmineTests, '*.js'), function(err, files) {
26+
if(err) throw err;
27+
28+
var file2cnt = {};
29+
30+
files.forEach(function(file) {
31+
var code = fs.readFileSync(file, 'utf-8');
32+
var bn = path.basename(file);
33+
34+
falafel(code, function(node) {
35+
if(isJasmineTestIt(node, tag)) {
36+
if(file2cnt[bn]) {
37+
file2cnt[bn]++;
38+
} else {
39+
file2cnt[bn] = 1;
40+
}
41+
}
42+
});
43+
});
44+
45+
var ranking = Object.keys(file2cnt);
46+
var runs = [];
47+
48+
// if 'it' count in file greater than threshold,
49+
// run only this file separately,
50+
// don't try to shard within file
51+
Object.keys(file2cnt).forEach(function(f) {
52+
if(file2cnt[f] > limit) {
53+
runs.push(f);
54+
ranking.splice(ranking.indexOf(f), 1);
55+
}
56+
});
57+
58+
// sort ranking in decreasing order
59+
ranking.sort(function(a, b) { return file2cnt[b] - file2cnt[a]; });
60+
61+
var runi;
62+
var cnt;
63+
64+
function newRun() {
65+
var r0 = ranking[0];
66+
runi = [r0];
67+
cnt = file2cnt[r0];
68+
ranking.shift();
69+
}
70+
71+
function concat() {
72+
runs.push(runi.join(','));
73+
}
74+
75+
// try to match files with many tests with files not-that-many,
76+
// by matching first rank with one or multiple trailing ranks.
77+
newRun();
78+
while(ranking.length) {
79+
var rn = ranking[ranking.length - 1];
80+
81+
if((cnt + file2cnt[rn]) > limit) {
82+
concat();
83+
newRun();
84+
} else {
85+
runi.push(rn);
86+
cnt += file2cnt[rn];
87+
ranking.pop();
88+
}
89+
}
90+
concat();
91+
92+
// print result to stdout
93+
console.log(runs.join('\n'));
94+
});

‎tasks/test_syntax.js‎

Lines changed: 43 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,11 @@ var readLastLines = require('read-last-lines');
88
var eslint = require('eslint');
99
var trueCasePath = require('true-case-path');
1010

11+
var common = require('./util/common');
12+
var isJasmineTestIt = common.isJasmineTestIt;
13+
var isJasmineTestDescribe = common.isJasmineTestDescribe;
14+
var hasJasmineTestTag = common.hasJasmineTestTag;
15+
1116
var constants = require('./util/constants');
1217
var srcGlob = path.join(constants.pathToSrc, '**/*.js');
1318
var libGlob = path.join(constants.pathToLib, '**/*.js');
@@ -28,26 +33,60 @@ assertES5();
2833
// check for for focus and exclude jasmine blocks
2934
function assertJasmineSuites() {
3035
var BLACK_LIST = ['fdescribe', 'fit', 'xdescribe', 'xit'];
36+
var TAGS = ['noCI', 'noCIdep', 'gl', 'flaky'];
37+
var IT_ONLY_TAGS = ['gl', 'flaky'];
3138
var logs = [];
3239

40+
var addTagPrefix = function(t) { return '@' + t; };
41+
3342
glob(combineGlobs([testGlob, bundleTestGlob]), function(err, files) {
3443
files.forEach(function(file) {
3544
var code = fs.readFileSync(file, 'utf-8');
45+
var bn = path.basename(file);
3646

3747
falafel(code, {locations: true}, function(node) {
48+
var lineInfo = '[line ' + node.loc.start.line + '] :';
49+
3850
if(node.type === 'Identifier' && BLACK_LIST.indexOf(node.name) !== -1) {
3951
logs.push([
40-
path.basename(file),
41-
'[line ' + node.loc.start.line + '] :',
52+
bn, lineInfo,
4253
'contains either a *fdescribe*, *fit*,',
4354
'*xdescribe* or *xit* block.'
4455
].join(' '));
4556
}
46-
});
4757

58+
if(isJasmineTestIt(node)) {
59+
if(hasJasmineTestTag(node)) {
60+
if(TAGS.every(function(t) { return !hasJasmineTestTag(node, t); })) {
61+
logs.push([
62+
bn, lineInfo,
63+
'contains an unrecognized tag,',
64+
'not one of: ' + TAGS.map(addTagPrefix).join(', ')
65+
].join(' '));
66+
}
67+
}
68+
69+
if(hasJasmineTestTag(node, 'gl') && hasJasmineTestTag(node, 'flaky')) {
70+
logs.push([
71+
bn, lineInfo,
72+
'contains a @gl tag AND a @flaky tag, which is not allowed'
73+
].join(' '));
74+
}
75+
}
76+
77+
IT_ONLY_TAGS.forEach(function(t) {
78+
if(isJasmineTestDescribe(node, t)) {
79+
logs.push([
80+
bn, lineInfo,
81+
'contains a', addTagPrefix(t), 'tag is a *describe* block,',
82+
addTagPrefix(t), 'tags are only allowed in jasmine *it* blocks.'
83+
].join(' '));
84+
}
85+
});
86+
});
4887
});
4988

50-
log('no jasmine suites focus/exclude blocks', logs);
89+
log('no jasmine suites focus/exclude blocks or wrong tag patterns', logs);
5190
});
5291
}
5392

‎tasks/util/common.js‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -105,3 +105,30 @@ exports.formatEnumeration = function(list) {
105105
return '`' + l + '`' + ending;
106106
}).join(' ');
107107
};
108+
109+
exports.hasJasmineTestTag = function(node, tag) {
110+
var re = tag ?
111+
new RegExp('@' + tag + '\\s') :
112+
new RegExp('@' + '\\w');
113+
return re.test(node.source());
114+
};
115+
116+
function isJasmineBase(block, node, tag) {
117+
return (
118+
node.type === 'Literal' &&
119+
node.parent &&
120+
node.parent.type === 'CallExpression' &&
121+
node.parent.callee &&
122+
node.parent.callee.type === 'Identifier' &&
123+
node.parent.callee.name === block &&
124+
(tag === undefined || exports.hasJasmineTestTag(node, tag))
125+
);
126+
}
127+
128+
exports.isJasmineTestIt = function(node, tag) {
129+
return isJasmineBase('it', node, tag);
130+
};
131+
132+
exports.isJasmineTestDescribe = function(node, tag) {
133+
return isJasmineBase('describe', node, tag);
134+
};

‎test/jasmine/karma.conf.js‎

Lines changed: 19 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -7,16 +7,18 @@ var constants = require('../../tasks/util/constants');
77
var isCI = !!process.env.CIRCLECI;
88
var argv = minimist(process.argv.slice(4), {
99
string: ['bundleTest', 'width', 'height'],
10-
'boolean': ['info', 'nowatch', 'verbose', 'Chrome', 'Firefox'],
10+
'boolean': ['info', 'nowatch', 'failFast', 'verbose', 'Chrome', 'Firefox'],
1111
alias: {
1212
'Chrome': 'chrome',
1313
'Firefox': ['firefox', 'FF'],
1414
'bundleTest': ['bundletest', 'bundle_test'],
15-
'nowatch': 'no-watch'
15+
'nowatch': 'no-watch',
16+
'failFast': 'fail-fast'
1617
},
1718
'default': {
1819
info: false,
1920
nowatch: isCI,
21+
failFast: false,
2022
verbose: false,
2123
width: '1035',
2224
height: '617'
@@ -53,6 +55,7 @@ if(argv.info) {
5355
' - `--Chrome` (alias `--chrome`): run test in (our custom) Chrome browser',
5456
' - `--Firefox` (alias `--FF`, `--firefox`): run test in (our custom) Firefox browser',
5557
' - `--nowatch (dflt: `false`, `true` on CI)`: run karma w/o `autoWatch` / multiple run mode',
58+
' - `--failFast` (dflt: `false`): exit karma upon first test failure',
5659
' - `--verbose` (dflt: `false`): show test result using verbose reporter',
5760
' - `--tags`: run only test with given tags (using the `jasmine-spec-tags` framework)',
5861
' - `--width`(dflt: 1035): set width of the browser window',
@@ -100,11 +103,14 @@ if(isFullSuite) {
100103

101104
var pathToShortcutPath = path.join(__dirname, '..', '..', 'tasks', 'util', 'shortcut_paths.js');
102105
var pathToStrictD3 = path.join(__dirname, '..', '..', 'tasks', 'util', 'strict_d3.js');
103-
var pathToMain = path.join(__dirname, '..', '..', 'lib', 'index.js');
104106
var pathToJQuery = path.join(__dirname, 'assets', 'jquery-1.8.3.min.js');
105107
var pathToIE9mock = path.join(__dirname, 'assets', 'ie9_mock.js');
106108
var pathToCustomMatchers = path.join(__dirname, 'assets', 'custom_matchers.js');
107109

110+
var reporters = (isFullSuite && !argv.tags) ? ['dots', 'spec'] : ['progress'];
111+
if(argv.failFast) reporters.push('fail-fast');
112+
if(argv.verbose) reporters.push('verbose');
113+
108114
function func(config) {
109115
// level of logging
110116
// possible values: config.LOG_DISABLE || config.LOG_ERROR || config.LOG_WARN || config.LOG_INFO || config.LOG_DEBUG
@@ -118,7 +124,7 @@ function func(config) {
118124
//
119125
// See https://github.com/karma-runner/karma/commit/89a7a1c#commitcomment-21009216
120126
func.defaultConfig.browserConsoleLogOptions = {
121-
level: 'log'
127+
level: 'debug'
122128
};
123129

124130
config.set(func.defaultConfig);
@@ -154,7 +160,7 @@ func.defaultConfig = {
154160
// See note in CONTRIBUTING.md about more verbose reporting via karma-verbose-reporter:
155161
// https://www.npmjs.com/package/karma-verbose-reporter ('verbose')
156162
//
157-
reporters: (isFullSuite && !argv.tags) ? ['dots', 'spec'] : ['progress'],
163+
reporters: reporters,
158164

159165
// web server port
160166
port: 9876,
@@ -235,16 +241,18 @@ func.defaultConfig = {
235241
suppressPassed: true,
236242
suppressSkipped: false,
237243
showSpecTiming: false,
244+
// use 'karma-fail-fast-reporter' to fail fast w/o conflicting
245+
// with other karma plugins
238246
failFast: false
239-
}
247+
},
248+
249+
// e.g. when a test file does not container a given spec tags
250+
failOnEmptyTestSuite: false
240251
};
241252

242253
func.defaultConfig.preprocessors[pathToCustomMatchers] = ['browserify'];
243254

244-
if(isFullSuite) {
245-
func.defaultConfig.files.push(pathToJQuery);
246-
func.defaultConfig.preprocessors[testFileGlob] = ['browserify'];
247-
} else if(isBundleTest) {
255+
if(isBundleTest) {
248256
switch(basename(testFileGlob)) {
249257
case 'requirejs':
250258
// browserified custom_matchers doesn't work with this route
@@ -271,14 +279,7 @@ if(isFullSuite) {
271279
break;
272280
}
273281
} else {
274-
// Add lib/index.js to non-full-suite runs,
275-
// to make sure the registry is set-up correctly.
276-
func.defaultConfig.files.push(
277-
pathToJQuery,
278-
pathToMain
279-
);
280-
281-
func.defaultConfig.preprocessors[pathToMain] = ['browserify'];
282+
func.defaultConfig.files.push(pathToJQuery);
282283
func.defaultConfig.preprocessors[testFileGlob] = ['browserify'];
283284
}
284285

@@ -291,9 +292,4 @@ if(argv.Chrome) browsers.push('_Chrome');
291292
if(argv.Firefox) browsers.push('_Firefox');
292293
if(browsers.length === 0) browsers.push('_Chrome');
293294

294-
// add verbose reporter if specified
295-
if(argv.verbose) {
296-
func.defaultConfig.reporters.push('verbose');
297-
}
298-
299295
module.exports = func;

0 commit comments

Comments
 (0)