Merge pull request #2373 from plotly/ci-retry

Auto-retry jasmine test on CI

Author: etpinard

.circleci/test.sh

@@ -5,17 +5,37 @@ set +e
 set +o pipefail
 
 EXIT_STATE=0
+MAX_AUTO_RETRY=5
+
+# inspired by https://unix.stackexchange.com/a/82602
+retry () {
+    local n=0
+
+    until [ $n -ge $MAX_AUTO_RETRY ]; do
+        "$@" && break
+        n=$[$n+1]
+        echo ''
+        echo run $n of $MAX_AUTO_RETRY failed, trying again ...
+        echo ''
+        sleep 15
+    done
+
+    if [ $n -eq $MAX_AUTO_RETRY ]; then
+        EXIT_STATE=1
+    fi
+}
 
 case $1 in
 
     jasmine)
-        npm run test-jasmine -- --skip-tags=gl,noCI || EXIT_STATE=$?
+        npm run test-jasmine -- --skip-tags=gl,noCI,flaky || EXIT_STATE=$?
         exit $EXIT_STATE
         ;;
 
     jasmine2)
-        npm run test-jasmine -- --tags=gl --skip-tags=noCI || EXIT_STATE=$?
-        npm run test-bundle                                || EXIT_STATE=$?
+        npm run test-jasmine -- --tags=gl --skip-tags=noCI,flaky || EXIT_STATE=$?
+        retry npm run test-jasmine -- --tags=flaky --skip-tags=noCI
+        npm run test-bundle || EXIT_STATE=$?
         exit $EXIT_STATE
         ;;
 

test/jasmine/tests/animate_test.js

@@ -720,7 +720,7 @@ describe('Animating multiple axes', function() {
         destroyGraphDiv();
     });
 
-    it('updates ranges of secondary axes', function(done) {
+    it('@flaky updates ranges of secondary axes', function(done) {
         Plotly.plot(gd, [
             {y: [1, 2, 3]},
             {y: [1, 2, 3], yaxis: 'y2'}

test/jasmine/tests/gl2d_plot_interact_test.js

@@ -250,7 +250,7 @@ describe('@gl Test gl2d plots', function() {
         });
     }
 
-    it('should respond to drag interactions', function(done) {
+    it('@flaky should respond to drag interactions', function(done) {
         var _mock = Lib.extendDeep({}, mock);
 
         var relayoutCallback = jasmine.createSpy('relayoutCallback');

test/jasmine/tests/parcoords_test.js

@@ -774,7 +774,7 @@ describe('@gl parcoords', function() {
 
         });
 
-        it('Calling `Plotly.animate` with patches targeting `dimensions` attributes should do the right thing', function(done) {
+        it('@flaky Calling `Plotly.animate` with patches targeting `dimensions` attributes should do the right thing', function(done) {
             Plotly.newPlot(gd, [{
                 type: 'parcoords',
                 line: {color: 'blue'},

test/jasmine/tests/select_test.js

@@ -107,7 +107,7 @@ var BOXEVENTS = [1, 2, 1];
 // assumes 5 points in the lasso path
 var LASSOEVENTS = [4, 2, 1];
 
-describe('Test select box and lasso in general:', function() {
+describe('@flaky Test select box and lasso in general:', function() {
     var mock = require('@mocks/14.json');
     var selectPath = [[93, 193], [143, 193]];
     var lassoPath = [[316, 171], [318, 239], [335, 243], [328, 169]];