Merge pull request #4321 from plotly/test-image-flaky-mocks

Flaky image tests

Author: etpinard

.circleci/test.sh

@@ -75,11 +75,12 @@ case $1 in
 
     image)
         SUITE=$(find $ROOT/test/image/mocks/ -type f -printf "%f\n" | circleci tests split)
-        npm run test-image -- $SUITE --filter || EXIT_STATE=$?
+        npm run test-image -- $SUITE --filter --skip-flaky || EXIT_STATE=$?
         exit $EXIT_STATE
         ;;
 
     image2)
+        retry npm run test-image -- --just-flaky
         npm run test-export     || EXIT_STATE=$?
         exit $EXIT_STATE
         ;;

test/image/compare_pixels_test.js

@@ -47,11 +47,13 @@ var QUEUE_WAIT = 10;
  *  Run all gl3d image test in queue:
  *
  *      npm run test-image -- gl3d_* --queue
+ *
+ *
  */
 
-var argv = minimist(process.argv.slice(2), {boolean: ['queue', 'filter' ]});
-var isInQueue = argv.queue;
-var filter = argv.filter;
+var argv = minimist(process.argv.slice(2), {
+    boolean: ['queue', 'filter', 'skip-flaky', 'just-flaky']
+});
 
 var allMock = false;
 // If no pattern is provided, all mocks are compared
@@ -80,40 +82,39 @@ allMockList = allMockList.filter(unique);
 
 // filter out untestable mocks if no pattern is specified (ie. we're testing all mocks)
 // or if flag '--filter' is provided
-if(allMock || filter) {
+console.log('');
+if(allMock || argv.filter) {
     console.log('Filtering out untestable mocks:');
-    allMockList = allMockList.filter(untestableFilter);
-    console.log('\n');
-}
-
-sortGl2dMockList(allMockList);
-
-// main
-if(isInQueue) {
-    runInQueue(allMockList);
-} else {
-    runInBatch(allMockList);
+    // Test cases:
+    // - font-wishlist
+    // - all mapbox
+    // don't behave consistently from run-to-run and/or
+    // machine-to-machine; skip over them for now.
+    allMockList = allMockList.filter(function(mockName) {
+        var cond = !(
+            mockName === 'font-wishlist' ||
+            mockName.indexOf('mapbox_') !== -1
+        );
+        if(!cond) console.log(' -', mockName);
+        return cond;
+    });
 }
 
-/* Test cases:
- *
- * - font-wishlist
- * - all mapbox
- *
- * don't behave consistently from run-to-run and/or
- * machine-to-machine; skip over them for now.
- *
- */
-function untestableFilter(mockName) {
-    var cond =
-    !(
-        mockName === 'font-wishlist' ||
-        mockName.indexOf('mapbox_') !== -1
-    );
-
-    if(!cond) console.log(' -', mockName);
+var FLAKY_LIST = [
+    'treemap_textposition'
+];
 
-    return cond;
+console.log('');
+if(argv['skip-flaky']) {
+    allMockList = allMockList.filter(function(mockName) {
+        var cond = FLAKY_LIST.indexOf(mockName) === -1;
+        if(!cond) console.log('Skipping flaky mock', mockName);
+        return cond;
+    });
+} else if(argv['just-flaky']) {
+    allMockList = allMockList.filter(function(mockName) {
+        return FLAKY_LIST.indexOf(mockName) !== -1;
+    });
 }
 
 /* gl2d pointcloud and other non-regl gl2d mock(s)
@@ -277,3 +278,13 @@ function comparePixels(mockName, cb) {
         .pipe(saveImageStream)
         .on('close', checkImage);
 }
+
+sortGl2dMockList(allMockList);
+console.log('');
+
+// main
+if(argv.queue) {
+    runInQueue(allMockList);
+} else {
+    runInBatch(allMockList);
+}

test/jasmine/tests/titles_test.js

@@ -1141,7 +1141,7 @@ describe('Editable titles', function() {
         .then(done);
     });
 
-    it('has no hover effects for titles that used to be blank', function(done) {
+    it('@flaky has no hover effects for titles that used to be blank', function(done) {
         Plotly.plot(gd, data, {
             xaxis: {title: {text: ''}},
             yaxis: {title: {text: ''}},