add flaky to tests that sometimes fail

Author: basnijholt

adaptive/tests/test_learners.py

@@ -9,6 +9,7 @@
 import shutil
 import tempfile
 
+import flaky
 import numpy as np
 import pytest
 import scipy.spatial
@@ -456,6 +457,7 @@ def test_learner_performance_is_invariant_under_scaling(
     assert math.isclose(learner.loss(), control.loss(), rel_tol=1e-10)
 
 
+@flaky.flaky(max_runs=3)
 @run_with(
     Learner1D,
     Learner2D,

adaptive/tests/test_runner.py

@@ -3,6 +3,7 @@
 import sys
 import time
 
+import flaky
 import pytest
 
 from adaptive.learner import Learner1D, Learner2D
@@ -125,6 +126,7 @@ def test_ipyparallel_executor(ipyparallel_executor):
     assert learner.npoints > 0
 
 
+@flaky.flaky(max_runs=3)
 @pytest.mark.timeout(60)
 @pytest.mark.skipif(not with_distributed, reason="dask.distributed is not installed")
 @pytest.mark.skipif(sys.version_info[:2] == (3, 8), reason="XXX: seems to always fail")

setup.py

@@ -43,6 +43,7 @@ def get_version_and_cmdclass(package_name):
         "plotly",
     ],
     "testing": [
+        "flaky",
         "pytest",
         "pytest-cov",
         "pytest-randomly",

tox.ini

@@ -65,4 +65,4 @@ include_trailing_comma=True
 force_grid_wrap=0
 use_parentheses=True
 line_length=88
-known_third_party=PIL,atomicwrites,holoviews,matplotlib,nbconvert,numpy,pytest,scipy,setuptools,skopt,sortedcollections,sortedcontainers
+known_third_party=PIL,atomicwrites,flaky,holoviews,matplotlib,nbconvert,numpy,pytest,scipy,setuptools,skopt,sortedcollections,sortedcontainers