Merge pull request #3262 from vkarak/feat/retries-threshold

[feat] Add new `--retries-threshold` option

Author: vkarak

docs/manpage.rst

@@ -694,6 +694,17 @@ Options controlling ReFrame execution
    .. versionchanged:: 3.6.1
       Multiple report files are now accepted.
 
+
+.. option:: --retries-threshold=VALUE[%]
+
+   Skip retries (see :option:`--max-retries`) if failures exceed the given threshold.
+
+   Threshold can be specified either as an absolute value or as a percentage using the ``%`` character, e.g., ``--retries-threshold=30%``.
+   Note that in certain shells the ``%`` character may need to be escaped.
+
+   .. versionadded:: 4.7
+
+
 .. option:: -S, --setvar=[TEST.]VAR=VAL
 
    Set variable ``VAR`` in all tests or optionally only in test ``TEST`` to ``VAL``.

reframe/frontend/cli.py

@@ -522,6 +522,11 @@ def main():
         metavar='REPORT',
         help='Restore a testing session from REPORT file'
     )
+    run_options.add_argument(
+        '--retries-threshold', action='store', default='1000%',
+        metavar='VALUE[%]',
+        help='Retry tests only if failures do not exceed threshold'
+    )
     run_options.add_argument(
         '-S', '--setvar', action='append', metavar='[TEST.]VAR=VAL',
         dest='vars', default=[],
@@ -1564,8 +1569,16 @@ def module_unuse(*paths):
                 f"{options.reruns}"
             )
 
+        # Parse retries threshold
+        if options.retries_threshold[-1] == '%':
+            ratio = int(options.retries_threshold[:-1]) / 100.
+            retries_threshold = int(len(testcases)*ratio)
+        else:
+            retries_threshold = int(options.retries_threshold)
+
         runner = Runner(exec_policy, printer, options.max_retries,
-                        options.maxfail, options.reruns, options.duration)
+                        options.maxfail, options.reruns, options.duration,
+                        retries_threshold)
         try:
             time_start = time.time()
             runner.runall(testcases, restored_cases)

reframe/frontend/executors/__init__.py

@@ -573,10 +573,12 @@ class Runner:
     _timeout = fields.TypedField(typ.Duration, type(None), allow_implicit=True)
 
     def __init__(self, policy, printer=None, max_retries=0,
-                 max_failures=sys.maxsize, reruns=0, timeout=None):
+                 max_failures=sys.maxsize, reruns=0, timeout=None,
+                 retries_threshold=sys.maxsize):
         self._policy = policy
         self._printer = printer or PrettyPrinter()
         self._max_retries = max_retries
+        self._retries_threshold = retries_threshold
         self._num_reruns = reruns
         self._timeout = timeout
         self._t_init = timeout
@@ -620,7 +622,8 @@ def runall(self, testcases, restored_cases=None):
                 self._policy.set_expiry(self._t_init + self._timeout)
 
             self._runall(testcases)
-            if self._max_retries:
+            if (self._max_retries and
+                len(self._stats.failed()) <= self._retries_threshold):
                 restored_cases = restored_cases or []
                 self._retry_failed(testcases + restored_cases)
 

unittests/test_policies.py

@@ -274,6 +274,13 @@ def test_retries_bad_check(make_runner, make_cases, common_exec_ctx):
     assert_runall(runner)
     assert runner.max_retries == rt.runtime().current_run
     assert 2 == len(runner.stats.failed())
+    assert 3 == runner.stats.num_runs
+
+
+def test_retries_threshold(make_runner, make_cases, common_exec_ctx):
+    runner = make_runner(max_retries=2, retries_threshold=1)
+    runner.runall(make_cases([BadSetupCheck(), BadSetupCheckEarly()]))
+    assert 1 == runner.stats.num_runs
 
 
 def test_retries_good_check(make_runner, make_cases, common_exec_ctx):