From fa7db2e2d4bfcc8962f010f5ec8dce326817f173 Mon Sep 17 00:00:00 2001 From: user202729 <25191436+user202729@users.noreply.github.com> Date: Mon, 17 Feb 2025 22:55:10 +0700 Subject: [PATCH 1/7] Implement # sage.doctest: flaky marker --- src/sage/doctest/forker.py | 134 +++++++++++++++++++++++++++++++----- src/sage/doctest/parsing.py | 22 +++++- 2 files changed, 138 insertions(+), 18 deletions(-) diff --git a/src/sage/doctest/forker.py b/src/sage/doctest/forker.py index f3950cde06b..81ad0cfa665 100644 --- a/src/sage/doctest/forker.py +++ b/src/sage/doctest/forker.py @@ -1724,6 +1724,21 @@ class DocTestDispatcher(SageObject): """ Create parallel :class:`DocTestWorker` processes and dispatches doctesting tasks. + + .. NOTE:: + + If this is run directly in the normal Sage command-line, + it calls :func:`init_sage` which in turn calls + :meth:`.switch_backend` to the doctest backend, which is + incompatible with the IPython-based command-line. + As such, if an error such as + ``TypeError: cannot unpack non-iterable NoneType object`` is seen, + a workaround is to run the following:: + + sage: # not tested + sage: from sage.repl.rich_output.backend_ipython import BackendIPythonCommandline + sage: backend = BackendIPythonCommandline() + sage: get_ipython().display_formatter.dm.switch_backend(backend, shell=get_ipython()) """ def __init__(self, controller: DocTestController): """ @@ -1851,6 +1866,67 @@ def parallel_dispatch(self): 1 of 1 in ... [1 test, 1 failure, ...s wall] Killing test ... + + TESTS: + + Test flaky files. This test should fail the first time and success the second time:: + + sage: with NTF(suffix='.py', mode='w+t') as f1: + ....: t = walltime() + ....: _ = f1.write(f"# sage.doctest: flaky\n'''\nsage: sleep(10 if walltime() < {t+1} else 0)\n'''") + ....: f1.flush() + ....: DC = DocTestController(DocTestDefaults(timeout=2, die_timeout=1), + ....: [f1.name]) + ....: DC.expand_files_into_sources() + ....: DD = DocTestDispatcher(DC) + ....: DR = DocTestReporter(DC) + ....: DC.reporter = DR + ....: DC.dispatcher = DD + ....: DC.timer = Timer().start() + ....: DD.parallel_dispatch() + sage -t ... + sage -t ... + [1 test, ...s wall] + + This test always fail, so even flaky can't help it:: + + sage: with NTF(suffix='.py', mode='w+t') as f1: + ....: _ = f1.write(f"# sage.doctest: flaky\n'''\nsage: sleep(10)\n'''") + ....: f1.flush() + ....: DC = DocTestController(DocTestDefaults(timeout=2, die_timeout=1), + ....: [f1.name]) + ....: DC.expand_files_into_sources() + ....: DD = DocTestDispatcher(DC) + ....: DR = DocTestReporter(DC) + ....: DC.reporter = DR + ....: DC.dispatcher = DD + ....: DC.timer = Timer().start() + ....: DD.parallel_dispatch() + sage -t ... + sage -t ... + Timed out + ********************************************************************** + ... + + Of course without flaky, the test should fail (since it timeouts the first time):: + + sage: with NTF(suffix='.py', mode='w+t') as f1: + ....: t = walltime() + ....: _ = f1.write(f"'''\nsage: sleep(10 if walltime() < {t+1} else 0)\n'''") + ....: f1.flush() + ....: DC = DocTestController(DocTestDefaults(timeout=2, die_timeout=1), + ....: [f1.name]) + ....: DC.expand_files_into_sources() + ....: DD = DocTestDispatcher(DC) + ....: DR = DocTestReporter(DC) + ....: DC.reporter = DR + ....: DC.dispatcher = DD + ....: DC.timer = Timer().start() + ....: DD.parallel_dispatch() + sage -t ... + Timed out + ********************************************************************** + ... """ opt = self.controller.options @@ -1937,6 +2013,28 @@ def sel_exit(): # precision. now = time.time() + def start_new_worker(source: DocTestSource, num_retries_left: typing.Optional[int] = None) -> DocTestWorker: + nonlocal opt, target_endtime, now, pending_tests, sel_exit + import copy + worker_options = copy.copy(opt) + baseline = self.controller.source_baseline(source) + if target_endtime is not None: + worker_options.target_walltime = (target_endtime - now) / (max(1, pending_tests / opt.nthreads)) + w = DocTestWorker(source, options=worker_options, funclist=[sel_exit], baseline=baseline) + if num_retries_left is not None: + w.num_retries_left = num_retries_left + elif 'flaky' in source.file_optional_tags: + w.num_retries_left = 1 + heading = self.controller.reporter.report_head(w.source) + if not self.controller.options.only_errors: + w.messages = heading + "\n" + # Store length of heading to detect if the + # worker has something interesting to report. + w.heading_len = len(w.messages) + w.start() # This might take some time + w.deadline = time.time() + opt.timeout + return w + # If there were any substantial changes in the state # (new worker started or finished worker reported), # restart this while loop instead of calling pselect(). @@ -1994,6 +2092,14 @@ def sel_exit(): # Similarly, process finished workers. new_finished = [] for w in finished: + if w.killed and w.num_retries_left > 0: + # in this case, the messages from w should be suppressed + # (better handling could be implemented later) + if follow is w: + follow = None + workers.append(start_new_worker(w.source, w.num_retries_left - 1)) + continue + if opt.exitfirst and w.result[1].failures: abort_now = True elif follow is not None and follow is not w: @@ -2026,28 +2132,13 @@ def sel_exit(): while (source_iter is not None and len(workers) < opt.nthreads and (not job_client or job_client.acquire())): try: - source = next(source_iter) + source: DocTestSource = next(source_iter) except StopIteration: source_iter = None if job_client: job_client.release() else: - # Start a new worker. - import copy - worker_options = copy.copy(opt) - baseline = self.controller.source_baseline(source) - if target_endtime is not None: - worker_options.target_walltime = (target_endtime - now) / (max(1, pending_tests / opt.nthreads)) - w = DocTestWorker(source, options=worker_options, funclist=[sel_exit], baseline=baseline) - heading = self.controller.reporter.report_head(w.source) - if not self.controller.options.only_errors: - w.messages = heading + "\n" - # Store length of heading to detect if the - # worker has something interesting to report. - w.heading_len = len(w.messages) - w.start() # This might take some time - w.deadline = time.time() + opt.timeout - workers.append(w) + workers.append(start_new_worker(source)) restart = True # Recompute state if needed @@ -2181,6 +2272,7 @@ class should be accessed by the child process. EXAMPLES:: + sage: # long time sage: from sage.doctest.forker import DocTestWorker, DocTestTask sage: from sage.doctest.sources import FileDocTestSource sage: from sage.doctest.reporting import DocTestReporter @@ -2223,6 +2315,10 @@ def __init__(self, source, options, funclist=[], baseline=None): self.funclist = funclist self.baseline = baseline + # This is not used by this class in any way, but DocTestDispatcher + # uses this to keep track of reruns for flaky tests. + self.num_retries_left = 0 + # Open pipe for messages. These are raw file descriptors, # not Python file objects! self.rmessages, self.wmessages = os.pipe() @@ -2305,6 +2401,7 @@ def start(self): TESTS:: + sage: # long time sage: from sage.doctest.forker import DocTestWorker, DocTestTask sage: from sage.doctest.sources import FileDocTestSource sage: from sage.doctest.reporting import DocTestReporter @@ -2344,6 +2441,7 @@ def read_messages(self): EXAMPLES:: + sage: # long time sage: from sage.doctest.forker import DocTestWorker, DocTestTask sage: from sage.doctest.sources import FileDocTestSource sage: from sage.doctest.reporting import DocTestReporter @@ -2378,6 +2476,7 @@ def save_result_output(self): EXAMPLES:: + sage: # long time sage: from sage.doctest.forker import DocTestWorker, DocTestTask sage: from sage.doctest.sources import FileDocTestSource sage: from sage.doctest.reporting import DocTestReporter @@ -2508,6 +2607,7 @@ class DocTestTask: EXAMPLES:: + sage: # long time sage: from sage.doctest.forker import DocTestTask sage: from sage.doctest.sources import FileDocTestSource sage: from sage.doctest.control import DocTestDefaults, DocTestController diff --git a/src/sage/doctest/parsing.py b/src/sage/doctest/parsing.py index 05662691d16..c6257ceedae 100644 --- a/src/sage/doctest/parsing.py +++ b/src/sage/doctest/parsing.py @@ -56,7 +56,7 @@ ansi_escape_sequence = re.compile(r"(\x1b[@-Z\\-~]|\x1b\[.*?[@-~]|\x9b.*?[@-~])") special_optional_regex = ( - "py2|long time|not implemented|not tested|optional|needs|known bug" + "py2|long time|not implemented|not tested|optional|needs|known bug|flaky" ) tag_with_explanation_regex = r"((?:!?\w|[.])*)\s*(?:\((?P.*?)\))?" optional_regex = re.compile( @@ -260,6 +260,11 @@ def parse_file_optional_tags(lines): sage: with open(filename, "r") as f: ....: parse_file_optional_tags(enumerate(f)) {'xyz': None} + sage: with open(filename, "w") as f: + ....: _ = f.write("# sage.doctest: flaky") + sage: with open(filename, "r") as f: + ....: parse_file_optional_tags(enumerate(f)) + {'flaky': None} """ tags = {} for line_count, line in lines: @@ -990,6 +995,14 @@ def parse(self, string, *args): 'C.minimum_distance(algorithm="guava") # optional - guava\n' sage: dte.want '...\n24\n' + + Test putting flaky tag in a single test:: + + sage: example5 = 'sage: 1 # flaky\n1' + sage: parsed5 = DTP.parse(example5) + Traceback (most recent call last): + ... + NotImplementedError: 'flaky' tag is only implemented for whole file, not for single test """ # Regular expressions find_sage_prompt = re.compile(r"^(\s*)sage: ", re.M) @@ -1063,6 +1076,8 @@ def check_and_clear_tag_counts(): for item in res: if isinstance(item, doctest.Example): optional_tags_with_values, _, is_persistent = parse_optional_tags(item.source, return_string_sans_tags=True) + if "flaky" in optional_tags_with_values: + raise NotImplementedError("'flaky' tag is only implemented for whole file, not for single test") optional_tags = set(optional_tags_with_values) if is_persistent: check_and_clear_tag_counts() @@ -1087,6 +1102,11 @@ def check_and_clear_tag_counts(): ('not tested' in optional_tags)): continue + if 'flaky' in optional_tags: + # since a single test cannot have the 'flaky' tag, + # this must come from file_optional_tags + optional_tags.remove('flaky') + if 'long time' in optional_tags: if self.long: optional_tags.remove('long time') From 359bd0d51b7e32b3c8ddc5ff9c2840784919071a Mon Sep 17 00:00:00 2001 From: user202729 <25191436+user202729@users.noreply.github.com> Date: Mon, 17 Feb 2025 23:13:56 +0700 Subject: [PATCH 2/7] Mark a few files as flaky --- src/sage/algebras/fusion_rings/fusion_ring.py | 1 + src/sage/rings/polynomial/multi_polynomial_libsingular.pyx | 1 + src/sage/rings/polynomial/polynomial_element.pyx | 1 + 3 files changed, 3 insertions(+) diff --git a/src/sage/algebras/fusion_rings/fusion_ring.py b/src/sage/algebras/fusion_rings/fusion_ring.py index e454b07dfcb..4450a1482ce 100644 --- a/src/sage/algebras/fusion_rings/fusion_ring.py +++ b/src/sage/algebras/fusion_rings/fusion_ring.py @@ -1,3 +1,4 @@ +# sage.doctest: flaky """ Fusion Rings """ diff --git a/src/sage/rings/polynomial/multi_polynomial_libsingular.pyx b/src/sage/rings/polynomial/multi_polynomial_libsingular.pyx index ddbd4bdbca2..cb632f9481e 100644 --- a/src/sage/rings/polynomial/multi_polynomial_libsingular.pyx +++ b/src/sage/rings/polynomial/multi_polynomial_libsingular.pyx @@ -1,3 +1,4 @@ +# sage.doctest: flaky r""" Multivariate Polynomials via libSINGULAR diff --git a/src/sage/rings/polynomial/polynomial_element.pyx b/src/sage/rings/polynomial/polynomial_element.pyx index 9181fb9ae3e..d61960b6a8e 100644 --- a/src/sage/rings/polynomial/polynomial_element.pyx +++ b/src/sage/rings/polynomial/polynomial_element.pyx @@ -1,3 +1,4 @@ +# sage.doctest: flaky """ Univariate polynomial base class From 8375fa5588a838351a17faf97b1bbe25cbe9838c Mon Sep 17 00:00:00 2001 From: user202729 <25191436+user202729@users.noreply.github.com> Date: Mon, 17 Feb 2025 23:38:43 +0700 Subject: [PATCH 3/7] Increase die_timeout --- src/sage/doctest/forker.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/sage/doctest/forker.py b/src/sage/doctest/forker.py index 81ad0cfa665..f84693a8f21 100644 --- a/src/sage/doctest/forker.py +++ b/src/sage/doctest/forker.py @@ -1875,7 +1875,7 @@ def parallel_dispatch(self): ....: t = walltime() ....: _ = f1.write(f"# sage.doctest: flaky\n'''\nsage: sleep(10 if walltime() < {t+1} else 0)\n'''") ....: f1.flush() - ....: DC = DocTestController(DocTestDefaults(timeout=2, die_timeout=1), + ....: DC = DocTestController(DocTestDefaults(timeout=2), ....: [f1.name]) ....: DC.expand_files_into_sources() ....: DD = DocTestDispatcher(DC) @@ -1893,7 +1893,7 @@ def parallel_dispatch(self): sage: with NTF(suffix='.py', mode='w+t') as f1: ....: _ = f1.write(f"# sage.doctest: flaky\n'''\nsage: sleep(10)\n'''") ....: f1.flush() - ....: DC = DocTestController(DocTestDefaults(timeout=2, die_timeout=1), + ....: DC = DocTestController(DocTestDefaults(timeout=2), ....: [f1.name]) ....: DC.expand_files_into_sources() ....: DD = DocTestDispatcher(DC) @@ -1914,7 +1914,7 @@ def parallel_dispatch(self): ....: t = walltime() ....: _ = f1.write(f"'''\nsage: sleep(10 if walltime() < {t+1} else 0)\n'''") ....: f1.flush() - ....: DC = DocTestController(DocTestDefaults(timeout=2, die_timeout=1), + ....: DC = DocTestController(DocTestDefaults(timeout=2), ....: [f1.name]) ....: DC.expand_files_into_sources() ....: DD = DocTestDispatcher(DC) From 299a76b62a555a843eed91988f9e74e52e80d109 Mon Sep 17 00:00:00 2001 From: user202729 <25191436+user202729@users.noreply.github.com> Date: Tue, 18 Feb 2025 07:15:39 +0700 Subject: [PATCH 4/7] Some missing documentation update --- src/sage/doctest/forker.py | 6 +++--- src/sage/doctest/parsing.py | 12 ++++++++++++ 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/src/sage/doctest/forker.py b/src/sage/doctest/forker.py index f84693a8f21..ca6602b107c 100644 --- a/src/sage/doctest/forker.py +++ b/src/sage/doctest/forker.py @@ -1729,9 +1729,9 @@ class DocTestDispatcher(SageObject): If this is run directly in the normal Sage command-line, it calls :func:`init_sage` which in turn calls - :meth:`.switch_backend` to the doctest backend, which is - incompatible with the IPython-based command-line. - As such, if an error such as + :meth:`~sage.repl.rich_output.display_manager.DisplayManager.switch_backend` + to the doctest backend, which is incompatible with the IPython-based + command-line. As such, if an error such as ``TypeError: cannot unpack non-iterable NoneType object`` is seen, a workaround is to run the following:: diff --git a/src/sage/doctest/parsing.py b/src/sage/doctest/parsing.py index c6257ceedae..0dab5ead0d5 100644 --- a/src/sage/doctest/parsing.py +++ b/src/sage/doctest/parsing.py @@ -97,6 +97,7 @@ def parse_optional_tags( - ``'not tested'`` - ``'known bug'`` (possible values are ``None``, ``linux`` and ``macos``) - ``'py2'`` + - ``'flaky'`` - ``'optional -- FEATURE...'`` or ``'needs FEATURE...'`` -- the dictionary will just have the key ``'FEATURE'`` @@ -166,6 +167,17 @@ def parse_optional_tags( sage: parse_optional_tags("sage: #this is not #needs scipy\n....: import scipy", ....: return_string_sans_tags=True) ({'scipy': None}, 'sage: #this is not \n....: import scipy', False) + + TESTS:: + + sage: parse_optional_tags("# flaky") + {'flaky': None} + + Remember to update the documentation above whenever the following changes:: + + sage: from sage.doctest.parsing import special_optional_regex + sage: special_optional_regex.pattern + 'py2|long time|not implemented|not tested|optional|needs|known bug|flaky' """ safe, literals, state = strip_string_literals(string) split = safe.split('\n', 1) From cc22c5a919a8da7eb4d3d45755f0fe5693186344 Mon Sep 17 00:00:00 2001 From: user202729 <25191436+user202729@users.noreply.github.com> Date: Tue, 18 Mar 2025 11:27:32 +0700 Subject: [PATCH 5/7] Also handle flaky segmentation fault etc. --- src/sage/doctest/forker.py | 46 +++++++++++++++++++++++++++++++++----- 1 file changed, 40 insertions(+), 6 deletions(-) diff --git a/src/sage/doctest/forker.py b/src/sage/doctest/forker.py index 2767fa15ef4..7c78f4e51ac 100644 --- a/src/sage/doctest/forker.py +++ b/src/sage/doctest/forker.py @@ -1889,6 +1889,29 @@ def parallel_dispatch(self): sage -t ... [1 test, ...s wall] + Segmentation fault and abort are also handled:: + + sage: with NTF(suffix='.py', mode='w+t') as f1: + ....: t = walltime() + ....: _ = f1.write(f"# sage.doctest: flaky\n'''\n" + ....: f"sage: from cysignals.tests import unguarded_abort\n" + ....: f"sage: sleep(1.5r)\n" + ....: f"sage: if walltime() < {t+1}: unguarded_abort()\n" + ....: f"'''") + ....: f1.flush() + ....: DC = DocTestController(DocTestDefaults(timeout=10), + ....: [f1.name]) + ....: DC.expand_files_into_sources() + ....: DD = DocTestDispatcher(DC) + ....: DR = DocTestReporter(DC) + ....: DC.reporter = DR + ....: DC.dispatcher = DD + ....: DC.timer = Timer().start() + ....: DD.parallel_dispatch() + sage -t ... + sage -t ... + [3 tests, ...s wall] + This test always fail, so even flaky can't help it:: sage: with NTF(suffix='.py', mode='w+t') as f1: @@ -1972,11 +1995,11 @@ def parallel_dispatch(self): # List of alive DocTestWorkers (child processes). Workers which # are done but whose messages have not been read are also # considered alive. - workers = [] + workers: list[DocTestWorker] = [] # List of DocTestWorkers which have finished running but # whose results have not been reported yet. - finished = [] + finished: list[DocTestWorker] = [] # If exitfirst is set and we got a failure. abort_now = False @@ -2048,7 +2071,7 @@ def start_new_worker(source: DocTestSource, num_retries_left: typing.Optional[in # "finished" list. # Create a new list "new_workers" containing the active # workers (to avoid updating "workers" in place). - new_workers = [] + new_workers: list[DocTestWorker] = [] for w in workers: if w.rmessages is not None or w.is_alive(): if now >= w.deadline: @@ -2091,11 +2114,14 @@ def start_new_worker(source: DocTestSource, num_retries_left: typing.Optional[in workers = new_workers # Similarly, process finished workers. - new_finished = [] + new_finished: list[DocTestWorker] = [] for w in finished: - if w.killed and w.num_retries_left > 0: + if w.num_retries_left > 0: # in this case, the messages from w should be suppressed # (better handling could be implemented later) + # should also check w.killed or w.result if want to only retry + # on timeout/segmentation fault + # in that case w.source.file_optional_tags['flaky'] can be checked if follow is w: follow = None workers.append(start_new_worker(w.source, w.num_retries_left - 1)) @@ -2326,7 +2352,10 @@ def __init__(self, source, options, funclist=[], baseline=None): # Create Queue for the result. Since we're running only one # doctest, this "queue" will contain only 1 element. - self.result_queue = multiprocessing.Queue(1) + self.result_queue: "Queue[tuple[int, DictAsObject]]" = multiprocessing.Queue(1) + + # See :meth:`DocTestTask.__call__` for more information of this. + self.result: tuple[int, DictAsObject] # Temporary file for stdout/stderr of the child process. # Normally, this isn't used in the master process except to @@ -2668,6 +2697,11 @@ def __call__(self, options, outtmpfile=None, msgfile=None, result_queue=None, *, doctests and ``result_dict`` is a dictionary annotated with timings and error information. + ``result_dict`` contains the key ``'err'`` (possibly ``None``), + and optionally contain the keys ``'walltime'``, ``'cputime'``, + ``'walltime_skips'``, ``'tb'``, ``'tab_linenos'``, ``'optionals'``, + ``'failures'``. + - Also put ``(doctests, result_dict)`` onto the ``result_queue`` if the latter isn't None. From 48f79d6a9775879323f5153fa83bd8403007a0ab Mon Sep 17 00:00:00 2001 From: user202729 <25191436+user202729@users.noreply.github.com> Date: Tue, 18 Mar 2025 11:35:28 +0700 Subject: [PATCH 6/7] Some more markers --- src/sage/algebras/fusion_rings/fusion_ring.py | 2 +- src/sage/libs/singular/function.pyx | 1 + src/sage/rings/polynomial/plural.pyx | 1 + src/sage/rings/polynomial/polynomial_element.pyx | 2 +- 4 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/sage/algebras/fusion_rings/fusion_ring.py b/src/sage/algebras/fusion_rings/fusion_ring.py index 4450a1482ce..55797d5809f 100644 --- a/src/sage/algebras/fusion_rings/fusion_ring.py +++ b/src/sage/algebras/fusion_rings/fusion_ring.py @@ -1,4 +1,4 @@ -# sage.doctest: flaky +# sage.doctest: flaky (:issue:`39538`) """ Fusion Rings """ diff --git a/src/sage/libs/singular/function.pyx b/src/sage/libs/singular/function.pyx index 87f0b7bab69..6e12cda556a 100644 --- a/src/sage/libs/singular/function.pyx +++ b/src/sage/libs/singular/function.pyx @@ -1,3 +1,4 @@ +# sage.doctest: flaky (:issue:`29528`) """ libSingular: Functions diff --git a/src/sage/rings/polynomial/plural.pyx b/src/sage/rings/polynomial/plural.pyx index 3dcfb16ce81..633090ea898 100644 --- a/src/sage/rings/polynomial/plural.pyx +++ b/src/sage/rings/polynomial/plural.pyx @@ -1,3 +1,4 @@ +# sage.doctest: flaky (:issue:`29528`) r""" Noncommutative polynomials via libSINGULAR/Plural diff --git a/src/sage/rings/polynomial/polynomial_element.pyx b/src/sage/rings/polynomial/polynomial_element.pyx index 62d677af197..292f567b941 100644 --- a/src/sage/rings/polynomial/polynomial_element.pyx +++ b/src/sage/rings/polynomial/polynomial_element.pyx @@ -1,4 +1,4 @@ -# sage.doctest: flaky +# sage.doctest: flaky (:issue:`39183`) """ Univariate polynomial base class From f8580bb1039b86c5f9c5617e7a5ac61ff0bc54d8 Mon Sep 17 00:00:00 2001 From: user202729 <25191436+user202729@users.noreply.github.com> Date: Thu, 20 Mar 2025 15:23:02 +0700 Subject: [PATCH 7/7] Fix some bugs --- src/sage/doctest/forker.py | 43 +++++++++++++++++++++++++++++++------- 1 file changed, 36 insertions(+), 7 deletions(-) diff --git a/src/sage/doctest/forker.py b/src/sage/doctest/forker.py index 7c78f4e51ac..36385f4c112 100644 --- a/src/sage/doctest/forker.py +++ b/src/sage/doctest/forker.py @@ -1872,6 +1872,7 @@ def parallel_dispatch(self): Test flaky files. This test should fail the first time and success the second time:: + sage: # long time sage: with NTF(suffix='.py', mode='w+t') as f1: ....: t = walltime() ....: _ = f1.write(f"# sage.doctest: flaky\n'''\nsage: sleep(10 if walltime() < {t+1} else 0)\n'''") @@ -1891,13 +1892,14 @@ def parallel_dispatch(self): Segmentation fault and abort are also handled:: + sage: # long time sage: with NTF(suffix='.py', mode='w+t') as f1: ....: t = walltime() - ....: _ = f1.write(f"# sage.doctest: flaky\n'''\n" + ....: _ = f1.write(f"# sage.doctest: flaky\n" + ....: f"'''\n" ....: f"sage: from cysignals.tests import unguarded_abort\n" - ....: f"sage: sleep(1.5r)\n" - ....: f"sage: if walltime() < {t+1}: unguarded_abort()\n" - ....: f"'''") + ....: f"sage: if walltime() < {t+0.5r}: sleep(1r); unguarded_abort()\n" + ....: f"'''\n") ....: f1.flush() ....: DC = DocTestController(DocTestDefaults(timeout=10), ....: [f1.name]) @@ -1910,10 +1912,11 @@ def parallel_dispatch(self): ....: DD.parallel_dispatch() sage -t ... sage -t ... - [3 tests, ...s wall] + [2 tests, ...s wall] This test always fail, so even flaky can't help it:: + sage: # long time sage: with NTF(suffix='.py', mode='w+t') as f1: ....: _ = f1.write(f"# sage.doctest: flaky\n'''\nsage: sleep(10)\n'''") ....: f1.flush() @@ -1934,6 +1937,7 @@ def parallel_dispatch(self): Of course without flaky, the test should fail (since it timeouts the first time):: + sage: # long time sage: with NTF(suffix='.py', mode='w+t') as f1: ....: t = walltime() ....: _ = f1.write(f"'''\nsage: sleep(10 if walltime() < {t+1} else 0)\n'''") @@ -1951,6 +1955,31 @@ def parallel_dispatch(self): Timed out ********************************************************************** ... + + If it doesn't fail the first time, it must not be retried:: + + sage: from contextlib import redirect_stdout + sage: from io import StringIO + sage: with NTF(suffix='.py', mode='w+t') as f1, StringIO() as f, redirect_stdout(f): + ....: t = walltime() + ....: _ = f1.write(f"'''\nsage: sleep(0.5)\n'''") + ....: f1.flush() + ....: DC = DocTestController(DocTestDefaults(timeout=2), + ....: [f1.name]) + ....: DC.expand_files_into_sources() + ....: DD = DocTestDispatcher(DC) + ....: DR = DocTestReporter(DC) + ....: DC.reporter = DR + ....: DC.dispatcher = DD + ....: DC.timer = Timer().start() + ....: DD.parallel_dispatch() + ....: s = f.getvalue() + sage: print(s) + sage -t ... + ********************************************************************** + ... + sage: s.count("sage -t") + 1 """ opt = self.controller.options @@ -2048,7 +2077,7 @@ def start_new_worker(source: DocTestSource, num_retries_left: typing.Optional[in if num_retries_left is not None: w.num_retries_left = num_retries_left elif 'flaky' in source.file_optional_tags: - w.num_retries_left = 1 + w.num_retries_left = 2 heading = self.controller.reporter.report_head(w.source) if not self.controller.options.only_errors: w.messages = heading + "\n" @@ -2116,7 +2145,7 @@ def start_new_worker(source: DocTestSource, num_retries_left: typing.Optional[in # Similarly, process finished workers. new_finished: list[DocTestWorker] = [] for w in finished: - if w.num_retries_left > 0: + if (w.killed or w.result[1].err) and w.num_retries_left > 0: # in this case, the messages from w should be suppressed # (better handling could be implemented later) # should also check w.killed or w.result if want to only retry