add a required reason parameter

Author: blink1073

CONTRIBUTING.md

@@ -406,10 +406,10 @@ If you are running one of the `no-responder` tests, omit the `run-server` step.
 
 ## Handling flaky tests
 
-We have a custom ``flaky`` decorator in [test/asynchronous/utils.py](test/asynchronous/utils.py) that can be used for
-tests that are ``flaky``.  By default the decorator only applies when not running on CPython on Linux, since other
-runtimes tend to have more variation.  When using the ``flaky`` decorator, open a corresponding ticket and
-a comment with the ticket number.
+We have a custom `flaky` decorator in [test/asynchronous/utils.py](test/asynchronous/utils.py) that can be used for
+tests that are `flaky`.  By default the decorator only applies when not running on CPython on Linux, since other
+runtimes tend to have more variation.  When using the `flaky` decorator, open a corresponding ticket and
+a use the ticket number as the "reason" parameter to the decorator, e.g. `@flaky(reason="PYTHON-1234")`.
 
 ## Specification Tests
 

test/asynchronous/test_client_bulk_write.py

@@ -627,8 +627,7 @@ async def asyncSetUp(self):
 
     @async_client_context.require_version_min(8, 0, 0, -24)
     @async_client_context.require_failCommand_fail_point
-    # PYTHON-5290
-    @flaky(max_runs=3, affects_cpython_linux=True)
+    @flaky(reason="PYTHON-5290", max_runs=3, affects_cpython_linux=True)
     async def test_timeout_in_multi_batch_bulk_write(self):
         if sys.platform != "linux":
             self.skipTest("PYTHON-3522 CSOT test runs too slow on Windows and MacOS")

test/asynchronous/test_csot.py

@@ -44,7 +44,7 @@
 class TestCSOT(AsyncIntegrationTest):
     RUN_ON_LOAD_BALANCER = True
 
-    @flaky  # PYTHON-3522
+    @flaky(reason="PYTHON-3522")
     async def test_timeout_nested(self):
         coll = self.db.coll
         self.assertEqual(_csot.get_timeout(), None)
@@ -82,7 +82,7 @@ async def test_timeout_nested(self):
         self.assertEqual(_csot.get_rtt(), 0.0)
 
     @async_client_context.require_change_streams
-    @flaky  # PYTHON-3522
+    @flaky(reason="PYTHON-3522")
     async def test_change_stream_can_resume_after_timeouts(self):
         coll = self.db.test
         await coll.insert_one({})

test/asynchronous/test_cursor.py

@@ -1416,7 +1416,7 @@ async def test_to_list_length(self):
         docs = await c.to_list(3)
         self.assertEqual(len(docs), 2)
 
-    @flaky  # PYTHON-3522
+    @flaky(reason="PYTHON-3522")
     async def test_to_list_csot_applied(self):
         client = await self.async_single_client(timeoutMS=500, w=1)
         coll = client.pymongo.test
@@ -1458,7 +1458,7 @@ async def test_command_cursor_to_list_length(self):
         self.assertEqual(len(await result.to_list(1)), 1)
 
     @async_client_context.require_failCommand_blockConnection
-    @flaky  # PYTHON-3522
+    @flaky(reason="PYTHON-3522")
     async def test_command_cursor_to_list_csot_applied(self):
         client = await self.async_single_client(timeoutMS=500, w=1)
         coll = client.pymongo.test

test/asynchronous/test_encryption.py

@@ -3262,7 +3262,7 @@ async def test_kms_retry(self):
 class TestAutomaticDecryptionKeys(AsyncEncryptionIntegrationTest):
     @async_client_context.require_no_standalone
     @async_client_context.require_version_min(7, 0, -1)
-    @flaky  # PYTHON-4982
+    @flaky(reason="PYTHON-4982")
     async def asyncSetUp(self):
         await super().asyncSetUp()
         self.key1_document = json_data("etc", "data", "keys", "key1-document.json")

test/asynchronous/test_retryable_writes.py

@@ -497,7 +497,7 @@ class TestPoolPausedError(AsyncIntegrationTest):
     @async_client_context.require_failCommand_blockConnection
     @async_client_context.require_retryable_writes
     @client_knobs(heartbeat_frequency=0.05, min_heartbeat_interval=0.05)
-    @flaky  # PYTHON-5291
+    @flaky(reason="PYTHON-5291")
     async def test_pool_paused_error_is_retryable(self):
         cmap_listener = CMAPListener()
         cmd_listener = OvertCommandListener()

test/asynchronous/test_server_selection_in_window.py

@@ -138,7 +138,7 @@ async def frequencies(self, client, listener, n_finds=10):
 
     @async_client_context.require_failCommand_appName
     @async_client_context.require_multiple_mongoses
-    @flaky  # PYTHON-3689
+    @flaky(reason="PYTHON-3689")
     async def test_load_balancing(self):
         listener = OvertCommandListener()
         cmap_listener = CMAPListener()

test/asynchronous/test_srv_polling.py

@@ -255,7 +255,7 @@ def final_callback():
             # Nodelist should reflect new valid DNS resolver response.
             await self.assert_nodelist_change(response_final, client)
 
-    @flaky  # PYTHON-5315
+    @flaky(reason="PYTHON-5315")
     async def test_recover_from_initially_empty_seedlist(self):
         def empty_seedlist():
             return []

test/asynchronous/unified_format.py

@@ -1390,23 +1390,17 @@ async def run_scenario(self, spec, uri=None):
 
         # Handle flaky tests.
         flaky_tests = [
-            # PYTHON-5170
-            ".*test_discovery_and_monitoring.*",
-            # PYTHON-5174
-            ".*Driver_extends_timeout_while_streaming",
-            # PYTHON-5315
-            ".*TestSrvPolling.test_recover_from_initially_.*",
-            # PYTHON-4987
-            ".*UnknownTransactionCommitResult_labels_to_connection_errors",
-            # PYTHON-3689
-            ".*TestProse.test_load_balancing",
-            # PYTHON-3522
-            ".*csot.*",
+            ("PYTHON-5170", ".*test_discovery_and_monitoring.*"),
+            ("PYTHON-5174", ".*Driver_extends_timeout_while_streaming"),
+            ("PYTHON-5315", ".*TestSrvPolling.test_recover_from_initially_.*"),
+            ("PYTHON-4987", ".*UnknownTransactionCommitResult_labels_to_connection_errors"),
+            ("PYTHON-3689", ".*TestProse.test_load_balancing"),
+            ("PYTHON-3522", ".*csot.*"),
         ]
-        for flaky_test in flaky_tests:
+        for reason, flaky_test in flaky_tests:
             if re.match(flaky_test, self.id()) is not None:
                 func_name = self.id()
-                options = dict(reset_func=self.asyncSetUp, func_name=func_name)
+                options = dict(reason=reason, reset_func=self.asyncSetUp, func_name=func_name)
                 if "csot" in func_name.lower():
                     options["max_runs"] = 3
                     options["affects_cpython_linux"] = True

test/asynchronous/utils.py

@@ -159,8 +159,8 @@ async def async_joinall(tasks):
 
 
 def flaky(
-    func=None,
     *,
+    reason=None,
     max_runs=2,
     min_passes=1,
     delay=1,
@@ -170,6 +170,7 @@ def flaky(
 ):
     """Decorate a test as flaky.
 
+    :param reason: the reason why the test is flaky
     :param max_runs: the maximum number of runs before raising an error
     :param min_passes: the minimum number of passing runs
     :param delay: the delay in seconds between retries
@@ -178,6 +179,8 @@ def flaky(
     :param reset_func: a function to call before retrying
 
     """
+    if reason is None:
+        raise ValueError("flaky requires a reason input")
     is_cpython_linux = sys.platform == "linux" and sys.implementation.name == "cpython"
     disable_flaky = "DISABLE_FLAKY" in os.environ
     if disable_flaky or (is_cpython_linux and not affects_cpython_linux):
@@ -198,7 +201,7 @@ async def wrapper(*args, **kwargs):
                     if i == max_runs - 1:
                         raise e
                     print(
-                        f"Retrying after attempt {i+1} of {func_name or target_func.__name__} failed with:\n"
+                        f"Retrying after attempt {i+1} of {func_name or target_func.__name__} failed with ({reason})):\n"
                         f"{traceback.format_exc()}",
                         file=sys.stderr,
                     )
@@ -208,11 +211,6 @@ async def wrapper(*args, **kwargs):
 
         return wrapper
 
-    # If `func` is callable, the decorator was used without arguments (`@flaky`).
-    if callable(func):
-        return decorator(func)
-
-    # Otherwise, return the decorator function, allowing arguments (`@flaky(max_runs=...)`).
     return decorator