skip more tests

Author: blink1073

CONTRIBUTING.md

@@ -404,6 +404,13 @@ If you are running one of the `no-responder` tests, omit the `run-server` step.
 - Regenerate the test variants and tasks using `pre-commit run --all-files generate-config`.
 - Make sure to add instructions for running the test suite to `CONTRIBUTING.md`.
 
+## Handling flaky tests
+
+We have a custom ``flaky`` decorator in [test/asynchronous/utils.py](test/asynchronous/utils.py) that can be used for
+tests that are ``flaky``.  By default the decorator only applies when not running on CPython on Linux, since other
+runtimes tend to have more variation.  When using the ``flaky`` decorator, open a corresponding ticket and
+a comment with the ticket number.
+
 ## Specification Tests
 
 The MongoDB [specifications repository](https://github.com/mongodb/specifications)

test/asynchronous/unified_format.py

@@ -534,17 +534,25 @@ def maybe_skip_test(self, spec):
         description = spec["description"].lower()
         if "csot" in class_name:
             # Skip tests that are too slow to run on a given platform.
-            slow_win32 = [
-                "maxtimems value in the command is less than timeoutms",
-                "non-tailable cursor lifetime remaining timeoutms applied to getmore if timeoutmode is unset",
-            ]
             slow_macos = [
-                "non-tailable cursor lifetime remaining timeoutms applied to getmore if timeoutmode is unset"
+                "operation fails after two consecutive socket timeouts.*",
+                "operation succeeds after one socket timeout.*",
+                "Non-tailable cursor lifetime remaining timeoutMS applied to getMore if timeoutMode is unset",
+            ]
+            slow_win32 = [
+                *slow_macos,
+                "maxTimeMS value in the command is less than timeoutMS",
             ]
-            if sys.platform == "win32" and description in slow_win32 or "gridfs" in class_name:
-                self.skipTest("PYTHON-3522 CSOT test runs too slow on Windows")
-            if sys.platform == "darwin" and description in slow_macos:
-                self.skipTest("PYTHON-3522 CSOT test runs too slow on MacOS")
+            if sys.platform == "win32" and "gridfs" in class_name:
+                self.skipTest("PYTHON-3522 CSOT GridFS test runs too slow on Windows")
+            if sys.platform == "win32":
+                for pat in slow_win32:
+                    if re.match(pat, description.lower()):
+                        self.skipTest("PYTHON-3522 CSOT test runs too slow on Windows")
+            if sys.platform == "darwin":
+                for pat in slow_macos:
+                    if re.match(pat, description.lower()):
+                        self.skipTest("PYTHON-3522 CSOT test runs too slow on MacOS")
             if async_client_context.storage_engine == "mmapv1":
                 self.skipTest(
                     "MMAPv1 does not support retryable writes which is required for CSOT tests"

test/asynchronous/utils.py

@@ -168,6 +168,16 @@ def flaky(
     func_name=None,
     reset_func=None,
 ):
+    """Decorate a test as flaky.
+
+    :param max_runs: the maximum number of runs before raising an error
+    :param min_passes: the minimum number of passing runs
+    :param delay: the delay in seconds between retries
+    :param affects_cpython_links: whether the test is flaky on CPython on Linux
+    :param func_name: the name of the function, used for the rety message
+    :param reset_func: a function to call before retrying
+
+    """
     is_cpython_linux = sys.platform == "linux" and sys.implementation.name == "cpython"
     disable_flaky = "DISABLE_FLAKY" in os.environ
     if disable_flaky or (is_cpython_linux and not affects_cpython_linux):

test/unified_format.py

@@ -533,17 +533,25 @@ def maybe_skip_test(self, spec):
         description = spec["description"].lower()
         if "csot" in class_name:
             # Skip tests that are too slow to run on a given platform.
-            slow_win32 = [
-                "maxtimems value in the command is less than timeoutms",
-                "non-tailable cursor lifetime remaining timeoutms applied to getmore if timeoutmode is unset",
-            ]
             slow_macos = [
-                "non-tailable cursor lifetime remaining timeoutms applied to getmore if timeoutmode is unset"
+                "operation fails after two consecutive socket timeouts.*",
+                "operation succeeds after one socket timeout.*",
+                "Non-tailable cursor lifetime remaining timeoutMS applied to getMore if timeoutMode is unset",
+            ]
+            slow_win32 = [
+                *slow_macos,
+                "maxTimeMS value in the command is less than timeoutMS",
             ]
-            if sys.platform == "win32" and description in slow_win32 or "gridfs" in class_name:
-                self.skipTest("PYTHON-3522 CSOT test runs too slow on Windows")
-            if sys.platform == "darwin" and description in slow_macos:
-                self.skipTest("PYTHON-3522 CSOT test runs too slow on MacOS")
+            if sys.platform == "win32" and "gridfs" in class_name:
+                self.skipTest("PYTHON-3522 CSOT GridFS test runs too slow on Windows")
+            if sys.platform == "win32":
+                for pat in slow_win32:
+                    if re.match(pat, description.lower()):
+                        self.skipTest("PYTHON-3522 CSOT test runs too slow on Windows")
+            if sys.platform == "darwin":
+                for pat in slow_macos:
+                    if re.match(pat, description.lower()):
+                        self.skipTest("PYTHON-3522 CSOT test runs too slow on MacOS")
             if client_context.storage_engine == "mmapv1":
                 self.skipTest(
                     "MMAPv1 does not support retryable writes which is required for CSOT tests"

test/utils.py

@@ -166,6 +166,16 @@ def flaky(
     func_name=None,
     reset_func=None,
 ):
+    """Decorate a test as flaky.
+
+    :param max_runs: the maximum number of runs before raising an error
+    :param min_passes: the minimum number of passing runs
+    :param delay: the delay in seconds between retries
+    :param affects_cpython_links: whether the test is flaky on CPython on Linux
+    :param func_name: the name of the function, used for the rety message
+    :param reset_func: a function to call before retrying
+
+    """
     is_cpython_linux = sys.platform == "linux" and sys.implementation.name == "cpython"
     disable_flaky = "DISABLE_FLAKY" in os.environ
     if disable_flaky or (is_cpython_linux and not affects_cpython_linux):