address review

Author: blink1073

.evergreen/generated_configs/functions.yml

@@ -145,6 +145,7 @@ functions:
           - MONGODB_API_VERSION
           - REQUIRE_API_VERSION
           - DEBUG_LOG
+          - DISABLE_FLAKY
           - ORCHESTRATION_FILE
           - OCSP_SERVER_TYPE
           - VERSION

.evergreen/scripts/generate_config.py

@@ -1084,6 +1084,7 @@ def create_run_tests_func():
         "MONGODB_API_VERSION",
         "REQUIRE_API_VERSION",
         "DEBUG_LOG",
+        "DISABLE_FLAKY",
         "ORCHESTRATION_FILE",
         "OCSP_SERVER_TYPE",
         "VERSION",

CONTRIBUTING.md

@@ -410,6 +410,8 @@ We have a custom `flaky` decorator in [test/asynchronous/utils.py](test/asynchro
 tests that are `flaky`.  By default the decorator only applies when not running on CPython on Linux, since other
 runtimes tend to have more variation.  When using the `flaky` decorator, open a corresponding ticket and
 a use the ticket number as the "reason" parameter to the decorator, e.g. `@flaky(reason="PYTHON-1234")`.
+When running tests locally (not in CI), the `flaky` decorator will be disabled unless `ENABLE_FLAKY` is set.
+To disable the `flaky` decorator in CI, you can use `evergreen patch --param DISABLE_FLAKY=1`.
 
 ## Specification Tests
 

test/asynchronous/test_client_bulk_write.py

@@ -629,7 +629,7 @@ async def asyncSetUp(self):
     @async_client_context.require_failCommand_fail_point
     @flaky(reason="PYTHON-5290", max_runs=3, affects_cpython_linux=True)
     async def test_timeout_in_multi_batch_bulk_write(self):
-        if sys.platform != "linux":
+        if sys.platform != "linux" and "CI" in os.environ:
             self.skipTest("PYTHON-3522 CSOT test runs too slow on Windows and MacOS")
         _OVERHEAD = 500
 

test/asynchronous/unified_format.py

@@ -536,17 +536,17 @@ def maybe_skip_test(self, spec):
             slow_pypy = [
                 "timeoutMS applies to whole operation.*",
             ]
-            if sys.platform == "win32" and "gridfs" in class_name:
+            if "CI" in os.environ and sys.platform == "win32" and "gridfs" in class_name:
                 self.skipTest("PYTHON-3522 CSOT GridFS test runs too slow on Windows")
-            if sys.platform == "win32":
+            if "CI" in os.environ and sys.platform == "win32":
                 for pat in slow_win32:
                     if re.match(pat.lower(), description):
                         self.skipTest("PYTHON-3522 CSOT test runs too slow on Windows")
-            if sys.platform == "darwin":
+            if "CI" in os.environ and sys.platform == "darwin":
                 for pat in slow_macos:
                     if re.match(pat.lower(), description):
                         self.skipTest("PYTHON-3522 CSOT test runs too slow on MacOS")
-            if sys.implementation.name.lower() == "pypy":
+            if "CI" in os.environ and sys.implementation.name.lower() == "pypy":
                 for pat in slow_pypy:
                     if re.match(pat.lower(), description):
                         self.skipTest("PYTHON-3522 CSOT test runs too slow on PyPy")

test/asynchronous/utils.py

@@ -183,6 +183,9 @@ def flaky(
         raise ValueError("flaky requires a reason input")
     is_cpython_linux = sys.platform == "linux" and sys.implementation.name == "cpython"
     disable_flaky = "DISABLE_FLAKY" in os.environ
+    if "CI" not in os.environ and "ENABLE_FLAKY" not in os.environ:
+        disable_flaky = True
+
     if disable_flaky or (is_cpython_linux and not affects_cpython_linux):
         max_runs = 1
         min_passes = 1

test/test_client_bulk_write.py

@@ -625,7 +625,7 @@ def setUp(self):
     @client_context.require_failCommand_fail_point
     @flaky(reason="PYTHON-5290", max_runs=3, affects_cpython_linux=True)
     def test_timeout_in_multi_batch_bulk_write(self):
-        if sys.platform != "linux":
+        if sys.platform != "linux" and "CI" in os.environ:
             self.skipTest("PYTHON-3522 CSOT test runs too slow on Windows and MacOS")
         _OVERHEAD = 500
 

test/unified_format.py

@@ -535,17 +535,17 @@ def maybe_skip_test(self, spec):
             slow_pypy = [
                 "timeoutMS applies to whole operation.*",
             ]
-            if sys.platform == "win32" and "gridfs" in class_name:
+            if "CI" in os.environ and sys.platform == "win32" and "gridfs" in class_name:
                 self.skipTest("PYTHON-3522 CSOT GridFS test runs too slow on Windows")
-            if sys.platform == "win32":
+            if "CI" in os.environ and sys.platform == "win32":
                 for pat in slow_win32:
                     if re.match(pat.lower(), description):
                         self.skipTest("PYTHON-3522 CSOT test runs too slow on Windows")
-            if sys.platform == "darwin":
+            if "CI" in os.environ and sys.platform == "darwin":
                 for pat in slow_macos:
                     if re.match(pat.lower(), description):
                         self.skipTest("PYTHON-3522 CSOT test runs too slow on MacOS")
-            if sys.implementation.name.lower() == "pypy":
+            if "CI" in os.environ and sys.implementation.name.lower() == "pypy":
                 for pat in slow_pypy:
                     if re.match(pat.lower(), description):
                         self.skipTest("PYTHON-3522 CSOT test runs too slow on PyPy")

test/utils.py

@@ -181,6 +181,9 @@ def flaky(
         raise ValueError("flaky requires a reason input")
     is_cpython_linux = sys.platform == "linux" and sys.implementation.name == "cpython"
     disable_flaky = "DISABLE_FLAKY" in os.environ
+    if "CI" not in os.environ and "ENABLE_FLAKY" not in os.environ:
+        disable_flaky = True
+
     if disable_flaky or (is_cpython_linux and not affects_cpython_linux):
         max_runs = 1
         min_passes = 1