Merge pull request #495 from pythonspeed/492-fil-profiler-fails-for-fairly-large-dataset

Disable OOM on macOS

Author: itamarst

CHANGELOG.md

@@ -1,4 +1,9 @@
 # Release notes
+## 2023.3.0 (2023-3-1)
+
+### Changes
+
+* Out-of-memory detection is disabled for now on macOS, since it is prone to false positives.
 
 ## 2023.1.0 (2023-1-20)
 

docs/src/oom.md

@@ -1,12 +1,15 @@
 # Debugging out-of-memory crashes using Fil
 
+> **Note:** Out-of-memory detection is currently disabled on macOS.
+> This [may change in a future release](https://github.com/pythonspeed/filprofiler/issues/494).
+
 Typically when your program runs out of memory, it will crash, or get killed mysteriously by the operating system, or [other unfortunate side-effects](https://pythonspeed.com/articles/python-out-of-memory/).
 
 To help you debug these problems, Fil will heuristically try to catch out-of-memory conditions, and dump a report if thinks your program is out of memory.
 It will then exit with exit code 53.
 
 ```console
-$ fil-profile run oom.py 
+$ fil-profile run oom.py
 ...
 =fil-profile= Wrote memory usage flamegraph to fil-result/2020-06-15T12:37:13.033/out-of-memory.svg
 ```

filprofiler/_ipython.py

@@ -8,7 +8,7 @@
 from tempfile import mkdtemp
 
 from IPython.core.magic import Magics, magics_class, cell_magic
-from IPython.display import IFrame, display
+from IPython.display import IFrame, display, HTML
 
 from .api import profile
 
@@ -66,4 +66,7 @@ def run_with_profile(code_to_profile):
         return profile(code_to_profile, tempdir)
     finally:
         svg_path = tempdir / "peak-memory.svg"
-        display(IFrame(svg_path, width="100%", height="600"))
+        if svg_path.exists():
+            display(IFrame(svg_path, width="100%", height="600"))
+        else:
+            display(HTML("No report generated, perhaps zero memory was allocated."))

filprofiler/_script.py

@@ -87,8 +87,12 @@
 PARSER.add_argument(
     "--disable-oom-detection",
     action="store_true",
-    default=False,
-    help="Disable the heuristic that tries to catch out-of-memory situations before they occur",
+    default=True if sys.platform == "darwin" else False,
+    help=(
+        "Disable the heuristic that tries to catch out-of-memory situations before"
+        "they occur. OOM detection is always disabled on macOS at the moment, see "
+        "https://github.com/pythonspeed/filprofiler/issues/494"
+    ),
 )
 PARSER.add_argument(
     "--no-browser",
@@ -258,12 +262,18 @@ def stage_2():
         ),
     )
 
-    msg_browser = ", and opened automatically in a browser" if not arguments.no_browser else ", and stored in {}.".format(arguments.output_path)
-    msg_fil_plan = "=fil-profile= Memory usage will be written out at exit{}.\n".format(msg_browser)
+    msg_browser = (
+        ", and opened automatically in a browser"
+        if not arguments.no_browser
+        else ", and stored in {}.".format(arguments.output_path)
+    )
+    msg_fil_plan = "=fil-profile= Memory usage will be written out at exit{}.\n".format(
+        msg_browser
+    )
     print(
-        msg_fil_plan +
-        "=fil-profile= You can also run the following command while the program is still running to write out peak memory usage up to that point: " +
-        "kill -s SIGUSR2 {}".format(getpid()),
+        msg_fil_plan
+        + "=fil-profile= You can also run the following command while the program is still running to write out peak memory usage up to that point: "
+        + "kill -s SIGUSR2 {}".format(getpid()),
         file=sys.stderr,
     )
     if not exists(arguments.output_path):

tests/test_endtoend.py

@@ -280,6 +280,10 @@ def test_out_of_memory():
     )
 
 
+@pytest.mark.skipif(
+    sys.platform.startswith("darwin"),
+    reason="macOS doesn't have OOM detection at the moment",
+)
 def test_out_of_memory_slow_leak():
     """
     If an allocation is run that runs out of memory slowly, current allocations are