Track also private memory size and run heap benchmarks for small benchmarks, on SVM, and on cpython

Author: timfel

ci.jsonnet

@@ -391,9 +391,13 @@
         for bench in ["warmup"]
     } + {
         [bench]: bench_task(bench) + platform_spec(no_jobs) + bench_variants({
-            "vm_name:graalpython_enterprise_interpreter"                : {"linux:amd64:jdk-latest" : post_merge     + t("00:30:00")},
+            "vm_name:graalvm_ee_default_interpreter"                    : {"linux:amd64:jdk-latest" : post_merge     + t("01:00:00")},
+            "vm_name:graalvm_ee_default_interpreter_bc_dsl"             : {"linux:amd64:jdk-latest" : post_merge     + t("01:00:00")},
+            "vm_name:graalpython_enterprise_interpreter"                : {"linux:amd64:jdk-latest" : daily          + t("01:00:00")},
+            "vm_name:graalpython_enterprise_interpreter_bc_dsl"         : {"linux:amd64:jdk-latest" : daily          + t("01:00:00")},
+            "vm_name:cpython"                                           : {"linux:amd64:jdk-latest" : weekly         + t("00:30:00")},
         }),
-        for bench in ["heap"]
+        for bench in ["heap", "micro_small_heap"]
     } + {
         // interop benchmarks only for graalpython, weekly is enough
         [bench]: bench_task(bench) + platform_spec(no_jobs) + bench_variants({

ci/python-bench.libsonnet

@@ -37,6 +37,7 @@
         java_embedding_meso_small: "java-embedding-meso-small:*",
         jmh: "python-jmh:GRAALPYTHON_BENCH",
         heap: "heap-graalpython:*",
+        micro_small_heap: "micro-small-heap-graalpython:*",
     },
 
     PY_BENCHMARKS:: {

mx.graalpython/live_heap_tracker.py

@@ -37,13 +37,42 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
 
-import sys
-import time
-
+from pathlib import Path
 import re
 import subprocess
+import sys
+import time
 
 TOTAL_RE = re.compile(r'^Total +\d+ +(\d+)', re.MULTILINE)
+PRIVATE_RE = re.compile(r'Private_(?:Clean|Dirty):\s+(\d+) kB')
+
+
+def jmap(jmap_binary, ppid):
+    if not jmap_binary:
+        return 0
+    try:
+        jmap_output = subprocess.check_output(
+            [jmap_binary, '-histo:live', str(ppid)],
+            universal_newlines=True,
+            stderr=subprocess.DEVNULL,
+        )
+        if match := TOTAL_RE.search(jmap_output):
+            heap_bytes = int(match.group(1))
+            return heap_bytes
+    except (subprocess.CalledProcessError, subprocess.TimeoutExpired):
+        pass
+    return 0
+
+
+def uss(ppid):
+    smap = Path(f"/proc/{ppid}/smaps")
+    try:
+        memory_map = smap.read_text()
+        total_bytes = sum(int(val) * 1024 for val in PRIVATE_RE.findall(memory_map))
+        return total_bytes
+    except FileNotFoundError:
+        pass
+    return 0
 
 
 def main():
@@ -54,19 +83,12 @@ def main():
     with open(output_file, 'w') as f:
         for _ in range(iterations):
             proc = subprocess.Popen(benchmark)
+            ppid = proc.pid
             while proc.poll() is None:
                 time.sleep(0.3)
-                try:
-                    jmap_output = subprocess.check_output(
-                        [jmap_binary, '-histo:live', str(proc.pid)],
-                        universal_newlines=True,
-                        stderr=subprocess.DEVNULL,
-                    )
-                    if match := TOTAL_RE.search(jmap_output):
-                        heap_bytes = int(match.group(1))
-                        f.write(f'{heap_bytes}\n')
-                except subprocess.CalledProcessError:
-                    pass
+                uss_bytes = uss(ppid)
+                heap_bytes = jmap(jmap_binary, ppid)
+                f.write(f"{heap_bytes} {uss_bytes}\n")
             if proc.returncode != 0:
                 sys.exit(proc.returncode)
 

mx.graalpython/mx_graalpython_bench_param.py

@@ -340,5 +340,6 @@ def _pickling_benchmarks(module='pickle'):
         "post-startup": [],
         "import-a-lot": [],
         "allocate-objects": [],
-    }]
+    }],
+    "micro-small-heap": [PATH_MICRO, MICRO_BENCHMARKS_SMALL],
 }

mx.graalpython/mx_graalpython_benchmark.py

@@ -949,9 +949,15 @@ def map_command(self, cmd):
         if self.bmSuite:
             bench_name = f"{self.bmSuite.name()}-{bench_name}"
         ts = datetime.now().strftime("%Y%m%d-%H%M%S")
-        jmap_command = mx.get_jdk().exe_path('jmap')
+        vm = self.bmSuite.execution_context.virtual_machine
+        if isinstance(vm, GraalPythonVm) and vm.launcher_type == "jvm":
+            jmap_command = mx.get_jdk().exe_path('jmap')
+        else:
+            jmap_command = ""
         self.out_file = os.path.join(os.getcwd(), f"heap_tracker_{bench_name}_{ts}.txt")
         iterations = 3
+        if "-i" in cmd:
+            cmd[cmd.index("-i") + 1] = "1"
         return [sys.executable, str(DIR / 'live_heap_tracker.py'), self.out_file, str(iterations), jmap_command, *cmd]
 
     def get_rules(self, bmSuiteArgs):
@@ -964,18 +970,41 @@ def __init__(self, tracker, bmSuiteArgs):
 
         def parse(self, text):
             with open(self.tracker.out_file) as f:
-                heap_mb = [int(line.strip()) / (1024 ** 2) for line in f if line]
+                heap_mb, uss_mb = zip(*(map(lambda i: int(i) / (1024 ** 2), line.split()) for line in f if line))
             os.unlink(self.tracker.out_file)
-            self.tracker.out_file = None
-            deciles = statistics.quantiles(heap_mb, n=10)
-            print(f"Heap size deciles (MiB): {deciles}")
+            heap_deciles = statistics.quantiles(heap_mb, n=10)
+            uss_deciles = statistics.quantiles(uss_mb, n=10)
+            print(f"Heap size deciles (MiB): {heap_deciles}")
+            print(f"USS size deciles (MiB): {uss_deciles}")
+            # The heap benchmarks are a separate suite, because they are run
+            # very differently, but we want to be able to conveniently query
+            # all data about the same suites that we have. So, if this suite
+            # name ends with "-heap", we drop that so it gets attributed to the
+            # base suite.
+            suite = self.tracker.bmSuite.benchSuiteName(self.bmSuiteArgs)
+            if suite.endswith("-heap"):
+                suite = suite[:-len("-heap")]
+            benchmark = f"{suite}.{self.tracker.bmSuite.currently_running_benchmark()}"
+            vm_flags = ' '.join(self.tracker.bmSuite.vmArgs(self.bmSuiteArgs))
             return [
                 PythonBaseBenchmarkSuite.with_branch_and_commit_dict({
-                    "benchmark": self.tracker.bmSuite.currently_running_benchmark(),
-                    "bench-suite": self.tracker.bmSuite.benchSuiteName(self.bmSuiteArgs),
-                    "config.vm-flags": ' '.join(self.tracker.bmSuite.vmArgs(self.bmSuiteArgs)),
+                    "benchmark": benchmark,
+                    "bench-suite": suite,
+                    "config.vm-flags": vm_flags,
                     "metric.name": "allocated-memory",
-                    "metric.value": deciles[-1],
+                    "metric.value": heap_deciles[-1],
+                    "metric.unit": "MB",
+                    "metric.type": "numeric",
+                    "metric.score-function": "id",
+                    "metric.better": "lower",
+                    "metric.iteration": 0
+                }),
+                PythonBaseBenchmarkSuite.with_branch_and_commit_dict({
+                    "benchmark": benchmark,
+                    "bench-suite": suite,
+                    "config.vm-flags": vm_flags,
+                    "metric.name": "memory",
+                    "metric.value": uss_deciles[-1],
                     "metric.unit": "MB",
                     "metric.type": "numeric",
                     "metric.score-function": "id",
@@ -1004,7 +1033,17 @@ def register_tracker(self, name, tracker_type):
     def createCommandLineArgs(self, benchmarks, bmSuiteArgs):
         benchmark = benchmarks[0]
         bench_path = os.path.join(self._bench_path, f'{benchmark}.py')
-        return [*self.vmArgs(bmSuiteArgs), bench_path, *self.runArgs(bmSuiteArgs)]
+        bench_args = self._benchmarks[benchmark]
+        run_args = self.runArgs(bmSuiteArgs)
+        cmd_args = []
+        if "-i" in bench_args:
+            # Need to use the harness to parse
+            cmd_args.append(HARNESS_PATH)
+        if "-i" not in run_args:
+            # Explicit iteration count overrides default
+            run_args += bench_args
+        cmd_args.append(bench_path)
+        return [*self.vmArgs(bmSuiteArgs), *cmd_args, *run_args]
 
     def successPatterns(self):
         return []