[GR-66890] Add PE microbenchmark.

PullRequest: graal/21379

Author: DSouzaM

truffle/ci/ci.jsonnet

@@ -33,7 +33,9 @@
       ["cd", "./compiler"],
       ["mx", "build" ],
       ["mx", "hsdis", "||", "true"],
-    ]
+    ],
+    notify_groups:: ["truffle"],
+    upload:: ["bench-uploader.py", "${BENCH_RESULTS_FILE_PATH}"],
   },
 
   local simple_tool_maven_project_gate = truffle_common + {
@@ -138,7 +140,7 @@
     ],
     timelimit: "3:00:00",
     teardown: [
-      ["bench-uploader.py", "${BENCH_RESULTS_FILE_PATH}"],
+      self.upload,
     ],
   },
 
@@ -149,6 +151,14 @@
     ],
   },
 
+  local truffle_compiler_benchmark = bench_common + {
+    name: self.name_prefix + 'truffle-compiler-benchmark-' + self.truffle_jdk_name + '-' + self.os + '-' + self.arch,
+    run+: [
+      ["mx", "--java-home", "$JAVA_HOME", "benchmark", "truffle:*", "--results-file", "${BENCH_RESULTS_FILE_PATH}", "--", "--jvm", "java-home", "--", "org.graalvm.truffle.compiler.benchmark", "-prof", "org.graalvm.truffle.compiler.benchmark.CompilationTimingsProfiler"],
+      self.upload,
+    ],
+  },
+
   local tier1  = common.tier1 + {
     name_prefix: "gate-",
     timelimit: "0:30:00"
@@ -244,7 +254,10 @@
 
     # Truffle Benchmarks
     [linux_amd64  + tier3  + jdk_latest_labs + jmh_benchmark_test],
-    [bench_hw.x52 + bench  + jdk_latest_labs + jmh_benchmark]
+    [bench_hw.x52 + bench  + jdk_latest_labs + jmh_benchmark],
+
+    # Truffle compilation benchmarks
+    [linux_amd64 + bench + jdk_latest_graalvm_ce + truffle_compiler_benchmark],
   ]),
   builds: utils.add_defined_in(_builds, std.thisFile),
 }

truffle/mx.truffle/mx_truffle.py

@@ -90,10 +90,91 @@ def subgroup(self):
 
     def extraVmArgs(self):
         extraVmArgs = super(JMHRunnerTruffleBenchmarkSuite, self).extraVmArgs()
-        # com.oracle.truffle.api.benchmark.InterpreterCallBenchmark$BenchmarkState needs DefaultTruffleRuntime
+        # org.graalvm.truffle.benchmark.InterpreterCallBenchmark$BenchmarkState needs DefaultTruffleRuntime
         extraVmArgs.append('--add-exports=org.graalvm.truffle/com.oracle.truffle.api.impl=ALL-UNNAMED')
+        # org.graalvm.truffle.compiler.benchmark.* needs OptimizedTruffleRuntime
+        extraVmArgs.append('--add-exports=org.graalvm.truffle.runtime/com.oracle.truffle.runtime=ALL-UNNAMED')
         return extraVmArgs
 
+    def rules(self, out, benchmarks, bmSuiteArgs):
+        result = super().rules(out, benchmarks, bmSuiteArgs)
+        result_file = self.get_jmh_result_file(bmSuiteArgs)
+        suite_name = self.benchSuiteName(bmSuiteArgs)
+        result.extend([
+            JMHJsonCompilationTimingRule(result_file, suite_name, "pe-time"),
+            JMHJsonCompilationTimingRule(result_file, suite_name, "compile-time"),
+            JMHJsonCompilationTimingRule(result_file, suite_name, "code-install-time"),
+        ])
+        return result
+
+class JMHJsonCompilationTimingRule(mx_benchmark.JMHJsonRule):
+    def __init__(self, filename, suite_name, metric_name):
+        super().__init__(filename, suite_name)
+        self.metric_name = metric_name
+
+    def parse(self, text):
+        r = []
+        with open(self._prepend_working_dir(self.filename)) as fp:
+            for result in json.load(fp):
+                benchmark = self.getBenchmarkNameFromResult(result)
+                metric = result.get("secondaryMetrics", {}).get(self.metric_name)
+                if metric is None:
+                    return []
+
+                unit = JMHJsonCompilationTimingRule.standardize_unit(metric["scoreUnit"])
+
+                template = {
+                    "bench-suite" : self.suiteName,
+                    "benchmark" : self.shortenPackageName(benchmark),
+                    "metric.unit": unit,
+                    "metric.score-function": "id",
+                    "metric.better": "lower",
+                    "metric.type": "numeric",
+                    # full name
+                    "extra.jmh.benchmark" : benchmark,
+                }
+
+                if "params" in result:
+                    # add all parameter as a single string
+                    template["extra.jmh.params"] = ", ".join(["=".join(kv) for kv in result["params"].items()])
+                    # and also the individual values
+                    for k, v in result["params"].items():
+                        template["extra.jmh.param." + k] = str(v)
+
+                for k in self.getExtraJmhKeys():
+                    extra_value = None
+                    if k in result:
+                        extra_value = result[k]
+                    template["extra.jmh." + k] = str(extra_value)
+
+
+                summary_datapoint = template.copy()
+                summary_datapoint.update({
+                    "metric.name": self.metric_name,
+                    "metric.value": float(metric["score"])
+                })
+                r.append(summary_datapoint)
+
+                score_percentiles = metric.get("scorePercentiles")
+                if score_percentiles:
+                    distribution_metric_name = f"{self.metric_name}-distribution"
+                    for percentile, score in score_percentiles.items():
+                        percentile_datapoint = template.copy()
+                        percentile_datapoint.update({
+                            "metric.name": distribution_metric_name,
+                            "metric.value": float(score),
+                            "metric.percentile": float(percentile)
+                        })
+                        r.append(percentile_datapoint)
+        return r
+
+    @staticmethod
+    def standardize_unit(unit: str) -> str:
+        if unit.endswith("/op"):
+            # JMH emits timings "per operation", e.g., "ms/op". Drop the suffix.
+            return unit[:-len("/op")]
+        return unit
+
 mx_benchmark.add_bm_suite(JMHRunnerTruffleBenchmarkSuite())
 #mx_benchmark.add_java_vm(mx_benchmark.DefaultJavaVm("server", "default"), priority=3)