Add configurable groups.

Author: Yhg1s

CHANGELOG.md

@@ -83,6 +83,39 @@ flags = ["TAILCALL"]
 runners = ["linux_clang", "darwin", "windows_clang"]
 ```
 
+### Configurable groups
+
+Runners can configure which groups they belong to, and benchmark runs can be
+started on whole groups. Unlike when using 'all' or individual runners, runs
+for which the result already exists are skipped. For example, a
+configuration like:
+
+```toml
+[runners.linux_clang]
+os = "linux"
+arch = "x86_64"
+hostname = "pyperf1"
+env.CC = "clang"
+groups = ["linux", "pyperf1", "clang"]
+
+[runners.linux_gcc]
+os = "linux"
+arch = "x86_64"
+hostname = "pyperf1"
+groups = ["linux", "pyperf1", "gcc"]
+
+[runners.linux2_gcc]
+os = "linux"
+arch = "x86_64"
+hostname = "pyperf2"
+groups = ["linux", "pyperf2", "gcc"]
+```
+
+... will add `group linux`, `group pyperf1`, `group pyperf2`, `group gcc`
+and `group clang` to the list of possible machines for benchmark runs.
+Selecting `group linux` will queue a run for all three runners, and `group
+pyperf1` only for the first two.
+
 ## v1.8.0
 
 ### bench_runner.toml change

bench_runner/runners.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 
+import collections
 import functools
 import os
 import socket
@@ -23,6 +24,7 @@ def __init__(
         # os-arch-nickname
         github_runner_name: str | None,
         include_in_all: bool = True,
+        groups: list[str] | None = None,
     ):
         self.nickname = nickname
         self.os = os
@@ -34,6 +36,7 @@ def __init__(
             github_runner_name = self.name
         self.github_runner_name = github_runner_name
         self.include_in_all = include_in_all
+        self.groups = groups
 
     @property
     def name(self) -> str:
@@ -62,6 +65,7 @@ def get_runners(cfgpath: PathLike | None = None) -> list[Runner]:
                 section.get("env", {}),
                 section.get("github_runner_name"),
                 section.get("include_in_all", True),
+                section.get("groups"),
             )
         )
 
@@ -102,3 +106,12 @@ def get_runner_for_hostname(
     if hostname is None:
         hostname = socket.gethostname()
     return get_runners_by_hostname(cfgpath).get(hostname, unknown_runner)
+
+
+def get_groups(cfgpath: PathLike | None = None) -> dict[str, list[Runner]]:
+    d = collections.defaultdict(list)
+    for runner in get_runners(cfgpath):
+        if runner.groups:
+            for group in runner.groups:
+                d[group].append(runner)
+    return dict(d)

bench_runner/scripts/get_merge_base.py

@@ -9,6 +9,7 @@
 from bench_runner import benchmark_definitions
 from bench_runner import flags as mflags
 from bench_runner import git
+from bench_runner import runners as mrunners
 from bench_runner.result import has_result
 from bench_runner.util import PathLike
 
@@ -40,29 +41,41 @@ def _main(
     if not need_to_run:
         print("ref=xxxxxxx")
         print("need_to_run=false")
-    else:
-        merge_base = git.get_git_merge_base(cpython)
+        return
+    merge_base = git.get_git_merge_base(cpython)
 
-        if merge_base is None:
-            print("ref=xxxxxxx")
-            print("need_to_run=false")
+    if merge_base is None:
+        print("ref=xxxxxxx")
+        print("need_to_run=false")
+        return
+
+    if machine in ("__really_all", "all"):
+        need_to_run = True
+    else:
+        if machine.startswith("group "):
+            group = machine.removeprefix("group ")
+            groups = mrunners.get_groups()
+            machines = [r.nickname for r in groups[group]]
         else:
-            need_to_run = (
-                machine in ("__really_all", "all")
-                or has_result(
+            machines = [machine]
+        for m in machines:
+            if (
+                has_result(
                     Path("results"),
                     merge_base,
-                    machine,
+                    m,
                     pystats,
                     flags,
                     benchmark_definitions.get_benchmark_hash(),
                     progress=False,
                 )
                 is None
-            )
+            ):
+                need_to_run = True
+                break
 
-            print(f"ref={merge_base}")
-            print(f"need_to_run={str(need_to_run).lower()}")
+    print(f"ref={merge_base}")
+    print(f"need_to_run={str(need_to_run).lower()}")
 
 
 def main():

bench_runner/scripts/install.py

@@ -165,6 +165,7 @@ def generate__benchmark(src: Any) -> Any:
             github_env = "$GITHUB_ENV"
         vars = copy.copy(runner.env)
         vars["BENCHMARK_MACHINE_NICKNAME"] = runner.nickname
+        vars["BENCHMARK_RUNNER_NAME"] = runner.name
         setup_environment = {
             "name": "Setup environment",
             "run": LiteralScalarString(
@@ -183,6 +184,11 @@ def generate__benchmark(src: Any) -> Any:
         ]
         if runner.include_in_all:
             machine_clauses.append("inputs.machine == 'all'")
+        if runner.groups:
+            for group in runner.groups:
+                if "'" in group:
+                    raise ValueError(f"group cannot contain `'` (runner {runner.name})")
+                machine_clauses.append(f"inputs.machine == 'group {group}'")
         runner_template["if"] = f"${{{{ ({' || '.join(machine_clauses)}) }}}}"
 
         dst["jobs"][f"benchmark-{runner.name}"] = runner_template
@@ -205,11 +211,19 @@ def generate_benchmark(dst: Any) -> Any:
     """
     Generates benchmark.yml from benchmark.src.yml.
 
-    Inserts the list of available machines to the drop-down presented to the
-    user.
+    Inserts the list of groups and available machines to the drop-down
+    presented to the user.
     """
     available_runners = [r for r in runners.get_runners() if r.available]
-    runner_choices = [*[x.name for x in available_runners], "all", "__really_all"]
+    groups = sorted(
+        set(f"group {g}" for r in available_runners if r.groups for g in r.groups)
+    )
+    runner_choices = [
+        *groups,
+        *[x.name for x in available_runners],
+        "all",
+        "__really_all",
+    ]
 
     dst["on"]["workflow_dispatch"]["inputs"]["machine"]["options"] = runner_choices
 

bench_runner/templates/_benchmark.src.yml

@@ -73,7 +73,9 @@ jobs:
           git gc
       - name: Building Python and running pyperformance
         run: |
-          python workflow_bootstrap.py ${{ inputs.fork }} ${{ inputs.ref }} ${{ inputs.machine }} ${{ inputs.benchmarks || 'all' }} "${{ env.flags }}" ${{ inputs.force && '--force' || '' }} ${{ inputs.pgo && '--pgo' || '' }} --run_id ${{ github.run_id }}
+          python workflow_bootstrap.py ${{ inputs.fork }} ${{ inputs.ref }} `
+              ${{ (inputs.machine == 'all' || inputs.machine == '__really_all') && inputs.machine || '$env:BENCHMARK_RUNNER_NAME' }} `
+              ${{ inputs.benchmarks || 'all' }} "${{ env.flags }}" ${{ inputs.force && '--force' || '' }} ${{ inputs.pgo && '--pgo' || '' }} --run_id ${{ github.run_id }}
       # Pull again, since another job may have committed results in the meantime
       - name: Pull benchmarking
         run: |
@@ -112,7 +114,9 @@ jobs:
           python-version: "3.11"
       - name: Building Python and running pyperformance
         run: |
-          python workflow_bootstrap.py ${{ inputs.fork }} ${{ inputs.ref }} ${{ inputs.machine }} ${{ inputs.benchmarks || 'all' }} ${{ env.flags }} ${{ inputs.force && '--force' || '' }} ${{ inputs.pgo && '--pgo' || '' }} ${{ inputs.perf && '--perf' || '' }} --run_id ${{ github.run_id }}
+          python workflow_bootstrap.py ${{ inputs.fork }} ${{ inputs.ref }} \
+              ${{ (inputs.machine == 'all' || inputs.machine == '__really_all') && inputs.machine || '"$BENCHMARK_RUNNER_NAME"' }} \
+              ${{ inputs.benchmarks || 'all' }} ${{ env.flags }} ${{ inputs.force && '--force' || '' }} ${{ inputs.pgo && '--pgo' || '' }} ${{ inputs.perf && '--perf' || '' }} --run_id ${{ github.run_id }}
       # Pull again, since another job may have committed results in the meantime
       - name: Pull benchmarking
         if: ${{ !inputs.perf }}
@@ -154,7 +158,9 @@ jobs:
           git gc
       - name: Building Python and running pyperformance
         run: |
-          python3 workflow_bootstrap.py ${{ inputs.fork }} ${{ inputs.ref }} ${{ inputs.machine }} ${{ inputs.benchmarks || 'all' }} ${{ env.flags }} ${{ inputs.force && '--force' || '' }} ${{ inputs.pgo && '--pgo' || '' }} --run_id ${{ github.run_id }}
+          python3 workflow_bootstrap.py ${{ inputs.fork }} ${{ inputs.ref }} \
+              ${{ (inputs.machine == 'all' || inputs.machine == '__really_all') && inputs.machine || '"$BENCHMARK_RUNNER_NAME"' }} \
+              ${{ inputs.benchmarks || 'all' }} ${{ env.flags }} ${{ inputs.force && '--force' || '' }} ${{ inputs.pgo && '--pgo' || '' }} --run_id ${{ github.run_id }}
       # Pull again, since another job may have committed results in the meantime
       - name: Pull benchmarking
         run: |

bench_runner/templates/benchmark.src.yml

@@ -68,7 +68,7 @@ jobs:
       - name: Determine base
         id: base
         run: |
-          python -m bench_runner get_merge_base ${{ inputs.benchmark_base }} ${{ inputs.machine }} ${{ inputs.pystats }} ${{ env.flags }} >> $GITHUB_OUTPUT
+          python -m bench_runner get_merge_base ${{ inputs.benchmark_base }} '${{ inputs.machine }}' ${{ inputs.pystats }} ${{ env.flags }} >> $GITHUB_OUTPUT
           cat $GITHUB_OUTPUT
 
   head:
@@ -80,7 +80,7 @@ jobs:
       benchmarks: ${{ inputs.benchmarks }}
       pgo: true
       perf: false
-      force: true
+      force: ${{ ! startsWith(inputs.machine, 'group ') }}
     secrets: inherit
 
   base: