From cf147dde9599625c377e2d40e79fd77891c2e1d1 Mon Sep 17 00:00:00 2001
From: Thomas Wouters <thomas@python.org>
Date: Sun, 20 Apr 2025 00:34:56 +0200
Subject: [PATCH 1/4] Add configurable groups.

---
 CHANGELOG.md                              | 33 ++++++++++++++++++++
 bench_runner/runners.py                   | 13 ++++++++
 bench_runner/scripts/get_merge_base.py    | 37 +++++++++++++++--------
 bench_runner/scripts/install.py           | 20 ++++++++++--
 bench_runner/templates/_benchmark.src.yml | 12 ++++++--
 bench_runner/templates/benchmark.src.yml  |  4 +--
 6 files changed, 99 insertions(+), 20 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 9d4d2de2..caa85018 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -100,6 +100,39 @@ See `README.md` for more information.
 python -m pytest -m "not long_running"
 ```
 
+### Configurable groups
+
+Runners can configure which groups they belong to, and benchmark runs can be
+started on whole groups. Unlike when using 'all' or individual runners, runs
+for which the result already exists are skipped. For example, a
+configuration like:
+
+```toml
+[runners.linux_clang]
+os = "linux"
+arch = "x86_64"
+hostname = "pyperf1"
+env.CC = "clang"
+groups = ["linux", "pyperf1", "clang"]
+
+[runners.linux_gcc]
+os = "linux"
+arch = "x86_64"
+hostname = "pyperf1"
+groups = ["linux", "pyperf1", "gcc"]
+
+[runners.linux2_gcc]
+os = "linux"
+arch = "x86_64"
+hostname = "pyperf2"
+groups = ["linux", "pyperf2", "gcc"]
+```
+
+... will add `group linux`, `group pyperf1`, `group pyperf2`, `group gcc`
+and `group clang` to the list of possible machines for benchmark runs.
+Selecting `group linux` will queue a run for all three runners, and `group
+pyperf1` only for the first two.
+
 ## v1.8.0
 
 ### bench_runner.toml change
diff --git a/bench_runner/runners.py b/bench_runner/runners.py
index 4ce9c697..054cc3bb 100644
--- a/bench_runner/runners.py
+++ b/bench_runner/runners.py
@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 
+import collections
 import functools
 import os
 import socket
@@ -34,6 +35,7 @@ def __init__(
         github_runner_name: str | None,
         include_in_all: bool = True,
         plot: dict[str, str] | None = None,
+        groups: list[str] | None = None,
     ):
         self.nickname = nickname
         self.os = os
@@ -48,6 +50,7 @@ def __init__(
         if plot is None:
             plot = {"name": nickname}
         self.plot = PlotConfig(**plot)
+        self.groups = groups
 
     @property
     def name(self) -> str:
@@ -77,6 +80,7 @@ def get_runners(cfgpath: PathLike | None = None) -> list[Runner]:
                 section.get("github_runner_name"),
                 section.get("include_in_all", True),
                 section.get("plot", None),
+                section.get("groups"),
             )
         )
 
@@ -117,3 +121,12 @@ def get_runner_for_hostname(
     if hostname is None:
         hostname = socket.gethostname()
     return get_runners_by_hostname(cfgpath).get(hostname, unknown_runner)
+
+
+def get_groups(cfgpath: PathLike | None = None) -> dict[str, list[Runner]]:
+    d = collections.defaultdict(list)
+    for runner in get_runners(cfgpath):
+        if runner.groups:
+            for group in runner.groups:
+                d[group].append(runner)
+    return dict(d)
diff --git a/bench_runner/scripts/get_merge_base.py b/bench_runner/scripts/get_merge_base.py
index 4209d4bd..8cd01c3d 100644
--- a/bench_runner/scripts/get_merge_base.py
+++ b/bench_runner/scripts/get_merge_base.py
@@ -9,6 +9,7 @@
 from bench_runner import benchmark_definitions
 from bench_runner import flags as mflags
 from bench_runner import git
+from bench_runner import runners as mrunners
 from bench_runner.result import has_result
 from bench_runner.util import PathLike
 
@@ -40,29 +41,41 @@ def _main(
     if not need_to_run:
         print("ref=xxxxxxx")
         print("need_to_run=false")
-    else:
-        merge_base = git.get_git_merge_base(cpython)
+        return
+    merge_base = git.get_git_merge_base(cpython)
 
-        if merge_base is None:
-            print("ref=xxxxxxx")
-            print("need_to_run=false")
+    if merge_base is None:
+        print("ref=xxxxxxx")
+        print("need_to_run=false")
+        return
+
+    if machine in ("__really_all", "all"):
+        need_to_run = True
+    else:
+        if machine.startswith("group "):
+            group = machine.removeprefix("group ")
+            groups = mrunners.get_groups()
+            machines = [r.nickname for r in groups[group]]
         else:
-            need_to_run = (
-                machine in ("__really_all", "all")
-                or has_result(
+            machines = [machine]
+        for m in machines:
+            if (
+                has_result(
                     Path("results"),
                     merge_base,
-                    machine,
+                    m,
                     pystats,
                     flags,
                     benchmark_definitions.get_benchmark_hash(),
                     progress=False,
                 )
                 is None
-            )
+            ):
+                need_to_run = True
+                break
 
-            print(f"ref={merge_base}")
-            print(f"need_to_run={str(need_to_run).lower()}")
+    print(f"ref={merge_base}")
+    print(f"need_to_run={str(need_to_run).lower()}")
 
 
 def main():
diff --git a/bench_runner/scripts/install.py b/bench_runner/scripts/install.py
index 8439106c..9243d6f6 100644
--- a/bench_runner/scripts/install.py
+++ b/bench_runner/scripts/install.py
@@ -165,6 +165,7 @@ def generate__benchmark(src: Any) -> Any:
             github_env = "$GITHUB_ENV"
         vars = copy.copy(runner.env)
         vars["BENCHMARK_MACHINE_NICKNAME"] = runner.nickname
+        vars["BENCHMARK_RUNNER_NAME"] = runner.name
         setup_environment = {
             "name": "Setup environment",
             "run": LiteralScalarString(
@@ -183,6 +184,11 @@ def generate__benchmark(src: Any) -> Any:
         ]
         if runner.include_in_all:
             machine_clauses.append("inputs.machine == 'all'")
+        if runner.groups:
+            for group in runner.groups:
+                if "'" in group:
+                    raise ValueError(f"group cannot contain `'` (runner {runner.name})")
+                machine_clauses.append(f"inputs.machine == 'group {group}'")
         runner_template["if"] = f"${{{{ ({' || '.join(machine_clauses)}) }}}}"
 
         dst["jobs"][f"benchmark-{runner.name}"] = runner_template
@@ -205,11 +211,19 @@ def generate_benchmark(dst: Any) -> Any:
     """
     Generates benchmark.yml from benchmark.src.yml.
 
-    Inserts the list of available machines to the drop-down presented to the
-    user.
+    Inserts the list of groups and available machines to the drop-down
+    presented to the user.
     """
     available_runners = [r for r in runners.get_runners() if r.available]
-    runner_choices = [*[x.name for x in available_runners], "all", "__really_all"]
+    groups = sorted(
+        set(f"group {g}" for r in available_runners if r.groups for g in r.groups)
+    )
+    runner_choices = [
+        *groups,
+        *[x.name for x in available_runners],
+        "all",
+        "__really_all",
+    ]
 
     dst["on"]["workflow_dispatch"]["inputs"]["machine"]["options"] = runner_choices
 
diff --git a/bench_runner/templates/_benchmark.src.yml b/bench_runner/templates/_benchmark.src.yml
index 6f7cc1a6..5f85cddb 100644
--- a/bench_runner/templates/_benchmark.src.yml
+++ b/bench_runner/templates/_benchmark.src.yml
@@ -73,7 +73,9 @@ jobs:
           git gc
       - name: Building Python and running pyperformance
         run: |
-          py workflow_bootstrap.py ${{ inputs.fork }} ${{ inputs.ref }} ${{ inputs.machine }} ${{ inputs.benchmarks || 'all' }} "${{ env.flags }}" ${{ inputs.force && '--force' || '' }} ${{ inputs.pgo && '--pgo' || '' }} --run_id ${{ github.run_id }}
+          py workflow_bootstrap.py ${{ inputs.fork }} ${{ inputs.ref }} `
+              ${{ (inputs.machine == 'all' || inputs.machine == '__really_all') && inputs.machine || '$env:BENCHMARK_RUNNER_NAME' }} `
+              ${{ inputs.benchmarks || 'all' }} "${{ env.flags }}" ${{ inputs.force && '--force' || '' }} ${{ inputs.pgo && '--pgo' || '' }} --run_id ${{ github.run_id }}
       # Pull again, since another job may have committed results in the meantime
       - name: Pull benchmarking
         run: |
@@ -112,7 +114,9 @@ jobs:
           python-version: "3.11"
       - name: Building Python and running pyperformance
         run: |
-          python workflow_bootstrap.py ${{ inputs.fork }} ${{ inputs.ref }} ${{ inputs.machine }} ${{ inputs.benchmarks || 'all' }} ${{ env.flags }} ${{ inputs.force && '--force' || '' }} ${{ inputs.pgo && '--pgo' || '' }} ${{ inputs.perf && '--perf' || '' }} --run_id ${{ github.run_id }}
+          python workflow_bootstrap.py ${{ inputs.fork }} ${{ inputs.ref }} \
+              ${{ (inputs.machine == 'all' || inputs.machine == '__really_all') && inputs.machine || '"$BENCHMARK_RUNNER_NAME"' }} \
+              ${{ inputs.benchmarks || 'all' }} ${{ env.flags }} ${{ inputs.force && '--force' || '' }} ${{ inputs.pgo && '--pgo' || '' }} ${{ inputs.perf && '--perf' || '' }} --run_id ${{ github.run_id }}
       # Pull again, since another job may have committed results in the meantime
       - name: Pull benchmarking
         if: ${{ !inputs.perf }}
@@ -154,7 +158,9 @@ jobs:
           git gc
       - name: Building Python and running pyperformance
         run: |
-          python3 workflow_bootstrap.py ${{ inputs.fork }} ${{ inputs.ref }} ${{ inputs.machine }} ${{ inputs.benchmarks || 'all' }} ${{ env.flags }} ${{ inputs.force && '--force' || '' }} ${{ inputs.pgo && '--pgo' || '' }} --run_id ${{ github.run_id }}
+          python3 workflow_bootstrap.py ${{ inputs.fork }} ${{ inputs.ref }} \
+              ${{ (inputs.machine == 'all' || inputs.machine == '__really_all') && inputs.machine || '"$BENCHMARK_RUNNER_NAME"' }} \
+              ${{ inputs.benchmarks || 'all' }} ${{ env.flags }} ${{ inputs.force && '--force' || '' }} ${{ inputs.pgo && '--pgo' || '' }} --run_id ${{ github.run_id }}
       # Pull again, since another job may have committed results in the meantime
       - name: Pull benchmarking
         run: |
diff --git a/bench_runner/templates/benchmark.src.yml b/bench_runner/templates/benchmark.src.yml
index 67ddc038..0177a2f2 100644
--- a/bench_runner/templates/benchmark.src.yml
+++ b/bench_runner/templates/benchmark.src.yml
@@ -68,7 +68,7 @@ jobs:
       - name: Determine base
         id: base
         run: |
-          python -m bench_runner get_merge_base ${{ inputs.benchmark_base }} ${{ inputs.machine }} ${{ inputs.pystats }} ${{ env.flags }} >> $GITHUB_OUTPUT
+          python -m bench_runner get_merge_base ${{ inputs.benchmark_base }} '${{ inputs.machine }}' ${{ inputs.pystats }} ${{ env.flags }} >> $GITHUB_OUTPUT
           cat $GITHUB_OUTPUT
 
   head:
@@ -80,7 +80,7 @@ jobs:
       benchmarks: ${{ inputs.benchmarks }}
       pgo: true
       perf: false
-      force: true
+      force: ${{ ! startsWith(inputs.machine, 'group ') }}
     secrets: inherit
 
   base:

From d1b03f29c9a065519124089a6794e03c517223a1 Mon Sep 17 00:00:00 2001
From: Thomas Wouters <thomas@python.org>
Date: Sun, 27 Apr 2025 23:20:25 +0200
Subject: [PATCH 2/4] Rename groups to tags, and change 'force' to false by
 default as suggested by Mike.

---
 CHANGELOG.md                             | 20 +++++++++-----------
 bench_runner/runners.py                  | 14 +++++++-------
 bench_runner/scripts/get_merge_base.py   |  8 ++++----
 bench_runner/scripts/install.py          | 18 +++++++++---------
 bench_runner/templates/benchmark.src.yml |  2 +-
 5 files changed, 30 insertions(+), 32 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index caa85018..fe1de97f 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -100,12 +100,10 @@ See `README.md` for more information.
 python -m pytest -m "not long_running"
 ```
 
-### Configurable groups
+### Configurable tags
 
-Runners can configure which groups they belong to, and benchmark runs can be
-started on whole groups. Unlike when using 'all' or individual runners, runs
-for which the result already exists are skipped. For example, a
-configuration like:
+Runners can have tags attached, and benchmark runs can be started on
+everything with a specific tag. For example, a configuration like:
 
 ```toml
 [runners.linux_clang]
@@ -113,24 +111,24 @@ os = "linux"
 arch = "x86_64"
 hostname = "pyperf1"
 env.CC = "clang"
-groups = ["linux", "pyperf1", "clang"]
+tags = ["linux", "pyperf1", "clang"]
 
 [runners.linux_gcc]
 os = "linux"
 arch = "x86_64"
 hostname = "pyperf1"
-groups = ["linux", "pyperf1", "gcc"]
+tags = ["linux", "pyperf1", "gcc"]
 
 [runners.linux2_gcc]
 os = "linux"
 arch = "x86_64"
 hostname = "pyperf2"
-groups = ["linux", "pyperf2", "gcc"]
+tags = ["linux", "pyperf2", "gcc"]
 ```
 
-... will add `group linux`, `group pyperf1`, `group pyperf2`, `group gcc`
-and `group clang` to the list of possible machines for benchmark runs.
-Selecting `group linux` will queue a run for all three runners, and `group
+... will add `tag linux`, `tag pyperf1`, `tag pyperf2`, `tag gcc`
+and `tag clang` to the list of possible machines for benchmark runs.
+Selecting `tag linux` will queue a run for all three runners, and `tag
 pyperf1` only for the first two.
 
 ## v1.8.0
diff --git a/bench_runner/runners.py b/bench_runner/runners.py
index 054cc3bb..38790962 100644
--- a/bench_runner/runners.py
+++ b/bench_runner/runners.py
@@ -35,7 +35,7 @@ def __init__(
         github_runner_name: str | None,
         include_in_all: bool = True,
         plot: dict[str, str] | None = None,
-        groups: list[str] | None = None,
+        tags: list[str] | None = None,
     ):
         self.nickname = nickname
         self.os = os
@@ -50,7 +50,7 @@ def __init__(
         if plot is None:
             plot = {"name": nickname}
         self.plot = PlotConfig(**plot)
-        self.groups = groups
+        self.tags = tags
 
     @property
     def name(self) -> str:
@@ -80,7 +80,7 @@ def get_runners(cfgpath: PathLike | None = None) -> list[Runner]:
                 section.get("github_runner_name"),
                 section.get("include_in_all", True),
                 section.get("plot", None),
-                section.get("groups"),
+                section.get("tags"),
             )
         )
 
@@ -123,10 +123,10 @@ def get_runner_for_hostname(
     return get_runners_by_hostname(cfgpath).get(hostname, unknown_runner)
 
 
-def get_groups(cfgpath: PathLike | None = None) -> dict[str, list[Runner]]:
+def get_tags(cfgpath: PathLike | None = None) -> dict[str, list[Runner]]:
     d = collections.defaultdict(list)
     for runner in get_runners(cfgpath):
-        if runner.groups:
-            for group in runner.groups:
-                d[group].append(runner)
+        if runner.tags:
+            for tag in runner.tags:
+                d[tag].append(runner)
     return dict(d)
diff --git a/bench_runner/scripts/get_merge_base.py b/bench_runner/scripts/get_merge_base.py
index 8cd01c3d..c4a52ea8 100644
--- a/bench_runner/scripts/get_merge_base.py
+++ b/bench_runner/scripts/get_merge_base.py
@@ -52,10 +52,10 @@ def _main(
     if machine in ("__really_all", "all"):
         need_to_run = True
     else:
-        if machine.startswith("group "):
-            group = machine.removeprefix("group ")
-            groups = mrunners.get_groups()
-            machines = [r.nickname for r in groups[group]]
+        if machine.startswith("tag "):
+            tag = machine.removeprefix("tag ")
+            tags = mrunners.get_tags()
+            machines = [r.nickname for r in tags[tag]]
         else:
             machines = [machine]
         for m in machines:
diff --git a/bench_runner/scripts/install.py b/bench_runner/scripts/install.py
index 9243d6f6..314a6674 100644
--- a/bench_runner/scripts/install.py
+++ b/bench_runner/scripts/install.py
@@ -184,11 +184,11 @@ def generate__benchmark(src: Any) -> Any:
         ]
         if runner.include_in_all:
             machine_clauses.append("inputs.machine == 'all'")
-        if runner.groups:
-            for group in runner.groups:
-                if "'" in group:
-                    raise ValueError(f"group cannot contain `'` (runner {runner.name})")
-                machine_clauses.append(f"inputs.machine == 'group {group}'")
+        if runner.tags:
+            for tag in runner.tags:
+                if "'" in tag:
+                    raise ValueError(f"tag cannot contain `'` (runner {runner.name})")
+                machine_clauses.append(f"inputs.machine == 'tag {tag}'")
         runner_template["if"] = f"${{{{ ({' || '.join(machine_clauses)}) }}}}"
 
         dst["jobs"][f"benchmark-{runner.name}"] = runner_template
@@ -211,15 +211,15 @@ def generate_benchmark(dst: Any) -> Any:
     """
     Generates benchmark.yml from benchmark.src.yml.
 
-    Inserts the list of groups and available machines to the drop-down
+    Inserts the list of tags and available machines to the drop-down
     presented to the user.
     """
     available_runners = [r for r in runners.get_runners() if r.available]
-    groups = sorted(
-        set(f"group {g}" for r in available_runners if r.groups for g in r.groups)
+    tags = sorted(
+        set(f"tag {g}" for r in available_runners if r.tags for g in r.tags)
     )
     runner_choices = [
-        *groups,
+        *tags,
         *[x.name for x in available_runners],
         "all",
         "__really_all",
diff --git a/bench_runner/templates/benchmark.src.yml b/bench_runner/templates/benchmark.src.yml
index 0177a2f2..9fef141f 100644
--- a/bench_runner/templates/benchmark.src.yml
+++ b/bench_runner/templates/benchmark.src.yml
@@ -80,7 +80,7 @@ jobs:
       benchmarks: ${{ inputs.benchmarks }}
       pgo: true
       perf: false
-      force: ${{ ! startsWith(inputs.machine, 'group ') }}
+      force: false
     secrets: inherit
 
   base:

From 4ed138654ccb89242ac6246b124e9fa91a5fe1b4 Mon Sep 17 00:00:00 2001
From: Thomas Wouters <thomas@python.org>
Date: Sun, 27 Apr 2025 23:58:58 +0200
Subject: [PATCH 3/4] Add the ability to list tags in the list of weekly
 runners.

---
 bench_runner/runners.py                | 19 +++++++++++++++++++
 bench_runner/scripts/get_merge_base.py |  9 +++------
 bench_runner/scripts/install.py        | 14 +++++---------
 3 files changed, 27 insertions(+), 15 deletions(-)

diff --git a/bench_runner/runners.py b/bench_runner/runners.py
index 38790962..4be71f36 100644
--- a/bench_runner/runners.py
+++ b/bench_runner/runners.py
@@ -130,3 +130,22 @@ def get_tags(cfgpath: PathLike | None = None) -> dict[str, list[Runner]]:
             for tag in runner.tags:
                 d[tag].append(runner)
     return dict(d)
+
+
+def get_runners_from_nicknames_and_tags(
+    nicknames: list[str], cfgpath: PathLike | None = None
+) -> list[Runner]:
+    result = []
+    tags = get_tags(cfgpath)
+    runners = get_runners_by_nickname(cfgpath)
+    for nickname in nicknames:
+        if nickname.startswith("tag "):
+            tag = nickname.removeprefix("tag ")
+            if tag not in tags:
+                raise ValueError(f"Tag {tag} not found in bench_runner.toml")
+            result.extend(tags[nickname.removeprefix("tag ")])
+        else:
+            if nickname not in runners:
+                raise ValueError(f"Runner {nickname} not found in bench_runner.toml")
+            result.append(runners[nickname])
+    return result
diff --git a/bench_runner/scripts/get_merge_base.py b/bench_runner/scripts/get_merge_base.py
index c4a52ea8..2986e009 100644
--- a/bench_runner/scripts/get_merge_base.py
+++ b/bench_runner/scripts/get_merge_base.py
@@ -52,12 +52,9 @@ def _main(
     if machine in ("__really_all", "all"):
         need_to_run = True
     else:
-        if machine.startswith("tag "):
-            tag = machine.removeprefix("tag ")
-            tags = mrunners.get_tags()
-            machines = [r.nickname for r in tags[tag]]
-        else:
-            machines = [machine]
+        machines = [
+            r.nickname for r in mrunners.get_runners_from_nicknames_and_tags([machine])
+        ]
         for m in machines:
             if (
                 has_result(
diff --git a/bench_runner/scripts/install.py b/bench_runner/scripts/install.py
index 314a6674..ecc2bec5 100644
--- a/bench_runner/scripts/install.py
+++ b/bench_runner/scripts/install.py
@@ -215,9 +215,7 @@ def generate_benchmark(dst: Any) -> Any:
     presented to the user.
     """
     available_runners = [r for r in runners.get_runners() if r.available]
-    tags = sorted(
-        set(f"tag {g}" for r in available_runners if r.tags for g in r.tags)
-    )
+    tags = sorted(set(f"tag {g}" for r in available_runners if r.tags for g in r.tags))
     runner_choices = [
         *tags,
         *[x.name for x in available_runners],
@@ -278,12 +276,10 @@ def generate__weekly(dst: Any) -> Any:
     all_jobs = []
 
     for name, weekly_cfg in weekly.items():
-        for runner_nickname in weekly_cfg.get("runners", []):
-            runner = runners.get_runner_by_nickname(runner_nickname)
-            if runner.nickname == "unknown":
-                raise ValueError(
-                    f"Runner {runner_nickname} not found in bench_runner.toml"
-                )
+        cfg_runners = runners.get_runners_from_nicknames_and_tags(
+            weekly_cfg.get("runners", [])
+        )
+        for runner_nickname in cfg_runners:
             weekly_flags = weekly_cfg.get("flags", [])
             job = {
                 "uses": "./.github/workflows/_benchmark.yml",

From 839617520f2d78d90bf8abc20ce1e592cc835c15 Mon Sep 17 00:00:00 2001
From: Thomas Wouters <thomas@python.org>
Date: Mon, 28 Apr 2025 00:11:52 +0200
Subject: [PATCH 4/4] Fix oopsie.

---
 bench_runner/scripts/install.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bench_runner/scripts/install.py b/bench_runner/scripts/install.py
index ecc2bec5..f114f1f5 100644
--- a/bench_runner/scripts/install.py
+++ b/bench_runner/scripts/install.py
@@ -279,7 +279,7 @@ def generate__weekly(dst: Any) -> Any:
         cfg_runners = runners.get_runners_from_nicknames_and_tags(
             weekly_cfg.get("runners", [])
         )
-        for runner_nickname in cfg_runners:
+        for runner in cfg_runners:
             weekly_flags = weekly_cfg.get("flags", [])
             job = {
                 "uses": "./.github/workflows/_benchmark.yml",