ci: Run semver checks even on non-host targets

Author: tgross35

.github/workflows/ci.yaml

@@ -17,6 +17,7 @@ env:
   RUSTDOCFLAGS: -Dwarnings
   RUSTFLAGS: -Dwarnings
   RUST_BACKTRACE: full
+  SERIES: "1.0" # 1.0 alpha series
 
 defaults:
   run:
@@ -79,6 +80,10 @@ jobs:
         uses: taiki-e/install-action@cargo-semver-checks
         if: matrix.toolchain == 'stable'
 
+      - name: Retrieve semver baseline
+        if: matrix.toolchain == 'stable'
+        run: ./ci/semver.sh
+
       # FIXME(ci): These `du` statements are temporary for debugging cache
       - name: Target size before restoring cache
         run: du -sh target | sort -k 2 || true
@@ -100,6 +105,7 @@ jobs:
 
           python3 ci/verify-build.py \
             --toolchain "$TOOLCHAIN" \
+            ${BASELINE_CRATE_PATH:+"--baseline-crate-path" "$BASELINE_CRATE_PATH"} \
             ${{ matrix.only && format('--only "{0}"', matrix.only) }} \
             ${{ matrix.half && format('--half "{0}"', matrix.half) }}
       - name: Target size after job completion

ci/semver.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+# Download a baseline crate to run semver checks against
+
+set -euxo pipefail
+
+# https://crates.io/data-access#api
+
+# Need to include versions to get the default and stable versions
+meta=$(curl -L https://crates.io/api/v1/crates/libc?include=versions)
+
+echo "$meta" | jq -r '.versions | map(select(.num | test("^0.2"))).[0].dl_path'
+
+# Versions to check against
+if [ "${SERIES:-}" = "0.2" ]; then
+    pat="^0.2"
+elif [ "${SERIES:-}" = "1.0" ]; then
+    pat="^1.0"
+else
+    echo "SERIES must be set to either '0.2' or '1.0'"
+    exit 1
+fi
+
+# Find the most recent version matching a pattern.
+release=$(
+    echo "$meta" |
+    jq -er --arg pat "$pat" '.versions | map(select(.num | test($pat))).[0]'
+)
+version=$(echo "$release" | jq -r '.num')
+dl_path=$(echo "$release" | jq -r '.dl_path')
+
+crate_dir="libc-$version"
+curl -L "https://crates.io$dl_path" | tar xf -
+
+# Need to convince Cargo it's not part of our workspace
+echo '[workspace]' >> "$crate_dir/Cargo.toml"
+
+echo "BASELINE_CRATE_PATH=$(realpath "$crate_dir")" >> "$GITHUB_ENV"

ci/verify-build.py

@@ -9,7 +9,8 @@
 import time
 from dataclasses import dataclass, field
 from enum import Enum, IntEnum
-from typing import Optional
+from pathlib import Path
+from typing import Optional, Sequence
 
 
 ESC_CYAN = "\033[1;36m"
@@ -35,6 +36,7 @@ class Cfg:
     toolchain: Toolchain = field(init=False)
     host_target: str = field(init=False)
     os_: Os = field(init=False)
+    baseline_crate_dir: Optional[Path]
 
     def __post_init__(self):
         rustc_output = check_output(["rustc", f"+{self.toolchain_name}", "-vV"])
@@ -204,9 +206,9 @@ def eprint(*args, **kw):
     print(*args, file=sys.stderr, **kw)
 
 
-def xtrace(args: list[str], /, env: Optional[dict[str, str]]):
+def xtrace(args: Sequence[str | Path], /, env: Optional[dict[str, str]]):
     """Print commands before running them."""
-    astr = " ".join(args)
+    astr = " ".join(str(arg) for arg in args)
     if env is None:
         eprint(f"+ {astr}")
     else:
@@ -215,17 +217,20 @@ def xtrace(args: list[str], /, env: Optional[dict[str, str]]):
         eprint(f"+ {estr} {astr}")
 
 
-def check_output(args: list[str], /, env: Optional[dict[str, str]] = None) -> str:
+def check_output(
+    args: Sequence[str | Path], /, env: Optional[dict[str, str]] = None
+) -> str:
     xtrace(args, env=env)
     return sp.check_output(args, env=env, encoding="utf8")
 
 
-def run(args: list[str], /, env: Optional[dict[str, str]] = None):
+def run(args: Sequence[str | Path], /, env: Optional[dict[str, str]] = None):
     xtrace(args, env=env)
     sp.run(args, env=env, check=True)
 
 
 def check_dup_targets():
+    """Ensure there are no duplicate targets in the list."""
     all = set()
     duplicates = set()
     for target in TARGETS:
@@ -235,7 +240,69 @@ def check_dup_targets():
     assert len(duplicates) == 0, f"duplicate targets: {duplicates}"
 
 
+def do_semver_checks(cfg: Cfg, target: Target):
+    tname = target.name
+    if cfg.toolchain != Toolchain.STABLE:
+        eprint("Skipping semver checks")
+        return
+
+    if tname == cfg.host_target:
+        # FIXME(semver): can't pass `--target` to `cargo-semver-checks` so we can only
+        # run directly on the host target
+        eprint("Running semver checks on host")
+        run(
+            [
+                "cargo",
+                "semver-checks",
+                "--only-explicit-features",
+                "--features=std,extra_traits",
+            ]
+        )
+        return
+
+    if cfg.baseline_crate_dir is None:
+        eprint(
+            "Non-host target: --baseline-crate-dir must be specified to \
+            run semver-checks"
+        )
+        return
+
+    if not target.dist:
+        eprint("Skipping semver checks on non-dist target")
+        return
+
+    # Since semver-checks doesn't work with `--target`, we build the json our self and
+    # hand it over.
+    eprint("Running semver checks with cross compilation")
+
+    env = os.environ.copy()
+    cmd = ["cargo", "rustdoc", "--target", tname]
+    rustdoc_args = ["--", "-Zunstable-options", "--output-format=json"]
+
+    run([*cmd, *rustdoc_args], env=env | {"RUSTC_BOOTSTRAP": "1"})
+    run(
+        [*cmd, "--manifest-path", cfg.baseline_crate_dir, *rustdoc_args],
+        env=env | {"RUSTC_BOOTSTRAP": "1"},
+    )
+    baseline = cfg.baseline_crate_dir / "target" / tname / "libc.json"
+    current = Path("target") / tname / "libc.json"
+
+    run(
+        [
+            "cargo",
+            "semver-checks",
+            "--only-explicit-features",
+            "--features=std,extra_traits",
+            "--baseline-rustdoc",
+            baseline,
+            "--current-rustdoc",
+            current,
+        ]
+    )
+
+
 def test_target(cfg: Cfg, target: Target):
+    """Run tests for a single target."""
     start = time.time()
     env = os.environ.copy()
     env.setdefault("RUSTFLAGS", "")
@@ -303,22 +370,7 @@ def test_target(cfg: Cfg, target: Target):
                 env=env | {"RUST_LIBC_UNSTABLE_FREEBSD_VERSION": str(version)},
             )
 
-    is_stable = cfg.toolchain == Toolchain.STABLE
-    # FIXME(semver): can't pass `--target` to `cargo-semver-checks` so we restrict to
-    # the host target
-    is_host = tname == cfg.host_target
-    if is_stable and is_host:
-        eprint("Running semver checks")
-        run(
-            [
-                "cargo",
-                "semver-checks",
-                "--only-explicit-features",
-                "--features=std,extra_traits",
-            ]
-        )
-    else:
-        eprint("Skipping semver checks")
+    do_semver_checks(cfg, target)
 
     elapsed = round(time.time() - start, 2)
     eprint(f"Finished checking target {tname} in {elapsed} seconds")
@@ -329,6 +381,10 @@ def main():
     p.add_argument("--toolchain", required=True, help="Rust toolchain")
     p.add_argument("--only", help="only targets matching this regex")
     p.add_argument("--skip", help="skip targets matching this regex")
+    p.add_argument(
+        "--baseline-crate-dir",
+        help="specify the directory of the crate to run semver checks against",
+    )
     p.add_argument(
         "--half",
         type=int,
@@ -337,7 +393,7 @@ def main():
     )
     args = p.parse_args()
 
-    cfg = Cfg(toolchain_name=args.toolchain)
+    cfg = Cfg(toolchain_name=args.toolchain, baseline_crate_dir=args.baseline_crate_dir)
     eprint(f"Config: {cfg}")
     eprint("Python version: ", sys.version)
     check_dup_targets()