Merge pull request #1475 from rust-lang/ci-runtime-test

Test runtime benchmarks on CI

Author: Kobzol

.github/workflows/ci.yml

@@ -108,7 +108,7 @@ jobs:
       - name: Build collector
         run: cargo build -p collector
 
-      - name: Check benchmarks
+      - name: Check compile benchmarks
         run: sh -x -c "ci/check-compile-benchmarks.sh"
         env:
           JEMALLOC_OVERRIDE: /usr/lib/x86_64-linux-gnu/libjemalloc.so
@@ -195,7 +195,7 @@ jobs:
       - name: Build collector
         run: cargo build -p collector
 
-      - name: Check benchmarks
+      - name: Check profiling
         run: sh -x -c "ci/check-profiling.sh"
 
   database-check:

ci/check-runtime-benchmarks.sh

@@ -6,11 +6,17 @@ bash -c "while true; do sleep 30; echo \$(date) - running ...; done" &
 PING_LOOP_PID=$!
 trap 'kill $PING_LOOP_PID' ERR 1 2 3 6
 
-# Check if the runtime benchmarks can be compiled.
-# Once we can actually run the benchmarks on CI, we will also execute them here.
-# Currently it is not possible because of `perf` permission issues when gathering perf. counters.
-cd collector/runtime-benchmarks
-cargo check
+# Install a toolchain.
+RUST_BACKTRACE=1 RUST_LOG=raw_cargo_messages=trace,collector=debug,rust_sysroot=debug \
+    bindir=`cargo run -p collector --bin collector install_next`
+
+# Do some benchmarking.
+RUST_LOG=raw_cargo_messages=trace,collector=debug,rust_sysroot=debug \
+    cargo run -p collector --bin collector -- \
+    bench_runtime_local $bindir/rustc \
+        --cargo $bindir/cargo \
+        --iterations 1 \
+        --id Test
 
 kill $PING_LOOP_PID
 exit 0

collector/src/bin/collector.rs

@@ -704,9 +704,16 @@ fn main_result() -> anyhow::Result<i32> {
 
     match args.command {
         Commands::BenchRuntimeLocal { local, iterations } => {
-            bench_runtime(
+            let toolchain = get_local_toolchain(
+                &[Profile::Opt],
                 &local.rustc,
+                None,
+                local.cargo.as_deref(),
                 local.id.as_deref(),
+                "",
+            )?;
+            bench_runtime(
+                toolchain,
                 BenchmarkFilter::new(local.exclude, local.include),
                 runtime_benchmark_dir,
                 iterations,

collector/src/runtime/mod.rs

@@ -1,7 +1,6 @@
 mod benchmark;
 
-use crate::benchmark::profile::Profile;
-use crate::toolchain::get_local_toolchain;
+use crate::toolchain::LocalToolchain;
 use benchlib::comm::messages::{BenchmarkMessage, BenchmarkResult, BenchmarkStats};
 use std::io::{BufRead, BufReader};
 use std::path::{Path, PathBuf};
@@ -14,13 +13,11 @@ pub use benchmark::BenchmarkFilter;
 /// to a Cargo crate. All binaries built by that crate will are expected to be runtime benchmark
 /// groups that leverage `benchlib`.
 pub fn bench_runtime(
-    rustc: &str,
-    id: Option<&str>,
+    toolchain: LocalToolchain,
     filter: BenchmarkFilter,
     benchmark_dir: PathBuf,
     iterations: u32,
 ) -> anyhow::Result<()> {
-    let toolchain = get_local_toolchain(&[Profile::Opt], rustc, None, None, id, "")?;
     let suite = benchmark::discover_benchmarks(&toolchain, &benchmark_dir)?;
 
     let total_benchmark_count = suite.total_benchmark_count();