Merge pull request #1692 from Kobzol/runtime-group-filter

Add `--group` flag to runtime benchmark/profiling command

Author: Kobzol

collector/README.md

@@ -183,6 +183,11 @@ by a selected version of rustc. You can run it using the following command:
 The following options alter the behaviour of the `bench_runtime_local` subcommand.
 - `--no-isolate`: you can use this flag to make repeated local benchmarks faster. It will cause the
   `collector` to reuse compiled artifacts of the runtime benchmark groups.
+- `--group`: Compile only the selected runtime benchmark group (i.e. only compile a crate inside the
+directory `collector/runtime-benchmarks/<group>`). This can be used to speed up local runtime benchmark
+experiments. Even with `--no-isolate`, it can take a few seconds to recompile all runtime benchmarks
+and discover all benchmarks within them. If you only want to run benchmark(s) from a single crate,
+you can use this to speed up the runtime benchmarking or profiling commands.
 
 The `bench_runtime_local` command also shares some options with the `bench_local` command, notably
 `--id`, `--db`, `--cargo`, `--include`, `--exclude` and `--iterations`. 

collector/src/bin/collector.rs

@@ -459,6 +459,14 @@ struct CompileTimeOptions {
     rustdoc: Option<PathBuf>,
 }
 
+#[derive(Debug, clap::Args)]
+struct RuntimeOptions {
+    /// Select a runtime benchmark group that should be compiled and used. If not specified, all
+    /// found groups will be compiled.
+    #[arg(long)]
+    group: Option<String>,
+}
+
 #[derive(Debug, clap::Args)]
 struct SelfProfileOption {
     /// Collect self-profile data
@@ -492,6 +500,9 @@ enum Commands {
         #[command(flatten)]
         local: LocalOptions,
 
+        #[command(flatten)]
+        runtime: RuntimeOptions,
+
         /// How many iterations of each benchmark should be executed.
         #[arg(long, default_value_t = DEFAULT_RUNTIME_ITERATIONS)]
         iterations: u32,
@@ -507,6 +518,9 @@ enum Commands {
 
     /// Profiles a runtime benchmark.
     ProfileRuntime {
+        #[command(flatten)]
+        runtime: RuntimeOptions,
+
         /// Profiler to use
         profiler: RuntimeProfiler,
 
@@ -660,6 +674,7 @@ fn main_result() -> anyhow::Result<i32> {
     match args.command {
         Commands::BenchRuntimeLocal {
             local,
+            runtime,
             iterations,
             db,
             no_isolate,
@@ -680,6 +695,7 @@ fn main_result() -> anyhow::Result<i32> {
                 conn.as_mut(),
                 &runtime_benchmark_dir,
                 isolation_mode,
+                runtime.group,
                 &toolchain,
                 &artifact_id,
             ))?;
@@ -697,6 +713,7 @@ fn main_result() -> anyhow::Result<i32> {
             Ok(0)
         }
         Commands::ProfileRuntime {
+            runtime,
             profiler,
             rustc,
             benchmark,
@@ -707,6 +724,7 @@ fn main_result() -> anyhow::Result<i32> {
                 &toolchain,
                 &runtime_benchmark_dir,
                 CargoIsolationMode::Cached,
+                runtime.group,
                 // Compile with debuginfo to have filenames and line numbers available in the
                 // generated profiles.
                 RuntimeCompilationOpts::default().debug_info("1"),
@@ -843,6 +861,7 @@ fn main_result() -> anyhow::Result<i32> {
                         conn.as_mut(),
                         &runtime_benchmark_dir,
                         CargoIsolationMode::Isolated,
+                        None,
                         &toolchain,
                         &artifact_id,
                     ))?;
@@ -1050,6 +1069,7 @@ async fn load_runtime_benchmarks(
     conn: &mut dyn Connection,
     benchmark_dir: &Path,
     isolation_mode: CargoIsolationMode,
+    group: Option<String>,
     toolchain: &Toolchain,
     artifact_id: &ArtifactId,
 ) -> anyhow::Result<BenchmarkSuite> {
@@ -1060,6 +1080,7 @@ async fn load_runtime_benchmarks(
         toolchain,
         benchmark_dir,
         isolation_mode,
+        group,
         RuntimeCompilationOpts::default(),
     )?;
 
@@ -1185,6 +1206,7 @@ fn bench_published_artifact(
         connection.as_mut(),
         dirs.runtime,
         CargoIsolationMode::Isolated,
+        None,
         &toolchain,
         &artifact_id,
     ))?;

collector/src/runtime/benchmark.rs

@@ -143,13 +143,16 @@ impl RuntimeCompilationOpts {
 /// We assume that each binary defines a benchmark suite using `benchlib`.
 /// We then execute each benchmark suite with the `list-benchmarks` command to find out its
 /// benchmark names.
+///
+/// If `group` is not `None`, only the benchmark group with the given name will be compiled.
 pub fn prepare_runtime_benchmark_suite(
     toolchain: &Toolchain,
     benchmark_dir: &Path,
     isolation_mode: CargoIsolationMode,
+    group: Option<String>,
     opts: RuntimeCompilationOpts,
 ) -> anyhow::Result<BenchmarkSuiteCompilation> {
-    let benchmark_crates = get_runtime_benchmark_groups(benchmark_dir)?;
+    let benchmark_crates = get_runtime_benchmark_groups(benchmark_dir, group)?;
 
     let temp_dir: Option<TempDir> = match isolation_mode {
         CargoIsolationMode::Cached => None,
@@ -167,7 +170,7 @@ pub fn prepare_runtime_benchmark_suite(
     };
 
     let group_count = benchmark_crates.len();
-    println!("Compiling {group_count} runtime benchmark groups");
+    println!("Compiling {group_count} runtime benchmark group(s)");
 
     let mut groups = Vec::new();
     let mut failed_to_compile = HashMap::new();
@@ -347,7 +350,10 @@ fn gather_benchmarks(binary: &Path) -> anyhow::Result<Vec<String>> {
 }
 
 /// Finds all runtime benchmarks (crates) in the given directory.
-fn get_runtime_benchmark_groups(directory: &Path) -> anyhow::Result<Vec<BenchmarkGroupCrate>> {
+fn get_runtime_benchmark_groups(
+    directory: &Path,
+    group: Option<String>,
+) -> anyhow::Result<Vec<BenchmarkGroupCrate>> {
     let mut groups = Vec::new();
     for entry in std::fs::read_dir(directory).with_context(|| {
         anyhow::anyhow!("Failed to list benchmark dir '{}'", directory.display())
@@ -363,6 +369,12 @@ fn get_runtime_benchmark_groups(directory: &Path) -> anyhow::Result<Vec<Benchmar
             .ok_or_else(|| anyhow::anyhow!("Cannot get filename of {}", path.display()))?
             .to_string();
 
+        if let Some(ref group) = group {
+            if group != &name {
+                continue;
+            }
+        }
+
         groups.push(BenchmarkGroupCrate { name, path });
     }
     groups.sort_unstable_by(|a, b| a.name.cmp(&b.name));