Merge pull request #1895 from Kobzol/binary-stats-ext

Add command to compare binary stats of two arbitrary binaries

Author: Kobzol

collector/README.md

@@ -516,23 +516,28 @@ Codegen diff is currently only implemented for runtime benchmarks.
 
 ## Binary size statistics
 You can use the `binary_stats` command to display size statistics (section and symbol sizes) of
-binary artifacts (executables, libraries) of selected compile benchmarks.
-
-```
-./target/release/collector binary_stats `<rustc>` --include <benchmark name> \
-    [--profile <Debug|Opt>] \
-    [--backend <Llvm|Cranelift>]
-```
-
-You can also compare (diff) the size statistics between two compilers:
-```
-./target/release/collector binary_stats `<rustc>` --include <benchmark name> --rustc2 <rustc2>
-```
-or between two codegen backends:
-```
-./target/release/collector binary_stats `<rustc>` --include <benchmark name> --rustc2 <rustc>
-    --backend <Llvm|Cranelift> --backend2 <Llvm|Cranelift>
-```
+binary artifacts (executables, libraries). You can compare the binary statistics of:
+
+- Selected compile benchmarks:
+  ```bash
+  ./target/release/collector binary_stats compile `<rustc>` --include <benchmark name> \
+      [--profile <Debug|Opt>] \
+      [--backend <Llvm|Cranelift>]
+  ```
+  
+  You can also compare (diff) the size statistics between two compilers:
+  ```bash
+  ./target/release/collector binary_stats compile `<rustc>` --include <benchmark name> --rustc2 <rustc2>
+  ```
+  or between two codegen backends:
+  ```bash
+  ./target/release/collector binary_stats compile `<rustc>` --include <benchmark name> --rustc2 <rustc>
+      --backend <Llvm|Cranelift> --backend2 <Llvm|Cranelift>
+  ```
+- Arbitrary binary artifacts on disk:
+  ```bash
+  ./target/release/collector binary_stats local `<artifact-path>` [<artifact-to-compare-to>]
+  ```
 
 ## How `rustc` wrapping works
 When a crate is benchmarked or profiled, the real `rustc` is replaced with the `rustc-fake` binary,

collector/src/artifact_stats.rs

@@ -2,6 +2,7 @@
 //! names and sizes.
 
 use std::collections::{HashMap, VecDeque};
+use std::ffi::OsStr;
 use std::path::{Path, PathBuf};
 use std::process::Command;
 use std::sync::OnceLock;
@@ -23,6 +24,17 @@ pub struct ArtifactStats {
 }
 
 impl ArtifactStats {
+    /// Try to auto-detect the artifact type from the given path.
+    /// If auto-detection fails, tries to load the artifact as a dynamic object.
+    pub fn from_path(path: &Path) -> anyhow::Result<Self> {
+        if path.extension() == Some(OsStr::new("a")) || path.extension() == Some(OsStr::new("rlib"))
+        {
+            Self::from_rlib(path)
+        } else {
+            Self::from_dynamic_object(path)
+        }
+    }
+
     /// Loads size statistics from an ELF file (either an executable or a shared library).
     pub fn from_dynamic_object(path: &Path) -> anyhow::Result<Self> {
         let data = std::fs::read(path)

collector/src/bin/collector.rs

@@ -26,7 +26,9 @@ use tabled::settings::{Alignment, Border, Color, Modify};
 use tokio::runtime::Runtime;
 
 use collector::api::next_artifact::NextArtifact;
-use collector::artifact_stats::{compile_and_get_stats, ArtifactWithStats, CargoProfile};
+use collector::artifact_stats::{
+    compile_and_get_stats, ArtifactStats, ArtifactWithStats, CargoProfile,
+};
 use collector::codegen::{codegen_diff, CodegenType};
 use collector::compile::benchmark::category::Category;
 use collector::compile::benchmark::codegen_backend::CodegenBackend;
@@ -408,39 +410,65 @@ struct PurgeOption {
     purge: Option<PurgeMode>,
 }
 
-// For each subcommand we list the mandatory arguments in the required
-// order, followed by the options in alphabetical order.
-#[derive(Debug, clap::Subcommand)]
+#[derive(Debug, clap::Args)]
 #[command(rename_all = "snake_case")]
-enum Commands {
-    /// Show binary (executable or library) section (and optionally symbol) size statistics of the
-    /// selected compile benchmark(s).
-    /// Optionally compares sizes between two compiler toolchains, if `--rustc2` is provided.
-    BinaryStats {
-        #[command(flatten)]
-        local: LocalOptions,
+struct BinaryStatsCompile {
+    #[command(flatten)]
+    local: LocalOptions,
 
-        /// Cargo profile to use.
-        #[arg(long, default_value = "Debug")]
-        profile: Profile,
+    /// Cargo profile to use.
+    #[arg(long, default_value = "Debug")]
+    profile: Profile,
 
-        /// Codegen backend to use.
-        #[arg(long = "backend", default_value = "Llvm")]
-        codegen_backend: CodegenBackend,
+    /// Codegen backend to use.
+    #[arg(long = "backend", default_value = "Llvm")]
+    codegen_backend: CodegenBackend,
 
-        /// An optional second toolchain to compare to.
-        #[arg(long)]
-        rustc2: Option<String>,
+    /// An optional second toolchain to compare to.
+    #[arg(long)]
+    rustc2: Option<String>,
+
+    /// Codegen backend to use for the second toolchain.
+    #[arg(long = "backend2")]
+    codegen_backend2: Option<CodegenBackend>,
+}
+
+#[derive(Debug, clap::Args)]
+#[command(rename_all = "snake_case")]
+struct BinaryStatsLocal {
+    /// Binary artifact to examine.
+    artifact: PathBuf,
 
-        /// Codegen backend to use for the second toolchain.
-        #[arg(long = "backend2")]
-        codegen_backend2: Option<CodegenBackend>,
+    /// Optional second artifact to compare with the first one.
+    artifact2: Option<PathBuf>,
+}
+
+#[derive(Debug, clap::Subcommand)]
+#[command(rename_all = "snake_case")]
+enum BinaryStatsMode {
+    /// Show size statistics for the selected compile benchmark(s).
+    /// Optionally compares sizes between two compiler toolchains, if `--rustc2` is provided.
+    Compile(BinaryStatsCompile),
+    /// Show size statistics for the selected binary artifact on disk.
+    /// Optionally compares sizes with a second provided artifact, if `--artifact2` is provided.
+    Local(BinaryStatsLocal),
+}
 
+// For each subcommand we list the mandatory arguments in the required
+// order, followed by the options in alphabetical order.
+#[derive(Debug, clap::Subcommand)]
+#[command(rename_all = "snake_case")]
+enum Commands {
+    /// Show binary (executable or library) section (and optionally symbol) size statistics.
+    BinaryStats {
         /// Also print symbol comparison in addition to section comparison.
         ///
         /// Warning: may generate *A LOT* of data.
-        #[arg(long, default_value_t = false)]
+        #[arg(long, default_value_t = false, global = true)]
         symbols: bool,
+
+        #[clap(subcommand)]
+        mode: BinaryStatsMode,
     },
 
     /// Benchmarks the performance of programs generated by a local rustc
@@ -649,89 +677,13 @@ fn main_result() -> anyhow::Result<i32> {
     let target_triple = format!("{}-unknown-linux-gnu", std::env::consts::ARCH);
 
     match args.command {
-        Commands::BinaryStats {
-            local,
-            codegen_backend,
-            profile,
-            rustc2,
-            codegen_backend2,
-            symbols,
-        } => {
-            let codegen_backend2 = codegen_backend2.unwrap_or(codegen_backend);
-            let toolchain = get_local_toolchain(
-                &[Profile::Debug, Profile::Opt],
-                &[codegen_backend],
-                &local.rustc,
-                *ToolchainConfig::default()
-                    .cargo(local.cargo.as_deref())
-                    .id(local.id.as_deref()),
-                "",
-                target_triple.clone(),
-            )?;
-            let toolchain2 = rustc2
-                .map(|rustc| {
-                    get_local_toolchain(
-                        &[Profile::Debug, Profile::Opt],
-                        &[codegen_backend2],
-                        &rustc,
-                        *ToolchainConfig::default()
-                            .cargo(local.cargo.as_deref())
-                            .id(local.id.as_deref()),
-                        "",
-                        target_triple,
-                    )
-                })
-                .transpose()?;
-            let profile = match profile {
-                Profile::Debug => CargoProfile::Debug,
-                Profile::Opt => CargoProfile::Release,
-                _ => return Err(anyhow::anyhow!("Only Debug and Opt profiles are supported")),
-            };
-            let benchmarks = get_compile_benchmarks(
-                &compile_benchmark_dir,
-                &local.include,
-                &local.exclude,
-                &local.exclude_suffix,
-            )?;
-            for benchmark in benchmarks {
-                println!("Stats for benchmark `{}`", benchmark.name);
-                println!("{}", "-".repeat(20));
-                let artifacts =
-                    compile_and_get_stats(&benchmark.path, &toolchain, profile, codegen_backend)?;
-                let archives2: HashMap<String, ArtifactWithStats> = toolchain2
-                    .as_ref()
-                    .map(|toolchain| {
-                        compile_and_get_stats(&benchmark.path, toolchain, profile, codegen_backend2)
-                    })
-                    .transpose()?
-                    .unwrap_or_default()
-                    .into_iter()
-                    .map(|artifact| (artifact.target_name.clone(), artifact))
-                    .collect();
-
-                for artifact in artifacts {
-                    let archive2 = archives2.get(&artifact.target_name);
-
-                    println!(
-                        "Target `{}` (artifact `{}`)",
-                        artifact.target_name,
-                        artifact
-                            .path
-                            .file_name()
-                            .and_then(|s| s.to_str())
-                            .unwrap_or(&artifact.target_name)
-                    );
-
-                    let sections = artifact.stats.sections;
-                    let sections2 = archive2.as_ref().map(|a| a.stats.sections.clone());
-                    print_binary_stats("Section", sections, sections2);
-
-                    if symbols {
-                        let symbols = artifact.stats.symbols;
-                        let symbols2 = archive2.as_ref().map(|a| a.stats.symbols.clone());
-                        print_binary_stats("Symbol", symbols, symbols2);
-                    }
-                    println!();
+        Commands::BinaryStats { mode, symbols } => {
+            match mode {
+                BinaryStatsMode::Compile(args) => {
+                    binary_stats_compile(args, symbols, &target_triple)?;
+                }
+                BinaryStatsMode::Local(args) => {
+                    binary_stats_local(args, symbols)?;
                 }
             }
 
@@ -1234,6 +1186,122 @@ Make sure to modify `{dir}/perf-config.json` if the category/artifact don't matc
     }
 }
 
+fn binary_stats_local(args: BinaryStatsLocal, symbols: bool) -> anyhow::Result<()> {
+    let stats = ArtifactStats::from_path(&args.artifact)
+        .with_context(|| format!("Cannot load artifact from {}", args.artifact.display()))?;
+    let stats2 = args
+        .artifact2
+        .as_ref()
+        .map(|path| {
+            ArtifactStats::from_path(path)
+                .with_context(|| format!("Cannot load artifact from {}", path.display()))
+        })
+        .transpose()?;
+    print_binary_stats(
+        "Sections",
+        stats.sections,
+        stats2.as_ref().map(|s| s.sections.clone()),
+    );
+    if symbols {
+        print_binary_stats("Symbols", stats.symbols, stats2.map(|s| s.symbols));
+    }
+
+    Ok(())
+}
+
+fn binary_stats_compile(
+    args: BinaryStatsCompile,
+    symbols: bool,
+    target_triple: &str,
+) -> anyhow::Result<()> {
+    let BinaryStatsCompile {
+        local,
+        profile,
+        codegen_backend,
+        rustc2,
+        codegen_backend2,
+    } = args;
+
+    let codegen_backend2 = codegen_backend2.unwrap_or(codegen_backend);
+    let toolchain = get_local_toolchain(
+        &[Profile::Debug, Profile::Opt],
+        &[codegen_backend],
+        &local.rustc,
+        *ToolchainConfig::default()
+            .cargo(local.cargo.as_deref())
+            .id(local.id.as_deref()),
+        "",
+        target_triple.to_string(),
+    )?;
+    let toolchain2 = rustc2
+        .map(|rustc| {
+            get_local_toolchain(
+                &[Profile::Debug, Profile::Opt],
+                &[codegen_backend2],
+                &rustc,
+                *ToolchainConfig::default()
+                    .cargo(local.cargo.as_deref())
+                    .id(local.id.as_deref()),
+                "",
+                target_triple.to_string(),
+            )
+        })
+        .transpose()?;
+    let profile = match profile {
+        Profile::Debug => CargoProfile::Debug,
+        Profile::Opt => CargoProfile::Release,
+        _ => return Err(anyhow::anyhow!("Only Debug and Opt profiles are supported")),
+    };
+    let benchmarks = get_compile_benchmarks(
+        &compile_benchmark_dir(),
+        &local.include,
+        &local.exclude,
+        &local.exclude_suffix,
+    )?;
+    for benchmark in benchmarks {
+        println!("Stats for benchmark `{}`", benchmark.name);
+        println!("{}", "-".repeat(20));
+        let artifacts =
+            compile_and_get_stats(&benchmark.path, &toolchain, profile, codegen_backend)?;
+        let archives2: HashMap<String, ArtifactWithStats> = toolchain2
+            .as_ref()
+            .map(|toolchain| {
+                compile_and_get_stats(&benchmark.path, toolchain, profile, codegen_backend2)
+            })
+            .transpose()?
+            .unwrap_or_default()
+            .into_iter()
+            .map(|artifact| (artifact.target_name.clone(), artifact))
+            .collect();
+
+        for artifact in artifacts {
+            let archive2 = archives2.get(&artifact.target_name);
+
+            println!(
+                "Target `{}` (artifact `{}`)",
+                artifact.target_name,
+                artifact
+                    .path
+                    .file_name()
+                    .and_then(|s| s.to_str())
+                    .unwrap_or(&artifact.target_name)
+            );
+
+            let sections = artifact.stats.sections;
+            let sections2 = archive2.as_ref().map(|a| a.stats.sections.clone());
+            print_binary_stats("Section", sections, sections2);
+
+            if symbols {
+                let symbols = artifact.stats.symbols;
+                let symbols2 = archive2.as_ref().map(|a| a.stats.symbols.clone());
+                print_binary_stats("Symbol", symbols, symbols2);
+            }
+            println!();
+        }
+    }
+    Ok(())
+}
+
 fn build_async_runtime() -> Runtime {
     let mut builder = tokio::runtime::Builder::new_multi_thread();
     // We want to minimize noise from the runtime