Merge pull request #2285 from Kobzol/runtime-benchmarks-target

Add target to runtime benchmarks

Author: Kobzol

collector/src/bin/collector.rs

@@ -121,14 +121,21 @@ struct RuntimeBenchmarkConfig {
     runtime_suite: BenchmarkSuite,
     filter: RuntimeBenchmarkFilter,
     iterations: u32,
+    target: Target,
 }
 
 impl RuntimeBenchmarkConfig {
-    fn new(suite: BenchmarkSuite, filter: RuntimeBenchmarkFilter, iterations: u32) -> Self {
+    fn new(
+        suite: BenchmarkSuite,
+        filter: RuntimeBenchmarkFilter,
+        iterations: u32,
+        target: Target,
+    ) -> Self {
         Self {
             runtime_suite: suite.filter(&filter),
             filter,
             iterations,
+            target,
         }
     }
 }
@@ -860,8 +867,9 @@ fn main_result() -> anyhow::Result<i32> {
             purge,
         } => {
             log_db(&db);
-            let toolchain =
-                get_local_toolchain_for_runtime_benchmarks(&local, &require_host_target_tuple())?;
+
+            let host_tuple = require_host_target_tuple();
+            let toolchain = get_local_toolchain_for_runtime_benchmarks(&local, &host_tuple)?;
             let pool = Pool::open(&db.db);
 
             let isolation_mode = if no_isolate {
@@ -899,6 +907,7 @@ fn main_result() -> anyhow::Result<i32> {
                 runtime_suite,
                 RuntimeBenchmarkFilter::new(local.exclude, local.include),
                 iterations,
+                Target::from_str(&host_tuple).expect("Found unexpected host target"),
             );
             rt.block_on(run_benchmarks(conn.as_mut(), shared, None, Some(config)))?;
             Ok(0)
@@ -1189,6 +1198,7 @@ fn main_result() -> anyhow::Result<i32> {
                             runtime_suite,
                             filter: RuntimeBenchmarkFilter::keep_all(),
                             iterations: DEFAULT_RUNTIME_ITERATIONS,
+                            target: Target::default(),
                         };
                         let shared = SharedBenchmarkConfig {
                             artifact_id,
@@ -1806,6 +1816,7 @@ async fn create_benchmark_configs(
             runtime_suite,
             filter: RuntimeBenchmarkFilter::keep_all(),
             iterations: DEFAULT_RUNTIME_ITERATIONS,
+            target: job.target().into(),
         })
     } else {
         None
@@ -2232,6 +2243,7 @@ async fn run_benchmarks(
             &collector,
             runtime.filter,
             runtime.iterations,
+            runtime.target,
         )
         .await
         .context("Runtime benchmarks failed")
@@ -2306,6 +2318,7 @@ async fn bench_published_artifact(
             runtime_suite,
             RuntimeBenchmarkFilter::keep_all(),
             DEFAULT_RUNTIME_ITERATIONS,
+            Target::default(),
         )),
     )
     .await

collector/src/runtime/mod.rs

@@ -20,6 +20,7 @@ use crate::{command_output, CollectorCtx};
 mod benchmark;
 mod profile;
 
+use crate::compile::benchmark::target::Target;
 pub use benchmark::RuntimeCompilationOpts;
 pub use profile::{profile_runtime, RuntimeProfiler};
 
@@ -35,6 +36,7 @@ pub async fn bench_runtime(
     collector: &CollectorCtx,
     filter: RuntimeBenchmarkFilter,
     iterations: u32,
+    target: Target,
 ) -> anyhow::Result<()> {
     let filtered = suite.filtered_benchmark_count(&filter);
     println!("Executing {filtered} benchmarks\n");
@@ -74,6 +76,7 @@ pub async fn bench_runtime(
                             tx.conn(),
                             collector.artifact_row_id,
                             &rustc_perf_version,
+                            target,
                             result,
                         )
                         .await;
@@ -126,13 +129,15 @@ async fn record_stats(
     conn: &dyn Connection,
     artifact_id: ArtifactIdNumber,
     rustc_perf_version: &str,
+    target: Target,
     result: BenchmarkResult,
 ) {
     async fn record<'a>(
         conn: &'a dyn Connection,
         artifact_id: ArtifactIdNumber,
         collection_id: CollectionId,
         result: &'a BenchmarkResult,
+        target: Target,
         value: Option<u64>,
         metric: &'a str,
     ) {
@@ -142,6 +147,7 @@ async fn record_stats(
                 artifact_id,
                 &result.name,
                 metric,
+                target.into(),
                 value as f64,
             )
             .await;
@@ -156,6 +162,7 @@ async fn record_stats(
             artifact_id,
             collection_id,
             &result,
+            target,
             stat.instructions,
             "instructions:u",
         )
@@ -165,6 +172,7 @@ async fn record_stats(
             artifact_id,
             collection_id,
             &result,
+            target,
             stat.cycles,
             "cycles:u",
         )
@@ -174,6 +182,7 @@ async fn record_stats(
             artifact_id,
             collection_id,
             &result,
+            target,
             stat.branch_misses,
             "branch-misses",
         )
@@ -183,6 +192,7 @@ async fn record_stats(
             artifact_id,
             collection_id,
             &result,
+            target,
             stat.cache_misses,
             "cache-misses",
         )
@@ -192,6 +202,7 @@ async fn record_stats(
             artifact_id,
             collection_id,
             &result,
+            target,
             Some(stat.wall_time.as_nanos() as u64),
             "wall-time",
         )

database/src/bin/import-sqlite.rs

@@ -82,7 +82,7 @@ async fn main() {
             }
         }
 
-        for (&(benchmark, metric), id) in sqlite_idx.runtime_statistic_descriptions() {
+        for (&(benchmark, target, metric), id) in sqlite_idx.runtime_statistic_descriptions() {
             let stat = sqlite_conn
                 .get_runtime_pstats(&[id], &[Some(sqlite_aid)])
                 .await
@@ -92,7 +92,14 @@ async fn main() {
                 .unwrap();
             if let Some(stat) = stat {
                 postgres_conn
-                    .record_runtime_statistic(cid, postgres_aid, &benchmark, metric.as_str(), stat)
+                    .record_runtime_statistic(
+                        cid,
+                        postgres_aid,
+                        &benchmark,
+                        metric.as_str(),
+                        target,
+                        stat,
+                    )
                     .await;
             }
         }

database/src/bin/postgres-to-sqlite.rs

@@ -397,11 +397,11 @@ impl Table for RuntimePstatSeries {
     }
 
     fn postgres_select_statement(&self, _since_weeks_ago: Option<u32>) -> String {
-        "select id, benchmark, metric from ".to_string() + self.name()
+        "SELECT id, benchmark, target, metric FROM ".to_string() + self.name()
     }
 
     fn sqlite_insert_statement(&self) -> &'static str {
-        "insert into runtime_pstat_series (id, benchmark, metric) VALUES (?, ?, ?)"
+        "INSERT INTO runtime_pstat_series (id, benchmark, target, metric) VALUES (?, ?, ?, ?)"
     }
 
     fn sqlite_execute_insert(&self, statement: &mut rusqlite::Statement, row: tokio_postgres::Row) {
@@ -410,6 +410,7 @@ impl Table for RuntimePstatSeries {
                 row.get::<_, i32>(0),
                 row.get::<_, &str>(1),
                 row.get::<_, &str>(2),
+                row.get::<_, &str>(3),
             ])
             .unwrap();
     }

database/src/bin/sqlite-to-postgres.rs

@@ -543,6 +543,7 @@ struct RuntimePstatSeries;
 struct RuntimePstatSeriesRow<'a> {
     id: i32,
     benchmark: &'a str,
+    target: &'a str,
     metric: &'a str,
 }
 
@@ -552,11 +553,11 @@ impl Table for RuntimePstatSeries {
     }
 
     fn sqlite_attributes() -> &'static str {
-        "id, benchmark, metric"
+        "id, benchmark, target, metric"
     }
 
     fn postgres_attributes() -> &'static str {
-        "id, benchmark, metric"
+        "id, benchmark, target, metric"
     }
 
     fn postgres_generated_id_attribute() -> Option<&'static str> {
@@ -568,7 +569,8 @@ impl Table for RuntimePstatSeries {
             .serialize(RuntimePstatSeriesRow {
                 id: row.get(0).unwrap(),
                 benchmark: row.get_ref(1).unwrap().as_str().unwrap(),
-                metric: row.get_ref(2).unwrap().as_str().unwrap(),
+                target: row.get_ref(2).unwrap().as_str().unwrap(),
+                metric: row.get_ref(3).unwrap().as_str().unwrap(),
             })
             .unwrap();
     }

database/src/lib.rs

@@ -522,7 +522,7 @@ pub struct Index {
     /// For legacy reasons called `pstat_series` in the database, and so the name is kept here.
     pstat_series: Indexed<(Benchmark, Profile, Scenario, CodegenBackend, Target, Metric)>,
     /// Id lookup of runtime stat description ids
-    runtime_pstat_series: Indexed<(Benchmark, Metric)>,
+    runtime_pstat_series: Indexed<(Benchmark, Target, Metric)>,
 }
 
 /// An index lookup
@@ -725,7 +725,7 @@ impl Index {
         self.runtime_pstat_series
             .map
             .keys()
-            .map(|(_, metric)| metric)
+            .map(|(_, _, metric)| metric)
             .collect::<std::collections::HashSet<_>>()
             .into_iter()
             .map(|s| s.to_string())
@@ -752,7 +752,7 @@ impl Index {
 
     pub fn runtime_statistic_descriptions(
         &self,
-    ) -> impl Iterator<Item = (&(Benchmark, Metric), StatisticalDescriptionId)> + '_ {
+    ) -> impl Iterator<Item = (&(Benchmark, Target, Metric), StatisticalDescriptionId)> + '_ {
         self.runtime_pstat_series
             .map
             .iter()

database/src/pool.rs

@@ -64,6 +64,7 @@ pub trait Connection: Send + Sync {
         artifact: ArtifactIdNumber,
         benchmark: &str,
         metric: &str,
+        target: Target,
         value: f64,
     );
     /// Records a self-profile artifact in S3.

database/src/pool/postgres.rs

@@ -432,6 +432,12 @@ static MIGRATIONS: &[&str] = &[
             benchmark_set
         );
     "#,
+    // Add target to runtime_pstat_series
+    r#"
+    ALTER TABLE runtime_pstat_series ADD target TEXT NOT NULL DEFAULT 'x86_64-unknown-linux-gnu';
+    ALTER TABLE runtime_pstat_series DROP CONSTRAINT runtime_pstat_series_benchmark_metric_key;
+    ALTER TABLE runtime_pstat_series ADD CONSTRAINT runtime_test_case UNIQUE(benchmark, target, metric);
+    "#,
 ];
 
 #[async_trait::async_trait]
@@ -660,8 +666,8 @@ impl PostgresConnection {
                     select name, category
                     from benchmark
                 ").await.unwrap(),
-                insert_runtime_pstat_series: conn.prepare("insert into runtime_pstat_series (benchmark, metric) VALUES ($1, $2) ON CONFLICT DO NOTHING RETURNING id").await.unwrap(),
-                select_runtime_pstat_series: conn.prepare("select id from runtime_pstat_series where benchmark = $1 and metric = $2").await.unwrap(),
+                insert_runtime_pstat_series: conn.prepare("INSERT INTO runtime_pstat_series (benchmark, target, metric) VALUES ($1, $2, $3) ON CONFLICT DO NOTHING RETURNING id").await.unwrap(),
+                select_runtime_pstat_series: conn.prepare("SELECT id FROM runtime_pstat_series WHERE benchmark = $1 AND target = $2 AND metric = $3").await.unwrap(),
                 insert_runtime_pstat: conn
                     .prepare("insert into runtime_pstat (series, aid, cid, value) VALUES ($1, $2, $3, $4)")
                     .await
@@ -883,7 +889,7 @@ where
             runtime_pstat_series: self
                 .conn()
                 .query(
-                    "select id, benchmark, metric from runtime_pstat_series;",
+                    "SELECT id, benchmark, target, metric FROM runtime_pstat_series;",
                     &[],
                 )
                 .await
@@ -893,8 +899,9 @@ where
                     (
                         row.get::<_, i32>(0) as u32,
                         (
-                            row.get::<_, String>(1).as_str().into(),
-                            row.get::<_, String>(2).as_str().into(),
+                            row.get::<_, &str>(1).into(),
+                            Target::from_str(row.get::<_, &str>(2)).unwrap(),
+                            row.get::<_, &str>(3).into(),
                         ),
                     )
                 })
@@ -1123,13 +1130,15 @@ where
         artifact: ArtifactIdNumber,
         benchmark: &str,
         metric: &str,
+        target: Target,
         value: f64,
     ) {
+        let target = target.to_string();
         let sid = self
             .conn()
             .query_opt(
                 &self.statements().select_runtime_pstat_series,
-                &[&benchmark, &metric],
+                &[&benchmark, &target, &metric],
             )
             .await
             .unwrap();
@@ -1139,14 +1148,14 @@ where
                 self.conn()
                     .query_opt(
                         &self.statements().insert_runtime_pstat_series,
-                        &[&benchmark, &metric],
+                        &[&benchmark, &target, &metric],
                     )
                     .await
                     .unwrap();
                 self.conn()
                     .query_one(
                         &self.statements().select_runtime_pstat_series,
-                        &[&benchmark, &metric],
+                        &[&benchmark, &target, &metric],
                     )
                     .await
                     .unwrap()