Skip to content

Commit 9c2f4de

Browse files
JamesbarfordJamesbarford
committed
Fix; Runtime & Rustc benchmarks to run once
1 parent edb9c07 commit 9c2f4de

File tree

8 files changed

+134
-47
lines changed

8 files changed

+134
-47
lines changed

‎collector/src/benchmark_set/mod.rs‎

Lines changed: 6 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -29,11 +29,6 @@ impl BenchmarkSetId {
2929
pub enum BenchmarkSetMember {
3030
/// Benchmark a specific compile-time benchmark
3131
CompileBenchmark(BenchmarkName),
32-
/// Benchmark *all* the runtime benchmarks.
33-
/// For simplicity, we currently always benchmark all of them on a single collector.
34-
RuntimeBenchmarks,
35-
/// Benchmark the rustc bootstrap
36-
Rustc,
3732
}
3833

3934
/// Return the number of benchmark sets for the given target.
@@ -102,8 +97,6 @@ pub fn expand_benchmark_set(id: BenchmarkSetId) -> Vec<BenchmarkSetMember> {
10297
compile(UNUSED_WARNINGS),
10398
compile(WF_PROJECTION_STRESS_65510),
10499
compile(WG_GRAMMAR),
105-
BenchmarkSetMember::Rustc,
106-
BenchmarkSetMember::RuntimeBenchmarks,
107100
]
108101
}
109102
(Target::X86_64UnknownLinuxGnu, 1..) => {
@@ -169,8 +162,6 @@ mod tests {
169162

170163
// Check that the union of all sets contains all the required benchmarks
171164
let all_members = sets.iter().flatten().collect::<HashSet<_>>();
172-
assert!(all_members.contains(&BenchmarkSetMember::Rustc));
173-
assert!(all_members.contains(&BenchmarkSetMember::RuntimeBenchmarks));
174165

175166
const BENCHMARK_DIR: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/compile-benchmarks");
176167
let all_compile_benchmarks =
@@ -187,13 +178,12 @@ mod tests {
187178
);
188179
}
189180
for benchmark in &all_members {
190-
if let BenchmarkSetMember::CompileBenchmark(name) = benchmark {
191-
assert!(
192-
all_compile_benchmarks.contains(name),
193-
"Compile-time benchmark {name} does not exist on disk or is a stable benchmark"
194-
);
195-
}
181+
let BenchmarkSetMember::CompileBenchmark(name) = benchmark;
182+
assert!(
183+
all_compile_benchmarks.contains(name),
184+
"Compile-time benchmark {name} does not exist on disk or is a stable benchmark"
185+
);
196186
}
197-
assert_eq!(all_members.len(), all_compile_benchmarks.len() + 2);
187+
assert_eq!(all_members.len(), all_compile_benchmarks.len());
198188
}
199189
}

‎collector/src/bin/collector.rs‎

Lines changed: 15 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1699,13 +1699,22 @@ async fn create_benchmark_configs(
16991699
let mut bench_rustc = false;
17001700
let mut bench_runtime = false;
17011701
let mut bench_compile_benchmarks = HashSet::new();
1702-
for member in benchmark_set_members {
1703-
match member {
1704-
BenchmarkSetMember::CompileBenchmark(benchmark) => {
1705-
bench_compile_benchmarks.insert(benchmark);
1702+
1703+
match job.kind() {
1704+
database::BenchmarkJobKind::Runtime => {
1705+
bench_runtime = true;
1706+
}
1707+
database::BenchmarkJobKind::Compiletime => {
1708+
for member in benchmark_set_members {
1709+
match member {
1710+
BenchmarkSetMember::CompileBenchmark(benchmark) => {
1711+
bench_compile_benchmarks.insert(benchmark);
1712+
}
1713+
}
17061714
}
1707-
BenchmarkSetMember::RuntimeBenchmarks => bench_runtime = true,
1708-
BenchmarkSetMember::Rustc => bench_rustc = true,
1715+
}
1716+
database::BenchmarkJobKind::Rustc => {
1717+
bench_rustc = true;
17091718
}
17101719
}
17111720

‎database/src/lib.rs‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1132,6 +1132,7 @@ pub struct BenchmarkJob {
11321132
created_at: DateTime<Utc>,
11331133
status: BenchmarkJobStatus,
11341134
deque_counter: u32,
1135+
kind: BenchmarkJobKind,
11351136
}
11361137

11371138
impl BenchmarkJob {
@@ -1179,6 +1180,10 @@ impl BenchmarkJob {
11791180
pub fn created_at(&self) -> DateTime<Utc> {
11801181
self.created_at
11811182
}
1183+
1184+
pub fn kind(&self) -> BenchmarkJobKind {
1185+
self.kind
1186+
}
11821187
}
11831188

11841189
/// Describes the final state of a job
@@ -1256,3 +1261,32 @@ pub struct BenchmarkRequestWithErrors {
12561261
/// Benchmark (name) -> error
12571262
pub errors: HashMap<String, String>,
12581263
}
1264+
1265+
#[derive(Debug, PartialEq, Clone, Copy, serde::Deserialize, serde::Serialize)]
1266+
pub enum BenchmarkJobKind {
1267+
Runtime,
1268+
Compiletime,
1269+
Rustc,
1270+
}
1271+
1272+
impl fmt::Display for BenchmarkJobKind {
1273+
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1274+
match self {
1275+
BenchmarkJobKind::Runtime => write!(f, "runtime"),
1276+
BenchmarkJobKind::Compiletime => write!(f, "compiletime"),
1277+
BenchmarkJobKind::Rustc => write!(f, "rustc"),
1278+
}
1279+
}
1280+
}
1281+
1282+
impl FromStr for BenchmarkJobKind {
1283+
type Err = String;
1284+
fn from_str(s: &str) -> Result<Self, Self::Err> {
1285+
Ok(match s.to_ascii_lowercase().as_str() {
1286+
"runtime" => BenchmarkJobKind::Runtime,
1287+
"compiletime" => BenchmarkJobKind::Compiletime,
1288+
"rustc" => BenchmarkJobKind::Rustc,
1289+
_ => return Err(format!("{s} is not a codegen backend")),
1290+
})
1291+
}
1292+
}

‎database/src/pool.rs‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
use crate::selector::CompileTestCase;
22
use crate::{
33
ArtifactCollection, ArtifactId, ArtifactIdNumber, BenchmarkJob, BenchmarkJobConclusion,
4-
BenchmarkRequest, BenchmarkRequestIndex, BenchmarkRequestStatus, BenchmarkRequestWithErrors,
5-
BenchmarkSet, CodegenBackend, CollectorConfig, CompileBenchmark, PendingBenchmarkRequests,
6-
Target,
4+
BenchmarkJobKind, BenchmarkRequest, BenchmarkRequestIndex, BenchmarkRequestStatus,
5+
BenchmarkRequestWithErrors, BenchmarkSet, CodegenBackend, CollectorConfig, CompileBenchmark,
6+
PendingBenchmarkRequests, Target,
77
};
88
use crate::{CollectionId, Index, Profile, QueuedCommit, Scenario, Step};
99
use chrono::{DateTime, Utc};
@@ -233,6 +233,7 @@ pub trait Connection: Send + Sync {
233233
backend: CodegenBackend,
234234
profile: Profile,
235235
benchmark_set: u32,
236+
kind: BenchmarkJobKind,
236237
) -> anyhow::Result<Option<u32>>;
237238

238239
/// Add a benchmark job which is explicitly using a `parent_sha` we split
@@ -245,6 +246,7 @@ pub trait Connection: Send + Sync {
245246
backend: CodegenBackend,
246247
profile: Profile,
247248
benchmark_set: u32,
249+
kind: BenchmarkJobKind,
248250
) -> (bool, anyhow::Result<u32>);
249251

250252
/// Returns a set of compile-time benchmark test cases that were already computed for the
@@ -706,6 +708,7 @@ mod tests {
706708
CodegenBackend::Llvm,
707709
Profile::Opt,
708710
0u32,
711+
BenchmarkJobKind::Runtime,
709712
)
710713
.await;
711714
assert!(result.is_ok());
@@ -860,6 +863,7 @@ mod tests {
860863
CodegenBackend::Llvm,
861864
Profile::Opt,
862865
1u32,
866+
BenchmarkJobKind::Runtime,
863867
)
864868
.await
865869
.unwrap();
@@ -956,6 +960,7 @@ mod tests {
956960
CodegenBackend::Llvm,
957961
Profile::Opt,
958962
benchmark_set.0,
963+
BenchmarkJobKind::Runtime,
959964
)
960965
.await
961966
.unwrap();
@@ -1213,6 +1218,7 @@ mod tests {
12131218
CodegenBackend::Llvm,
12141219
Profile::Debug,
12151220
0,
1221+
BenchmarkJobKind::Runtime,
12161222
)
12171223
.await;
12181224

‎database/src/pool/postgres.rs‎

Lines changed: 28 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -2,13 +2,13 @@ use crate::pool::{Connection, ConnectionManager, ManagedConnection, Transaction}
22
use crate::selector::CompileTestCase;
33
use crate::{
44
ArtifactCollection, ArtifactId, ArtifactIdNumber, Benchmark, BenchmarkJob,
5-
BenchmarkJobConclusion, BenchmarkJobStatus, BenchmarkRequest, BenchmarkRequestIndex,
6-
BenchmarkRequestStatus, BenchmarkRequestType, BenchmarkRequestWithErrors, BenchmarkSet,
7-
CodegenBackend, CollectionId, CollectorConfig, Commit, CommitType, CompileBenchmark, Date,
8-
Index, PendingBenchmarkRequests, Profile, QueuedCommit, Scenario, Target,
9-
BENCHMARK_JOB_STATUS_FAILURE_STR, BENCHMARK_JOB_STATUS_IN_PROGRESS_STR,
10-
BENCHMARK_JOB_STATUS_QUEUED_STR, BENCHMARK_JOB_STATUS_SUCCESS_STR,
11-
BENCHMARK_REQUEST_MASTER_STR, BENCHMARK_REQUEST_RELEASE_STR,
5+
BenchmarkJobConclusion, BenchmarkJobKind, BenchmarkJobStatus, BenchmarkRequest,
6+
BenchmarkRequestIndex, BenchmarkRequestStatus, BenchmarkRequestType,
7+
BenchmarkRequestWithErrors, BenchmarkSet, CodegenBackend, CollectionId, CollectorConfig,
8+
Commit, CommitType, CompileBenchmark, Date, Index, PendingBenchmarkRequests, Profile,
9+
QueuedCommit, Scenario, Target, BENCHMARK_JOB_STATUS_FAILURE_STR,
10+
BENCHMARK_JOB_STATUS_IN_PROGRESS_STR, BENCHMARK_JOB_STATUS_QUEUED_STR,
11+
BENCHMARK_JOB_STATUS_SUCCESS_STR, BENCHMARK_REQUEST_MASTER_STR, BENCHMARK_REQUEST_RELEASE_STR,
1212
BENCHMARK_REQUEST_STATUS_ARTIFACTS_READY_STR, BENCHMARK_REQUEST_STATUS_COMPLETED_STR,
1313
BENCHMARK_REQUEST_STATUS_IN_PROGRESS_STR, BENCHMARK_REQUEST_STATUS_WAITING_FOR_ARTIFACTS_STR,
1414
BENCHMARK_REQUEST_TRY_STR,
@@ -417,6 +417,9 @@ static MIGRATIONS: &[&str] = &[
417417
r#"
418418
CREATE INDEX benchmark_request_completed_idx ON benchmark_request(completed_at);
419419
"#,
420+
r#"
421+
ALTER TABLE job_queue ADD COLUMN kind TEXT NOT NULL;
422+
"#,
420423
];
421424

422425
#[async_trait::async_trait]
@@ -1746,6 +1749,7 @@ where
17461749
backend: CodegenBackend,
17471750
profile: Profile,
17481751
benchmark_set: u32,
1752+
kind: BenchmarkJobKind,
17491753
) -> (bool, anyhow::Result<u32>) {
17501754
let row_result = self
17511755
.conn()
@@ -1757,9 +1761,10 @@ where
17571761
backend,
17581762
profile,
17591763
benchmark_set,
1760-
status
1764+
status,
1765+
kind
17611766
)
1762-
VALUES ($1, $2, $3, $4, $5, $6)
1767+
VALUES ($1, $2, $3, $4, $5, $6, $7)
17631768
ON CONFLICT DO NOTHING
17641769
RETURNING job_queue.id
17651770
"#,
@@ -1770,6 +1775,7 @@ where
17701775
&profile,
17711776
&(benchmark_set as i32),
17721777
&BENCHMARK_JOB_STATUS_QUEUED_STR,
1778+
&kind,
17731779
],
17741780
)
17751781
.await;
@@ -1804,6 +1810,7 @@ where
18041810
backend: CodegenBackend,
18051811
profile: Profile,
18061812
benchmark_set: u32,
1813+
kind: BenchmarkJobKind,
18071814
) -> anyhow::Result<Option<u32>> {
18081815
// This will return zero rows if the job already exists
18091816
let rows = self
@@ -1816,9 +1823,10 @@ where
18161823
backend,
18171824
profile,
18181825
benchmark_set,
1819-
status
1826+
status,
1827+
kind
18201828
)
1821-
VALUES ($1, $2, $3, $4, $5, $6)
1829+
VALUES ($1, $2, $3, $4, $5, $6, $7)
18221830
ON CONFLICT DO NOTHING
18231831
RETURNING job_queue.id
18241832
"#,
@@ -1829,6 +1837,7 @@ where
18291837
&profile,
18301838
&(benchmark_set as i32),
18311839
&BENCHMARK_JOB_STATUS_QUEUED_STR,
1840+
&kind,
18321841
],
18331842
)
18341843
.await
@@ -2014,6 +2023,7 @@ where
20142023
updated.created_at,
20152024
updated.started_at,
20162025
updated.retry,
2026+
updated.kind,
20172027
br.commit_type,
20182028
br.commit_date
20192029
FROM updated
@@ -2047,9 +2057,11 @@ where
20472057
collector_name: collector_name.into(),
20482058
},
20492059
deque_counter: row.get::<_, i32>(6) as u32,
2060+
kind: BenchmarkJobKind::from_str(row.get::<_, &str>(7))
2061+
.map_err(|e| anyhow::anyhow!(e))?,
20502062
};
2051-
let commit_type = row.get::<_, &str>(7);
2052-
let commit_date = row.get::<_, Option<DateTime<Utc>>>(8);
2063+
let commit_type = row.get::<_, &str>(8);
2064+
let commit_date = row.get::<_, Option<DateTime<Utc>>>(9);
20532065

20542066
let commit_date = Date(commit_date.ok_or_else(|| {
20552067
anyhow::anyhow!("Dequeuing job for a benchmark request without commit date")
@@ -2255,6 +2267,8 @@ where
22552267
created_at: row.get::<_, DateTime<Utc>>(6),
22562268
status,
22572269
deque_counter: row.get::<_, i32>(10) as u32,
2270+
kind: BenchmarkJobKind::from_str(row.get::<_, &str>(12))
2271+
.map_err(|e| anyhow::anyhow!(e))?,
22582272
};
22592273
request_to_jobs
22602274
.entry(job.request_tag.clone())
@@ -2422,6 +2436,7 @@ impl_to_postgresql_via_to_string!(BenchmarkRequestType);
24222436
impl_to_postgresql_via_to_string!(Target);
24232437
impl_to_postgresql_via_to_string!(CodegenBackend);
24242438
impl_to_postgresql_via_to_string!(Profile);
2439+
impl_to_postgresql_via_to_string!(BenchmarkJobKind);
24252440

24262441
#[cfg(test)]
24272442
mod tests {

‎database/src/pool/sqlite.rs‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,9 @@ use crate::pool::{Connection, ConnectionManager, ManagedConnection, Transaction}
22
use crate::selector::CompileTestCase;
33
use crate::{
44
ArtifactCollection, ArtifactId, Benchmark, BenchmarkJob, BenchmarkJobConclusion,
5-
BenchmarkRequest, BenchmarkRequestIndex, BenchmarkRequestStatus, BenchmarkRequestWithErrors,
6-
BenchmarkSet, CodegenBackend, CollectionId, CollectorConfig, Commit, CommitType,
7-
CompileBenchmark, Date, PendingBenchmarkRequests, Profile, Target,
5+
BenchmarkJobKind, BenchmarkRequest, BenchmarkRequestIndex, BenchmarkRequestStatus,
6+
BenchmarkRequestWithErrors, BenchmarkSet, CodegenBackend, CollectionId, CollectorConfig,
7+
Commit, CommitType, CompileBenchmark, Date, PendingBenchmarkRequests, Profile, Target,
88
};
99
use crate::{ArtifactIdNumber, Index, QueuedCommit};
1010
use chrono::{DateTime, TimeZone, Utc};
@@ -1335,6 +1335,7 @@ impl Connection for SqliteConnection {
13351335
_backend: CodegenBackend,
13361336
_profile: Profile,
13371337
_benchmark_set: u32,
1338+
_kind: BenchmarkJobKind,
13381339
) -> anyhow::Result<Option<u32>> {
13391340
no_queue_implementation_abort!()
13401341
}
@@ -1346,6 +1347,7 @@ impl Connection for SqliteConnection {
13461347
_backend: CodegenBackend,
13471348
_profile: Profile,
13481349
_benchmark_set: u32,
1350+
_kind: BenchmarkJobKind,
13491351
) -> (bool, anyhow::Result<u32>) {
13501352
no_queue_implementation_abort!()
13511353
}

‎database/src/tests/builder.rs‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
use crate::{
2-
BenchmarkJob, BenchmarkJobConclusion, BenchmarkRequest, BenchmarkRequestStatus, BenchmarkSet,
3-
CodegenBackend, CollectorConfig, Connection, Profile, Target,
2+
BenchmarkJob, BenchmarkJobConclusion, BenchmarkJobKind, BenchmarkRequest,
3+
BenchmarkRequestStatus, BenchmarkSet, CodegenBackend, CollectorConfig, Connection, Profile,
4+
Target,
45
};
56
use chrono::Utc;
67
use hashbrown::{HashMap, HashSet};
@@ -46,6 +47,7 @@ impl RequestBuilder {
4647
job.backend,
4748
job.profile,
4849
job.benchmark_set,
50+
job.kind,
4951
)
5052
.await
5153
.unwrap();
@@ -109,6 +111,7 @@ pub struct JobBuilder {
109111
profile: Profile,
110112
benchmark_set: u32,
111113
conclusion: BenchmarkJobConclusion,
114+
kind: BenchmarkJobKind,
112115
}
113116

114117
impl JobBuilder {
@@ -126,6 +129,7 @@ impl Default for JobBuilder {
126129
profile: Profile::Check,
127130
benchmark_set: 0,
128131
conclusion: BenchmarkJobConclusion::Success,
132+
kind: BenchmarkJobKind::Compiletime,
129133
}
130134
}
131135
}

0 commit comments

Comments
 (0)