Skip to content

Commit e982ff3

Browse files
KobzolKobzol
authored
Merge pull request #1800 from Kobzol/collector-catch-panic
Catch panics in `collector`
2 parents 67d16be + dca8f85 commit e982ff3

File tree

1 file changed

+104
-89
lines changed

1 file changed

+104
-89
lines changed

‎collector/src/bin/collector.rs‎

Lines changed: 104 additions & 89 deletions
Original file line numberDiff line numberDiff line change
@@ -645,15 +645,6 @@ fn main_result() -> anyhow::Result<i32> {
645645
runtime: &runtime_benchmark_dir,
646646
};
647647

648-
let mut builder = tokio::runtime::Builder::new_multi_thread();
649-
// We want to minimize noise from the runtime
650-
builder
651-
.worker_threads(1)
652-
.max_blocking_threads(1)
653-
.enable_time()
654-
.enable_io();
655-
let mut rt = builder.build().expect("built runtime");
656-
657648
// XXX: This doesn't necessarily work for all archs
658649
let target_triple = format!("{}-unknown-linux-gnu", std::env::consts::ARCH);
659650

@@ -765,6 +756,7 @@ fn main_result() -> anyhow::Result<i32> {
765756
CargoIsolationMode::Isolated
766757
};
767758

759+
let mut rt = build_async_runtime();
768760
let mut conn = rt.block_on(pool.connection());
769761
let artifact_id = ArtifactId::Tag(toolchain.id.clone());
770762
rt.block_on(purge_old_data(conn.as_mut(), &artifact_id, purge.purge));
@@ -916,6 +908,7 @@ fn main_result() -> anyhow::Result<i32> {
916908
benchmarks.retain(|b| local.category.0.contains(&b.category()));
917909

918910
let artifact_id = ArtifactId::Tag(toolchain.id.clone());
911+
let mut rt = build_async_runtime();
919912
let mut conn = rt.block_on(pool.connection());
920913
rt.block_on(purge_old_data(conn.as_mut(), &artifact_id, purge.purge));
921914

@@ -964,96 +957,106 @@ fn main_result() -> anyhow::Result<i32> {
964957
return Ok(0);
965958
};
966959

967-
let pool = database::Pool::open(&db.db);
968-
969-
match next {
970-
NextArtifact::Release(tag) => {
971-
let toolchain = create_toolchain_from_published_version(&tag, &target_triple)?;
972-
let res = bench_published_artifact(
973-
rt.block_on(pool.connection()),
974-
&mut rt,
975-
toolchain,
976-
&benchmark_dirs,
977-
);
978-
979-
client.post(format!("{}/perf/onpush", site_url)).send()?;
980-
981-
res?;
960+
let res = std::panic::catch_unwind(|| {
961+
let pool = database::Pool::open(&db.db);
962+
let mut rt = build_async_runtime();
963+
964+
match next {
965+
NextArtifact::Release(tag) => {
966+
let toolchain =
967+
create_toolchain_from_published_version(&tag, &target_triple)?;
968+
bench_published_artifact(
969+
rt.block_on(pool.connection()),
970+
&mut rt,
971+
toolchain,
972+
&benchmark_dirs,
973+
)
974+
}
975+
NextArtifact::Commit {
976+
commit,
977+
include,
978+
exclude,
979+
runs,
980+
} => {
981+
let sha = commit.sha.to_string();
982+
let sysroot = Sysroot::install(
983+
sha.clone(),
984+
&target_triple,
985+
vec![CodegenBackend::Llvm],
986+
)
987+
.with_context(|| format!("failed to install sysroot for {:?}", commit))?;
988+
989+
let mut benchmarks = get_compile_benchmarks(
990+
&compile_benchmark_dir,
991+
include.as_deref(),
992+
exclude.as_deref(),
993+
None,
994+
)?;
995+
benchmarks.retain(|b| b.category().is_primary_or_secondary());
996+
997+
let artifact_id = ArtifactId::Commit(commit);
998+
let mut conn = rt.block_on(pool.connection());
999+
let toolchain = Toolchain::from_sysroot(&sysroot, sha);
1000+
1001+
let compile_config = CompileBenchmarkConfig {
1002+
benchmarks,
1003+
profiles: vec![
1004+
Profile::Check,
1005+
Profile::Debug,
1006+
Profile::Doc,
1007+
Profile::Opt,
1008+
],
1009+
scenarios: Scenario::all(),
1010+
backends: vec![CodegenBackend::Llvm],
1011+
iterations: runs.map(|v| v as usize),
1012+
is_self_profile: self_profile.self_profile,
1013+
bench_rustc: bench_rustc.bench_rustc,
1014+
};
1015+
let runtime_suite = rt.block_on(load_runtime_benchmarks(
1016+
conn.as_mut(),
1017+
&runtime_benchmark_dir,
1018+
CargoIsolationMode::Isolated,
1019+
None,
1020+
&toolchain,
1021+
&artifact_id,
1022+
))?;
1023+
1024+
let runtime_config = RuntimeBenchmarkConfig {
1025+
runtime_suite,
1026+
filter: BenchmarkFilter::keep_all(),
1027+
iterations: DEFAULT_RUNTIME_ITERATIONS,
1028+
};
1029+
let shared = SharedBenchmarkConfig {
1030+
artifact_id,
1031+
toolchain,
1032+
};
1033+
1034+
run_benchmarks(
1035+
&mut rt,
1036+
conn,
1037+
shared,
1038+
Some(compile_config),
1039+
Some(runtime_config),
1040+
)
1041+
}
9821042
}
983-
NextArtifact::Commit {
984-
commit,
985-
include,
986-
exclude,
987-
runs,
988-
} => {
989-
let sha = commit.sha.to_string();
990-
let sysroot = Sysroot::install(
991-
sha.clone(),
992-
&target_triple,
993-
vec![CodegenBackend::Llvm],
994-
)
995-
.with_context(|| format!("failed to install sysroot for {:?}", commit))?;
996-
997-
let mut benchmarks = get_compile_benchmarks(
998-
&compile_benchmark_dir,
999-
include.as_deref(),
1000-
exclude.as_deref(),
1001-
None,
1002-
)?;
1003-
benchmarks.retain(|b| b.category().is_primary_or_secondary());
1004-
1005-
let artifact_id = ArtifactId::Commit(commit);
1006-
let mut conn = rt.block_on(pool.connection());
1007-
let toolchain = Toolchain::from_sysroot(&sysroot, sha);
1008-
1009-
let compile_config = CompileBenchmarkConfig {
1010-
benchmarks,
1011-
profiles: vec![Profile::Check, Profile::Debug, Profile::Doc, Profile::Opt],
1012-
scenarios: Scenario::all(),
1013-
backends: vec![CodegenBackend::Llvm],
1014-
iterations: runs.map(|v| v as usize),
1015-
is_self_profile: self_profile.self_profile,
1016-
bench_rustc: bench_rustc.bench_rustc,
1017-
};
1018-
let runtime_suite = rt.block_on(load_runtime_benchmarks(
1019-
conn.as_mut(),
1020-
&runtime_benchmark_dir,
1021-
CargoIsolationMode::Isolated,
1022-
None,
1023-
&toolchain,
1024-
&artifact_id,
1025-
))?;
1026-
1027-
let runtime_config = RuntimeBenchmarkConfig {
1028-
runtime_suite,
1029-
filter: BenchmarkFilter::keep_all(),
1030-
iterations: DEFAULT_RUNTIME_ITERATIONS,
1031-
};
1032-
let shared = SharedBenchmarkConfig {
1033-
artifact_id,
1034-
toolchain,
1035-
};
1036-
1037-
let res = run_benchmarks(
1038-
&mut rt,
1039-
conn,
1040-
shared,
1041-
Some(compile_config),
1042-
Some(runtime_config),
1043-
);
1044-
1045-
client.post(format!("{}/perf/onpush", site_url)).send()?;
1043+
});
1044+
// We need to send a message to this endpoint even if the collector panics
1045+
client.post(format!("{}/perf/onpush", site_url)).send()?;
10461046

1047-
res?;
1047+
match res {
1048+
Ok(res) => res?,
1049+
Err(error) => {
1050+
log::error!("The collector has crashed\n{error:?}");
10481051
}
10491052
}
1050-
10511053
Ok(0)
10521054
}
10531055

10541056
Commands::BenchPublished { toolchain, db } => {
10551057
log_db(&db);
10561058
let pool = database::Pool::open(&db.db);
1059+
let mut rt = build_async_runtime();
10571060
let conn = rt.block_on(pool.connection());
10581061
let toolchain = create_toolchain_from_published_version(&toolchain, &target_triple)?;
10591062
bench_published_artifact(conn, &mut rt, toolchain, &benchmark_dirs)?;
@@ -1211,6 +1214,7 @@ Make sure to modify `{dir}/perf-config.json` if the category/artifact don't matc
12111214
}
12121215
Commands::PurgeArtifact { name, db } => {
12131216
let pool = Pool::open(&db.db);
1217+
let rt = build_async_runtime();
12141218
let conn = rt.block_on(pool.connection());
12151219
rt.block_on(conn.purge_artifact(&ArtifactId::Tag(name.clone())));
12161220

@@ -1220,6 +1224,17 @@ Make sure to modify `{dir}/perf-config.json` if the category/artifact don't matc
12201224
}
12211225
}
12221226

1227+
fn build_async_runtime() -> Runtime {
1228+
let mut builder = tokio::runtime::Builder::new_multi_thread();
1229+
// We want to minimize noise from the runtime
1230+
builder
1231+
.worker_threads(1)
1232+
.max_blocking_threads(1)
1233+
.enable_time()
1234+
.enable_io();
1235+
builder.build().expect("built runtime")
1236+
}
1237+
12231238
fn print_binary_stats(
12241239
name_header: &str,
12251240
items: HashMap<String, u64>,

0 commit comments

Comments
 (0)