Skip to content

Commit 41f480b

Browse files
KobzolKobzol
committed
Refactor self profile downloading
1 parent ce83d3c commit 41f480b

File tree

2 files changed

+154
-111
lines changed

2 files changed

+154
-111
lines changed

‎site/src/request_handlers/self_profile.rs‎

Lines changed: 39 additions & 111 deletions
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,23 @@
11
use std::collections::HashSet;
22
use std::io::Read;
3+
use std::num::NonZeroUsize;
34
use std::sync::Arc;
4-
use std::time::{Duration, Instant};
5+
use std::time::Instant;
56

67
use bytes::Buf;
78
use database::CommitType;
89
use headers::{ContentType, Header};
910
use hyper::StatusCode;
11+
use lru::LruCache;
1012

11-
use crate::api::self_profile::{ArtifactSize, ArtifactSizeDelta};
13+
use crate::api::self_profile::ArtifactSizeDelta;
1214
use crate::api::{self_profile, self_profile_processed, self_profile_raw, ServerResult};
1315
use crate::comparison::Metric;
1416
use crate::db::ArtifactId;
1517
use crate::load::SiteCtxt;
1618
use crate::selector::{self};
1719
use crate::self_profile::{
18-
extract_profiling_data, fetch_raw_self_profile_data, get_self_profile_raw_data,
20+
download_and_analyze_self_profile, get_self_profile_raw_data, SelfProfileWithAnalysis,
1921
};
2022
use crate::server::{Response, ResponseHeaders};
2123

@@ -154,85 +156,12 @@ pub async fn handle_self_profile_processed_download(
154156
builder.body(hyper::Body::from(output.data)).unwrap()
155157
}
156158

157-
fn get_self_profile_data(
158-
cpu_clock: Option<f64>,
159-
profile: &analyzeme::AnalysisResults,
160-
) -> ServerResult<self_profile::SelfProfile> {
161-
let total_time: Duration = profile.query_data.iter().map(|qd| qd.self_time).sum();
162-
163-
let query_data = profile
164-
.query_data
165-
.iter()
166-
.map(|qd| self_profile::QueryData {
167-
label: qd.label.as_str().into(),
168-
self_time: qd.self_time.as_nanos() as u64,
169-
percent_total_time: ((qd.self_time.as_secs_f64() / total_time.as_secs_f64()) * 100.0)
170-
as f32,
171-
number_of_cache_misses: qd.number_of_cache_misses as u32,
172-
number_of_cache_hits: qd.number_of_cache_hits as u32,
173-
invocation_count: qd.invocation_count as u32,
174-
blocked_time: qd.blocked_time.as_nanos() as u64,
175-
incremental_load_time: qd.incremental_load_time.as_nanos() as u64,
176-
})
177-
.collect();
178-
179-
let totals = self_profile::QueryData {
180-
label: "Totals".into(),
181-
self_time: total_time.as_nanos() as u64,
182-
// TODO: check against wall-time from perf stats
183-
percent_total_time: cpu_clock
184-
.map(|w| ((total_time.as_secs_f64() / w) * 100.0) as f32)
185-
// sentinel "we couldn't compute this time"
186-
.unwrap_or(-100.0),
187-
number_of_cache_misses: profile
188-
.query_data
189-
.iter()
190-
.map(|qd| qd.number_of_cache_misses as u32)
191-
.sum(),
192-
number_of_cache_hits: profile
193-
.query_data
194-
.iter()
195-
.map(|qd| qd.number_of_cache_hits as u32)
196-
.sum(),
197-
invocation_count: profile
198-
.query_data
199-
.iter()
200-
.map(|qd| qd.invocation_count as u32)
201-
.sum(),
202-
blocked_time: profile
203-
.query_data
204-
.iter()
205-
.map(|qd| qd.blocked_time.as_nanos() as u64)
206-
.sum(),
207-
incremental_load_time: profile
208-
.query_data
209-
.iter()
210-
.map(|qd| qd.incremental_load_time.as_nanos() as u64)
211-
.sum(),
212-
};
213-
214-
let artifact_sizes = profile
215-
.artifact_sizes
216-
.iter()
217-
.map(|a| ArtifactSize {
218-
label: a.label.as_str().into(),
219-
bytes: a.value,
220-
})
221-
.collect();
222-
223-
Ok(self_profile::SelfProfile {
224-
query_data,
225-
totals,
226-
artifact_sizes: Some(artifact_sizes),
227-
})
228-
}
229-
230159
// Add query data entries to `profile` for any queries in `base_profile` which are not present in
231160
// `profile` (i.e. queries not invoked during the compilation that generated `profile`). This is
232161
// bit of a hack to enable showing rows for these queries in the table on the self-profile page.
233162
fn add_uninvoked_base_profile_queries(
234163
profile: &mut self_profile::SelfProfile,
235-
base_profile: &Option<self_profile::SelfProfile>,
164+
base_profile: Option<&self_profile::SelfProfile>,
236165
) {
237166
let base_profile = match base_profile.as_ref() {
238167
Some(bp) => bp,
@@ -265,7 +194,7 @@ fn add_uninvoked_base_profile_queries(
265194

266195
fn get_self_profile_delta(
267196
profile: &self_profile::SelfProfile,
268-
base_profile: &Option<self_profile::SelfProfile>,
197+
base_profile: Option<&self_profile::SelfProfile>,
269198
profiling_data: &analyzeme::AnalysisResults,
270199
base_profiling_data: Option<&analyzeme::AnalysisResults>,
271200
) -> Option<self_profile::SelfProfileDelta> {
@@ -606,45 +535,44 @@ pub async fn handle_self_profile(
606535
assert_eq!(cpu_responses.len(), 1, "all selectors are exact");
607536
let mut cpu_response = cpu_responses.remove(0).series;
608537

609-
let mut self_profile_data = Vec::new();
610-
let conn = ctxt.conn().await;
611-
for commit in commits.iter() {
612-
let aids_and_cids = conn
613-
.list_self_profile(commit.clone(), bench_name, profile, &body.scenario)
614-
.await;
615-
if let Some((aid, cid)) = aids_and_cids.first() {
616-
match fetch_raw_self_profile_data(*aid, bench_name, profile, scenario, *cid).await {
617-
Ok(d) => self_profile_data.push(
618-
extract_profiling_data(d)
619-
.map_err(|e| format!("error extracting self profiling data: {}", e))?,
620-
),
621-
Err(e) => return Err(format!("could not fetch raw profile data: {e:?}")),
622-
};
623-
}
624-
}
625-
let profiling_data = self_profile_data.remove(0).perform_analysis();
626-
let mut profile = get_self_profile_data(cpu_response.next().unwrap().1, &profiling_data)
627-
.map_err(|e| format!("{}: {}", body.commit, e))?;
628-
let (base_profile, base_raw_data) = if body.base_commit.is_some() {
629-
let base_profiling_data = self_profile_data.remove(0).perform_analysis();
630-
let profile = get_self_profile_data(cpu_response.next().unwrap().1, &base_profiling_data)
631-
.map_err(|e| format!("{}: {}", body.base_commit.as_ref().unwrap(), e))?;
632-
(Some(profile), Some(base_profiling_data))
633-
} else {
634-
(None, None)
538+
let mut self_profile = download_and_analyze_self_profile(
539+
ctxt,
540+
commits.get(0).unwrap().clone(),
541+
bench_name,
542+
profile,
543+
scenario,
544+
cpu_response.next().unwrap().1,
545+
)
546+
.await?;
547+
let base_self_profile = match commits.get(1) {
548+
Some(aid) => Some(
549+
download_and_analyze_self_profile(
550+
ctxt,
551+
aid.clone(),
552+
bench_name,
553+
profile,
554+
scenario,
555+
cpu_response.next().unwrap().1,
556+
)
557+
.await?,
558+
),
559+
None => None,
635560
};
561+
add_uninvoked_base_profile_queries(
562+
&mut self_profile.profile,
563+
base_self_profile.as_ref().map(|p| &p.profile),
564+
);
636565

637-
add_uninvoked_base_profile_queries(&mut profile, &base_profile);
638566
let mut base_profile_delta = get_self_profile_delta(
639-
&profile,
640-
&base_profile,
641-
&profiling_data,
642-
base_raw_data.as_ref(),
567+
&self_profile.profile,
568+
base_self_profile.as_ref().map(|p| &p.profile),
569+
&self_profile.profiling_data,
570+
base_self_profile.as_ref().map(|p| &p.profiling_data),
643571
);
644-
sort_self_profile(&mut profile, &mut base_profile_delta, sort_idx);
572+
sort_self_profile(&mut self_profile.profile, &mut base_profile_delta, sort_idx);
645573

646574
Ok(self_profile::Response {
647575
base_profile_delta,
648-
profile,
576+
profile: self_profile.profile,
649577
})
650578
}

‎site/src/self_profile.rs‎

Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,13 @@
11
//! This module handles self-profile "rich" APIs (e.g., chrome profiler JSON)
22
//! generation from the raw artifacts on demand.
33
4+
use crate::api::self_profile::ArtifactSize;
5+
use crate::api::{self_profile, ServerResult};
6+
use crate::load::SiteCtxt;
47
use anyhow::Context;
58
use bytes::Buf;
9+
use database::ArtifactId;
10+
use std::time::Duration;
611
use std::{collections::HashMap, io::Read, time::Instant};
712

813
mod codegen_schedule;
@@ -117,6 +122,116 @@ pub(crate) async fn get_self_profile_raw_data(url: &str) -> anyhow::Result<Vec<u
117122
extract(&compressed)
118123
}
119124

125+
#[derive(Clone)]
126+
pub struct SelfProfileWithAnalysis {
127+
pub profile: self_profile::SelfProfile,
128+
pub profiling_data: analyzeme::AnalysisResults,
129+
}
130+
131+
pub(crate) async fn download_and_analyze_self_profile(
132+
ctxt: &SiteCtxt,
133+
aid: ArtifactId,
134+
benchmark: &str,
135+
profile: &str,
136+
scenario: database::Scenario,
137+
metric: Option<f64>,
138+
) -> ServerResult<SelfProfileWithAnalysis> {
139+
let conn = ctxt.conn().await;
140+
let aids_and_cids = conn
141+
.list_self_profile(aid.clone(), benchmark, profile, &scenario.to_string())
142+
.await;
143+
144+
let Some((anum, cid)) = aids_and_cids.first() else {
145+
return Err(format!("no self-profile found for {aid}"));
146+
};
147+
let profiling_data =
148+
match fetch_raw_self_profile_data(*anum, benchmark, profile, scenario, *cid).await {
149+
Ok(d) => extract_profiling_data(d)
150+
.map_err(|e| format!("error extracting self profiling data: {}", e))?,
151+
Err(e) => return Err(format!("could not fetch raw profile data: {e:?}")),
152+
};
153+
let profiling_data = profiling_data.perform_analysis();
154+
let profile =
155+
get_self_profile_data(metric, &profiling_data).map_err(|e| format!("{}: {}", aid, e))?;
156+
Ok(SelfProfileWithAnalysis {
157+
profile,
158+
profiling_data,
159+
})
160+
}
161+
162+
fn get_self_profile_data(
163+
cpu_clock: Option<f64>,
164+
profile: &analyzeme::AnalysisResults,
165+
) -> ServerResult<self_profile::SelfProfile> {
166+
let total_time: Duration = profile.query_data.iter().map(|qd| qd.self_time).sum();
167+
168+
let query_data = profile
169+
.query_data
170+
.iter()
171+
.map(|qd| self_profile::QueryData {
172+
label: qd.label.as_str().into(),
173+
self_time: qd.self_time.as_nanos() as u64,
174+
percent_total_time: ((qd.self_time.as_secs_f64() / total_time.as_secs_f64()) * 100.0)
175+
as f32,
176+
number_of_cache_misses: qd.number_of_cache_misses as u32,
177+
number_of_cache_hits: qd.number_of_cache_hits as u32,
178+
invocation_count: qd.invocation_count as u32,
179+
blocked_time: qd.blocked_time.as_nanos() as u64,
180+
incremental_load_time: qd.incremental_load_time.as_nanos() as u64,
181+
})
182+
.collect();
183+
184+
let totals = self_profile::QueryData {
185+
label: "Totals".into(),
186+
self_time: total_time.as_nanos() as u64,
187+
// TODO: check against wall-time from perf stats
188+
percent_total_time: cpu_clock
189+
.map(|w| ((total_time.as_secs_f64() / w) * 100.0) as f32)
190+
// sentinel "we couldn't compute this time"
191+
.unwrap_or(-100.0),
192+
number_of_cache_misses: profile
193+
.query_data
194+
.iter()
195+
.map(|qd| qd.number_of_cache_misses as u32)
196+
.sum(),
197+
number_of_cache_hits: profile
198+
.query_data
199+
.iter()
200+
.map(|qd| qd.number_of_cache_hits as u32)
201+
.sum(),
202+
invocation_count: profile
203+
.query_data
204+
.iter()
205+
.map(|qd| qd.invocation_count as u32)
206+
.sum(),
207+
blocked_time: profile
208+
.query_data
209+
.iter()
210+
.map(|qd| qd.blocked_time.as_nanos() as u64)
211+
.sum(),
212+
incremental_load_time: profile
213+
.query_data
214+
.iter()
215+
.map(|qd| qd.incremental_load_time.as_nanos() as u64)
216+
.sum(),
217+
};
218+
219+
let artifact_sizes = profile
220+
.artifact_sizes
221+
.iter()
222+
.map(|a| ArtifactSize {
223+
label: a.label.as_str().into(),
224+
bytes: a.value,
225+
})
226+
.collect();
227+
228+
Ok(self_profile::SelfProfile {
229+
query_data,
230+
totals,
231+
artifact_sizes: Some(artifact_sizes),
232+
})
233+
}
234+
120235
fn extract(compressed: &[u8]) -> anyhow::Result<Vec<u8>> {
121236
let mut data = Vec::new();
122237
match snap::read::FrameDecoder::new(compressed.reader()).read_to_end(&mut data) {

0 commit comments

Comments
 (0)