Auto merge of #605 - Mark-Simulacrum:endpoint-time-metric, r=Mark-Simulacrum

bors · bors · commit e3ea7fdd8ca9 · 2022-03-05T03:45:18.000Z
Record endpoint time for record_progress
diff --git a/src/server/metrics.rs b/src/server/metrics.rs
@@ -5,21 +5,23 @@ use crate::server::agents::Agent;
 use chrono::{DateTime, Utc};
 use prometheus::proto::{Metric, MetricFamily};
 use prometheus::{
-    IntCounterVec, IntGauge, IntGaugeVec, __register_counter_vec, __register_gauge,
+    HistogramVec, IntCounterVec, IntGauge, IntGaugeVec, __register_counter_vec, __register_gauge,
     __register_gauge_vec,
 };
 
 const JOBS_METRIC: &str = "crater_completed_jobs_total";
 const AGENT_WORK_METRIC: &str = "crater_agent_supposed_to_work";
 const AGENT_FAILED: &str = "crater_agent_failure";
 const LAST_CRATES_UPDATE_METRIC: &str = "crater_last_crates_update";
+const ENDPOINT_TIME: &str = "crater_endpoint_time_seconds";
 
 #[derive(Clone)]
 pub struct Metrics {
     crater_completed_jobs_total: IntCounterVec,
     crater_agent_failure: IntCounterVec,
     crater_work_status: IntGaugeVec,
     crater_last_crates_update: IntGauge,
+    pub crater_endpoint_time: HistogramVec,
 }
 
 impl Metrics {
@@ -35,12 +37,20 @@ impl Metrics {
         let crates_update_opts =
             prometheus::opts!(LAST_CRATES_UPDATE_METRIC, "last update of crates lists");
         let crater_last_crates_update = prometheus::register_int_gauge!(crates_update_opts)?;
+        let crater_endpoint_time = prometheus::register_histogram_vec!(
+            prometheus::HistogramOpts::new(ENDPOINT_TIME, "duration of endpoint requests")
+                // Exponential buckets, with 5ms as start and top bucket ending at
+                // approximately 5 seconds.
+                .buckets(prometheus::exponential_buckets(0.005, 1.5, 17).unwrap()),
+            &["endpoint"]
+        )?;
 
         Ok(Metrics {
             crater_completed_jobs_total,
             crater_agent_failure,
             crater_work_status,
             crater_last_crates_update,
+            crater_endpoint_time,
         })
     }
 
diff --git a/src/server/routes/agent.rs b/src/server/routes/agent.rs
@@ -159,6 +159,7 @@ fn endpoint_record_progress(
     mutex: Arc<Mutex<Data>>,
     auth: AuthDetails,
 ) -> Fallible<Response<Body>> {
+    let start = std::time::Instant::now();
     let data = mutex.lock().unwrap();
     let ex = Experiment::get(&data.db, &result.experiment_name)?
         .ok_or_else(|| err_msg("no experiment run by this agent"))?;
@@ -169,7 +170,12 @@ fn endpoint_record_progress(
     let db = DatabaseDB::new(&data.db);
     db.store(&ex, &result.data, EncodingType::Gzip)?;
 
-    Ok(ApiResponse::Success { result: true }.into_response()?)
+    let ret = Ok(ApiResponse::Success { result: true }.into_response()?);
+    data.metrics
+        .crater_endpoint_time
+        .with_label_values(&["record_progress"])
+        .observe(start.elapsed().as_secs_f64());
+    ret
 }
 
 fn endpoint_heartbeat(data: Arc<Data>, auth: AuthDetails) -> Fallible<Response<Body>> {
