Merge pull request #1064 from rust-lang/throttling

Track how many people are waiting for a one-off coordinator

Author: shepmaster

ui/src/metrics.rs

@@ -17,6 +17,11 @@ lazy_static! {
         vec![0.1, 1.0, 2.5, 5.0, 10.0, 15.0]
     )
     .unwrap();
+    pub(crate) static ref ONE_OFF_QUEUE_DEPTH: IntGauge = register_int_gauge!(
+        "playground_one_off_coordinator_queue_depth",
+        "Number of clients waiting for a one-off coordinator"
+    )
+    .unwrap();
     pub(crate) static ref LIVE_WS: IntGauge = register_int_gauge!(
         "playground_active_websocket_connections_count",
         "Number of active WebSocket connections"

ui/src/server_axum.rs

@@ -2,7 +2,7 @@ use crate::{
     gist,
     metrics::{
         record_metric, track_metric_no_request_async, Endpoint, HasLabelsCore, Outcome,
-        UNAVAILABLE_WS,
+        ONE_OFF_QUEUE_DEPTH, UNAVAILABLE_WS,
     },
     request_database::{Handle, How},
     sandbox::DOCKER_PROCESS_TIMEOUT_SOFT,
@@ -442,7 +442,9 @@ where
     for<'f> F:
         FnOnce(&'f coordinator::Coordinator<DockerBackend>, Req) -> BoxFuture<'f, Result<Resp>>,
 {
+    ONE_OFF_QUEUE_DEPTH.inc();
     let coordinator = factory.build().await;
+    ONE_OFF_QUEUE_DEPTH.dec();
 
     let job = async {
         let req = req.try_into()?;