feat(metrics): add connection latency tracking (#3606)

## Description

<!-- A summary of what this pull request achieves and a rough list of
changes. -->

## Breaking Changes

<!-- Optional, if there are any breaking changes document them,
including how to migrate older code. -->

## Notes & open questions

<!-- Any notes, remarks or open questions you have to make about the PR.
-->

## Change checklist
<!-- Remove any that are not relevant. -->
- [ ] Self-review.
- [ ] Documentation updates following the [style
guide](https://rust-lang.github.io/rfcs/1574-more-api-documentation-conventions.html#appendix-a-full-conventions-text),
if relevant.
- [ ] Tests if relevant.
- [ ] All breaking changes documented.
- [ ] List all breaking changes in the above "Breaking Changes" section.
- [ ] Open an issue or PR on any number0 repos that are affected by this
breaking change. Give guidance on how the updates should be handled or
do the actual updates themselves. The major ones are:
    - [ ] [`quic-rpc`](https://github.com/n0-computer/quic-rpc)
    - [ ] [`iroh-gossip`](https://github.com/n0-computer/iroh-gossip)
    - [ ] [`iroh-blobs`](https://github.com/n0-computer/iroh-blobs)
    - [ ] [`dumbpipe`](https://github.com/n0-computer/dumbpipe)
    - [ ] [`sendme`](https://github.com/n0-computer/sendme)

Author: Arqu

iroh/src/magicsock/endpoint_map/path_validity.rs

@@ -285,6 +285,9 @@ impl PathValidity {
 
         let quality = state.congestion_metrics.quality_score();
         metrics.path_quality_score.observe(quality);
+
+        let latency_ms = state.latency.as_secs_f64() * 1000.0;
+        metrics.connection_latency_ms.observe(latency_ms);
     }
 }
 
@@ -395,4 +398,36 @@ mod tests {
         let quality = validity.quality_score();
         assert!(quality < 0.9); // Should be penalized
     }
+
+    #[tokio::test]
+    async fn test_connection_latency_histogram() {
+        use crate::magicsock::Metrics as MagicsockMetrics;
+
+        let metrics = MagicsockMetrics::default();
+        let mut validity = PathValidity::new(Instant::now(), Duration::from_millis(10));
+
+        validity.record_metrics(&metrics);
+        assert_eq!(metrics.connection_latency_ms.count(), 1);
+
+        validity.update_pong(Instant::now(), Duration::from_millis(25));
+        validity.record_metrics(&metrics);
+        assert_eq!(metrics.connection_latency_ms.count(), 2);
+
+        validity.update_pong(Instant::now(), Duration::from_millis(50));
+        validity.record_metrics(&metrics);
+        assert_eq!(metrics.connection_latency_ms.count(), 3);
+
+        validity.update_pong(Instant::now(), Duration::from_millis(100));
+        validity.record_metrics(&metrics);
+        assert_eq!(metrics.connection_latency_ms.count(), 4);
+
+        let buckets = metrics.connection_latency_ms.buckets();
+        assert!(!buckets.is_empty());
+
+        let p50 = metrics.connection_latency_ms.percentile(0.5);
+        assert!(p50 > 10.0 && p50 < 100.0);
+
+        let p95 = metrics.connection_latency_ms.percentile(0.95);
+        assert!(p95 >= p50);
+    }
 }

iroh/src/magicsock/metrics.rs

@@ -77,6 +77,9 @@ pub struct Metrics {
     pub connection_handshake_success: Counter,
     /// Number of connections with a successful handshake that became direct.
     pub connection_became_direct: Counter,
+    /// Histogram of connection latency in milliseconds across all endpoint connections.
+    #[default(Histogram::new(vec![1.0, 5.0, 10.0, 25.0, 50.0, 100.0, 250.0, 500.0, 1000.0, f64::INFINITY]))]
+    pub connection_latency_ms: Histogram,
 
     /*
      * Path Congestion Metrics