From abcec6abca2933ce617f52cbc1d2c5f892119a16 Mon Sep 17 00:00:00 2001 From: Jan Steinke Date: Fri, 22 Aug 2025 17:47:26 +0200 Subject: [PATCH 01/36] include server trace information in tower instrumentation --- .../Cargo.toml | 9 +- opentelemetry-instrumentation-tower/README.md | 102 ++++- .../src/lib.rs | 395 +++++++++++++++--- 3 files changed, 436 insertions(+), 70 deletions(-) diff --git a/opentelemetry-instrumentation-tower/Cargo.toml b/opentelemetry-instrumentation-tower/Cargo.toml index d50b964f2..66ff7fd56 100644 --- a/opentelemetry-instrumentation-tower/Cargo.toml +++ b/opentelemetry-instrumentation-tower/Cargo.toml @@ -5,7 +5,7 @@ rust-version = "1.75.0" version = "0.16.0" license = "Apache-2.0" -description = "OpenTelemetry Metrics Middleware for Tower-compatible Rust HTTP servers" +description = "OpenTelemetry Metrics and Tracing Middleware for Tower-compatible Rust HTTP servers" homepage = "https://github.com/open-telemetry/opentelemetry-rust-contrib" repository = "https://github.com/open-telemetry/opentelemetry-rust-contrib" documentation = "https://docs.rs/tower-otel-http-metrics" @@ -21,12 +21,17 @@ axum = { features = ["matched-path", "macros"], version = "0.8", default-feature futures-util = { version = "0.3", default-features = false } http = { version = "1", features = ["std"], default-features = false } http-body = { version = "1", default-features = false } -opentelemetry = { workspace = true, features = ["futures", "metrics"]} +opentelemetry = { workspace = true, features = ["futures", "metrics", "trace"] } +opentelemetry-semantic-conventions = { workspace = true, features = ["semconv_experimental"] } # Corrected feature name pin-project-lite = { version = "0.2", default-features = false } tower-service = { version = "0.3", default-features = false } tower-layer = { version = "0.3", default-features = false } [dev-dependencies] +tokio = { version = "1" } +tower = { version = "0.3" } +tower-test = { version = "0.3" } +opentelemetry_sdk = { workspace = true, features = ["testing"] } [lints] workspace = true \ No newline at end of file diff --git a/opentelemetry-instrumentation-tower/README.md b/opentelemetry-instrumentation-tower/README.md index 60f2ea709..09a879c56 100644 --- a/opentelemetry-instrumentation-tower/README.md +++ b/opentelemetry-instrumentation-tower/README.md @@ -1,6 +1,104 @@ -# Tower OTEL Metrics Middleware +# Tower OTEL HTTP Instrumentation Middleware -OpenTelemetry Metrics Middleware for Tower-compatible Rust HTTP servers. +OpenTelemetry HTTP Metrics and Tracing Middleware for Tower-compatible Rust HTTP servers. + +This middleware provides both metrics and distributed tracing for HTTP requests, following OpenTelemetry semantic conventions. + +## Features + +- **HTTP Metrics**: Request duration, active requests, request/response body sizes +- **Distributed Tracing**: HTTP spans with semantic attributes +- **Semantic Conventions**: Uses OpenTelemetry semantic conventions for consistent attribute naming +- **Flexible Configuration**: Support for custom attribute extractors and tracer configuration +- **Framework Support**: Works with any Tower-compatible HTTP framework (Axum, Hyper, etc.) + +## Usage + +### Basic Usage (Metrics Only) + +```rust +use opentelemetry_instrumentation_tower::HTTPLayerBuilder; + +let meter = global::meter("my-service"); +let http_layer = HTTPLayerBuilder::builder() + .with_meter(meter) + .build()?; + +let app = Router::new() + .route("/", get(handler)) + .layer(http_layer); +``` + +### With Tracing + +```rust +use opentelemetry_instrumentation_tower::HTTPLayerBuilder; + +let meter = global::meter("my-service"); +let http_layer = HTTPLayerBuilder::builder() + .with_meter(meter) + .with_tracing() // Uses global tracer provider + .build()?; + +let app = Router::new() + .route("/", get(handler)) + .layer(http_layer); +``` + +### With Custom Tracer + +```rust +use opentelemetry_instrumentation_tower::HTTPLayerBuilder; + +let meter = global::meter("my-service"); +let tracer = global::tracer_provider().tracer("my-service"); + +let http_layer = HTTPLayerBuilder::builder() + .with_meter(meter) + .with_tracer(tracer) + .build()?; + +let app = Router::new() + .route("/", get(handler)) + .layer(http_layer); +``` + +### With Custom Attribute Extractors + +```rust +use opentelemetry_instrumentation_tower::HTTPLayerBuilder; + +let http_layer = HTTPLayerBuilder::builder() + .with_meter(meter) + .with_tracing() + .with_request_extractor_fn(|req| { + vec![KeyValue::new("custom.header", + req.headers().get("x-custom").unwrap_or_default().to_str().unwrap_or(""))] + }) + .build()?; +``` + +## Metrics + +The middleware exports the following metrics: + +- `http.server.request.duration` - Duration of HTTP requests +- `http.server.active_requests` - Number of active HTTP requests +- `http.server.request.body.size` - Size of HTTP request bodies +- `http.server.response.body.size` - Size of HTTP response bodies + +## Tracing + +HTTP spans are created with the following attributes (following OpenTelemetry semantic conventions): + +- `http.request.method` - HTTP method +- `url.scheme` - URL scheme (http/https) +- `http.target` - Request target (path + query) +- `url.full` - Full URL +- `http.route` - Matched route pattern (when available) +- `server.address` - Server host +- `user_agent.original` - User agent string +- `http.response.status_code` - HTTP response status code ## Examples diff --git a/opentelemetry-instrumentation-tower/src/lib.rs b/opentelemetry-instrumentation-tower/src/lib.rs index efd84d6a1..bc13248da 100644 --- a/opentelemetry-instrumentation-tower/src/lib.rs +++ b/opentelemetry-instrumentation-tower/src/lib.rs @@ -11,8 +11,12 @@ use std::{fmt, result}; #[cfg(feature = "axum")] use axum::extract::MatchedPath; use futures_util::ready; +use opentelemetry::global::BoxedTracer; use opentelemetry::metrics::{Histogram, Meter, UpDownCounter}; -use opentelemetry::KeyValue; +use opentelemetry::trace::noop::NoopTracer; +use opentelemetry::trace::{SpanKind, Status, TraceContextExt, Tracer}; +use opentelemetry::{Context as OtelContext, KeyValue}; +use opentelemetry_semantic_conventions as semconv; use pin_project_lite::pin_project; use tower_layer::Layer; use tower_service::Service; @@ -31,23 +35,23 @@ const _OTEL_DEFAULT_HTTP_SERVER_DURATION_BOUNDARIES: [f64; 14] = [ const LIBRARY_DEFAULT_HTTP_SERVER_DURATION_BOUNDARIES: [f64; 14] = [ 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0, 60.0, 120.0, 300.0, ]; -const HTTP_SERVER_ACTIVE_REQUESTS_METRIC: &str = "http.server.active_requests"; +const HTTP_SERVER_ACTIVE_REQUESTS_METRIC: &str = semconv::metric::HTTP_SERVER_ACTIVE_REQUESTS; const HTTP_SERVER_ACTIVE_REQUESTS_UNIT: &str = "{request}"; -const HTTP_SERVER_REQUEST_BODY_SIZE_METRIC: &str = "http.server.request.body.size"; +const HTTP_SERVER_REQUEST_BODY_SIZE_METRIC: &str = semconv::metric::HTTP_SERVER_REQUEST_BODY_SIZE; const HTTP_SERVER_REQUEST_BODY_SIZE_UNIT: &str = "By"; -const HTTP_SERVER_RESPONSE_BODY_SIZE_METRIC: &str = "http.server.response.body.size"; +const HTTP_SERVER_RESPONSE_BODY_SIZE_METRIC: &str = semconv::metric::HTTP_SERVER_RESPONSE_BODY_SIZE; const HTTP_SERVER_RESPONSE_BODY_SIZE_UNIT: &str = "By"; -const NETWORK_PROTOCOL_NAME_LABEL: &str = "network.protocol.name"; -const NETWORK_PROTOCOL_VERSION_LABEL: &str = "network.protocol.version"; -const URL_SCHEME_LABEL: &str = "url.scheme"; +const NETWORK_PROTOCOL_NAME_LABEL: &str = semconv::trace::NETWORK_PROTOCOL_NAME; +const NETWORK_PROTOCOL_VERSION_LABEL: &str = semconv::trace::NETWORK_PROTOCOL_VERSION; +const URL_SCHEME_LABEL: &str = semconv::trace::URL_SCHEME; -const HTTP_REQUEST_METHOD_LABEL: &str = "http.request.method"; +const HTTP_REQUEST_METHOD_LABEL: &str = semconv::trace::HTTP_REQUEST_METHOD; #[allow(dead_code)] // cargo check is not smart -const HTTP_ROUTE_LABEL: &str = "http.route"; -const HTTP_RESPONSE_STATUS_CODE_LABEL: &str = "http.response.status_code"; +const HTTP_ROUTE_LABEL: &str = semconv::trace::HTTP_ROUTE; +const HTTP_RESPONSE_STATUS_CODE_LABEL: &str = semconv::trace::HTTP_RESPONSE_STATUS_CODE; /// Trait for extracting custom attributes from HTTP requests pub trait RequestAttributeExtractor: Clone + Send + Sync + 'static { @@ -119,35 +123,44 @@ where /// State scoped to the entire middleware Layer. /// -/// For now the only global state we hold onto is the metrics instruments. -/// The OTEL SDKs do support calling for the global meter provider instead of holding a reference -/// but it seems ideal to avoid extra access to the global meter, which sits behind a RWLock. -struct HTTPMetricsLayerState { +/// Holds both metrics instruments and tracing configuration. +struct HTTPLayerState { pub server_request_duration: Histogram, pub server_active_requests: UpDownCounter, pub server_request_body_size: Histogram, pub server_response_body_size: Histogram, + pub tracer: BoxedTracer, } #[derive(Clone)] -/// [`Service`] used by [`HTTPMetricsLayer`] -pub struct HTTPMetricsService { - pub(crate) state: Arc, +/// [`Service`] used by [`HTTPLayer`] +pub struct HTTPService { + pub(crate) state: Arc, request_extractor: ReqExt, response_extractor: ResExt, inner_service: S, } #[derive(Clone)] -/// [`Layer`] which applies the OTEL HTTP server metrics middleware -pub struct HTTPMetricsLayer { - state: Arc, +/// [`Layer`] which applies the OTEL HTTP server metrics and tracing middleware +pub struct HTTPLayer { + state: Arc, request_extractor: ReqExt, response_extractor: ResExt, } -pub struct HTTPMetricsLayerBuilder { +// Type aliases for backward compatibility +pub type HTTPMetricsLayer = + HTTPLayer; +pub type HTTPMetricsService = + HTTPService; +pub type HTTPMetricsResponseFuture = HTTPResponseFuture; +pub type HTTPMetricsLayerBuilder = + HTTPLayerBuilder; + +pub struct HTTPLayerBuilder { meter: Option, + tracer: Option, req_dur_bounds: Option>, request_extractor: ReqExt, response_extractor: ResExt, @@ -189,10 +202,11 @@ impl fmt::Debug for Error { } } -impl HTTPMetricsLayerBuilder { +impl HTTPLayerBuilder { pub fn builder() -> Self { - HTTPMetricsLayerBuilder { + HTTPLayerBuilder { meter: None, + tracer: None, req_dur_bounds: Some(LIBRARY_DEFAULT_HTTP_SERVER_DURATION_BOUNDARIES.to_vec()), request_extractor: NoOpExtractor, response_extractor: NoOpExtractor, @@ -200,17 +214,18 @@ impl HTTPMetricsLayerBuilder { } } -impl HTTPMetricsLayerBuilder { +impl HTTPLayerBuilder { /// Set a request attribute extractor pub fn with_request_extractor( self, extractor: NewReqExt, - ) -> HTTPMetricsLayerBuilder + ) -> HTTPLayerBuilder where NewReqExt: RequestAttributeExtractor, { - HTTPMetricsLayerBuilder { + HTTPLayerBuilder { meter: self.meter, + tracer: self.tracer, req_dur_bounds: self.req_dur_bounds, request_extractor: extractor, response_extractor: self.response_extractor, @@ -221,12 +236,13 @@ impl HTTPMetricsLayerBuilder { pub fn with_response_extractor( self, extractor: NewResExt, - ) -> HTTPMetricsLayerBuilder + ) -> HTTPLayerBuilder where NewResExt: ResponseAttributeExtractor, { - HTTPMetricsLayerBuilder { + HTTPLayerBuilder { meter: self.meter, + tracer: self.tracer, req_dur_bounds: self.req_dur_bounds, request_extractor: self.request_extractor, response_extractor: extractor, @@ -237,7 +253,7 @@ impl HTTPMetricsLayerBuilder { pub fn with_request_extractor_fn( self, f: F, - ) -> HTTPMetricsLayerBuilder, ResExt> + ) -> HTTPLayerBuilder, ResExt> where F: Fn(&http::Request) -> Vec + Clone + Send + Sync + 'static, { @@ -248,20 +264,24 @@ impl HTTPMetricsLayerBuilder { pub fn with_response_extractor_fn( self, f: F, - ) -> HTTPMetricsLayerBuilder> + ) -> HTTPLayerBuilder> where F: Fn(&http::Response) -> Vec + Clone + Send + Sync + 'static, { self.with_response_extractor(FnResponseExtractor::new(f)) } - pub fn build(self) -> Result> { + pub fn build(self) -> Result> { let req_dur_bounds = self .req_dur_bounds .unwrap_or_else(|| LIBRARY_DEFAULT_HTTP_SERVER_DURATION_BOUNDARIES.to_vec()); + let tracer = match self.tracer { + Some(t) => t, + None => BoxedTracer::new(Box::new(NoopTracer::new())), + }; match self.meter { Some(meter) => Ok(HTTPMetricsLayer { - state: Arc::from(Self::make_state(meter, req_dur_bounds)), + state: Arc::from(Self::make_state(meter, req_dur_bounds, tracer)), request_extractor: self.request_extractor, response_extractor: self.response_extractor, }), @@ -281,8 +301,13 @@ impl HTTPMetricsLayerBuilder { self } - fn make_state(meter: Meter, req_dur_bounds: Vec) -> HTTPMetricsLayerState { - HTTPMetricsLayerState { + pub fn with_tracer(mut self, tracer: BoxedTracer) -> Self { + self.tracer = Some(tracer); + self + } + + fn make_state(meter: Meter, req_dur_bounds: Vec, tracer: BoxedTracer) -> HTTPLayerState { + HTTPLayerState { server_request_duration: meter .f64_histogram(Cow::from(HTTP_SERVER_DURATION_METRIC)) .with_description("Duration of HTTP server requests.") @@ -304,19 +329,20 @@ impl HTTPMetricsLayerBuilder { .with_description("Size of HTTP server response bodies.") .with_unit(HTTP_SERVER_RESPONSE_BODY_SIZE_UNIT) .build(), + tracer, } } } -impl Layer for HTTPMetricsLayer +impl Layer for HTTPLayer where ReqExt: Clone, ResExt: Clone, { - type Service = HTTPMetricsService; + type Service = HTTPService; fn layer(&self, service: S) -> Self::Service { - HTTPMetricsService { + HTTPService { state: self.state.clone(), request_extractor: self.request_extractor.clone(), response_extractor: self.response_extractor.clone(), @@ -325,14 +351,13 @@ where } } -/// ResponseFutureMetricsState holds request-scoped data for metrics and their attributes. +/// ResponseFutureState holds request-scoped data for metrics, tracing and their attributes. /// -/// ResponseFutureMetricsState lives inside the response future, as it needs to hold data +/// ResponseFutureState lives inside the response future, as it needs to hold data /// initialized or extracted from the request before it is forwarded to the inner Service. /// The rest of the data (e.g. status code, error) can be extracted from the response /// or calculated with respect to the data held here (e.g., duration = now - duration start). -#[derive(Clone)] -struct ResponseFutureMetricsState { +struct ResponseFutureState { // fields for the metric values // https://opentelemetry.io/docs/specs/semconv/http/http-metrics/#metric-httpserverrequestduration duration_start: Instant, @@ -348,21 +373,24 @@ struct ResponseFutureMetricsState { // Custom attributes from request custom_request_attributes: Vec, + + // Tracing fields + otel_context: OtelContext, } pin_project! { - /// Response [`Future`] for [`HTTPMetricsService`]. - pub struct HTTPMetricsResponseFuture { + /// Response [`Future`] for [`HTTPService`]. + pub struct HTTPResponseFuture { #[pin] inner_response_future: F, - layer_state: Arc, - metrics_state: ResponseFutureMetricsState, + layer_state: Arc, + future_state: ResponseFutureState, response_extractor: ResExt, } } impl Service> - for HTTPMetricsService + for HTTPService where S: Service, Response = http::Response>, ResBody: http_body::Body, @@ -371,7 +399,7 @@ where { type Response = S::Response; type Error = S::Error; - type Future = HTTPMetricsResponseFuture; + type Future = HTTPResponseFuture; fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll> { self.inner_service.poll_ready(cx) @@ -393,29 +421,73 @@ where let url_scheme_kv = KeyValue::new(URL_SCHEME_LABEL, scheme); let method = req.method().as_str().to_owned(); - let method_kv = KeyValue::new(HTTP_REQUEST_METHOD_LABEL, method); + let method_kv = KeyValue::new(HTTP_REQUEST_METHOD_LABEL, method.clone()); #[allow(unused_mut)] let mut route_kv_opt = None; + let mut route_str = None; #[cfg(feature = "axum")] if let Some(matched_path) = req.extensions().get::() { - route_kv_opt = Some(KeyValue::new( - HTTP_ROUTE_LABEL, - matched_path.as_str().to_owned(), - )); + let route = matched_path.as_str().to_owned(); + route_kv_opt = Some(KeyValue::new(HTTP_ROUTE_LABEL, route.clone())); + route_str = Some(route.clone()); }; // Extract custom request attributes let custom_request_attributes = self.request_extractor.extract_attributes(&req); + // Start tracing span + let span_name = route_str + .clone() + .unwrap_or_else(|| format!("{} {}", method, req.uri().path())); + + let mut span_attributes = vec![ + KeyValue::new(semconv::trace::HTTP_REQUEST_METHOD, method.clone()), + url_scheme_kv.clone(), + KeyValue::new(semconv::attribute::URL_PATH, req.uri().path().to_string()), + KeyValue::new(semconv::trace::URL_FULL, req.uri().to_string()), + ]; + + if let Some(user_agent) = req + .headers() + .get("user-agent") + .and_then(|v| v.to_str().ok()) + { + span_attributes.push(KeyValue::new( + semconv::trace::USER_AGENT_ORIGINAL, + user_agent.to_string(), + )); + } + + if let Some(host) = req.headers().get("host").and_then(|v| v.to_str().ok()) { + span_attributes.push(KeyValue::new( + semconv::trace::SERVER_ADDRESS, + host.to_string(), + )); + } + + if let Some(route) = &route_str { + span_attributes.push(KeyValue::new(semconv::trace::HTTP_ROUTE, route.clone())); + } + + span_attributes.extend(custom_request_attributes.clone()); + + let tracer = &self.state.tracer; + let span = tracer + .span_builder(span_name) + .with_kind(SpanKind::Server) + .with_attributes(span_attributes) + .start(tracer); + let ctx = OtelContext::current_with_span(span); + self.state .server_active_requests .add(1, &[url_scheme_kv.clone(), method_kv.clone()]); - HTTPMetricsResponseFuture { + HTTPResponseFuture { inner_response_future: self.inner_service.call(req), layer_state: self.state.clone(), - metrics_state: ResponseFutureMetricsState { + future_state: ResponseFutureState { duration_start, req_body_size: content_length, @@ -425,13 +497,15 @@ where method_kv, route_kv_opt, custom_request_attributes, + + otel_context: ctx, }, response_extractor: self.response_extractor.clone(), } } } -impl Future for HTTPMetricsResponseFuture +impl Future for HTTPResponseFuture where F: Future, E>>, ResBody: http_body::Body, @@ -446,30 +520,56 @@ where // Build base label set let mut label_superset = vec![ - this.metrics_state.protocol_name_kv.clone(), - this.metrics_state.protocol_version_kv.clone(), - this.metrics_state.url_scheme_kv.clone(), - this.metrics_state.method_kv.clone(), + this.future_state.protocol_name_kv.clone(), + this.future_state.protocol_version_kv.clone(), + this.future_state.url_scheme_kv.clone(), + this.future_state.method_kv.clone(), KeyValue::new(HTTP_RESPONSE_STATUS_CODE_LABEL, i64::from(status.as_u16())), ]; - if let Some(route_kv) = this.metrics_state.route_kv_opt.clone() { + if let Some(route_kv) = this.future_state.route_kv_opt.clone() { label_superset.push(route_kv); } // Add custom request attributes - label_superset.extend(this.metrics_state.custom_request_attributes.clone()); + label_superset.extend(this.future_state.custom_request_attributes.clone()); // Extract and add custom response attributes let custom_response_attributes = this.response_extractor.extract_attributes(&response); - label_superset.extend(custom_response_attributes); + label_superset.extend(custom_response_attributes.clone()); + + // Update span + let span = this.future_state.otel_context.span(); + span.set_attribute(KeyValue::new( + semconv::trace::HTTP_RESPONSE_STATUS_CODE, + status.as_u16() as i64, + )); + + // Add custom response attributes to span + for attr in &custom_response_attributes { + span.set_attribute(attr.clone()); + } + + // Set span status based on HTTP status code + // Following server-side semantic conventions: + // - 5xx server errors indicate server failure and should be marked as span errors + // - 4xx client errors indicate client mistakes, not server failures + if status.is_server_error() { + span.set_status(Status::Error { + description: format!("HTTP {}", status.as_u16()).into(), + }); + } else { + span.set_status(Status::Ok); + } + + span.end(); this.layer_state.server_request_duration.record( - this.metrics_state.duration_start.elapsed().as_secs_f64(), + this.future_state.duration_start.elapsed().as_secs_f64(), &label_superset, ); - if let Some(req_content_length) = this.metrics_state.req_body_size { + if let Some(req_content_length) = this.future_state.req_body_size { this.layer_state .server_request_body_size .record(req_content_length, &label_superset); @@ -485,8 +585,8 @@ where this.layer_state.server_active_requests.add( -1, &[ - this.metrics_state.url_scheme_kv.clone(), - this.metrics_state.method_kv.clone(), + this.future_state.url_scheme_kv.clone(), + this.future_state.method_kv.clone(), ], ); @@ -505,3 +605,166 @@ fn split_and_format_protocol_version(http_version: http::Version) -> (String, St }; (String::from("http"), String::from(version_str)) } + +#[cfg(test)] +mod tests { + use super::*; + use opentelemetry::metrics::MeterProvider; + use opentelemetry::trace::{FutureExt, TraceContextExt, Tracer, TracerProvider}; + use opentelemetry::Key; + use opentelemetry_sdk::metrics::data::{Metric, ResourceMetrics}; + use opentelemetry_sdk::metrics::InMemoryMetricExporterBuilder; + use opentelemetry_sdk::metrics::SdkMeterProvider; + use opentelemetry_sdk::trace::InMemorySpanExporterBuilder; + use opentelemetry_sdk::trace::SdkTracerProvider; + use opentelemetry_sdk::Resource; + use std::result::Result; + use tower::Service; + use tower::ServiceBuilder; + use tower::ServiceExt; + + #[tokio::test(flavor = "current_thread")] + async fn test_tracing_with_in_memory_tracer() { + let trace_exporter = InMemorySpanExporterBuilder::new().build(); + let tracer_provider = SdkTracerProvider::builder() + .with_simple_exporter(trace_exporter.clone()) + .build(); + let tracer = tracer_provider.tracer("test_tracer"); + + let metric_exporter = InMemoryMetricExporterBuilder::new().build(); + let meter_provider = SdkMeterProvider::builder() + .with_periodic_exporter(metric_exporter.clone()) + .build(); + let meter = meter_provider.meter("test_meter"); + + let layer = HTTPLayerBuilder::builder() + .with_tracer(BoxedTracer::new(Box::new(tracer.clone()))) + .with_meter(meter) + .build() + .unwrap(); + + let mut service = ServiceBuilder::new() + .layer(layer) + .service(tower::service_fn(echo)); + + // Create a parent span and set it as the current context + let parent_span = tracer.start("parent_operation"); + let cx = OtelContext::current_with_span(parent_span); + + let request_body = "test".to_string(); + let request = http::Request::builder() + .uri("http://example.com") + .header("Content-Length", request_body.len().to_string()) + .body(request_body) + .unwrap(); + + // Execute the service call within the parent span context + let _response = async { + service + .ready_and() + .await + .unwrap() + .call(request) + .await + .unwrap() + } + .with_context(cx) + .await; + + tracer_provider.force_flush().unwrap(); + meter_provider.force_flush().unwrap(); + + let spans = trace_exporter.get_finished_spans().unwrap(); + assert_eq!( + spans.len(), + 2, + "Expected exactly two spans to be recorded (parent + HTTP)" + ); + + // Find the HTTP span (should be the child) + let http_span = spans + .iter() + .find(|span| span.name == "GET /") + .expect("Should find HTTP span"); + + // Find the parent span + let parent_span = spans + .iter() + .find(|span| span.name == "parent_operation") + .expect("Should find parent span"); + + // Verify the HTTP span has the correct parent + assert_eq!( + http_span.parent_span_id, + parent_span.span_context.span_id(), + "HTTP span should have parent span as parent" + ); + + // Verify they share the same trace ID + assert_eq!( + http_span.span_context.trace_id(), + parent_span.span_context.trace_id(), + "Parent and child spans should share the same trace ID" + ); + + assert_eq!( + http_span.name, "GET /", + "Span name should match the request" + ); + assert_eq!( + http_span.attributes, + vec![ + KeyValue::new(semconv::trace::HTTP_REQUEST_METHOD, "GET".to_string()), + KeyValue::new(semconv::trace::URL_SCHEME, "http".to_string()), + KeyValue::new(semconv::trace::URL_PATH, "/".to_string()), + KeyValue::new(semconv::trace::URL_FULL, "http://example.com/".to_string()), + KeyValue::new(semconv::trace::HTTP_RESPONSE_STATUS_CODE, 200), + ] + ); + + // Verify metrics are recorded + let resource_metrics = metric_exporter.get_finished_metrics().unwrap(); + assert_eq!( + resource_metrics.len(), + 1, + "Expected 1 ResourceMetrics entry" + ); + + let resource_metric = &resource_metrics[0]; + let service_name = Key::new(semconv::resource::SERVICE_NAME); + assert_eq!( + "unknown_service", + resource_metric + .resource() + .get(&service_name) + .unwrap() + .to_string() + ); + + // Count total metrics across all scopes + let total_metrics: usize = resource_metric + .scope_metrics() + .map(|scope| scope.metrics().count()) + .sum(); + assert_eq!(total_metrics, 4); + + assert_eq!(resource_metric.scope_metrics().count(), 1); + let scope_metric = resource_metric.scope_metrics().next().unwrap(); + assert_eq!(scope_metric.scope().name(), "test_meter"); + + let metric_names = vec![ + semconv::metric::HTTP_SERVER_REQUEST_DURATION, + semconv::metric::HTTP_SERVER_ACTIVE_REQUESTS, + semconv::metric::HTTP_SERVER_REQUEST_BODY_SIZE, + semconv::metric::HTTP_SERVER_RESPONSE_BODY_SIZE, + ]; + assert_eq!(scope_metric.metrics().count(), metric_names.len()); + for (idx, metric) in scope_metric.metrics().enumerate() { + assert_eq!(metric.name(), metric_names[idx]); + } + } + + async fn echo(req: http::Request) -> Result, Error> { + Ok(http::Response::new(req.into_body())) + } +} From 20f580f2746ded0d3bacf75c1fd561400b32265e Mon Sep 17 00:00:00 2001 From: Jan Steinke Date: Mon, 25 Aug 2025 10:22:34 +0200 Subject: [PATCH 02/36] remove useless comment --- opentelemetry-instrumentation-tower/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opentelemetry-instrumentation-tower/Cargo.toml b/opentelemetry-instrumentation-tower/Cargo.toml index 66ff7fd56..aec3bc52b 100644 --- a/opentelemetry-instrumentation-tower/Cargo.toml +++ b/opentelemetry-instrumentation-tower/Cargo.toml @@ -22,7 +22,7 @@ futures-util = { version = "0.3", default-features = false } http = { version = "1", features = ["std"], default-features = false } http-body = { version = "1", default-features = false } opentelemetry = { workspace = true, features = ["futures", "metrics", "trace"] } -opentelemetry-semantic-conventions = { workspace = true, features = ["semconv_experimental"] } # Corrected feature name +opentelemetry-semantic-conventions = { workspace = true, features = ["semconv_experimental"] } pin-project-lite = { version = "0.2", default-features = false } tower-service = { version = "0.3", default-features = false } tower-layer = { version = "0.3", default-features = false } From 84c2860e14e80b984d29a08379939c1d509ce0a3 Mon Sep 17 00:00:00 2001 From: Jan Steinke Date: Mon, 25 Aug 2025 10:35:02 +0200 Subject: [PATCH 03/36] update README --- opentelemetry-instrumentation-tower/README.md | 66 +------------------ 1 file changed, 1 insertion(+), 65 deletions(-) diff --git a/opentelemetry-instrumentation-tower/README.md b/opentelemetry-instrumentation-tower/README.md index 09a879c56..dfe847c05 100644 --- a/opentelemetry-instrumentation-tower/README.md +++ b/opentelemetry-instrumentation-tower/README.md @@ -10,74 +10,10 @@ This middleware provides both metrics and distributed tracing for HTTP requests, - **Distributed Tracing**: HTTP spans with semantic attributes - **Semantic Conventions**: Uses OpenTelemetry semantic conventions for consistent attribute naming - **Flexible Configuration**: Support for custom attribute extractors and tracer configuration -- **Framework Support**: Works with any Tower-compatible HTTP framework (Axum, Hyper, etc.) +- **Framework Support**: Works with any Tower-compatible HTTP framework (Axum, Hyper, Tonic etc.) ## Usage -### Basic Usage (Metrics Only) - -```rust -use opentelemetry_instrumentation_tower::HTTPLayerBuilder; - -let meter = global::meter("my-service"); -let http_layer = HTTPLayerBuilder::builder() - .with_meter(meter) - .build()?; - -let app = Router::new() - .route("/", get(handler)) - .layer(http_layer); -``` - -### With Tracing - -```rust -use opentelemetry_instrumentation_tower::HTTPLayerBuilder; - -let meter = global::meter("my-service"); -let http_layer = HTTPLayerBuilder::builder() - .with_meter(meter) - .with_tracing() // Uses global tracer provider - .build()?; - -let app = Router::new() - .route("/", get(handler)) - .layer(http_layer); -``` - -### With Custom Tracer - -```rust -use opentelemetry_instrumentation_tower::HTTPLayerBuilder; - -let meter = global::meter("my-service"); -let tracer = global::tracer_provider().tracer("my-service"); - -let http_layer = HTTPLayerBuilder::builder() - .with_meter(meter) - .with_tracer(tracer) - .build()?; - -let app = Router::new() - .route("/", get(handler)) - .layer(http_layer); -``` - -### With Custom Attribute Extractors - -```rust -use opentelemetry_instrumentation_tower::HTTPLayerBuilder; - -let http_layer = HTTPLayerBuilder::builder() - .with_meter(meter) - .with_tracing() - .with_request_extractor_fn(|req| { - vec![KeyValue::new("custom.header", - req.headers().get("x-custom").unwrap_or_default().to_str().unwrap_or(""))] - }) - .build()?; -``` - ## Metrics The middleware exports the following metrics: From dc535a594e0423912a59e5c8c50c3716d0354d7f Mon Sep 17 00:00:00 2001 From: Jan Steinke Date: Mon, 25 Aug 2025 14:45:34 +0200 Subject: [PATCH 04/36] remove http.route --- .../src/lib.rs | 52 +++++++++---------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/opentelemetry-instrumentation-tower/src/lib.rs b/opentelemetry-instrumentation-tower/src/lib.rs index bc13248da..76d592e67 100644 --- a/opentelemetry-instrumentation-tower/src/lib.rs +++ b/opentelemetry-instrumentation-tower/src/lib.rs @@ -279,6 +279,7 @@ impl HTTPLayerBuilder { Some(t) => t, None => BoxedTracer::new(Box::new(NoopTracer::new())), }; + match self.meter { Some(meter) => Ok(HTTPMetricsLayer { state: Arc::from(Self::make_state(meter, req_dur_bounds, tracer)), @@ -425,22 +426,16 @@ where #[allow(unused_mut)] let mut route_kv_opt = None; - let mut route_str = None; #[cfg(feature = "axum")] if let Some(matched_path) = req.extensions().get::() { let route = matched_path.as_str().to_owned(); route_kv_opt = Some(KeyValue::new(HTTP_ROUTE_LABEL, route.clone())); - route_str = Some(route.clone()); }; // Extract custom request attributes let custom_request_attributes = self.request_extractor.extract_attributes(&req); // Start tracing span - let span_name = route_str - .clone() - .unwrap_or_else(|| format!("{} {}", method, req.uri().path())); - let mut span_attributes = vec![ KeyValue::new(semconv::trace::HTTP_REQUEST_METHOD, method.clone()), url_scheme_kv.clone(), @@ -466,12 +461,9 @@ where )); } - if let Some(route) = &route_str { - span_attributes.push(KeyValue::new(semconv::trace::HTTP_ROUTE, route.clone())); - } - span_attributes.extend(custom_request_attributes.clone()); + let span_name = format!("{} {}", method, req.uri().path()); let tracer = &self.state.tracer; let span = tracer .span_builder(span_name) @@ -609,15 +601,15 @@ fn split_and_format_protocol_version(http_version: http::Version) -> (String, St #[cfg(test)] mod tests { use super::*; + #[cfg(feature = "axum")] + use axum::extract::MatchedPath; use opentelemetry::metrics::MeterProvider; use opentelemetry::trace::{FutureExt, TraceContextExt, Tracer, TracerProvider}; use opentelemetry::Key; - use opentelemetry_sdk::metrics::data::{Metric, ResourceMetrics}; use opentelemetry_sdk::metrics::InMemoryMetricExporterBuilder; use opentelemetry_sdk::metrics::SdkMeterProvider; use opentelemetry_sdk::trace::InMemorySpanExporterBuilder; use opentelemetry_sdk::trace::SdkTracerProvider; - use opentelemetry_sdk::Resource; use std::result::Result; use tower::Service; use tower::ServiceBuilder; @@ -653,8 +645,9 @@ mod tests { let request_body = "test".to_string(); let request = http::Request::builder() - .uri("http://example.com") + .uri("http://example.com/api/users/123") .header("Content-Length", request_body.len().to_string()) + .header("User-Agent", "tower-test-client/1.0") .body(request_body) .unwrap(); @@ -684,7 +677,7 @@ mod tests { // Find the HTTP span (should be the child) let http_span = spans .iter() - .find(|span| span.name == "GET /") + .find(|span| span.name == "GET /api/users/123") .expect("Should find HTTP span"); // Find the parent span @@ -708,19 +701,26 @@ mod tests { ); assert_eq!( - http_span.name, "GET /", + http_span.name, "GET /api/users/123", "Span name should match the request" ); - assert_eq!( - http_span.attributes, - vec![ - KeyValue::new(semconv::trace::HTTP_REQUEST_METHOD, "GET".to_string()), - KeyValue::new(semconv::trace::URL_SCHEME, "http".to_string()), - KeyValue::new(semconv::trace::URL_PATH, "/".to_string()), - KeyValue::new(semconv::trace::URL_FULL, "http://example.com/".to_string()), - KeyValue::new(semconv::trace::HTTP_RESPONSE_STATUS_CODE, 200), - ] - ); + // Build expected attributes + let expected_attributes = vec![ + KeyValue::new(semconv::trace::HTTP_REQUEST_METHOD, "GET".to_string()), + KeyValue::new(semconv::trace::URL_SCHEME, "http".to_string()), + KeyValue::new(semconv::trace::URL_PATH, "/api/users/123".to_string()), + KeyValue::new( + semconv::trace::URL_FULL, + "http://example.com/api/users/123".to_string(), + ), + KeyValue::new( + semconv::trace::USER_AGENT_ORIGINAL, + "tower-test-client/1.0".to_string(), + ), + KeyValue::new(semconv::trace::HTTP_RESPONSE_STATUS_CODE, 200), + ]; + + assert_eq!(http_span.attributes, expected_attributes); // Verify metrics are recorded let resource_metrics = metric_exporter.get_finished_metrics().unwrap(); @@ -752,7 +752,7 @@ mod tests { let scope_metric = resource_metric.scope_metrics().next().unwrap(); assert_eq!(scope_metric.scope().name(), "test_meter"); - let metric_names = vec![ + let metric_names = [ semconv::metric::HTTP_SERVER_REQUEST_DURATION, semconv::metric::HTTP_SERVER_ACTIVE_REQUESTS, semconv::metric::HTTP_SERVER_REQUEST_BODY_SIZE, From 2497a7fbd606ecfc82401d7c750cadd576789759 Mon Sep 17 00:00:00 2001 From: Jan Steinke Date: Mon, 25 Aug 2025 14:49:56 +0200 Subject: [PATCH 05/36] clean up code and documentation --- opentelemetry-instrumentation-tower/README.md | 4 +--- opentelemetry-instrumentation-tower/src/lib.rs | 7 ------- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/opentelemetry-instrumentation-tower/README.md b/opentelemetry-instrumentation-tower/README.md index dfe847c05..ea04d9b99 100644 --- a/opentelemetry-instrumentation-tower/README.md +++ b/opentelemetry-instrumentation-tower/README.md @@ -29,10 +29,8 @@ HTTP spans are created with the following attributes (following OpenTelemetry se - `http.request.method` - HTTP method - `url.scheme` - URL scheme (http/https) -- `http.target` - Request target (path + query) +- `url.path` - Request path - `url.full` - Full URL -- `http.route` - Matched route pattern (when available) -- `server.address` - Server host - `user_agent.original` - User agent string - `http.response.status_code` - HTTP response status code diff --git a/opentelemetry-instrumentation-tower/src/lib.rs b/opentelemetry-instrumentation-tower/src/lib.rs index 76d592e67..652b6f6b2 100644 --- a/opentelemetry-instrumentation-tower/src/lib.rs +++ b/opentelemetry-instrumentation-tower/src/lib.rs @@ -454,13 +454,6 @@ where )); } - if let Some(host) = req.headers().get("host").and_then(|v| v.to_str().ok()) { - span_attributes.push(KeyValue::new( - semconv::trace::SERVER_ADDRESS, - host.to_string(), - )); - } - span_attributes.extend(custom_request_attributes.clone()); let span_name = format!("{} {}", method, req.uri().path()); From 46d451016ee9eeb76403e8998d4c1bfd5d9de034 Mon Sep 17 00:00:00 2001 From: Jan Steinke Date: Mon, 25 Aug 2025 14:52:18 +0200 Subject: [PATCH 06/36] add CHANGELOG --- opentelemetry-instrumentation-tower/CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 opentelemetry-instrumentation-tower/CHANGELOG.md diff --git a/opentelemetry-instrumentation-tower/CHANGELOG.md b/opentelemetry-instrumentation-tower/CHANGELOG.md new file mode 100644 index 000000000..59af936b9 --- /dev/null +++ b/opentelemetry-instrumentation-tower/CHANGELOG.md @@ -0,0 +1,7 @@ +# Changelog + +## vNext + +### Changed + +* Added OpenTelemetry trace support From 1d61504912f100538f14161f7fc577598650f6de Mon Sep 17 00:00:00 2001 From: Jan Steinke Date: Mon, 25 Aug 2025 15:24:37 +0200 Subject: [PATCH 07/36] remove unused import --- opentelemetry-instrumentation-tower/src/lib.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/opentelemetry-instrumentation-tower/src/lib.rs b/opentelemetry-instrumentation-tower/src/lib.rs index 652b6f6b2..89b1ee5d8 100644 --- a/opentelemetry-instrumentation-tower/src/lib.rs +++ b/opentelemetry-instrumentation-tower/src/lib.rs @@ -594,8 +594,6 @@ fn split_and_format_protocol_version(http_version: http::Version) -> (String, St #[cfg(test)] mod tests { use super::*; - #[cfg(feature = "axum")] - use axum::extract::MatchedPath; use opentelemetry::metrics::MeterProvider; use opentelemetry::trace::{FutureExt, TraceContextExt, Tracer, TracerProvider}; use opentelemetry::Key; From 0e9cc0b5790dfbbf8e30f9a35a5baaf0666c6426 Mon Sep 17 00:00:00 2001 From: Jan Steinke Date: Wed, 27 Aug 2025 18:09:53 +0200 Subject: [PATCH 08/36] use global providers --- .../CHANGELOG.md | 23 ++++++ .../examples/axum-http-service/src/main.rs | 4 +- .../examples/hyper-http-service/src/main.rs | 4 +- .../src/lib.rs | 80 +++++++------------ 4 files changed, 54 insertions(+), 57 deletions(-) diff --git a/opentelemetry-instrumentation-tower/CHANGELOG.md b/opentelemetry-instrumentation-tower/CHANGELOG.md index 59af936b9..72ebcbd05 100644 --- a/opentelemetry-instrumentation-tower/CHANGELOG.md +++ b/opentelemetry-instrumentation-tower/CHANGELOG.md @@ -4,4 +4,27 @@ ### Changed +* **BREAKING**: Removed `with_meter()` method from `HTTPLayerBuilder`. The middleware now uses global providers. * Added OpenTelemetry trace support + +### Migration Guide + +Before: +```rust +let layer = HTTPLayerBuilder::builder() + .with_meter(meter) + .build() + .unwrap(); +``` + +After: +```rust +// Set global providers first +global::set_meter_provider(meter_provider); +global::set_tracer_provider(tracer_provider); // for tracing support + +// Then create the layer without explicit meter +let layer = HTTPLayerBuilder::builder() + .build() + .unwrap(); +``` diff --git a/opentelemetry-instrumentation-tower/examples/axum-http-service/src/main.rs b/opentelemetry-instrumentation-tower/examples/axum-http-service/src/main.rs index 559c74c2e..d1901595b 100644 --- a/opentelemetry-instrumentation-tower/examples/axum-http-service/src/main.rs +++ b/opentelemetry-instrumentation-tower/examples/axum-http-service/src/main.rs @@ -56,10 +56,8 @@ async fn main() { .build(); global::set_meter_provider(meter_provider); - // init our otel metrics middleware - let global_meter = global::meter(SERVICE_NAME); + let otel_metrics_service_layer = otel_tower_metrics::HTTPMetricsLayerBuilder::builder() - .with_meter(global_meter) .build() .unwrap(); diff --git a/opentelemetry-instrumentation-tower/examples/hyper-http-service/src/main.rs b/opentelemetry-instrumentation-tower/examples/hyper-http-service/src/main.rs index ccd31bd5b..2055e78d0 100644 --- a/opentelemetry-instrumentation-tower/examples/hyper-http-service/src/main.rs +++ b/opentelemetry-instrumentation-tower/examples/hyper-http-service/src/main.rs @@ -61,10 +61,8 @@ async fn main() { .build(); global::set_meter_provider(meter_provider); - // init our otel metrics middleware - let global_meter = global::meter(SERVICE_NAME); + let otel_metrics_service_layer = otel_tower_metrics::HTTPMetricsLayerBuilder::builder() - .with_meter(global_meter) .build() .unwrap(); diff --git a/opentelemetry-instrumentation-tower/src/lib.rs b/opentelemetry-instrumentation-tower/src/lib.rs index 89b1ee5d8..b9a118e98 100644 --- a/opentelemetry-instrumentation-tower/src/lib.rs +++ b/opentelemetry-instrumentation-tower/src/lib.rs @@ -11,9 +11,8 @@ use std::{fmt, result}; #[cfg(feature = "axum")] use axum::extract::MatchedPath; use futures_util::ready; -use opentelemetry::global::BoxedTracer; -use opentelemetry::metrics::{Histogram, Meter, UpDownCounter}; -use opentelemetry::trace::noop::NoopTracer; +use opentelemetry::global; +use opentelemetry::metrics::{Histogram, UpDownCounter}; use opentelemetry::trace::{SpanKind, Status, TraceContextExt, Tracer}; use opentelemetry::{Context as OtelContext, KeyValue}; use opentelemetry_semantic_conventions as semconv; @@ -123,13 +122,12 @@ where /// State scoped to the entire middleware Layer. /// -/// Holds both metrics instruments and tracing configuration. +/// Holds metrics instruments. struct HTTPLayerState { pub server_request_duration: Histogram, pub server_active_requests: UpDownCounter, pub server_request_body_size: Histogram, pub server_response_body_size: Histogram, - pub tracer: BoxedTracer, } #[derive(Clone)] @@ -159,8 +157,6 @@ pub type HTTPMetricsLayerBuilder HTTPLayerBuilder; pub struct HTTPLayerBuilder { - meter: Option, - tracer: Option, req_dur_bounds: Option>, request_extractor: ReqExt, response_extractor: ResExt, @@ -205,8 +201,6 @@ impl fmt::Debug for Error { impl HTTPLayerBuilder { pub fn builder() -> Self { HTTPLayerBuilder { - meter: None, - tracer: None, req_dur_bounds: Some(LIBRARY_DEFAULT_HTTP_SERVER_DURATION_BOUNDARIES.to_vec()), request_extractor: NoOpExtractor, response_extractor: NoOpExtractor, @@ -224,8 +218,6 @@ impl HTTPLayerBuilder { NewReqExt: RequestAttributeExtractor, { HTTPLayerBuilder { - meter: self.meter, - tracer: self.tracer, req_dur_bounds: self.req_dur_bounds, request_extractor: extractor, response_extractor: self.response_extractor, @@ -241,8 +233,6 @@ impl HTTPLayerBuilder { NewResExt: ResponseAttributeExtractor, { HTTPLayerBuilder { - meter: self.meter, - tracer: self.tracer, req_dur_bounds: self.req_dur_bounds, request_extractor: self.request_extractor, response_extractor: extractor, @@ -275,26 +265,12 @@ impl HTTPLayerBuilder { let req_dur_bounds = self .req_dur_bounds .unwrap_or_else(|| LIBRARY_DEFAULT_HTTP_SERVER_DURATION_BOUNDARIES.to_vec()); - let tracer = match self.tracer { - Some(t) => t, - None => BoxedTracer::new(Box::new(NoopTracer::new())), - }; - - match self.meter { - Some(meter) => Ok(HTTPMetricsLayer { - state: Arc::from(Self::make_state(meter, req_dur_bounds, tracer)), - request_extractor: self.request_extractor, - response_extractor: self.response_extractor, - }), - None => Err(Error { - inner: ErrorKind::Config(String::from("no meter provided")), - }), - } - } - pub fn with_meter(mut self, meter: Meter) -> Self { - self.meter = Some(meter); - self + Ok(HTTPMetricsLayer { + state: Arc::from(Self::make_state(req_dur_bounds)), + request_extractor: self.request_extractor, + response_extractor: self.response_extractor, + }) } pub fn with_request_duration_bounds(mut self, bounds: Vec) -> Self { @@ -302,12 +278,8 @@ impl HTTPLayerBuilder { self } - pub fn with_tracer(mut self, tracer: BoxedTracer) -> Self { - self.tracer = Some(tracer); - self - } - - fn make_state(meter: Meter, req_dur_bounds: Vec, tracer: BoxedTracer) -> HTTPLayerState { + fn make_state(req_dur_bounds: Vec) -> HTTPLayerState { + let meter = global::meter("opentelemetry-instrumentation-tower"); HTTPLayerState { server_request_duration: meter .f64_histogram(Cow::from(HTTP_SERVER_DURATION_METRIC)) @@ -330,7 +302,6 @@ impl HTTPLayerBuilder { .with_description("Size of HTTP server response bodies.") .with_unit(HTTP_SERVER_RESPONSE_BODY_SIZE_UNIT) .build(), - tracer, } } } @@ -457,12 +428,12 @@ where span_attributes.extend(custom_request_attributes.clone()); let span_name = format!("{} {}", method, req.uri().path()); - let tracer = &self.state.tracer; + let tracer = global::tracer("opentelemetry-instrumentation-tower"); let span = tracer .span_builder(span_name) .with_kind(SpanKind::Server) .with_attributes(span_attributes) - .start(tracer); + .start(&tracer); let ctx = OtelContext::current_with_span(span); self.state @@ -594,8 +565,8 @@ fn split_and_format_protocol_version(http_version: http::Version) -> (String, St #[cfg(test)] mod tests { use super::*; - use opentelemetry::metrics::MeterProvider; - use opentelemetry::trace::{FutureExt, TraceContextExt, Tracer, TracerProvider}; + + use opentelemetry::trace::{FutureExt, TraceContextExt, Tracer}; use opentelemetry::Key; use opentelemetry_sdk::metrics::InMemoryMetricExporterBuilder; use opentelemetry_sdk::metrics::SdkMeterProvider; @@ -612,19 +583,19 @@ mod tests { let tracer_provider = SdkTracerProvider::builder() .with_simple_exporter(trace_exporter.clone()) .build(); - let tracer = tracer_provider.tracer("test_tracer"); let metric_exporter = InMemoryMetricExporterBuilder::new().build(); let meter_provider = SdkMeterProvider::builder() .with_periodic_exporter(metric_exporter.clone()) .build(); - let meter = meter_provider.meter("test_meter"); - let layer = HTTPLayerBuilder::builder() - .with_tracer(BoxedTracer::new(Box::new(tracer.clone()))) - .with_meter(meter) - .build() - .unwrap(); + // Set global providers + global::set_tracer_provider(tracer_provider.clone()); + global::set_meter_provider(meter_provider.clone()); + + let tracer = global::tracer("test_tracer"); + + let layer = HTTPLayerBuilder::builder().build().unwrap(); let mut service = ServiceBuilder::new() .layer(layer) @@ -741,7 +712,10 @@ mod tests { assert_eq!(resource_metric.scope_metrics().count(), 1); let scope_metric = resource_metric.scope_metrics().next().unwrap(); - assert_eq!(scope_metric.scope().name(), "test_meter"); + assert_eq!( + scope_metric.scope().name(), + "opentelemetry-instrumentation-tower" + ); let metric_names = [ semconv::metric::HTTP_SERVER_REQUEST_DURATION, @@ -753,6 +727,10 @@ mod tests { for (idx, metric) in scope_metric.metrics().enumerate() { assert_eq!(metric.name(), metric_names[idx]); } + + // Reset to noop providers to avoid test interference + global::set_tracer_provider(opentelemetry_sdk::trace::SdkTracerProvider::default()); + global::set_meter_provider(opentelemetry_sdk::metrics::SdkMeterProvider::default()); } async fn echo(req: http::Request) -> Result, Error> { From 8a5e5331df5243bcb36d3de6a0f35f28fcaaf151 Mon Sep 17 00:00:00 2001 From: Jan Steinke Date: Wed, 27 Aug 2025 18:20:01 +0200 Subject: [PATCH 09/36] break backwards compatibility --- .../CHANGELOG.md | 20 +++++++++++++++++-- .../examples/axum-http-service/src/main.rs | 2 +- .../examples/hyper-http-service/src/main.rs | 2 +- .../src/lib.rs | 11 +--------- 4 files changed, 21 insertions(+), 14 deletions(-) diff --git a/opentelemetry-instrumentation-tower/CHANGELOG.md b/opentelemetry-instrumentation-tower/CHANGELOG.md index 72ebcbd05..6ac80384c 100644 --- a/opentelemetry-instrumentation-tower/CHANGELOG.md +++ b/opentelemetry-instrumentation-tower/CHANGELOG.md @@ -4,14 +4,22 @@ ### Changed -* **BREAKING**: Removed `with_meter()` method from `HTTPLayerBuilder`. The middleware now uses global providers. +* **BREAKING**: Removed `with_meter()` method. The middleware now uses global meter and tracer providers via `opentelemetry::global::meter()` and `opentelemetry::global::tracer()`. +* **BREAKING**: Renamed types. Use the new names: + - `HTTPMetricsLayer` → `HTTPLayer` + - `HTTPMetricsService` → `HTTPService` + - `HTTPMetricsResponseFuture` → `HTTPResponseFuture` + - `HTTPMetricsLayerBuilder` → `HTTPLayerBuilder` * Added OpenTelemetry trace support ### Migration Guide +#### API Changes Before: ```rust -let layer = HTTPLayerBuilder::builder() +use opentelemetry_instrumentation_tower::HTTPMetricsLayerBuilder; + +let layer = HTTPMetricsLayerBuilder::builder() .with_meter(meter) .build() .unwrap(); @@ -19,6 +27,8 @@ let layer = HTTPLayerBuilder::builder() After: ```rust +use opentelemetry_instrumentation_tower::HTTPLayerBuilder; + // Set global providers first global::set_meter_provider(meter_provider); global::set_tracer_provider(tracer_provider); // for tracing support @@ -28,3 +38,9 @@ let layer = HTTPLayerBuilder::builder() .build() .unwrap(); ``` + +#### Type Name Changes +- Replace `HTTPMetricsLayerBuilder` with `HTTPLayerBuilder` +- Replace `HTTPMetricsLayer` with `HTTPLayer` +- Replace `HTTPMetricsService` with `HTTPService` +- Replace `HTTPMetricsResponseFuture` with `HTTPResponseFuture` diff --git a/opentelemetry-instrumentation-tower/examples/axum-http-service/src/main.rs b/opentelemetry-instrumentation-tower/examples/axum-http-service/src/main.rs index d1901595b..47d39265f 100644 --- a/opentelemetry-instrumentation-tower/examples/axum-http-service/src/main.rs +++ b/opentelemetry-instrumentation-tower/examples/axum-http-service/src/main.rs @@ -57,7 +57,7 @@ async fn main() { global::set_meter_provider(meter_provider); - let otel_metrics_service_layer = otel_tower_metrics::HTTPMetricsLayerBuilder::builder() + let otel_metrics_service_layer = otel_tower_metrics::HTTPLayerBuilder::builder() .build() .unwrap(); diff --git a/opentelemetry-instrumentation-tower/examples/hyper-http-service/src/main.rs b/opentelemetry-instrumentation-tower/examples/hyper-http-service/src/main.rs index 2055e78d0..ad01a6209 100644 --- a/opentelemetry-instrumentation-tower/examples/hyper-http-service/src/main.rs +++ b/opentelemetry-instrumentation-tower/examples/hyper-http-service/src/main.rs @@ -62,7 +62,7 @@ async fn main() { global::set_meter_provider(meter_provider); - let otel_metrics_service_layer = otel_tower_metrics::HTTPMetricsLayerBuilder::builder() + let otel_metrics_service_layer = otel_tower_metrics::HTTPLayerBuilder::builder() .build() .unwrap(); diff --git a/opentelemetry-instrumentation-tower/src/lib.rs b/opentelemetry-instrumentation-tower/src/lib.rs index b9a118e98..4e797ff4c 100644 --- a/opentelemetry-instrumentation-tower/src/lib.rs +++ b/opentelemetry-instrumentation-tower/src/lib.rs @@ -147,15 +147,6 @@ pub struct HTTPLayer { response_extractor: ResExt, } -// Type aliases for backward compatibility -pub type HTTPMetricsLayer = - HTTPLayer; -pub type HTTPMetricsService = - HTTPService; -pub type HTTPMetricsResponseFuture = HTTPResponseFuture; -pub type HTTPMetricsLayerBuilder = - HTTPLayerBuilder; - pub struct HTTPLayerBuilder { req_dur_bounds: Option>, request_extractor: ReqExt, @@ -266,7 +257,7 @@ impl HTTPLayerBuilder { .req_dur_bounds .unwrap_or_else(|| LIBRARY_DEFAULT_HTTP_SERVER_DURATION_BOUNDARIES.to_vec()); - Ok(HTTPMetricsLayer { + Ok(HTTPLayer { state: Arc::from(Self::make_state(req_dur_bounds)), request_extractor: self.request_extractor, response_extractor: self.response_extractor, From 7b976777ea2740c07eaa1ff9cd8a006d3d7fdf47 Mon Sep 17 00:00:00 2001 From: Jan Steinke Date: Wed, 27 Aug 2025 18:23:37 +0200 Subject: [PATCH 10/36] undo semconv for metrics, it's done in a separate PR --- .../src/lib.rs | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/opentelemetry-instrumentation-tower/src/lib.rs b/opentelemetry-instrumentation-tower/src/lib.rs index 4e797ff4c..d4c04630d 100644 --- a/opentelemetry-instrumentation-tower/src/lib.rs +++ b/opentelemetry-instrumentation-tower/src/lib.rs @@ -34,23 +34,23 @@ const _OTEL_DEFAULT_HTTP_SERVER_DURATION_BOUNDARIES: [f64; 14] = [ const LIBRARY_DEFAULT_HTTP_SERVER_DURATION_BOUNDARIES: [f64; 14] = [ 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0, 60.0, 120.0, 300.0, ]; -const HTTP_SERVER_ACTIVE_REQUESTS_METRIC: &str = semconv::metric::HTTP_SERVER_ACTIVE_REQUESTS; +const HTTP_SERVER_ACTIVE_REQUESTS_METRIC: &str = "http.server.active_requests"; const HTTP_SERVER_ACTIVE_REQUESTS_UNIT: &str = "{request}"; -const HTTP_SERVER_REQUEST_BODY_SIZE_METRIC: &str = semconv::metric::HTTP_SERVER_REQUEST_BODY_SIZE; +const HTTP_SERVER_REQUEST_BODY_SIZE_METRIC: &str = "http.server.request.body.size"; const HTTP_SERVER_REQUEST_BODY_SIZE_UNIT: &str = "By"; -const HTTP_SERVER_RESPONSE_BODY_SIZE_METRIC: &str = semconv::metric::HTTP_SERVER_RESPONSE_BODY_SIZE; +const HTTP_SERVER_RESPONSE_BODY_SIZE_METRIC: &str = "http.server.response.body.size"; const HTTP_SERVER_RESPONSE_BODY_SIZE_UNIT: &str = "By"; -const NETWORK_PROTOCOL_NAME_LABEL: &str = semconv::trace::NETWORK_PROTOCOL_NAME; -const NETWORK_PROTOCOL_VERSION_LABEL: &str = semconv::trace::NETWORK_PROTOCOL_VERSION; -const URL_SCHEME_LABEL: &str = semconv::trace::URL_SCHEME; +const NETWORK_PROTOCOL_NAME_LABEL: &str = "network.protocol.name"; +const NETWORK_PROTOCOL_VERSION_LABEL: &str = "network.protocol.version"; +const URL_SCHEME_LABEL: &str = "url.scheme"; -const HTTP_REQUEST_METHOD_LABEL: &str = semconv::trace::HTTP_REQUEST_METHOD; +const HTTP_REQUEST_METHOD_LABEL: &str = "http.request.method"; #[allow(dead_code)] // cargo check is not smart -const HTTP_ROUTE_LABEL: &str = semconv::trace::HTTP_ROUTE; -const HTTP_RESPONSE_STATUS_CODE_LABEL: &str = semconv::trace::HTTP_RESPONSE_STATUS_CODE; +const HTTP_ROUTE_LABEL: &str = "http.route"; +const HTTP_RESPONSE_STATUS_CODE_LABEL: &str = "http.response.status_code"; /// Trait for extracting custom attributes from HTTP requests pub trait RequestAttributeExtractor: Clone + Send + Sync + 'static { @@ -718,7 +718,7 @@ mod tests { for (idx, metric) in scope_metric.metrics().enumerate() { assert_eq!(metric.name(), metric_names[idx]); } - + // Reset to noop providers to avoid test interference global::set_tracer_provider(opentelemetry_sdk::trace::SdkTracerProvider::default()); global::set_meter_provider(opentelemetry_sdk::metrics::SdkMeterProvider::default()); From 47300fffbf54a8947874e65803f5211be6afa8ff Mon Sep 17 00:00:00 2001 From: Jan Steinke Date: Thu, 28 Aug 2025 14:29:59 +0200 Subject: [PATCH 11/36] fix trait --- opentelemetry-instrumentation-tower/src/lib.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/opentelemetry-instrumentation-tower/src/lib.rs b/opentelemetry-instrumentation-tower/src/lib.rs index 7cb69c525..21ec2fddb 100644 --- a/opentelemetry-instrumentation-tower/src/lib.rs +++ b/opentelemetry-instrumentation-tower/src/lib.rs @@ -142,6 +142,7 @@ pub struct HTTPService { inner_service: S, } +#[derive(Clone)] /// [`Layer`] which applies the OTEL HTTP server metrics and tracing middleware pub struct HTTPLayer { state: Arc, From 8c4081b219a5908d08b51e1a7995d1a5f499639b Mon Sep 17 00:00:00 2001 From: Jan Steinke Date: Thu, 28 Aug 2025 14:43:55 +0200 Subject: [PATCH 12/36] fmt --- .../examples/axum-http-service/src/main.rs | 2 +- .../examples/hyper-http-service/src/main.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/opentelemetry-instrumentation-tower/examples/axum-http-service/src/main.rs b/opentelemetry-instrumentation-tower/examples/axum-http-service/src/main.rs index 47d39265f..df419d383 100644 --- a/opentelemetry-instrumentation-tower/examples/axum-http-service/src/main.rs +++ b/opentelemetry-instrumentation-tower/examples/axum-http-service/src/main.rs @@ -56,7 +56,7 @@ async fn main() { .build(); global::set_meter_provider(meter_provider); - + let otel_metrics_service_layer = otel_tower_metrics::HTTPLayerBuilder::builder() .build() .unwrap(); diff --git a/opentelemetry-instrumentation-tower/examples/hyper-http-service/src/main.rs b/opentelemetry-instrumentation-tower/examples/hyper-http-service/src/main.rs index ad01a6209..e5560d9f5 100644 --- a/opentelemetry-instrumentation-tower/examples/hyper-http-service/src/main.rs +++ b/opentelemetry-instrumentation-tower/examples/hyper-http-service/src/main.rs @@ -61,7 +61,7 @@ async fn main() { .build(); global::set_meter_provider(meter_provider); - + let otel_metrics_service_layer = otel_tower_metrics::HTTPLayerBuilder::builder() .build() .unwrap(); From b4d597a7d2ae0a0ec8f5145b1c9cc62e23076e0b Mon Sep 17 00:00:00 2001 From: Jan Steinke Date: Thu, 28 Aug 2025 14:56:06 +0200 Subject: [PATCH 13/36] update CHANGELOG --- opentelemetry-instrumentation-tower/CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opentelemetry-instrumentation-tower/CHANGELOG.md b/opentelemetry-instrumentation-tower/CHANGELOG.md index f4014e1a8..4fb8f4514 100644 --- a/opentelemetry-instrumentation-tower/CHANGELOG.md +++ b/opentelemetry-instrumentation-tower/CHANGELOG.md @@ -4,7 +4,7 @@ ### Changed -* **BREAKING**: Removed `with_meter()` method. The middleware now uses global meter and tracer providers via `opentelemetry::global::meter()` and `opentelemetry::global::tracer()`. +* **BREAKING**: Removed `with_meter()` method. The middleware now uses global meter and tracer providers by default via `opentelemetry::global::meter()` and `opentelemetry::global::tracer()`, with optional overrides via `with_tracer_provider()` and `with_meter_provider()` methods. * **BREAKING**: Renamed types. Use the new names: - `HTTPMetricsLayer` → `HTTPLayer` - `HTTPMetricsService` → `HTTPService` From 0fd11c5a33fbe7565172f4ecac714be4db06919c Mon Sep 17 00:00:00 2001 From: Jan Steinke Date: Thu, 28 Aug 2025 15:08:58 +0200 Subject: [PATCH 14/36] update examples --- opentelemetry-instrumentation-tower/CHANGELOG.md | 8 +++----- .../examples/axum-http-service/src/main.rs | 14 +++++++------- .../examples/hyper-http-service/src/main.rs | 14 +++++++------- opentelemetry-instrumentation-tower/src/lib.rs | 13 +++++++++++++ 4 files changed, 30 insertions(+), 19 deletions(-) diff --git a/opentelemetry-instrumentation-tower/CHANGELOG.md b/opentelemetry-instrumentation-tower/CHANGELOG.md index 4fb8f4514..a6de97f72 100644 --- a/opentelemetry-instrumentation-tower/CHANGELOG.md +++ b/opentelemetry-instrumentation-tower/CHANGELOG.md @@ -42,16 +42,14 @@ let layer = HTTPMetricsLayerBuilder::builder() After: ```rust -use opentelemetry_instrumentation_tower::HTTPLayerBuilder; +use opentelemetry_instrumentation_tower::HTTPLayer; // Set global providers first global::set_meter_provider(meter_provider); global::set_tracer_provider(tracer_provider); // for tracing support -// Then create the layer without explicit meter -let layer = HTTPLayerBuilder::builder() - .build() - .unwrap(); +// Then create the layer - simple API using global providers +let layer = HTTPLayer::new(); ``` #### Type Name Changes diff --git a/opentelemetry-instrumentation-tower/examples/axum-http-service/src/main.rs b/opentelemetry-instrumentation-tower/examples/axum-http-service/src/main.rs index df419d383..42521cc5c 100644 --- a/opentelemetry-instrumentation-tower/examples/axum-http-service/src/main.rs +++ b/opentelemetry-instrumentation-tower/examples/axum-http-service/src/main.rs @@ -1,7 +1,9 @@ use axum::routing::{get, post, put, Router}; use bytes::Bytes; use opentelemetry::global; -use opentelemetry_instrumentation_tower as otel_tower_metrics; +use opentelemetry_instrumentation_tower::HTTPLayer; +use opentelemetry_otlp::MetricExporter; +use opentelemetry_sdk::metrics::{PeriodicReader, SdkMeterProvider}; use std::time::Duration; const SERVICE_NAME: &str = "example-axum-http-service"; @@ -40,26 +42,24 @@ async fn handle() -> Bytes { #[tokio::main] async fn main() { - let exporter = opentelemetry_otlp::MetricExporter::builder() + let exporter = MetricExporter::builder() .with_tonic() // .with_endpoint("http://localhost:4317") // default; leave out in favor of env var OTEL_EXPORTER_OTLP_ENDPOINT .build() .unwrap(); - let reader = opentelemetry_sdk::metrics::PeriodicReader::builder(exporter) + let reader = PeriodicReader::builder(exporter) .with_interval(_OTEL_METRIC_EXPORT_INTERVAL) .build(); - let meter_provider = opentelemetry_sdk::metrics::SdkMeterProvider::builder() + let meter_provider = SdkMeterProvider::builder() .with_reader(reader) .with_resource(init_otel_resource()) .build(); global::set_meter_provider(meter_provider); - let otel_metrics_service_layer = otel_tower_metrics::HTTPLayerBuilder::builder() - .build() - .unwrap(); + let otel_metrics_service_layer = HTTPLayer::new(); let app = Router::new() .route("/", get(handle)) diff --git a/opentelemetry-instrumentation-tower/examples/hyper-http-service/src/main.rs b/opentelemetry-instrumentation-tower/examples/hyper-http-service/src/main.rs index e5560d9f5..0f4e195dc 100644 --- a/opentelemetry-instrumentation-tower/examples/hyper-http-service/src/main.rs +++ b/opentelemetry-instrumentation-tower/examples/hyper-http-service/src/main.rs @@ -2,7 +2,9 @@ use http_body_util::Full; use hyper::body::Bytes; use hyper::{Request, Response}; use opentelemetry::global; -use opentelemetry_instrumentation_tower as otel_tower_metrics; +use opentelemetry_instrumentation_tower::HTTPLayer; +use opentelemetry_otlp::MetricExporter; +use opentelemetry_sdk::metrics::{PeriodicReader, SdkMeterProvider}; use std::convert::Infallible; use std::net::SocketAddr; use std::time::Duration; @@ -45,26 +47,24 @@ async fn handle(_req: Request) -> Result { response_extractor: ResExt, } +impl HTTPLayer { + /// Create a new HTTP layer with default configuration using global providers + pub fn new() -> Self { + HTTPLayerBuilder::builder().build().unwrap() + } +} + +impl Default for HTTPLayer { + fn default() -> Self { + Self::new() + } +} + pub struct HTTPLayerBuilder { tracer_provider: Option, meter_provider: Option>, From 62425e911f5f42ce72e579d8d246ffec46e71f8d Mon Sep 17 00:00:00 2001 From: Jan Steinke Date: Thu, 28 Aug 2025 15:12:23 +0200 Subject: [PATCH 15/36] update remaining labels to semconv --- opentelemetry-instrumentation-tower/src/lib.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/opentelemetry-instrumentation-tower/src/lib.rs b/opentelemetry-instrumentation-tower/src/lib.rs index 3cac0b93a..4c34b8997 100644 --- a/opentelemetry-instrumentation-tower/src/lib.rs +++ b/opentelemetry-instrumentation-tower/src/lib.rs @@ -46,8 +46,8 @@ const HTTP_SERVER_RESPONSE_BODY_SIZE_METRIC: &str = semconv::metric::HTTP_SERVER const HTTP_SERVER_RESPONSE_BODY_SIZE_UNIT: &str = "By"; const NETWORK_PROTOCOL_NAME_LABEL: &str = semconv::attribute::NETWORK_PROTOCOL_NAME; -const NETWORK_PROTOCOL_VERSION_LABEL: &str = "network.protocol.version"; -const URL_SCHEME_LABEL: &str = "url.scheme"; +const NETWORK_PROTOCOL_VERSION_LABEL: &str = semconv::attribute::NETWORK_PROTOCOL_VERSION; +const URL_SCHEME_LABEL: &str = semconv::attribute::URL_SCHEME; const HTTP_REQUEST_METHOD_LABEL: &str = semconv::attribute::HTTP_REQUEST_METHOD; #[cfg(feature = "axum")] From 4d10b92cd2110430a53936fa4774bafdafaee204 Mon Sep 17 00:00:00 2001 From: Jan Steinke Date: Fri, 29 Aug 2025 09:27:14 +0200 Subject: [PATCH 16/36] upgrade tower-test --- opentelemetry-instrumentation-tower/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opentelemetry-instrumentation-tower/Cargo.toml b/opentelemetry-instrumentation-tower/Cargo.toml index 411dda766..4258d49dd 100644 --- a/opentelemetry-instrumentation-tower/Cargo.toml +++ b/opentelemetry-instrumentation-tower/Cargo.toml @@ -32,7 +32,7 @@ tower-layer = { version = "0.3", default-features = false } opentelemetry_sdk = { workspace = true, features = ["metrics", "testing"] } tokio = { version = "1.0", features = ["macros", "rt"] } tower = { version = "0.5", features = ["util"] } -tower-test = { version = "0.3" } +tower-test = { version = "0.4" } [lints] workspace = true From 42e5223132a0ae8aba09fcce813d3e27aec97ed7 Mon Sep 17 00:00:00 2001 From: Jan Steinke Date: Wed, 29 Oct 2025 11:02:22 +0100 Subject: [PATCH 17/36] move migration guide to vNext --- .../CHANGELOG.md | 46 ++++++++++--------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/opentelemetry-instrumentation-tower/CHANGELOG.md b/opentelemetry-instrumentation-tower/CHANGELOG.md index 01d936fc7..f8b42ddeb 100644 --- a/opentelemetry-instrumentation-tower/CHANGELOG.md +++ b/opentelemetry-instrumentation-tower/CHANGELOG.md @@ -12,30 +12,10 @@ - `HTTPMetricsLayerBuilder` → `HTTPLayerBuilder` * Added OpenTelemetry trace support -## v0.17.0 - -### Changed - -* Update to OpenTelemetry v0.31 -* Migrate to use `opentelemetry-semantic-conventions` package for metric names and attribute keys instead of hardcoded strings -* Add dependency on otel semantic conventions crate and use constants from it instead of hardcoded attribute names. The values are unchanged - - `HTTP_SERVER_ACTIVE_REQUESTS_METRIC` now uses `semconv::metric::HTTP_SERVER_ACTIVE_REQUESTS` - - `HTTP_SERVER_REQUEST_BODY_SIZE_METRIC` now uses `semconv::metric::HTTP_SERVER_REQUEST_BODY_SIZE` - - `HTTP_SERVER_RESPONSE_BODY_SIZE_METRIC` now uses `semconv::metric::HTTP_SERVER_RESPONSE_BODY_SIZE` - - `HTTP_SERVER_DURATION_METRIC` now uses `semconv::metric::HTTP_SERVER_REQUEST_DURATION` -* Update attribute keys to use semantic conventions constants: - - `NETWORK_PROTOCOL_NAME_LABEL` now uses `semconv::attribute::NETWORK_PROTOCOL_NAME` - - `HTTP_REQUEST_METHOD_LABEL` now uses `semconv::attribute::HTTP_REQUEST_METHOD` - - `HTTP_ROUTE_LABEL` now uses `semconv::attribute::HTTP_ROUTE` - - `HTTP_RESPONSE_STATUS_CODE_LABEL` now uses `semconv::attribute::HTTP_RESPONSE_STATUS_CODE` - -### Added - -* Add comprehensive test coverage for all HTTP server metrics with attribute validation - ### Migration Guide #### API Changes + Before: ```rust use opentelemetry_instrumentation_tower::HTTPMetricsLayerBuilder; @@ -50,7 +30,7 @@ After: ```rust use opentelemetry_instrumentation_tower::HTTPLayer; -// Set global providers first +// Set global providers global::set_meter_provider(meter_provider); global::set_tracer_provider(tracer_provider); // for tracing support @@ -59,11 +39,33 @@ let layer = HTTPLayer::new(); ``` #### Type Name Changes + - Replace `HTTPMetricsLayerBuilder` with `HTTPLayerBuilder` - Replace `HTTPMetricsLayer` with `HTTPLayer` - Replace `HTTPMetricsService` with `HTTPService` - Replace `HTTPMetricsResponseFuture` with `HTTPResponseFuture` +## v0.17.0 + +### Changed + +* Update to OpenTelemetry v0.31 +* Migrate to use `opentelemetry-semantic-conventions` package for metric names and attribute keys instead of hardcoded strings +* Add dependency on otel semantic conventions crate and use constants from it instead of hardcoded attribute names. The values are unchanged + - `HTTP_SERVER_ACTIVE_REQUESTS_METRIC` now uses `semconv::metric::HTTP_SERVER_ACTIVE_REQUESTS` + - `HTTP_SERVER_REQUEST_BODY_SIZE_METRIC` now uses `semconv::metric::HTTP_SERVER_REQUEST_BODY_SIZE` + - `HTTP_SERVER_RESPONSE_BODY_SIZE_METRIC` now uses `semconv::metric::HTTP_SERVER_RESPONSE_BODY_SIZE` + - `HTTP_SERVER_DURATION_METRIC` now uses `semconv::metric::HTTP_SERVER_REQUEST_DURATION` +* Update attribute keys to use semantic conventions constants: + - `NETWORK_PROTOCOL_NAME_LABEL` now uses `semconv::attribute::NETWORK_PROTOCOL_NAME` + - `HTTP_REQUEST_METHOD_LABEL` now uses `semconv::attribute::HTTP_REQUEST_METHOD` + - `HTTP_ROUTE_LABEL` now uses `semconv::attribute::HTTP_ROUTE` + - `HTTP_RESPONSE_STATUS_CODE_LABEL` now uses `semconv::attribute::HTTP_RESPONSE_STATUS_CODE` + +### Added + +* Add comprehensive test coverage for all HTTP server metrics with attribute validation + ## v0.16.0 Initial release of OpenTelemetry Tower instrumentation middleware for HTTP metrics collection. From d0ec3cccda449f8e2a4f2f6a40e1478ed87f1e3e Mon Sep 17 00:00:00 2001 From: Jan Steinke Date: Tue, 25 Nov 2025 21:31:05 +0100 Subject: [PATCH 18/36] don't set Ok --- opentelemetry-instrumentation-tower/src/lib.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/opentelemetry-instrumentation-tower/src/lib.rs b/opentelemetry-instrumentation-tower/src/lib.rs index 7951a46a4..73c105eb8 100644 --- a/opentelemetry-instrumentation-tower/src/lib.rs +++ b/opentelemetry-instrumentation-tower/src/lib.rs @@ -566,8 +566,6 @@ where span.set_status(Status::Error { description: format!("HTTP {}", status.as_u16()).into(), }); - } else { - span.set_status(Status::Ok); } span.end(); From e03beb23e3e61b3794ba5f59c71a8766dda7d3ea Mon Sep 17 00:00:00 2001 From: Jan Steinke Date: Tue, 25 Nov 2025 21:36:05 +0100 Subject: [PATCH 19/36] rename layer in example --- .../examples/axum-http-service/src/main.rs | 4 ++-- .../examples/hyper-http-service/src/main.rs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/opentelemetry-instrumentation-tower/examples/axum-http-service/src/main.rs b/opentelemetry-instrumentation-tower/examples/axum-http-service/src/main.rs index 42521cc5c..9ce6b8723 100644 --- a/opentelemetry-instrumentation-tower/examples/axum-http-service/src/main.rs +++ b/opentelemetry-instrumentation-tower/examples/axum-http-service/src/main.rs @@ -59,13 +59,13 @@ async fn main() { global::set_meter_provider(meter_provider); - let otel_metrics_service_layer = HTTPLayer::new(); + let otel_service_layer = HTTPLayer::new(); let app = Router::new() .route("/", get(handle)) .route("/", post(handle)) .route("/", put(handle)) - .layer(otel_metrics_service_layer); + .layer(otel_service_layer); let listener = tokio::net::TcpListener::bind("0.0.0.0:5000").await.unwrap(); let server = axum::serve(listener, app); diff --git a/opentelemetry-instrumentation-tower/examples/hyper-http-service/src/main.rs b/opentelemetry-instrumentation-tower/examples/hyper-http-service/src/main.rs index 0f4e195dc..2e0d0c745 100644 --- a/opentelemetry-instrumentation-tower/examples/hyper-http-service/src/main.rs +++ b/opentelemetry-instrumentation-tower/examples/hyper-http-service/src/main.rs @@ -64,10 +64,10 @@ async fn main() { global::set_meter_provider(meter_provider); - let otel_metrics_service_layer = HTTPLayer::new(); + let otel_service_layer = HTTPLayer::new(); let tower_service = ServiceBuilder::new() - .layer(otel_metrics_service_layer) + .layer(otel_service_layer) .service_fn(handle); let hyper_service = hyper_util::service::TowerToHyperService::new(tower_service); From eaf22d3d8a7f518283f71c34374984f567082a3c Mon Sep 17 00:00:00 2001 From: Jan Steinke Date: Tue, 25 Nov 2025 21:45:53 +0100 Subject: [PATCH 20/36] setup also tracing in examples --- .../examples/axum-http-service/Cargo.toml | 4 +- .../examples/axum-http-service/src/main.rs | 50 +++++++++++++------ .../examples/hyper-http-service/Cargo.toml | 4 +- .../examples/hyper-http-service/src/main.rs | 46 +++++++++++------ 4 files changed, 71 insertions(+), 33 deletions(-) diff --git a/opentelemetry-instrumentation-tower/examples/axum-http-service/Cargo.toml b/opentelemetry-instrumentation-tower/examples/axum-http-service/Cargo.toml index 986bb69d3..f62e6f8de 100644 --- a/opentelemetry-instrumentation-tower/examples/axum-http-service/Cargo.toml +++ b/opentelemetry-instrumentation-tower/examples/axum-http-service/Cargo.toml @@ -12,9 +12,9 @@ axum = { features = ["http1", "tokio"], version = "0.8", default-features = fals bytes = { version = "1", default-features = false } opentelemetry = { workspace = true} opentelemetry_sdk = { workspace = true, default-features = false } -opentelemetry-otlp = { version = "0.31.0", features = ["grpc-tonic", "metrics"], default-features = false } +opentelemetry-otlp = { version = "0.31.0", features = ["grpc-tonic", "metrics", "trace"], default-features = false } tokio = { version = "1", features = ["rt-multi-thread"], default-features = false } rand_09 = { package = "rand", version = "0.9" } [lints] -workspace = true \ No newline at end of file +workspace = true diff --git a/opentelemetry-instrumentation-tower/examples/axum-http-service/src/main.rs b/opentelemetry-instrumentation-tower/examples/axum-http-service/src/main.rs index 9ce6b8723..1718387aa 100644 --- a/opentelemetry-instrumentation-tower/examples/axum-http-service/src/main.rs +++ b/opentelemetry-instrumentation-tower/examples/axum-http-service/src/main.rs @@ -2,8 +2,11 @@ use axum::routing::{get, post, put, Router}; use bytes::Bytes; use opentelemetry::global; use opentelemetry_instrumentation_tower::HTTPLayer; -use opentelemetry_otlp::MetricExporter; -use opentelemetry_sdk::metrics::{PeriodicReader, SdkMeterProvider}; +use opentelemetry_otlp::{MetricExporter, SpanExporter}; +use opentelemetry_sdk::{ + metrics::{PeriodicReader, SdkMeterProvider}, + trace::SdkTracerProvider, +}; use std::time::Duration; const SERVICE_NAME: &str = "example-axum-http-service"; @@ -42,22 +45,39 @@ async fn handle() -> Bytes { #[tokio::main] async fn main() { - let exporter = MetricExporter::builder() - .with_tonic() - // .with_endpoint("http://localhost:4317") // default; leave out in favor of env var OTEL_EXPORTER_OTLP_ENDPOINT - .build() - .unwrap(); + { + let exporter = MetricExporter::builder() + .with_tonic() + // .with_endpoint("http://localhost:4317") // default; leave out in favor of env var OTEL_EXPORTER_OTLP_ENDPOINT + .build() + .unwrap(); + + let reader = PeriodicReader::builder(exporter) + .with_interval(_OTEL_METRIC_EXPORT_INTERVAL) + .build(); + + let provider = SdkMeterProvider::builder() + .with_reader(reader) + .with_resource(init_otel_resource()) + .build(); - let reader = PeriodicReader::builder(exporter) - .with_interval(_OTEL_METRIC_EXPORT_INTERVAL) - .build(); + global::set_meter_provider(provider); + } + + { + let exporter = SpanExporter::builder() + .with_tonic() + // .with_endpoint("http://localhost:4317") // default; leave out in favor of env var OTEL_EXPORTER_OTLP_ENDPOINT + .build() + .unwrap(); - let meter_provider = SdkMeterProvider::builder() - .with_reader(reader) - .with_resource(init_otel_resource()) - .build(); + let provider = SdkTracerProvider::builder() + .with_batch_exporter(exporter) + .with_resource(init_otel_resource()) + .build(); - global::set_meter_provider(meter_provider); + global::set_tracer_provider(provider); + } let otel_service_layer = HTTPLayer::new(); diff --git a/opentelemetry-instrumentation-tower/examples/hyper-http-service/Cargo.toml b/opentelemetry-instrumentation-tower/examples/hyper-http-service/Cargo.toml index b04da5798..943b4ff5a 100644 --- a/opentelemetry-instrumentation-tower/examples/hyper-http-service/Cargo.toml +++ b/opentelemetry-instrumentation-tower/examples/hyper-http-service/Cargo.toml @@ -13,10 +13,10 @@ http-body-util = { version = "0.1", default-features = false } hyper-util = { version = "0.1", features = ["http1", "service", "server", "tokio"], default-features = false } opentelemetry = { workspace = true} opentelemetry_sdk = { workspace = true, default-features = false } -opentelemetry-otlp = { version = "0.31.0", features = ["grpc-tonic", "metrics"], default-features = false } +opentelemetry-otlp = { version = "0.31.0", features = ["grpc-tonic", "metrics", "trace"], default-features = false } tokio = { version = "1", features = ["rt-multi-thread", "macros"], default-features = false } tower = { version = "0.5", default-features = false } rand_09 = { package = "rand", version = "0.9" } [lints] -workspace = true \ No newline at end of file +workspace = true diff --git a/opentelemetry-instrumentation-tower/examples/hyper-http-service/src/main.rs b/opentelemetry-instrumentation-tower/examples/hyper-http-service/src/main.rs index 2e0d0c745..8134f3fcb 100644 --- a/opentelemetry-instrumentation-tower/examples/hyper-http-service/src/main.rs +++ b/opentelemetry-instrumentation-tower/examples/hyper-http-service/src/main.rs @@ -3,8 +3,9 @@ use hyper::body::Bytes; use hyper::{Request, Response}; use opentelemetry::global; use opentelemetry_instrumentation_tower::HTTPLayer; -use opentelemetry_otlp::MetricExporter; +use opentelemetry_otlp::{MetricExporter, SpanExporter}; use opentelemetry_sdk::metrics::{PeriodicReader, SdkMeterProvider}; +use opentelemetry_sdk::trace::SdkTracerProvider; use std::convert::Infallible; use std::net::SocketAddr; use std::time::Duration; @@ -47,22 +48,39 @@ async fn handle(_req: Request) -> Result Date: Tue, 25 Nov 2025 22:49:07 +0100 Subject: [PATCH 21/36] extract parent span --- opentelemetry-instrumentation-tower/src/lib.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/opentelemetry-instrumentation-tower/src/lib.rs b/opentelemetry-instrumentation-tower/src/lib.rs index 73c105eb8..c88aa1f35 100644 --- a/opentelemetry-instrumentation-tower/src/lib.rs +++ b/opentelemetry-instrumentation-tower/src/lib.rs @@ -450,7 +450,11 @@ where // Extract custom request attributes let custom_request_attributes = self.request_extractor.extract_attributes(&req); - // Start tracing span + // Extract the context from the incoming request headers + let parent_cx = global::get_text_map_propagator(|propagator| { + propagator.extract(&HeaderExtractor(req.headers())) + }); + let mut span_attributes = vec![ KeyValue::new(semconv::trace::HTTP_REQUEST_METHOD, method.clone()), url_scheme_kv.clone(), @@ -484,7 +488,7 @@ where .span_builder(span_name) .with_kind(SpanKind::Server) .with_attributes(span_attributes) - .start(&tracer); + .start_with_context(&tracer, &parent_cx); let cx = OtelContext::current_with_span(span); From 76702fb9300e08f2ed4b2e3143c24e94fd437c4f Mon Sep 17 00:00:00 2001 From: Jan Steinke Date: Tue, 25 Nov 2025 22:50:17 +0100 Subject: [PATCH 22/36] add imports and Cargo dependency for HTTP extractor --- opentelemetry-instrumentation-tower/Cargo.toml | 2 +- opentelemetry-instrumentation-tower/src/lib.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/opentelemetry-instrumentation-tower/Cargo.toml b/opentelemetry-instrumentation-tower/Cargo.toml index d5ea153c7..4951754a7 100644 --- a/opentelemetry-instrumentation-tower/Cargo.toml +++ b/opentelemetry-instrumentation-tower/Cargo.toml @@ -22,7 +22,7 @@ futures-util = { version = "0.3", default-features = false } http = { version = "1", features = ["std"], default-features = false } http-body = { version = "1", default-features = false } opentelemetry = { workspace = true, features = ["futures", "metrics", "trace"] } -opentelemetry_sdk = { workspace = true, features = ["trace"] } +opentelemetry-http = "0.31" opentelemetry-semantic-conventions = { workspace = true, features = ["semconv_experimental"] } pin-project-lite = { version = "0.2", default-features = false } tower-service = { version = "0.3", default-features = false } diff --git a/opentelemetry-instrumentation-tower/src/lib.rs b/opentelemetry-instrumentation-tower/src/lib.rs index c88aa1f35..540df1b53 100644 --- a/opentelemetry-instrumentation-tower/src/lib.rs +++ b/opentelemetry-instrumentation-tower/src/lib.rs @@ -16,7 +16,7 @@ use opentelemetry::metrics::{Histogram, MeterProvider, UpDownCounter}; use opentelemetry::trace::{SpanKind, Status, TraceContextExt, Tracer, TracerProvider}; use opentelemetry::Context as OtelContext; use opentelemetry::KeyValue; -use opentelemetry_sdk::trace::SdkTracerProvider; +use opentelemetry_http::HeaderExtractor; use opentelemetry_semantic_conventions as semconv; use pin_project_lite::pin_project; use tower_layer::Layer; From 54397098f9fb4ad0bc99a3ae3886c89ec7d123d4 Mon Sep 17 00:00:00 2001 From: Jan Steinke Date: Tue, 25 Nov 2025 22:58:27 +0100 Subject: [PATCH 23/36] attach and activate Context --- opentelemetry-instrumentation-tower/src/lib.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/opentelemetry-instrumentation-tower/src/lib.rs b/opentelemetry-instrumentation-tower/src/lib.rs index 540df1b53..4ce703c2b 100644 --- a/opentelemetry-instrumentation-tower/src/lib.rs +++ b/opentelemetry-instrumentation-tower/src/lib.rs @@ -391,7 +391,7 @@ struct ResponseFutureState { custom_request_attributes: Vec, // Tracing fields - otel_context: OtelContext, + guard: ContextGuard, } pin_project! { @@ -491,6 +491,7 @@ where .start_with_context(&tracer, &parent_cx); let cx = OtelContext::current_with_span(span); + let guard = cx.attach(); self.state .server_active_requests @@ -510,7 +511,7 @@ where route_kv_opt, custom_request_attributes, - otel_context: cx, + guard, }, response_extractor: self.response_extractor.clone(), } @@ -551,7 +552,8 @@ where label_superset.extend(custom_response_attributes.clone()); // Update span - let span = this.future_state.otel_context.span(); + let cx = OtelContext::current(); + let span = cx.span(); span.set_attribute(KeyValue::new( semconv::trace::HTTP_RESPONSE_STATUS_CODE, status.as_u16() as i64, From 1a0a77ee07cc3dc68301b224ebed4559ca516c3e Mon Sep 17 00:00:00 2001 From: Jan Steinke Date: Tue, 25 Nov 2025 23:01:48 +0100 Subject: [PATCH 24/36] add ContextGuard import --- opentelemetry-instrumentation-tower/src/lib.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/opentelemetry-instrumentation-tower/src/lib.rs b/opentelemetry-instrumentation-tower/src/lib.rs index 4ce703c2b..0b893afee 100644 --- a/opentelemetry-instrumentation-tower/src/lib.rs +++ b/opentelemetry-instrumentation-tower/src/lib.rs @@ -15,6 +15,7 @@ use opentelemetry::global::{self, BoxedTracer}; use opentelemetry::metrics::{Histogram, MeterProvider, UpDownCounter}; use opentelemetry::trace::{SpanKind, Status, TraceContextExt, Tracer, TracerProvider}; use opentelemetry::Context as OtelContext; +use opentelemetry::ContextGuard; use opentelemetry::KeyValue; use opentelemetry_http::HeaderExtractor; use opentelemetry_semantic_conventions as semconv; From 6a56a2ed1d7aea16dd1d3b90ae19374968dacfcf Mon Sep 17 00:00:00 2001 From: Jan Steinke Date: Tue, 25 Nov 2025 23:03:01 +0100 Subject: [PATCH 25/36] re-add sdk for now to keep it compiling --- opentelemetry-instrumentation-tower/Cargo.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/opentelemetry-instrumentation-tower/Cargo.toml b/opentelemetry-instrumentation-tower/Cargo.toml index 4951754a7..2c4ac4b02 100644 --- a/opentelemetry-instrumentation-tower/Cargo.toml +++ b/opentelemetry-instrumentation-tower/Cargo.toml @@ -22,6 +22,7 @@ futures-util = { version = "0.3", default-features = false } http = { version = "1", features = ["std"], default-features = false } http-body = { version = "1", default-features = false } opentelemetry = { workspace = true, features = ["futures", "metrics", "trace"] } +opentelemetry_sdk = { workspace = true, features = ["trace"] } opentelemetry-http = "0.31" opentelemetry-semantic-conventions = { workspace = true, features = ["semconv_experimental"] } pin-project-lite = { version = "0.2", default-features = false } From 72481392d9174076f64adb3f3cbe73e09a96491a Mon Sep 17 00:00:00 2001 From: Jan Steinke Date: Tue, 25 Nov 2025 23:04:27 +0100 Subject: [PATCH 26/36] re-add sdk for now to keep it compiling --- opentelemetry-instrumentation-tower/src/lib.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/opentelemetry-instrumentation-tower/src/lib.rs b/opentelemetry-instrumentation-tower/src/lib.rs index 0b893afee..ba4d8e0ad 100644 --- a/opentelemetry-instrumentation-tower/src/lib.rs +++ b/opentelemetry-instrumentation-tower/src/lib.rs @@ -18,6 +18,7 @@ use opentelemetry::Context as OtelContext; use opentelemetry::ContextGuard; use opentelemetry::KeyValue; use opentelemetry_http::HeaderExtractor; +use opentelemetry_sdk::trace::SdkTracerProvider; use opentelemetry_semantic_conventions as semconv; use pin_project_lite::pin_project; use tower_layer::Layer; From bb9b92f3d63b4f76e7bff66cee4080a72cd3ec5a Mon Sep 17 00:00:00 2001 From: Jan Steinke Date: Tue, 25 Nov 2025 23:18:03 +0100 Subject: [PATCH 27/36] Revert "attach and activate Context" This reverts commit 54397098f9fb4ad0bc99a3ae3886c89ec7d123d4. --- opentelemetry-instrumentation-tower/src/lib.rs | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/opentelemetry-instrumentation-tower/src/lib.rs b/opentelemetry-instrumentation-tower/src/lib.rs index ba4d8e0ad..9d0a7ed2b 100644 --- a/opentelemetry-instrumentation-tower/src/lib.rs +++ b/opentelemetry-instrumentation-tower/src/lib.rs @@ -393,7 +393,7 @@ struct ResponseFutureState { custom_request_attributes: Vec, // Tracing fields - guard: ContextGuard, + otel_context: OtelContext, } pin_project! { @@ -493,7 +493,6 @@ where .start_with_context(&tracer, &parent_cx); let cx = OtelContext::current_with_span(span); - let guard = cx.attach(); self.state .server_active_requests @@ -513,7 +512,7 @@ where route_kv_opt, custom_request_attributes, - guard, + otel_context: cx, }, response_extractor: self.response_extractor.clone(), } @@ -554,8 +553,7 @@ where label_superset.extend(custom_response_attributes.clone()); // Update span - let cx = OtelContext::current(); - let span = cx.span(); + let span = this.future_state.otel_context.span(); span.set_attribute(KeyValue::new( semconv::trace::HTTP_RESPONSE_STATUS_CODE, status.as_u16() as i64, From 36fe7846e00c90bfcaa611115134179c66dd160a Mon Sep 17 00:00:00 2001 From: Jan Steinke Date: Tue, 25 Nov 2025 23:18:43 +0100 Subject: [PATCH 28/36] Revert "add ContextGuard import" This reverts commit 1a0a77ee07cc3dc68301b224ebed4559ca516c3e. --- opentelemetry-instrumentation-tower/src/lib.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/opentelemetry-instrumentation-tower/src/lib.rs b/opentelemetry-instrumentation-tower/src/lib.rs index 9d0a7ed2b..feb822c8d 100644 --- a/opentelemetry-instrumentation-tower/src/lib.rs +++ b/opentelemetry-instrumentation-tower/src/lib.rs @@ -15,7 +15,6 @@ use opentelemetry::global::{self, BoxedTracer}; use opentelemetry::metrics::{Histogram, MeterProvider, UpDownCounter}; use opentelemetry::trace::{SpanKind, Status, TraceContextExt, Tracer, TracerProvider}; use opentelemetry::Context as OtelContext; -use opentelemetry::ContextGuard; use opentelemetry::KeyValue; use opentelemetry_http::HeaderExtractor; use opentelemetry_sdk::trace::SdkTracerProvider; From d9189c10cf69865165e798d4d2577312ec6926dc Mon Sep 17 00:00:00 2001 From: Jan Steinke Date: Tue, 25 Nov 2025 23:37:06 +0100 Subject: [PATCH 29/36] remove sdk dependency --- .../Cargo.toml | 1 - .../src/lib.rs | 82 ++++++++----------- 2 files changed, 34 insertions(+), 49 deletions(-) diff --git a/opentelemetry-instrumentation-tower/Cargo.toml b/opentelemetry-instrumentation-tower/Cargo.toml index 2c4ac4b02..4951754a7 100644 --- a/opentelemetry-instrumentation-tower/Cargo.toml +++ b/opentelemetry-instrumentation-tower/Cargo.toml @@ -22,7 +22,6 @@ futures-util = { version = "0.3", default-features = false } http = { version = "1", features = ["std"], default-features = false } http-body = { version = "1", default-features = false } opentelemetry = { workspace = true, features = ["futures", "metrics", "trace"] } -opentelemetry_sdk = { workspace = true, features = ["trace"] } opentelemetry-http = "0.31" opentelemetry-semantic-conventions = { workspace = true, features = ["semconv_experimental"] } pin-project-lite = { version = "0.2", default-features = false } diff --git a/opentelemetry-instrumentation-tower/src/lib.rs b/opentelemetry-instrumentation-tower/src/lib.rs index feb822c8d..6f6b9a671 100644 --- a/opentelemetry-instrumentation-tower/src/lib.rs +++ b/opentelemetry-instrumentation-tower/src/lib.rs @@ -12,12 +12,13 @@ use std::{fmt, result}; use axum::extract::MatchedPath; use futures_util::ready; use opentelemetry::global::{self, BoxedTracer}; -use opentelemetry::metrics::{Histogram, MeterProvider, UpDownCounter}; -use opentelemetry::trace::{SpanKind, Status, TraceContextExt, Tracer, TracerProvider}; +use opentelemetry::metrics::Meter; +use opentelemetry::metrics::MeterProvider; +use opentelemetry::metrics::{Histogram, UpDownCounter}; +use opentelemetry::trace::{SpanKind, Status, TraceContextExt, Tracer}; use opentelemetry::Context as OtelContext; use opentelemetry::KeyValue; use opentelemetry_http::HeaderExtractor; -use opentelemetry_sdk::trace::SdkTracerProvider; use opentelemetry_semantic_conventions as semconv; use pin_project_lite::pin_project; use tower_layer::Layer; @@ -124,14 +125,11 @@ where } /// State scoped to the entire middleware Layer. -/// -/// Holds metrics instruments. struct HTTPLayerState { pub server_request_duration: Histogram, pub server_active_requests: UpDownCounter, pub server_request_body_size: Histogram, pub server_response_body_size: Histogram, - pub tracer_provider: Option, } #[derive(Clone)] @@ -141,6 +139,7 @@ pub struct HTTPService { request_extractor: ReqExt, response_extractor: ResExt, inner_service: S, + tracer: Arc, } #[derive(Clone)] @@ -149,6 +148,7 @@ pub struct HTTPLayer { state: Arc, request_extractor: ReqExt, response_extractor: ResExt, + tracer: Arc, } impl HTTPLayer { @@ -165,8 +165,8 @@ impl Default for HTTPLayer { } pub struct HTTPLayerBuilder { - tracer_provider: Option, - meter_provider: Option>, + tracer_provider: Option, + meter: Option, req_dur_bounds: Option>, request_extractor: ReqExt, response_extractor: ResExt, @@ -212,7 +212,7 @@ impl HTTPLayerBuilder { pub fn builder() -> Self { HTTPLayerBuilder { tracer_provider: None, - meter_provider: None, + meter: None, req_dur_bounds: Some(LIBRARY_DEFAULT_HTTP_SERVER_DURATION_BOUNDARIES.to_vec()), request_extractor: NoOpExtractor, response_extractor: NoOpExtractor, @@ -230,7 +230,7 @@ impl HTTPLayerBuilder { NewReqExt: RequestAttributeExtractor, { HTTPLayerBuilder { - meter_provider: self.meter_provider, + meter: self.meter, tracer_provider: self.tracer_provider, req_dur_bounds: self.req_dur_bounds, request_extractor: extractor, @@ -247,7 +247,7 @@ impl HTTPLayerBuilder { NewResExt: ResponseAttributeExtractor, { HTTPLayerBuilder { - meter_provider: self.meter_provider, + meter: self.meter, tracer_provider: self.tracer_provider, req_dur_bounds: self.req_dur_bounds, request_extractor: self.request_extractor, @@ -282,14 +282,20 @@ impl HTTPLayerBuilder { .req_dur_bounds .unwrap_or_else(|| LIBRARY_DEFAULT_HTTP_SERVER_DURATION_BOUNDARIES.to_vec()); + let tracer: BoxedTracer = self + .tracer_provider + .unwrap_or_else(|| global::tracer("opentelemetry-instrumentation-tower")); + let tracer = Arc::new(tracer); + + let meter: Meter = self + .meter + .unwrap_or_else(|| global::meter("opentelemetry-instrumentation-tower")); + Ok(HTTPLayer { - state: Arc::from(Self::make_state( - self.meter_provider, - self.tracer_provider, - req_dur_bounds, - )), + state: Arc::from(Self::make_state(meter, req_dur_bounds)), request_extractor: self.request_extractor, response_extractor: self.response_extractor, + tracer: tracer, }) } @@ -304,26 +310,11 @@ impl HTTPLayerBuilder { where M: MeterProvider + Send + Sync + 'static, { - self.meter_provider = Some(Box::new(provider)); - self - } - - /// Set a meter provider to use for creating a meter. - /// If none is specified, the global provider is used. - pub fn with_tracer_provider(mut self, provider: SdkTracerProvider) -> Self { - self.tracer_provider = Some(provider); + self.meter = Some(provider.meter("opentelemetry-instrumentation-tower")); self } - fn make_state( - meter_provider: Option>, - tracer_provider: Option, - req_dur_bounds: Vec, - ) -> HTTPLayerState { - let meter = match meter_provider { - Some(provider) => provider.meter("opentelemetry-instrumentation-tower"), - None => global::meter("opentelemetry-instrumentation-tower"), - }; + fn make_state(meter: Meter, req_dur_bounds: Vec) -> HTTPLayerState { HTTPLayerState { server_request_duration: meter .f64_histogram(Cow::from(HTTP_SERVER_DURATION_METRIC)) @@ -346,7 +337,6 @@ impl HTTPLayerBuilder { .with_description("Size of HTTP server response bodies.") .with_unit(HTTP_SERVER_RESPONSE_BODY_SIZE_UNIT) .build(), - tracer_provider, } } } @@ -364,6 +354,7 @@ where request_extractor: self.request_extractor.clone(), response_extractor: self.response_extractor.clone(), inner_service: service, + tracer: self.tracer.clone(), } } } @@ -478,18 +469,12 @@ where let span_name = format!("{} {}", method, req.uri().path()); - let tracer = match &self.state.tracer_provider { - Some(tp) => { - BoxedTracer::new(Box::new(tp.tracer("opentelemetry-instrumentation-tower"))) - } - None => global::tracer("opentelemetry-instrumentation-tower"), - }; - - let span = tracer + let span = self + .tracer .span_builder(span_name) .with_kind(SpanKind::Server) .with_attributes(span_attributes) - .start_with_context(&tracer, &parent_cx); + .start_with_context(self.tracer.as_ref(), &parent_cx); let cx = OtelContext::current_with_span(span); @@ -623,6 +608,7 @@ mod tests { use super::*; use http::{Request, Response, StatusCode}; + use opentelemetry::trace::TracerProvider; use opentelemetry::trace::{FutureExt, TraceContextExt, Tracer}; use opentelemetry_sdk::metrics::SdkMeterProvider; use opentelemetry_sdk::metrics::{ @@ -641,12 +627,12 @@ mod tests { .with_simple_exporter(trace_exporter.clone()) .build(); - let tracer = tracer_provider.tracer("test_tracer"); + let tracer = Arc::new(BoxedTracer::new(Box::new( + tracer_provider.tracer("test_tracer"), + ))); - let layer = HTTPLayerBuilder::builder() - .with_tracer_provider(tracer_provider.clone()) - .build() - .unwrap(); + let mut layer = HTTPLayerBuilder::builder().build().unwrap(); + layer.tracer = tracer.clone(); let mut service = ServiceBuilder::new() .layer(layer) From 0394821274fa094703e497c0cc87c81023cf1c0b Mon Sep 17 00:00:00 2001 From: Jan Steinke Date: Tue, 25 Nov 2025 23:44:39 +0100 Subject: [PATCH 30/36] remove tracer from builder --- opentelemetry-instrumentation-tower/src/lib.rs | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/opentelemetry-instrumentation-tower/src/lib.rs b/opentelemetry-instrumentation-tower/src/lib.rs index 6f6b9a671..54f85170b 100644 --- a/opentelemetry-instrumentation-tower/src/lib.rs +++ b/opentelemetry-instrumentation-tower/src/lib.rs @@ -165,7 +165,6 @@ impl Default for HTTPLayer { } pub struct HTTPLayerBuilder { - tracer_provider: Option, meter: Option, req_dur_bounds: Option>, request_extractor: ReqExt, @@ -211,7 +210,6 @@ impl fmt::Debug for Error { impl HTTPLayerBuilder { pub fn builder() -> Self { HTTPLayerBuilder { - tracer_provider: None, meter: None, req_dur_bounds: Some(LIBRARY_DEFAULT_HTTP_SERVER_DURATION_BOUNDARIES.to_vec()), request_extractor: NoOpExtractor, @@ -231,7 +229,6 @@ impl HTTPLayerBuilder { { HTTPLayerBuilder { meter: self.meter, - tracer_provider: self.tracer_provider, req_dur_bounds: self.req_dur_bounds, request_extractor: extractor, response_extractor: self.response_extractor, @@ -248,7 +245,6 @@ impl HTTPLayerBuilder { { HTTPLayerBuilder { meter: self.meter, - tracer_provider: self.tracer_provider, req_dur_bounds: self.req_dur_bounds, request_extractor: self.request_extractor, response_extractor: extractor, @@ -282,10 +278,7 @@ impl HTTPLayerBuilder { .req_dur_bounds .unwrap_or_else(|| LIBRARY_DEFAULT_HTTP_SERVER_DURATION_BOUNDARIES.to_vec()); - let tracer: BoxedTracer = self - .tracer_provider - .unwrap_or_else(|| global::tracer("opentelemetry-instrumentation-tower")); - let tracer = Arc::new(tracer); + let tracer = Arc::new(global::tracer("opentelemetry-instrumentation-tower")); let meter: Meter = self .meter From 6da62ae5fdfbcb12e6f00621312a95fd463d4254 Mon Sep 17 00:00:00 2001 From: Jan Steinke Date: Tue, 25 Nov 2025 23:52:03 +0100 Subject: [PATCH 31/36] undo meter changes --- opentelemetry-instrumentation-tower/src/lib.rs | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/opentelemetry-instrumentation-tower/src/lib.rs b/opentelemetry-instrumentation-tower/src/lib.rs index 54f85170b..dd593c321 100644 --- a/opentelemetry-instrumentation-tower/src/lib.rs +++ b/opentelemetry-instrumentation-tower/src/lib.rs @@ -13,7 +13,6 @@ use axum::extract::MatchedPath; use futures_util::ready; use opentelemetry::global::{self, BoxedTracer}; use opentelemetry::metrics::Meter; -use opentelemetry::metrics::MeterProvider; use opentelemetry::metrics::{Histogram, UpDownCounter}; use opentelemetry::trace::{SpanKind, Status, TraceContextExt, Tracer}; use opentelemetry::Context as OtelContext; @@ -292,18 +291,13 @@ impl HTTPLayerBuilder { }) } - pub fn with_request_duration_bounds(mut self, bounds: Vec) -> Self { - self.req_dur_bounds = Some(bounds); + pub fn with_meter(mut self, meter: Meter) -> Self { + self.meter = Some(meter); self } - /// Set a meter provider to use for creating a meter. - /// If none is specified, the global provider is used. - pub fn with_meter_provider(mut self, provider: M) -> Self - where - M: MeterProvider + Send + Sync + 'static, - { - self.meter = Some(provider.meter("opentelemetry-instrumentation-tower")); + pub fn with_request_duration_bounds(mut self, bounds: Vec) -> Self { + self.req_dur_bounds = Some(bounds); self } @@ -717,10 +711,10 @@ mod tests { .with_interval(Duration::from_millis(100)) .build(); let meter_provider = SdkMeterProvider::builder().with_reader(reader).build(); + let meter = meter_provider.meter("test"); - // Use the new API with optional provider override instead of global providers let layer = HTTPLayerBuilder::builder() - .with_meter_provider(meter_provider.clone()) + .with_meter(meter) .build() .unwrap(); From ae516e6aacfb238ecf3127ae0d121cb21f8a606a Mon Sep 17 00:00:00 2001 From: Jan Steinke Date: Wed, 26 Nov 2025 10:28:11 +0100 Subject: [PATCH 32/36] smaller fixups --- opentelemetry-instrumentation-tower/src/lib.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/opentelemetry-instrumentation-tower/src/lib.rs b/opentelemetry-instrumentation-tower/src/lib.rs index dd593c321..48b53fac0 100644 --- a/opentelemetry-instrumentation-tower/src/lib.rs +++ b/opentelemetry-instrumentation-tower/src/lib.rs @@ -287,7 +287,7 @@ impl HTTPLayerBuilder { state: Arc::from(Self::make_state(meter, req_dur_bounds)), request_extractor: self.request_extractor, response_extractor: self.response_extractor, - tracer: tracer, + tracer, }) } @@ -595,6 +595,7 @@ mod tests { use super::*; use http::{Request, Response, StatusCode}; + use opentelemetry::metrics::MeterProvider; use opentelemetry::trace::TracerProvider; use opentelemetry::trace::{FutureExt, TraceContextExt, Tracer}; use opentelemetry_sdk::metrics::SdkMeterProvider; From 89c6d2daf55b360d5bc8ef51c67377c2ee88f0ce Mon Sep 17 00:00:00 2001 From: Jan Steinke Date: Wed, 26 Nov 2025 14:41:14 +0100 Subject: [PATCH 33/36] use the opentelemetry with_context future extension for attaching --- .../src/lib.rs | 297 ++++++++++-------- 1 file changed, 158 insertions(+), 139 deletions(-) diff --git a/opentelemetry-instrumentation-tower/src/lib.rs b/opentelemetry-instrumentation-tower/src/lib.rs index 48b53fac0..db4bd1905 100644 --- a/opentelemetry-instrumentation-tower/src/lib.rs +++ b/opentelemetry-instrumentation-tower/src/lib.rs @@ -3,23 +3,21 @@ use std::future::Future; use std::pin::Pin; use std::string::String; use std::sync::Arc; -use std::task::Poll::Ready; use std::task::{Context, Poll}; use std::time::Instant; use std::{fmt, result}; #[cfg(feature = "axum")] use axum::extract::MatchedPath; -use futures_util::ready; +use futures_util::FutureExt; use opentelemetry::global::{self, BoxedTracer}; use opentelemetry::metrics::Meter; use opentelemetry::metrics::{Histogram, UpDownCounter}; -use opentelemetry::trace::{SpanKind, Status, TraceContextExt, Tracer}; +use opentelemetry::trace::{FutureExt as OtelFutureExt, SpanKind, Status, TraceContextExt, Tracer}; use opentelemetry::Context as OtelContext; use opentelemetry::KeyValue; use opentelemetry_http::HeaderExtractor; use opentelemetry_semantic_conventions as semconv; -use pin_project_lite::pin_project; use tower_layer::Layer; use tower_service::Service; @@ -131,6 +129,19 @@ struct HTTPLayerState { pub server_response_body_size: Histogram, } +/// Request data extracted before the inner service call. +/// This data is needed for metrics and span finalization after the response is received. +struct RequestData { + duration_start: Instant, + req_body_size: Option, + protocol_name_kv: KeyValue, + protocol_version_kv: KeyValue, + url_scheme_kv: KeyValue, + method_kv: KeyValue, + route_kv_opt: Option, + custom_request_attributes: Vec, +} + #[derive(Clone)] /// [`Service`] used by [`HTTPLayer`] pub struct HTTPService { @@ -346,55 +357,18 @@ where } } -/// ResponseFutureState holds request-scoped data for metrics, tracing and their attributes. -/// -/// ResponseFutureState lives inside the response future, as it needs to hold data -/// initialized or extracted from the request before it is forwarded to the inner Service. -/// The rest of the data (e.g. status code, error) can be extracted from the response -/// or calculated with respect to the data held here (e.g., duration = now - duration start). -struct ResponseFutureState { - // fields for the metric values - // https://opentelemetry.io/docs/specs/semconv/http/http-metrics/#metric-httpserverrequestduration - duration_start: Instant, - // https://opentelemetry.io/docs/specs/semconv/http/http-metrics/#metric-httpserverrequestbodysize - req_body_size: Option, - - // fields for metric labels - protocol_name_kv: KeyValue, - protocol_version_kv: KeyValue, - url_scheme_kv: KeyValue, - method_kv: KeyValue, - route_kv_opt: Option, - - // Custom attributes from request - custom_request_attributes: Vec, - - // Tracing fields - otel_context: OtelContext, -} - -pin_project! { - /// Response [`Future`] for [`HTTPService`]. - pub struct HTTPResponseFuture { - #[pin] - inner_response_future: F, - layer_state: Arc, - future_state: ResponseFutureState, - response_extractor: ResExt, - } -} - impl Service> for HTTPService where S: Service, Response = http::Response>, + S::Future: Send + 'static, ResBody: http_body::Body, ReqExt: RequestAttributeExtractor, ResExt: ResponseAttributeExtractor, { type Response = S::Response; type Error = S::Error; - type Future = HTTPResponseFuture; + type Future = Pin> + Send>>; fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll> { self.inner_service.poll_ready(cx) @@ -418,13 +392,14 @@ where let method = req.method().as_str().to_owned(); let method_kv = KeyValue::new(HTTP_REQUEST_METHOD_LABEL, method.clone()); - #[allow(unused_mut)] - let mut route_kv_opt = None; #[cfg(feature = "axum")] - if let Some(matched_path) = req.extensions().get::() { - let route = matched_path.as_str().to_owned(); - route_kv_opt = Some(KeyValue::new(HTTP_ROUTE_LABEL, route.clone())); - }; + let route_kv_opt = req + .extensions() + .get::() + .map(|matched_path| KeyValue::new(HTTP_ROUTE_LABEL, matched_path.as_str().to_owned())); + + #[cfg(not(feature = "axum"))] + let route_kv_opt = None; // Extract custom request attributes let custom_request_attributes = self.request_extractor.extract_attributes(&req); @@ -463,118 +438,114 @@ where .with_attributes(span_attributes) .start_with_context(self.tracer.as_ref(), &parent_cx); - let cx = OtelContext::current_with_span(span); + let cx = parent_cx.with_span(span); self.state .server_active_requests .add(1, &[url_scheme_kv.clone(), method_kv.clone()]); - HTTPResponseFuture { - inner_response_future: self.inner_service.call(req), - layer_state: self.state.clone(), - future_state: ResponseFutureState { - duration_start, - req_body_size: content_length, - - protocol_name_kv, - protocol_version_kv, - url_scheme_kv, - method_kv, - route_kv_opt, - custom_request_attributes, - - otel_context: cx, - }, - response_extractor: self.response_extractor.clone(), - } + let request_data = RequestData { + duration_start, + req_body_size: content_length, + protocol_name_kv, + protocol_version_kv, + url_scheme_kv, + method_kv, + route_kv_opt, + custom_request_attributes, + }; + + let layer_state = self.state.clone(); + let response_extractor = self.response_extractor.clone(); + + Box::pin( + self.inner_service + .call(req) + .with_context(cx.clone()) + .map(move |result| { + finalize_request(result, cx, request_data, layer_state, response_extractor) + }), + ) } } -impl Future for HTTPResponseFuture +/// Finalizes the request by updating the span and recording metrics after the response is received. +fn finalize_request( + result: result::Result, E>, + cx: OtelContext, + request_data: RequestData, + layer_state: Arc, + response_extractor: ResExt, +) -> result::Result, E> where - F: Future, E>>, ResBody: http_body::Body, ResExt: ResponseAttributeExtractor, { - type Output = F::Output; - - fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { - let this = self.project(); - let response = ready!(this.inner_response_future.poll(cx))?; - let status = response.status(); - - // Build base label set - let mut label_superset = vec![ - this.future_state.protocol_name_kv.clone(), - this.future_state.protocol_version_kv.clone(), - this.future_state.url_scheme_kv.clone(), - this.future_state.method_kv.clone(), - KeyValue::new(HTTP_RESPONSE_STATUS_CODE_LABEL, i64::from(status.as_u16())), - ]; - - if let Some(route_kv) = this.future_state.route_kv_opt.clone() { - label_superset.push(route_kv); - } - - // Add custom request attributes - label_superset.extend(this.future_state.custom_request_attributes.clone()); + let response = result?; + let status = response.status(); + + // Build base label set + let mut label_superset = vec![ + request_data.protocol_name_kv, + request_data.protocol_version_kv, + request_data.url_scheme_kv.clone(), + request_data.method_kv.clone(), + KeyValue::new(HTTP_RESPONSE_STATUS_CODE_LABEL, i64::from(status.as_u16())), + ]; + + if let Some(route_kv) = request_data.route_kv_opt { + label_superset.push(route_kv); + } - // Extract and add custom response attributes - let custom_response_attributes = this.response_extractor.extract_attributes(&response); - label_superset.extend(custom_response_attributes.clone()); + // Add custom request attributes + label_superset.extend(request_data.custom_request_attributes); - // Update span - let span = this.future_state.otel_context.span(); - span.set_attribute(KeyValue::new( - semconv::trace::HTTP_RESPONSE_STATUS_CODE, - status.as_u16() as i64, - )); + // Extract and add custom response attributes + let custom_response_attributes = response_extractor.extract_attributes(&response); + label_superset.extend(custom_response_attributes.clone()); - // Add custom response attributes to span - for attr in &custom_response_attributes { - span.set_attribute(attr.clone()); - } + // Update span + let span = cx.span(); + span.set_attribute(KeyValue::new( + semconv::trace::HTTP_RESPONSE_STATUS_CODE, + status.as_u16() as i64, + )); - // Set span status based on HTTP status code - // Following server-side semantic conventions: - // - 5xx server errors indicate server failure and should be marked as span errors - // - 4xx client errors indicate client mistakes, not server failures - if status.is_server_error() { - span.set_status(Status::Error { - description: format!("HTTP {}", status.as_u16()).into(), - }); - } + // Add custom response attributes to span + for attr in &custom_response_attributes { + span.set_attribute(attr.clone()); + } - span.end(); + // Set span status based on HTTP status code + if status.is_server_error() { + span.set_status(Status::Error { + description: format!("HTTP {}", status.as_u16()).into(), + }); + } - this.layer_state.server_request_duration.record( - this.future_state.duration_start.elapsed().as_secs_f64(), - &label_superset, - ); + // Record metrics + layer_state.server_request_duration.record( + request_data.duration_start.elapsed().as_secs_f64(), + &label_superset, + ); - if let Some(req_content_length) = this.future_state.req_body_size { - this.layer_state - .server_request_body_size - .record(req_content_length, &label_superset); - } + if let Some(req_content_length) = request_data.req_body_size { + layer_state + .server_request_body_size + .record(req_content_length, &label_superset); + } - // use same approach for `http.server.response.body.size` as hyper does to set content-length - if let Some(resp_content_length) = response.body().size_hint().exact() { - this.layer_state - .server_response_body_size - .record(resp_content_length, &label_superset); - } + if let Some(resp_content_length) = response.body().size_hint().exact() { + layer_state + .server_response_body_size + .record(resp_content_length, &label_superset); + } - this.layer_state.server_active_requests.add( - -1, - &[ - this.future_state.url_scheme_kv.clone(), - this.future_state.method_kv.clone(), - ], - ); + layer_state + .server_active_requests + .add(-1, &[request_data.url_scheme_kv, request_data.method_kv]); - Ready(Ok(response)) - } + Ok(response) } fn split_and_format_protocol_version(http_version: http::Version) -> (String, String) { @@ -947,4 +918,52 @@ mod tests { } } } + + #[tokio::test(flavor = "current_thread")] + async fn test_context_available_in_handler() { + let trace_exporter = InMemorySpanExporterBuilder::new().build(); + let tracer_provider = SdkTracerProvider::builder() + .with_simple_exporter(trace_exporter.clone()) + .build(); + + let tracer = Arc::new(BoxedTracer::new(Box::new( + tracer_provider.tracer("test_tracer"), + ))); + + let mut layer = HTTPLayerBuilder::builder().build().unwrap(); + layer.tracer = tracer.clone(); + + let service = tower::service_fn(|_req: Request| async { + // Access the current context - this should have the HTTP span + let cx = OtelContext::current(); + let span = cx.span(); + + // Verify we can get span context (means context is attached) + let span_context = span.span_context(); + assert!(span_context.is_valid(), "Span context should be valid"); + + Ok::<_, std::convert::Infallible>( + Response::builder() + .status(StatusCode::OK) + .body(String::from("OK")) + .unwrap(), + ) + }); + + let mut service = layer.layer(service); + + let request = Request::builder() + .method("GET") + .uri("http://example.com/test") + .body("test".to_string()) + .unwrap(); + + let _response = service.call(request).await.unwrap(); + + tracer_provider.force_flush().unwrap(); + + let spans = trace_exporter.get_finished_spans().unwrap(); + assert_eq!(spans.len(), 1, "Expected one HTTP span"); + assert_eq!(spans[0].name, "GET /test"); + } } From c1f6a0418db3319245fcc626b3881f07e9823bbb Mon Sep 17 00:00:00 2001 From: Jan Steinke Date: Wed, 26 Nov 2025 15:02:40 +0100 Subject: [PATCH 34/36] clean up around the context handling and response parsing --- .../src/lib.rs | 168 ++++++++++-------- 1 file changed, 98 insertions(+), 70 deletions(-) diff --git a/opentelemetry-instrumentation-tower/src/lib.rs b/opentelemetry-instrumentation-tower/src/lib.rs index db4bd1905..8a6f04e2e 100644 --- a/opentelemetry-instrumentation-tower/src/lib.rs +++ b/opentelemetry-instrumentation-tower/src/lib.rs @@ -9,7 +9,6 @@ use std::{fmt, result}; #[cfg(feature = "axum")] use axum::extract::MatchedPath; -use futures_util::FutureExt; use opentelemetry::global::{self, BoxedTracer}; use opentelemetry::metrics::Meter; use opentelemetry::metrics::{Histogram, UpDownCounter}; @@ -362,6 +361,7 @@ impl Service> where S: Service, Response = http::Response>, S::Future: Send + 'static, + S::Error: std::fmt::Debug, ResBody: http_body::Body, ReqExt: RequestAttributeExtractor, ResExt: ResponseAttributeExtractor, @@ -458,94 +458,122 @@ where let layer_state = self.state.clone(); let response_extractor = self.response_extractor.clone(); + let inner_future = self.inner_service.call(req); + Box::pin( - self.inner_service - .call(req) - .with_context(cx.clone()) - .map(move |result| { - finalize_request(result, cx, request_data, layer_state, response_extractor) - }), + async move { + let result = inner_future.await; + finalize_request(&result, &request_data, &layer_state, &response_extractor); + result + } + .with_context(cx), ) } } /// Finalizes the request by updating the span and recording metrics after the response is received. fn finalize_request( - result: result::Result, E>, - cx: OtelContext, - request_data: RequestData, - layer_state: Arc, - response_extractor: ResExt, -) -> result::Result, E> -where + result: &result::Result, E>, + request_data: &RequestData, + layer_state: &Arc, + response_extractor: &ResExt, +) where ResBody: http_body::Body, ResExt: ResponseAttributeExtractor, + E: std::fmt::Debug, { - let response = result?; - let status = response.status(); - - // Build base label set - let mut label_superset = vec![ - request_data.protocol_name_kv, - request_data.protocol_version_kv, - request_data.url_scheme_kv.clone(), - request_data.method_kv.clone(), - KeyValue::new(HTTP_RESPONSE_STATUS_CODE_LABEL, i64::from(status.as_u16())), - ]; - - if let Some(route_kv) = request_data.route_kv_opt { - label_superset.push(route_kv); - } + let cx = OtelContext::current(); + let span = cx.span(); - // Add custom request attributes - label_superset.extend(request_data.custom_request_attributes); + match result { + Ok(response) => { + let status = response.status(); + + // Build base label set + let mut label_superset = vec![ + request_data.protocol_name_kv.clone(), + request_data.protocol_version_kv.clone(), + request_data.url_scheme_kv.clone(), + request_data.method_kv.clone(), + KeyValue::new(HTTP_RESPONSE_STATUS_CODE_LABEL, i64::from(status.as_u16())), + ]; + + if let Some(route_kv) = &request_data.route_kv_opt { + label_superset.push(route_kv.clone()); + } - // Extract and add custom response attributes - let custom_response_attributes = response_extractor.extract_attributes(&response); - label_superset.extend(custom_response_attributes.clone()); + // Add custom request attributes + label_superset.extend(request_data.custom_request_attributes.clone()); - // Update span - let span = cx.span(); - span.set_attribute(KeyValue::new( - semconv::trace::HTTP_RESPONSE_STATUS_CODE, - status.as_u16() as i64, - )); - - // Add custom response attributes to span - for attr in &custom_response_attributes { - span.set_attribute(attr.clone()); - } + // Extract and add custom response attributes + let custom_response_attributes = response_extractor.extract_attributes(response); + label_superset.extend(custom_response_attributes.clone()); - // Set span status based on HTTP status code - if status.is_server_error() { - span.set_status(Status::Error { - description: format!("HTTP {}", status.as_u16()).into(), - }); - } + // Update span + span.set_attribute(KeyValue::new( + semconv::trace::HTTP_RESPONSE_STATUS_CODE, + status.as_u16() as i64, + )); - // Record metrics - layer_state.server_request_duration.record( - request_data.duration_start.elapsed().as_secs_f64(), - &label_superset, - ); + // Add custom response attributes to span + for attr in &custom_response_attributes { + span.set_attribute(attr.clone()); + } - if let Some(req_content_length) = request_data.req_body_size { - layer_state - .server_request_body_size - .record(req_content_length, &label_superset); - } + // Set span status based on HTTP status code + if status.is_server_error() { + span.set_status(Status::Error { + description: format!("HTTP {}", status.as_u16()).into(), + }); + } - if let Some(resp_content_length) = response.body().size_hint().exact() { - layer_state - .server_response_body_size - .record(resp_content_length, &label_superset); - } + // Record metrics + layer_state.server_request_duration.record( + request_data.duration_start.elapsed().as_secs_f64(), + &label_superset, + ); - layer_state - .server_active_requests - .add(-1, &[request_data.url_scheme_kv, request_data.method_kv]); + if let Some(req_content_length) = request_data.req_body_size { + layer_state + .server_request_body_size + .record(req_content_length, &label_superset); + } - Ok(response) + if let Some(resp_content_length) = response.body().size_hint().exact() { + layer_state + .server_response_body_size + .record(resp_content_length, &label_superset); + } + } + Err(error) => { + // Mark span as error + span.set_status(Status::Error { + description: format!("{:?}", error).into(), + }); + + // Still record duration metric with error label + let label_superset = vec![ + request_data.protocol_name_kv.clone(), + request_data.protocol_version_kv.clone(), + request_data.url_scheme_kv.clone(), + request_data.method_kv.clone(), + ]; + + layer_state.server_request_duration.record( + request_data.duration_start.elapsed().as_secs_f64(), + &label_superset, + ); + } + } + + // Always decrement active requests counter + layer_state.server_active_requests.add( + -1, + &[ + request_data.url_scheme_kv.clone(), + request_data.method_kv.clone(), + ], + ); } fn split_and_format_protocol_version(http_version: http::Version) -> (String, String) { From 9c0210820e082d36952bfa1ec6b523b2b3e7eeb8 Mon Sep 17 00:00:00 2001 From: Jan Steinke Date: Wed, 26 Nov 2025 15:13:20 +0100 Subject: [PATCH 35/36] Add field comments and move RequestData before Service impl --- .../src/lib.rs | 33 +++++++++++-------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/opentelemetry-instrumentation-tower/src/lib.rs b/opentelemetry-instrumentation-tower/src/lib.rs index 8a6f04e2e..66059f331 100644 --- a/opentelemetry-instrumentation-tower/src/lib.rs +++ b/opentelemetry-instrumentation-tower/src/lib.rs @@ -128,19 +128,6 @@ struct HTTPLayerState { pub server_response_body_size: Histogram, } -/// Request data extracted before the inner service call. -/// This data is needed for metrics and span finalization after the response is received. -struct RequestData { - duration_start: Instant, - req_body_size: Option, - protocol_name_kv: KeyValue, - protocol_version_kv: KeyValue, - url_scheme_kv: KeyValue, - method_kv: KeyValue, - route_kv_opt: Option, - custom_request_attributes: Vec, -} - #[derive(Clone)] /// [`Service`] used by [`HTTPLayer`] pub struct HTTPService { @@ -356,6 +343,26 @@ where } } +/// Request data extracted before the inner service call. +/// This data is needed for metrics and span finalization after the response is received. +struct RequestData { + // fields for the metric values + // https://opentelemetry.io/docs/specs/semconv/http/http-metrics/#metric-httpserverrequestduration + duration_start: Instant, + // https://opentelemetry.io/docs/specs/semconv/http/http-metrics/#metric-httpserverrequestbodysize + req_body_size: Option, + + // fields for metric labels + protocol_name_kv: KeyValue, + protocol_version_kv: KeyValue, + url_scheme_kv: KeyValue, + method_kv: KeyValue, + route_kv_opt: Option, + + // Custom attributes from request + custom_request_attributes: Vec, +} + impl Service> for HTTPService where From 7cc2289f9606d78aa5f326e2109875f0672bd664 Mon Sep 17 00:00:00 2001 From: Jan Steinke Date: Wed, 26 Nov 2025 15:20:03 +0100 Subject: [PATCH 36/36] Remove unused dependencies futures-util and pin-project-lite --- opentelemetry-instrumentation-tower/Cargo.toml | 2 -- 1 file changed, 2 deletions(-) diff --git a/opentelemetry-instrumentation-tower/Cargo.toml b/opentelemetry-instrumentation-tower/Cargo.toml index 4951754a7..8bded4822 100644 --- a/opentelemetry-instrumentation-tower/Cargo.toml +++ b/opentelemetry-instrumentation-tower/Cargo.toml @@ -18,13 +18,11 @@ axum = ["dep:axum"] [dependencies] axum = { features = ["matched-path", "macros"], version = "0.8", default-features = false, optional = true } -futures-util = { version = "0.3", default-features = false } http = { version = "1", features = ["std"], default-features = false } http-body = { version = "1", default-features = false } opentelemetry = { workspace = true, features = ["futures", "metrics", "trace"] } opentelemetry-http = "0.31" opentelemetry-semantic-conventions = { workspace = true, features = ["semconv_experimental"] } -pin-project-lite = { version = "0.2", default-features = false } tower-service = { version = "0.3", default-features = false } tower-layer = { version = "0.3", default-features = false }