Add http client url template customizer (open-telemetry#15217)

Author: laurit

instrumentation-api-incubator/javaagent-testing/build.gradle.kts

@@ -0,0 +1,17 @@
+plugins {
+  id("otel.javaagent-testing")
+}
+
+dependencies {
+  testInstrumentation(project(":instrumentation:http-url-connection:javaagent"))
+
+  testImplementation(project(":testing-common"))
+}
+
+tasks {
+  test {
+    jvmArgs(
+      "-Dotel.experimental.config.file=$projectDir/src/test/resources/declarative-config.yaml"
+    )
+  }
+}

instrumentation-api-incubator/javaagent-testing/src/test/java/io/opentelemetry/instrumentation/api/incubator/semconv/http/HttpClientUrlTemplateCustomizerTest.java

@@ -0,0 +1,64 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package io.opentelemetry.instrumentation.api.incubator.semconv.http;
+
+import static io.opentelemetry.semconv.incubating.UrlIncubatingAttributes.URL_TEMPLATE;
+import static org.assertj.core.api.Assertions.assertThat;
+
+import io.opentelemetry.api.trace.SpanKind;
+import io.opentelemetry.instrumentation.testing.junit.AgentInstrumentationExtension;
+import io.opentelemetry.instrumentation.testing.junit.InstrumentationExtension;
+import io.opentelemetry.instrumentation.testing.junit.http.HttpClientTestServer;
+import io.opentelemetry.sdk.trace.data.StatusData;
+import java.net.HttpURLConnection;
+import java.net.URI;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.RegisterExtension;
+
+class HttpClientUrlTemplateCustomizerTest {
+  private static HttpClientTestServer server;
+
+  @RegisterExtension
+  static InstrumentationExtension testing = AgentInstrumentationExtension.create();
+
+  @BeforeAll
+  static void setUp() {
+    server = new HttpClientTestServer(testing.getOpenTelemetry());
+    server.start();
+  }
+
+  @AfterAll
+  static void tearDown() {
+    server.stop();
+  }
+
+  @Test
+  void test() throws Exception {
+    URI uri = URI.create("http://localhost:" + server.httpPort() + "/hello/world");
+    HttpURLConnection connection = (HttpURLConnection) uri.toURL().openConnection();
+    connection.getInputStream().close();
+    int responseCode = connection.getResponseCode();
+    connection.disconnect();
+
+    assertThat(responseCode).isEqualTo(200);
+
+    testing.waitAndAssertTraces(
+        trace ->
+            trace.hasSpansSatisfyingExactly(
+                span ->
+                    span.hasName("GET /hello/*")
+                        .hasNoParent()
+                        .hasKind(SpanKind.CLIENT)
+                        .hasAttribute(URL_TEMPLATE, "/hello/*")
+                        .hasStatus(StatusData.unset()),
+                span ->
+                    span.hasName("test-http-server")
+                        .hasParent(trace.getSpan(0))
+                        .hasKind(SpanKind.SERVER)));
+  }
+}

instrumentation-api-incubator/javaagent-testing/src/test/resources/declarative-config.yaml

@@ -0,0 +1,11 @@
+file_format: "1.0-rc.1"
+propagator:
+  composite_list: "tracecontext"
+instrumentation/development:
+  java:
+    http:
+      client:
+        emit_experimental_telemetry: true
+        url_template_rules:
+          - pattern: "http://localhost:.*/hello/.*"
+            template: "/hello/*"

instrumentation-api-incubator/src/main/java/io/opentelemetry/instrumentation/api/incubator/builder/internal/DefaultHttpClientInstrumenterBuilder.java

@@ -11,11 +11,10 @@
 import io.opentelemetry.context.Context;
 import io.opentelemetry.context.propagation.TextMapSetter;
 import io.opentelemetry.instrumentation.api.incubator.config.internal.CommonConfig;
-import io.opentelemetry.instrumentation.api.incubator.semconv.http.HttpClientExperimentalAttributesGetter;
 import io.opentelemetry.instrumentation.api.incubator.semconv.http.HttpClientExperimentalMetrics;
 import io.opentelemetry.instrumentation.api.incubator.semconv.http.HttpClientPeerServiceAttributesExtractor;
-import io.opentelemetry.instrumentation.api.incubator.semconv.http.HttpClientUrlTemplate;
 import io.opentelemetry.instrumentation.api.incubator.semconv.http.HttpExperimentalAttributesExtractor;
+import io.opentelemetry.instrumentation.api.incubator.semconv.http.internal.HttpClientUrlTemplateUtil;
 import io.opentelemetry.instrumentation.api.incubator.semconv.net.PeerServiceResolver;
 import io.opentelemetry.instrumentation.api.instrumenter.AttributesExtractor;
 import io.opentelemetry.instrumentation.api.instrumenter.Instrumenter;
@@ -37,7 +36,6 @@
 import java.util.List;
 import java.util.Objects;
 import java.util.function.Consumer;
-import java.util.function.Function;
 import java.util.function.Supplier;
 import java.util.function.UnaryOperator;
 import javax.annotation.Nullable;
@@ -222,19 +220,11 @@ public DefaultHttpClientInstrumenterBuilder<REQUEST, RESPONSE> setBuilderCustomi
 
   public Instrumenter<REQUEST, RESPONSE> build() {
     if (emitExperimentalHttpClientTelemetry) {
-      Function<REQUEST, String> urlTemplateExtractorFunction = unused -> null;
-      if (attributesGetter instanceof HttpClientExperimentalAttributesGetter) {
-        HttpClientExperimentalAttributesGetter<REQUEST, RESPONSE> experimentalAttributesGetter =
-            (HttpClientExperimentalAttributesGetter<REQUEST, RESPONSE>) attributesGetter;
-        urlTemplateExtractorFunction = experimentalAttributesGetter::getUrlTemplate;
-      }
-      Function<REQUEST, String> urlTemplateExtractor = urlTemplateExtractorFunction;
       Experimental.setUrlTemplateExtractor(
           httpSpanNameExtractorBuilder,
-          request -> {
-            String urlTemplate = HttpClientUrlTemplate.get(Context.current());
-            return urlTemplate != null ? urlTemplate : urlTemplateExtractor.apply(request);
-          });
+          request ->
+              HttpClientUrlTemplateUtil.getUrlTemplate(
+                  Context.current(), request, attributesGetter));
     }
     SpanNameExtractor<? super REQUEST> spanNameExtractor =
         spanNameExtractorTransformer.apply(httpSpanNameExtractorBuilder.build());

instrumentation-api-incubator/src/main/java/io/opentelemetry/instrumentation/api/incubator/semconv/http/HttpClientUrlTemplateCustomizer.java

@@ -0,0 +1,28 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package io.opentelemetry.instrumentation.api.incubator.semconv.http;
+
+import io.opentelemetry.instrumentation.api.semconv.http.HttpClientAttributesGetter;
+import javax.annotation.Nullable;
+
+/** A service provider interface (SPI) for customizing http client url template. */
+public interface HttpClientUrlTemplateCustomizer {
+
+  /**
+   * Customize url template for given request. Typically, the customizer will extract full url from
+   * the request and apply some logic (e.g. regex matching) to generate url template. The customizer
+   * can choose to override existing url template or skip customization when a url template is
+   * already set.
+   *
+   * @param urlTemplate existing url template, can be null
+   * @param request current request
+   * @param getter request attributes getter
+   * @return customized url template, or null
+   */
+  @Nullable
+  <REQUEST> String customize(
+      @Nullable String urlTemplate, REQUEST request, HttpClientAttributesGetter<REQUEST, ?> getter);
+}

instrumentation-api-incubator/src/main/java/io/opentelemetry/instrumentation/api/incubator/semconv/http/HttpExperimentalAttributesExtractor.java

@@ -10,6 +10,7 @@
 import io.opentelemetry.api.common.AttributeKey;
 import io.opentelemetry.api.common.AttributesBuilder;
 import io.opentelemetry.context.Context;
+import io.opentelemetry.instrumentation.api.incubator.semconv.http.internal.HttpClientUrlTemplateUtil;
 import io.opentelemetry.instrumentation.api.instrumenter.AttributesExtractor;
 import io.opentelemetry.instrumentation.api.semconv.http.HttpClientAttributesGetter;
 import io.opentelemetry.instrumentation.api.semconv.http.HttpCommonAttributesGetter;
@@ -47,7 +48,15 @@ private HttpExperimentalAttributesExtractor(
   }
 
   @Override
-  public void onStart(AttributesBuilder attributes, Context parentContext, REQUEST request) {}
+  public void onStart(AttributesBuilder attributes, Context parentContext, REQUEST request) {
+    if (getter instanceof HttpClientAttributesGetter) {
+      HttpClientAttributesGetter<REQUEST, RESPONSE> clientGetter =
+          (HttpClientAttributesGetter<REQUEST, RESPONSE>) getter;
+      String urlTemplate =
+          HttpClientUrlTemplateUtil.getUrlTemplate(parentContext, request, clientGetter);
+      internalSet(attributes, URL_TEMPLATE, urlTemplate);
+    }
+  }
 
   @Override
   public void onEnd(
@@ -64,15 +73,6 @@ public void onEnd(
       Long responseBodySize = responseBodySize(request, response);
       internalSet(attributes, HTTP_RESPONSE_BODY_SIZE, responseBodySize);
     }
-
-    String urlTemplate = HttpClientUrlTemplate.get(context);
-    if (urlTemplate != null) {
-      internalSet(attributes, URL_TEMPLATE, urlTemplate);
-    } else if (getter instanceof HttpClientExperimentalAttributesGetter) {
-      HttpClientExperimentalAttributesGetter<REQUEST, RESPONSE> experimentalGetter =
-          (HttpClientExperimentalAttributesGetter<REQUEST, RESPONSE>) getter;
-      internalSet(attributes, URL_TEMPLATE, experimentalGetter.getUrlTemplate(request));
-    }
   }
 
   @Nullable

instrumentation-api-incubator/src/main/java/io/opentelemetry/instrumentation/api/incubator/semconv/http/internal/HttpClientUrlTemplateUtil.java

@@ -0,0 +1,75 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package io.opentelemetry.instrumentation.api.incubator.semconv.http.internal;
+
+import io.opentelemetry.context.Context;
+import io.opentelemetry.instrumentation.api.incubator.semconv.http.HttpClientExperimentalAttributesGetter;
+import io.opentelemetry.instrumentation.api.incubator.semconv.http.HttpClientUrlTemplate;
+import io.opentelemetry.instrumentation.api.incubator.semconv.http.HttpClientUrlTemplateCustomizer;
+import io.opentelemetry.instrumentation.api.internal.InstrumenterContext;
+import io.opentelemetry.instrumentation.api.internal.ServiceLoaderUtil;
+import io.opentelemetry.instrumentation.api.semconv.http.HttpClientAttributesGetter;
+import java.util.ArrayList;
+import java.util.List;
+import javax.annotation.Nullable;
+
+/**
+ * This class is internal and is hence not for public use. Its APIs are unstable and can change at
+ * any time.
+ */
+public final class HttpClientUrlTemplateUtil {
+
+  private static final List<HttpClientUrlTemplateCustomizer> customizers = new ArrayList<>();
+
+  static {
+    for (HttpClientUrlTemplateCustomizer customizer :
+        ServiceLoaderUtil.load(HttpClientUrlTemplateCustomizer.class)) {
+      customizers.add(customizer);
+    }
+  }
+
+  @Nullable
+  public static <REQUEST> String getUrlTemplate(
+      Context context, REQUEST request, HttpClientAttributesGetter<REQUEST, ?> getter) {
+    // first, try to get url template from context
+    String urlTemplate = HttpClientUrlTemplate.get(context);
+    if (urlTemplate == null && getter instanceof HttpClientExperimentalAttributesGetter) {
+      HttpClientExperimentalAttributesGetter<REQUEST, ?> experimentalGetter =
+          (HttpClientExperimentalAttributesGetter<REQUEST, ?>) getter;
+      // next, try to get url template from getter
+      urlTemplate = experimentalGetter.getUrlTemplate(request);
+    }
+
+    return customizeUrlTemplate(urlTemplate, request, getter);
+  }
+
+  @Nullable
+  private static <REQUEST> String customizeUrlTemplate(
+      @Nullable String urlTemplate,
+      REQUEST request,
+      HttpClientAttributesGetter<REQUEST, ?> getter) {
+    if (customizers.isEmpty()) {
+      return urlTemplate;
+    }
+
+    // we cache the computation in InstrumenterContext because url template is used by both
+    // HttpSpanNameExtractor and HttpExperimentalAttributesExtractor
+    return InstrumenterContext.computeIfAbsent(
+        "url.template",
+        unused -> {
+          for (HttpClientUrlTemplateCustomizer customizer : customizers) {
+            String result = customizer.customize(urlTemplate, request, getter);
+            if (result != null) {
+              return result;
+            }
+          }
+
+          return urlTemplate;
+        });
+  }
+
+  private HttpClientUrlTemplateUtil() {}
+}

instrumentation-api-incubator/src/test/java/io/opentelemetry/instrumentation/api/incubator/semconv/http/HttpExperimentalAttributesExtractorTest.java

@@ -60,7 +60,7 @@ void runTest(
 
     AttributesBuilder attributes = Attributes.builder();
     extractor.onStart(attributes, Context.root(), "request");
-    assertThat(attributes.build()).isEmpty();
+    assertThat(attributes.build().asMap()).containsExactlyInAnyOrderEntriesOf(expected);
 
     extractor.onEnd(attributes, Context.root(), "request", "response", null);
     Map<AttributeKey<?>, Object> expectedAttributes = new HashMap<>(expected);

javaagent-tooling/src/main/java/io/opentelemetry/javaagent/tooling/instrumentation/http/RegexUrlTemplateCustomizer.java

@@ -0,0 +1,46 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package io.opentelemetry.javaagent.tooling.instrumentation.http;
+
+import static io.opentelemetry.javaagent.tooling.instrumentation.http.UrlTemplateRules.getRules;
+
+import com.google.auto.service.AutoService;
+import io.opentelemetry.instrumentation.api.incubator.semconv.http.HttpClientUrlTemplateCustomizer;
+import io.opentelemetry.instrumentation.api.semconv.http.HttpClientAttributesGetter;
+import io.opentelemetry.javaagent.tooling.instrumentation.http.UrlTemplateRules.Rule;
+import java.util.regex.Pattern;
+import javax.annotation.Nullable;
+
+@AutoService(HttpClientUrlTemplateCustomizer.class)
+public final class RegexUrlTemplateCustomizer implements HttpClientUrlTemplateCustomizer {
+
+  @Override
+  @Nullable
+  public <REQUEST> String customize(
+      @Nullable String urlTemplate,
+      REQUEST request,
+      HttpClientAttributesGetter<REQUEST, ?> getter) {
+    String url = getter.getUrlFull(request);
+    if (url == null) {
+      return null;
+    }
+
+    for (Rule rule : getRules()) {
+      if (urlTemplate != null && !rule.getOverride()) {
+        continue;
+      }
+
+      Pattern pattern = rule.getPattern();
+      // to generate the url template, we apply the regex replacement on the full url
+      String result = pattern.matcher(url).replaceFirst(rule.getReplacement());
+      if (!url.equals(result)) {
+        return result;
+      }
+    }
+
+    return null;
+  }
+}