Support Reactor-Netty v0.9 too (and thereby more Spring webclients)

Unfortunately we can't test this without doing something *much* more
complicated, since the versions conflict, but it works nicely when
tested manually against Spring PetClinic Microservices when using
either v1 or v0.9.

Author: pimterry

README.md

@@ -16,9 +16,9 @@ Traffic can be captured from at least:
   [x] Apache HttpAsyncClient v4 & v5
 - [x] OkHttp v2, v3 & v4
 - [x] Retrofit
-- [x] Jetty-Client v9, 10 & 11
+- [x] Jetty-Client v9, v10 & v11
 - [x] Async-Http-Client
-- [x] Reactor-Netty
+- [x] Reactor-Netty v0.9 & v1+
 - [x] Spring WebClient
 
 This will also capture HTTP(S) from any downstream libraries based on each of these clients, and many other untested clients sharing similar implementations, and so should cover a very large percentage of HTTP client usage.

build.gradle

@@ -20,6 +20,10 @@ jar {
         attributes 'Can-Redefine-Classes': 'true'
         attributes 'Can-Retransform-Classes': 'true'
     }
+
+    // We include classes in our package. We *don't* include stub classes used to support multiple
+    // dependency versions, which are under their corresponding real package names.
+    include('tech/httptoolkit/javaagent/**/*')
 }
 
 dependencies {
@@ -63,6 +67,10 @@ shadowJar {
     exclude '**/*.kotlin_builtins'
     exclude '**/module_info.class'
     exclude 'META-INF/maven/**'
+
+    // We have to specifically exclude packages here, because we *do* want to include lots of non-local code, just not
+    // these specific client stubs:
+    exclude 'reactor/'
 }
 
 import com.github.jengelman.gradle.plugins.shadow.tasks.ConfigureShadowRelocation

src/main/java/reactor/netty/tcp/ProxyProvider.java

@@ -0,0 +1,51 @@
+package reactor.netty.tcp;
+
+import java.net.InetSocketAddress;
+
+/**
+ * A stub with the parts of the interface we need to support v0.9 of Reactor-Netty. We compile against this, but we
+ * *don't* include this in the resulting JAR, so references instead resolve to the real implementation, when that
+ * is present. This is required because we can't depend on both v0.9 and v1 in the same module, and this class has
+ * moved packages between the two.
+ */
+public final class ProxyProvider {
+
+    public static ProxyProvider.TypeSpec builder() {
+        return new ProxyProvider.Build();
+    }
+
+    ProxyProvider(ProxyProvider.Build builder) {}
+
+    public enum Proxy {
+        HTTP
+    }
+
+    static final class Build implements TypeSpec, AddressSpec, Builder {
+
+        Build() {}
+
+        public final Builder address(InetSocketAddress address) {
+            return this;
+        }
+
+        public final AddressSpec type(Proxy type) {
+            return this;
+        }
+
+        public ProxyProvider build() {
+            return new ProxyProvider(this);
+        }
+    }
+
+    public interface TypeSpec {
+        AddressSpec type(Proxy type);
+    }
+
+    public interface AddressSpec {
+        Builder address(InetSocketAddress address);
+    }
+
+    public interface Builder {
+        ProxyProvider build();
+    }
+}

src/main/java/tech/httptoolkit/javaagent/ReturnProxyAddressAdvice.java

@@ -0,0 +1,16 @@
+package tech.httptoolkit.javaagent;
+
+import net.bytebuddy.asm.Advice;
+
+import java.net.InetSocketAddress;
+import java.net.SocketAddress;
+
+public class ReturnProxyAddressAdvice {
+    @Advice.OnMethodExit
+    public static void proxy(@Advice.Return(readOnly = false) SocketAddress returnValue) {
+        returnValue = new InetSocketAddress(
+                HttpProxyAgent.getAgentProxyHost(),
+                HttpProxyAgent.getAgentProxyPort()
+        );
+    }
+}

src/main/java/tech/httptoolkit/javaagent/reactornetty/ReactorNettyResetHttpClientSecureSslAdvice.java

@@ -0,0 +1,36 @@
+package tech.httptoolkit.javaagent.reactornetty;
+
+import io.netty.handler.ssl.SslContextBuilder;
+import net.bytebuddy.asm.Advice;
+import reactor.netty.tcp.SslProvider;
+import tech.httptoolkit.javaagent.HttpProxyAgent;
+
+public class ReactorNettyResetHttpClientSecureSslAdvice {
+
+    public static final SslProvider agentSslProvider;
+
+    static {
+        try {
+            // Initialize our intercepted SSL provider:
+            agentSslProvider = SslProvider.builder()
+                    .sslContext(
+                            SslContextBuilder
+                                    .forClient()
+                                    .trustManager(HttpProxyAgent.getInterceptedTrustManagerFactory())
+                                    .build()
+                    ).build();
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    // In v0.9 versions of Reactor Netty, the sslProvider is stored on HttpClientSecure. Here we hook that class's
+    // constructor and replace the SSL provider as soon as it's set.
+
+    @Advice.OnMethodExit
+    public static void afterConstructor(
+        @Advice.FieldValue(value = "sslProvider", readOnly = false) SslProvider sslProviderField
+    ) {
+        sslProviderField = agentSslProvider;
+    }
+}

src/main/java/tech/httptoolkit/javaagent/reactornetty/ReactorNettyV09ResetProxyProviderFieldAdvice.java

@@ -0,0 +1,29 @@
+package tech.httptoolkit.javaagent.reactornetty;
+
+import net.bytebuddy.asm.Advice;
+import reactor.netty.tcp.ProxyProvider;
+import tech.httptoolkit.javaagent.HttpProxyAgent;
+
+import java.net.InetSocketAddress;
+
+
+// Reset the proxyProvider field to use our own intercepted proxy after certain constructors
+// complete. Note that this uses the v0.9 proxyProvider class, so it would fail if applied to
+// when v1 is loaded in the target app.
+public class ReactorNettyV09ResetProxyProviderFieldAdvice {
+
+    public static final ProxyProvider agentProxyProvider = ProxyProvider.builder()
+            .type(ProxyProvider.Proxy.HTTP)
+            .address(new InetSocketAddress(
+                    HttpProxyAgent.getAgentProxyHost(),
+                    HttpProxyAgent.getAgentProxyPort()
+            ))
+            .build();
+
+    @Advice.OnMethodExit
+    public static void afterConstructor(
+        @Advice.FieldValue(value = "proxyProvider", readOnly = false) ProxyProvider proxyProviderField
+    ) {
+        proxyProviderField = agentProxyProvider;
+    }
+}

src/main/kotlin/tech/httptoolkit/javaagent/AgentMain.kt

@@ -89,7 +89,10 @@ fun interceptAllHttps(config: Config, instrumentation: Instrumentation) {
         JettyClientTransformer(logger),
         AsyncHttpClientConfigTransformer(logger),
         AsyncHttpChannelManagerTransformer(logger),
-        ReactorNettyClientConfigTransformer(logger)
+        ReactorNettyClientConfigTransformer(logger),
+        ReactorNettyProxyProviderTransformer(logger),
+        ReactorNettyOverrideRequestAddressTransformer(logger),
+        ReactorNettyHttpClientSecureTransformer(logger)
     ).forEach { matchingAgentTransformer ->
         agentBuilder = matchingAgentTransformer.register(agentBuilder)
     }

src/main/kotlin/tech/httptoolkit/javaagent/ReactorNettyTransformers.kt

@@ -2,33 +2,121 @@ package tech.httptoolkit.javaagent
 
 import net.bytebuddy.agent.builder.AgentBuilder
 import net.bytebuddy.asm.Advice
+import net.bytebuddy.description.field.FieldDescription
 import net.bytebuddy.description.method.MethodDescription
 import net.bytebuddy.description.type.TypeDescription
 import net.bytebuddy.dynamic.DynamicType
 import net.bytebuddy.matcher.ElementMatchers.*
-import net.bytebuddy.utility.JavaModule
-import reactor.netty.http.client.HttpClientConfig
 import tech.httptoolkit.javaagent.reactornetty.ReactorNettyResetAllConfigAdvice
+import tech.httptoolkit.javaagent.reactornetty.ReactorNettyResetHttpClientSecureSslAdvice
+import tech.httptoolkit.javaagent.reactornetty.ReactorNettyV09ResetProxyProviderFieldAdvice
 
-// To patch Reactor-Netty's HTTP client, we hook the constructor of the client itself. It has a constructor
+// To patch Reactor-Netty's v1 HTTP client, we hook the constructor of the client itself. It has a constructor
 // that receives the config as part of every single HTTP request - we hook that to reset the relevant
 // config props every time they're used.
 
+private val matchConfigConstructor = isConstructor<MethodDescription>()
+    .and(takesArguments(1))
+    .and(takesArgument(0,
+        named("reactor.netty.http.client.HttpClientConfig")
+    ))
+
 class ReactorNettyClientConfigTransformer(logger: TransformationLogger): MatchingAgentTransformer(logger) {
+
     override fun register(builder: AgentBuilder): AgentBuilder {
         return builder
             .type(
                 hasSuperType(named("reactor.netty.http.client.HttpClient"))
             ).and(
                 not(isInterface())
+            ).and(
+                // This matches v1+ only, where the config is passed into the constructor repeatedly, and can
+                // be mutated there. v0.9 is handled separately below.
+                declaresMethod(matchConfigConstructor)
             ).transform(this)
     }
 
     override fun transform(builder: DynamicType.Builder<*>): DynamicType.Builder<*> {
         return builder
             .visit(Advice.to(ReactorNettyResetAllConfigAdvice::class.java)
-                .on(isConstructor<MethodDescription>().and(
-                    takesArguments(HttpClientConfig::class.java)
-                )))
+                .on(matchConfigConstructor)
+            )
+    }
+}
+
+// In v0.9, that wasn't the case. Instead, the SSL provider and proxy provider are passed as arguments to
+// and stored within various client classes. Here, we patch all their constructors to reset those fields
+// immediately after instantiation, ensuring our values replace the given arguments.
+
+// First, the sslProvider field:
+class ReactorNettyHttpClientSecureTransformer(logger: TransformationLogger): MatchingAgentTransformer(logger) {
+    override fun register(builder: AgentBuilder): AgentBuilder {
+        return builder
+            .type(
+                named("reactor.netty.http.client.HttpClientSecure")
+            ).and(
+                declaresField(named("sslProvider"))
+            ).transform(this)
+    }
+
+    override fun transform(builder: DynamicType.Builder<*>): DynamicType.Builder<*> {
+        return builder
+            .visit(Advice.to(ReactorNettyResetHttpClientSecureSslAdvice::class.java)
+                .on(isConstructor()))
     }
 }
+
+// Then each of the important cases where a proxy provider is stored:
+class ReactorNettyProxyProviderTransformer(logger: TransformationLogger): MatchingAgentTransformer(logger) {
+    override fun register(builder: AgentBuilder): AgentBuilder {
+        return builder
+            .type(
+                declaresField(named<FieldDescription>("proxyProvider").and(
+                    // This only applies to v0.9+ which uses this package name, not v1+ where ProxyProvider
+                    // lives in reactor.netty.transport (handled by the other transformer above)
+                    fieldType(named("reactor.netty.tcp.ProxyProvider")))
+                )
+            ).and(
+                named<TypeDescription>(
+                    "reactor.netty.http.client.HttpClientConnect\$MonoHttpConnect"
+                ).or(
+                    named(
+                        "reactor.netty.http.client.HttpClientConnect\$HttpClientHandler"
+                    )
+                )
+            ).transform(this)
+    }
+
+    override fun transform(builder: DynamicType.Builder<*>): DynamicType.Builder<*> {
+        return builder
+            .visit(Advice.to(ReactorNettyV09ResetProxyProviderFieldAdvice::class.java)
+                .on(isConstructor()))
+    }
+}
+
+// Then, on top of all that, we also forcibly set the socket address for all outgoing HTTP connections, because that's
+// that's the goal, and the above proxyProvider logic doesn't properly cover everything as proxy logic is spread across
+// a few places including the generic TCP clients (which we shouldn't touch). This is a bit messy/risky, but only
+// applies to v0.9, since v1+ stores the config in a properly structured way.
+class ReactorNettyOverrideRequestAddressTransformer(logger: TransformationLogger): MatchingAgentTransformer(logger) {
+    override fun register(builder: AgentBuilder): AgentBuilder {
+        return builder
+            .type(
+                declaresField(named<FieldDescription>("proxyProvider").and(
+                    // This ensures this only applies to v0.9+ which uses this package name, not v1+ where
+                    // ProxyProvider lives in reactor.netty.transport (handled separately above).
+                    fieldType(named("reactor.netty.tcp.ProxyProvider")))
+                )
+            ).and(
+                named(
+                    "reactor.netty.http.client.HttpClientConnect\$HttpClientHandler"
+                )
+            ).transform(this)
+    }
+
+    override fun transform(builder: DynamicType.Builder<*>): DynamicType.Builder<*> {
+        return builder
+            .visit(Advice.to(ReturnProxyAddressAdvice::class.java)
+                .on(hasMethodName("get")))
+    }
+}

src/main/kotlin/tech/httptoolkit/javaagent/TransformationLogger.kt

@@ -5,7 +5,7 @@ import net.bytebuddy.description.type.TypeDescription
 import net.bytebuddy.dynamic.DynamicType
 import net.bytebuddy.utility.JavaModule
 
-class TransformationLogger(val debugMode: Boolean) : AgentBuilder.Listener.Adapter() {
+class TransformationLogger(private val debugMode: Boolean) : AgentBuilder.Listener.Adapter() {
 
     private val transformingTypes: ArrayList<String> = ArrayList()