Add support for Apache Commons HttpClient v3

Super old now, but it's a classic, and still used in some notable places
including Intellij itself (but only for plugins AFAICT).

Author: pimterry

README.md

@@ -12,8 +12,8 @@ Traffic can be captured from at least:
 
 - [x] Java's built-in HttpURLConnection
 - [x] Java 11's built-in HttpClient
-- [x] Apache HttpClient v4 & v5
-  [x] Apache HttpAsyncClient v4 & v5
+- [x] Apache HttpClient v3, v4 & v5
+- [x] Apache HttpAsyncClient v4 & v5
 - [x] OkHttp v2, v3 & v4
 - [x] Retrofit
 - [x] Jetty-Client v9, v10 & v11

build.gradle

@@ -32,6 +32,7 @@ dependencies {
     implementation group: 'net.bytebuddy', name: 'byte-buddy-dep', version: '1.10.20'
 
     // Dependencies we load only as part of rewriting them, iff the target app includes them:
+    compileOnly group: 'commons-httpclient', name: 'commons-httpclient', version: '3.1'
     compileOnly group: 'org.apache.httpcomponents', name: 'httpclient', version: '4.5'
     compileOnly group: 'org.apache.httpcomponents.client5', name: 'httpclient5', version: '5.0.3'
     compileOnly group: 'org.eclipse.jetty', name: 'jetty-client', version: '11.0.1'

src/main/java/tech/httptoolkit/javaagent/advice/apacheclient/ApacheCustomSslProtocolSocketFactory.java

@@ -0,0 +1,56 @@
+package tech.httptoolkit.javaagent.advice.apacheclient;
+
+import org.apache.commons.httpclient.params.HttpConnectionParams;
+import org.apache.commons.httpclient.protocol.SecureProtocolSocketFactory;
+import tech.httptoolkit.javaagent.HttpProxyAgent;
+
+import javax.net.SocketFactory;
+import javax.net.ssl.SSLSocketFactory;
+import java.io.IOException;
+import java.net.*;
+
+public class ApacheCustomSslProtocolSocketFactory implements SecureProtocolSocketFactory {
+
+    private final SSLSocketFactory interceptedSocketFactory = HttpProxyAgent
+            .getInterceptedSslContext()
+            .getSocketFactory();
+
+    @Override
+    public Socket createSocket(String host, int port) throws IOException {
+        return interceptedSocketFactory.createSocket(host, port);
+    }
+
+    @Override
+    public Socket createSocket(String host, int port, InetAddress localAddress, int localPort) throws IOException {
+        return interceptedSocketFactory.createSocket(host, port, localAddress, localPort);
+    }
+
+    @Override
+    public Socket createSocket(Socket socket, String host, int port, boolean autoClose) throws IOException {
+        return interceptedSocketFactory.createSocket(socket, host, port, autoClose);
+    }
+
+    @Override
+    public Socket createSocket(String host, int port, InetAddress localAddress, int localPort, HttpConnectionParams params) throws IOException {
+        // Marginally more complicated logic here unfortunately, since timeout isn't natively
+        // supported. Minimal implementation taken from the existing lib implementations:
+        if (params == null) {
+            throw new IllegalArgumentException("Parameters may not be null");
+        }
+        int timeout = params.getConnectionTimeout();
+        Socket socket;
+
+        SocketFactory socketfactory = SSLSocketFactory.getDefault();
+        if (timeout == 0) {
+            socket = socketfactory.createSocket(host, port, localAddress, localPort);
+        } else {
+            socket = socketfactory.createSocket();
+            SocketAddress localAddr = new InetSocketAddress(localAddress, localPort);
+            SocketAddress remoteAddr = new InetSocketAddress(host, port);
+            socket.bind(localAddr);
+            socket.connect(remoteAddr, timeout);
+        }
+
+        return socket;
+    }
+}

src/main/java/tech/httptoolkit/javaagent/advice/apacheclient/ApacheOverrideProxyHostFieldAdvice.java

@@ -0,0 +1,19 @@
+package tech.httptoolkit.javaagent.advice.apacheclient;
+
+import net.bytebuddy.asm.Advice;
+import org.apache.commons.httpclient.ProxyHost;
+import tech.httptoolkit.javaagent.HttpProxyAgent;
+
+public class ApacheOverrideProxyHostFieldAdvice {
+
+    @Advice.OnMethodExit
+    public static void resetProxyHost(
+        @Advice.FieldValue(value = "proxyHost", readOnly = false) ProxyHost proxyHostField
+    ) {
+        // After creating/changing HostConfiguration we override the proxy field:
+        proxyHostField = new ProxyHost(
+            HttpProxyAgent.getAgentProxyHost(),
+            HttpProxyAgent.getAgentProxyPort()
+        );
+    }
+}

src/main/java/tech/httptoolkit/javaagent/advice/apacheclient/ApacheReturnCustomSslProtocolSocketFactoryAdvice.java

@@ -0,0 +1,17 @@
+package tech.httptoolkit.javaagent.advice.apacheclient;
+
+import net.bytebuddy.asm.Advice;
+import org.apache.commons.httpclient.protocol.ProtocolSocketFactory;
+
+public class ApacheReturnCustomSslProtocolSocketFactoryAdvice {
+    
+    @Advice.OnMethodExit
+    public static void getSocketFactory(
+        @Advice.FieldValue(value = "secure") boolean isSecure,
+        @Advice.Return(readOnly = false) ProtocolSocketFactory returnValue
+    ) {
+        if (isSecure) {
+            returnValue = new ApacheCustomSslProtocolSocketFactory();
+        }
+    }
+}

src/main/java/tech/httptoolkit/javaagent/advice/apacheclient/ApacheSetConfigProxyHostAdvice.java

@@ -0,0 +1,16 @@
+package tech.httptoolkit.javaagent.advice.apacheclient;
+
+import net.bytebuddy.asm.Advice;
+import org.apache.commons.httpclient.HostConfiguration;
+
+public class ApacheSetConfigProxyHostAdvice {
+
+    @Advice.OnMethodEnter
+    public static void beforeMakingRequests(
+        @Advice.FieldValue(value = "hostConfiguration") HostConfiguration hostConfiguration
+    ) {
+        // Elsewhere, we hook setProxyHost to reset the proxy to our configured version whenever it's called.
+        // Then, here we hook various methods to call it before they use the config:
+        hostConfiguration.setProxyHost(null); // null here is ignored as this method is already hooked
+    }
+}

src/main/kotlin/tech/httptoolkit/javaagent/AgentMain.kt

@@ -13,6 +13,7 @@ import java.lang.instrument.Instrumentation
 import javax.net.ssl.SSLContext
 import java.net.*
 import javax.net.ssl.HttpsURLConnection
+import javax.net.ssl.SSLSocketFactory
 import javax.net.ssl.TrustManagerFactory
 
 
@@ -85,6 +86,9 @@ fun interceptAllHttps(config: Config, instrumentation: Instrumentation) {
         ApacheClientRoutingV5Transformer(logger),
         ApacheSslSocketFactoryTransformer(logger),
         ApacheClientTlsStrategyTransformer(logger),
+        ApacheHostConfigurationTransformer(logger),
+        ApacheHttpMethodDirectorTransformer(logger),
+        ApacheProtocolTransformer(logger),
         JavaClientTransformer(logger),
         UrlConnectionTransformer(logger),
         HttpsUrlConnectionTransformer(logger),

src/main/kotlin/tech/httptoolkit/javaagent/ApacheClientTransformers.kt

@@ -2,11 +2,10 @@ package tech.httptoolkit.javaagent
 
 import net.bytebuddy.agent.builder.AgentBuilder
 import net.bytebuddy.asm.Advice
+import net.bytebuddy.description.method.MethodDescription
 import net.bytebuddy.dynamic.DynamicType
 import net.bytebuddy.matcher.ElementMatchers.*
-import tech.httptoolkit.javaagent.advice.apacheclient.ApacheSetSslSocketFactoryAdvice
-import tech.httptoolkit.javaagent.advice.apacheclient.ApacheV4ReturnProxyRouteAdvice
-import tech.httptoolkit.javaagent.advice.apacheclient.ApacheV5ReturnProxyRouteAdvice
+import tech.httptoolkit.javaagent.advice.apacheclient.*
 
 // For both v4 & v5 we override all implementations of the RoutePlanner interface, and we redefine all routes
 // to go via our proxy instead of their existing configuration.
@@ -68,3 +67,71 @@ class ApacheSslSocketFactoryTransformer(logger: TransformationLogger) : Matching
         );
     }
 }
+
+// Meanwhile, for V3 we need to do something totally different: we patch HostConfiguration to apply a proxy to
+// all new configurations (and ignore changes), we patch HttpMethodDirector to update existing configurations
+// as they're used, and we patch Protocol to change the SslSocketFactory on all secure protocols.
+
+class ApacheHostConfigurationTransformer(logger: TransformationLogger) : MatchingAgentTransformer(logger) {
+    override fun register(builder: AgentBuilder): AgentBuilder {
+        return builder
+            .type(
+                named("org.apache.commons.httpclient.HostConfiguration")
+            ).transform(this)
+    }
+
+    override fun transform(builder: DynamicType.Builder<*>): DynamicType.Builder<*> {
+        return builder
+            // Override the proxy field value for all new configurations, and for any attempts to call
+            // setProxy/ProxyHost. We don't no-op these, because we want to call them ourselves later on
+            // existing configs to reset them - we don't just want to ignore this.
+            .visit(
+                Advice.to(ApacheOverrideProxyHostFieldAdvice::class.java)
+                    .on(isConstructor<MethodDescription>()
+                        .or(hasMethodName("setProxy"))
+                        .or(hasMethodName("setProxyHost"))
+                    )
+            )
+    }
+}
+
+// Whenever an HttpMethodDirector is used, we reset the proxy in the passed configuration. This uses the above
+// hooks, which ensure that setProxyHost(anything) automatically loads & sets our intercepted proxy.
+// We *don't* want to reset all proxy hosts in all existing configurations, because that's a) quite tricky and
+// b) some are used as keys in existing direct connections in pools, and we don't want to match those later.
+class ApacheHttpMethodDirectorTransformer(logger: TransformationLogger) : MatchingAgentTransformer(logger) {
+    override fun register(builder: AgentBuilder): AgentBuilder {
+        return builder
+            .type(
+                named("org.apache.commons.httpclient.HttpMethodDirector")
+            ).transform(this)
+    }
+
+    override fun transform(builder: DynamicType.Builder<*>): DynamicType.Builder<*> {
+        return builder
+            .visit(
+                Advice.to(ApacheSetConfigProxyHostAdvice::class.java)
+                    .on(hasMethodName("executeMethod"))
+            )
+    }
+}
+
+// Every v3 configuration has a protocol, and each one can build sockets in its own unique way. Here, we patch
+// all of them so that all _secure_ protocols trust only our certificate, and nothing else. This would
+// be an issue for a generic TCP client, but for HTTPS we know we should be the only authority present.
+class ApacheProtocolTransformer(logger: TransformationLogger) : MatchingAgentTransformer(logger) {
+    override fun register(builder: AgentBuilder): AgentBuilder {
+        return builder
+            .type(
+                named("org.apache.commons.httpclient.protocol.Protocol")
+            ).transform(this)
+    }
+
+    override fun transform(builder: DynamicType.Builder<*>): DynamicType.Builder<*> {
+        return builder
+            .visit(
+                Advice.to(ApacheReturnCustomSslProtocolSocketFactoryAdvice::class.java)
+                    .on(hasMethodName("getSocketFactory"))
+            )
+    }
+}

test-app/build.gradle

@@ -11,12 +11,15 @@ repositories {
 }
 
 dependencies {
+    implementation group: 'commons-httpclient', name: 'commons-httpclient', version: '3.1'
     implementation group: 'org.apache.httpcomponents', name: 'httpclient', version: '4.5'
     implementation group: 'org.apache.httpcomponents.client5', name: 'httpclient5', version: '5.0.3'
     implementation group: 'org.apache.httpcomponents', name: 'httpasyncclient', version: '4.1.4'
+
     implementation group: 'com.squareup.okhttp3', name: 'okhttp', version: '4.9.1'
     implementation group: 'com.squareup.okhttp', name: 'okhttp', version: '2.7.5'
     implementation group: 'com.squareup.retrofit2', name: 'retrofit', version: '2.9.0'
+
     implementation group: 'org.eclipse.jetty', name: 'jetty-client', version: '10.0.0'
     implementation group: 'org.asynchttpclient', name: 'async-http-client', version: '2.12.2'
     implementation group: 'org.springframework', name: 'spring-webflux', version: '5.3.4'

test-app/src/main/java/tech/httptoolkit/testapp/Main.java

@@ -15,6 +15,7 @@
 public class Main {
 
     private static final Map<String, ClientCase<?>> cases = Map.ofEntries(
+        entry("apache-v3", new ApacheHttpClientV3Case()),
         entry("apache-v4", new ApacheHttpClientV4Case()),
         entry("apache-v5", new ApacheHttpClientV5Case()),
         entry("apache-async-v4", new ApacheHttpAsyncClientV4Case()),