More forcefully override built-in HttpUrlConnections

Attempting to intercept Intellij shows that applications which manage their
own proxy config aren't always captured. This helps to lock down some of
those gaps, ensuring HTTP(S) traffic is _always_ sent to our proxy.

Author: pimterry

src/main/java/tech/httptoolkit/javaagent/advice/OverrideAllProxySelectionAdvice.java

@@ -0,0 +1,32 @@
+package tech.httptoolkit.javaagent.advice;
+
+import net.bytebuddy.asm.Advice;
+
+import java.net.InetSocketAddress;
+import java.net.Proxy;
+import java.net.URI;
+import java.util.Collections;
+import java.util.List;
+
+public class OverrideAllProxySelectionAdvice {
+
+    @Advice.OnMethodExit
+    public static void selectProxy(
+            @Advice.Argument(value = 0) URI uri,
+            @Advice.Return(readOnly = false) List<Proxy> returnedProxies
+    ) {
+        String scheme = uri.getScheme();
+
+        // For HTTP URIs only, we override all proxy selection globally to select our proxy instead:
+        if (scheme.equals("http") || scheme.equals("https")) {
+            returnedProxies = Collections.singletonList(
+                new Proxy(Proxy.Type.HTTP, new InetSocketAddress(
+                    // We read from our custom variables, since we can't access HttpProxyAgent from a bootstrapped
+                    // class, and we use namespaced properties to make this extra reliable:
+                    System.getProperty("tech.httptoolkit.proxyHost"),
+                    Integer.parseInt(System.getProperty("tech.httptoolkit.proxyPort"))
+                ))
+            );
+        }
+    }
+}

src/main/java/tech/httptoolkit/javaagent/advice/OverrideUrlConnectionProxyAdvice.java

@@ -0,0 +1,24 @@
+package tech.httptoolkit.javaagent.advice;
+
+import net.bytebuddy.asm.Advice;
+
+import java.net.Proxy;
+import java.net.ProxySelector;
+import java.net.URI;
+
+public class OverrideUrlConnectionProxyAdvice {
+
+    @Advice.OnMethodEnter
+    public static void openConnection(
+        @Advice.FieldValue(value = "protocol") String urlProtocol,
+        @Advice.Argument(value = 0, readOnly = false) Proxy proxyArgument
+    ) {
+        if (urlProtocol.equals("http") || urlProtocol.equals("https")) {
+            // We can't access HttpProxyAgent here or even thisd class, since we're in the bootstrap loader, but
+            // we've already stored a proxy on ProxySelector for all URLs, so we can just use that directly:
+            proxyArgument = ProxySelector.getDefault().select(
+                    URI.create("http://example.com")
+            ).get(0);
+        }
+    }
+}

src/main/java/tech/httptoolkit/javaagent/advice/ReturnSslSocketFactoryAdvice.java

@@ -1,13 +1,18 @@
 package tech.httptoolkit.javaagent.advice;
 
 import net.bytebuddy.asm.Advice;
-import tech.httptoolkit.javaagent.HttpProxyAgent;
 
+import javax.net.ssl.SSLContext;
 import javax.net.ssl.SSLSocketFactory;
+import java.security.NoSuchAlgorithmException;
 
 public class ReturnSslSocketFactoryAdvice {
     @Advice.OnMethodExit
     public static void sslSocketFactory(@Advice.Return(readOnly = false) SSLSocketFactory returnValue) {
-        returnValue = HttpProxyAgent.getInterceptedSslContext().getSocketFactory();
+        try {
+            returnValue = SSLContext.getDefault().getSocketFactory();
+        } catch (NoSuchAlgorithmException e) {
+            throw new RuntimeException(e);
+        }
     }
 }

src/main/java/tech/httptoolkit/javaagent/advice/SkipMethodAdvice.java

@@ -0,0 +1,12 @@
+package tech.httptoolkit.javaagent.advice;
+
+import net.bytebuddy.asm.Advice;
+
+// General purpose advice which skips a given method, returning the default value for its type
+// (so usually null) if there is a return value, and silently doing nothing.
+public class SkipMethodAdvice {
+    @Advice.OnMethodEnter(skipOn = Advice.OnNonDefaultValue.class) // => skip if we return true (or similar)
+    public static boolean skipMethod() {
+        return true; // Skip the method body entirely
+    }
+}

src/main/kotlin/tech/httptoolkit/javaagent/AgentMain.kt

@@ -86,6 +86,10 @@ fun interceptAllHttps(config: Config, instrumentation: Instrumentation) {
         ApacheSslSocketFactoryTransformer(logger),
         ApacheClientTlsStrategyTransformer(logger),
         JavaClientTransformer(logger),
+        UrlConnectionTransformer(logger),
+        HttpsUrlConnectionTransformer(logger),
+        ProxySelectorTransformer(logger),
+        SslContextTransformer(logger),
         JettyClientTransformer(logger),
         AsyncHttpClientConfigTransformer(logger),
         AsyncHttpChannelManagerTransformer(logger),
@@ -123,6 +127,10 @@ private fun setDefaultProxy(proxyHost: String, proxyPort: Int) {
     System.setProperty("https.proxyHost", proxyHost)
     System.setProperty("https.proxyPort", proxyPort.toString())
 
+    // We back up the properties in our namespace too, in case anybody manually overrides the above:
+    System.setProperty("tech.httptoolkit.proxyHost", proxyHost)
+    System.setProperty("tech.httptoolkit.proxyPort", proxyPort.toString())
+
     val proxySelector = ConstantProxySelector(proxyHost, proxyPort)
     AgentProxySelector = proxySelector
     ProxySelector.setDefault(proxySelector)

src/main/kotlin/tech/httptoolkit/javaagent/HttpsUrlConnectionTransformer.kt

@@ -0,0 +1,26 @@
+package tech.httptoolkit.javaagent
+
+import net.bytebuddy.agent.builder.AgentBuilder
+import net.bytebuddy.asm.Advice
+import net.bytebuddy.dynamic.DynamicType
+import net.bytebuddy.matcher.ElementMatchers.*
+import tech.httptoolkit.javaagent.advice.ReturnSslSocketFactoryAdvice
+
+// We override the SSLSocketFactory field for HttpsURLConnections. This is the only way to access the
+// configured field, so this effectively reconfigured every such connection to trust our certificate.
+// Without this, connections still work as our SSLContext is the default, but this ensures they work
+// even for connections that are explicitly configured with their own settings.
+class HttpsUrlConnectionTransformer(logger: TransformationLogger): MatchingAgentTransformer(logger) {
+    override fun register(builder: AgentBuilder): AgentBuilder {
+        return builder
+            .type(
+                named("javax.net.ssl.HttpsURLConnection")
+            ).transform(this)
+    }
+
+    override fun transform(builder: DynamicType.Builder<*>): DynamicType.Builder<*> {
+        return builder
+            .visit(Advice.to(ReturnSslSocketFactoryAdvice::class.java)
+                .on(hasMethodName("getSSLSocketFactory")))
+    }
+}

src/main/kotlin/tech/httptoolkit/javaagent/ProxySelectorTransformer.kt

@@ -0,0 +1,39 @@
+package tech.httptoolkit.javaagent
+
+import net.bytebuddy.agent.builder.AgentBuilder
+import net.bytebuddy.asm.Advice
+import net.bytebuddy.description.method.MethodDescription
+import net.bytebuddy.dynamic.DynamicType
+import net.bytebuddy.matcher.ElementMatchers.*
+import tech.httptoolkit.javaagent.advice.OverrideAllProxySelectionAdvice
+import tech.httptoolkit.javaagent.advice.OverrideUrlConnectionProxyAdvice
+import tech.httptoolkit.javaagent.advice.SkipMethodAdvice
+
+// To ensure that target applications don't override our ProxySelector (which we configure as the default), we
+// also patch the ProxySelector class itself, to guarantee that our proxy is always always selected, and
+// to stop anybody else changing the default.
+class ProxySelectorTransformer(logger: TransformationLogger): MatchingAgentTransformer(logger) {
+    override fun register(builder: AgentBuilder): AgentBuilder {
+        return builder
+            .type(
+                hasSuperType(named("java.net.ProxySelector"))
+            ).and(
+                not(isInterface())
+            ).transform(this)
+    }
+
+    override fun transform(builder: DynamicType.Builder<*>): DynamicType.Builder<*> {
+        return builder
+            // We patch *all* proxy selectors, so that even code which doesn't use the default
+            // still returns our proxy regardless.
+            .visit(Advice.to(OverrideAllProxySelectionAdvice::class.java)
+                .on(
+                    hasMethodName<MethodDescription>("select")
+                    .and(takesArguments(1))
+                    .and(takesArgument(0, named("java.net.URI")))))
+            // We already set the default ProxySelector on startup, before we intercept anything.
+            // Here we patch ProxySelector so nobody can overwrite that later.
+            .visit(Advice.to(SkipMethodAdvice::class.java)
+                .on(hasMethodName("setDefault")));
+    }
+}

src/main/kotlin/tech/httptoolkit/javaagent/SslContextTransformer.kt

@@ -0,0 +1,26 @@
+package tech.httptoolkit.javaagent
+
+import net.bytebuddy.agent.builder.AgentBuilder
+import net.bytebuddy.asm.Advice
+import net.bytebuddy.dynamic.DynamicType
+import net.bytebuddy.matcher.ElementMatchers.*
+import tech.httptoolkit.javaagent.advice.SkipMethodAdvice
+
+// We patch SSL context purely to ensure that the default context that we set isn't changed later
+// by anybody else. The default context is already set in AgentMain before we begin patching.
+class SslContextTransformer(logger: TransformationLogger): MatchingAgentTransformer(logger) {
+    override fun register(builder: AgentBuilder): AgentBuilder {
+        return builder
+            .type(
+                named("javax.net.ssl.SSLContext")
+            ).transform(this)
+    }
+
+    override fun transform(builder: DynamicType.Builder<*>): DynamicType.Builder<*> {
+        return builder
+            // We set the default SSLContext on startup, before we intercepted anything.
+            // Here we patch SSLContext itself so nobody can overwrite that later.
+            .visit(Advice.to(SkipMethodAdvice::class.java)
+                .on(hasMethodName("setDefault")));
+    }
+}

src/main/kotlin/tech/httptoolkit/javaagent/UrlConnectionTransformer.kt

@@ -0,0 +1,28 @@
+package tech.httptoolkit.javaagent
+
+import net.bytebuddy.agent.builder.AgentBuilder
+import net.bytebuddy.asm.Advice
+import net.bytebuddy.description.method.MethodDescription
+import net.bytebuddy.dynamic.DynamicType
+import net.bytebuddy.matcher.ElementMatchers.*
+import tech.httptoolkit.javaagent.advice.OverrideUrlConnectionProxyAdvice
+
+// We override URL.openConnection() so that even if a proxy setting is passed explicitly, it's
+// overridden and ignored (for HTTP(S) traffic only) so all such traffic goes to our proxy.
+class UrlConnectionTransformer(logger: TransformationLogger): MatchingAgentTransformer(logger) {
+    override fun register(builder: AgentBuilder): AgentBuilder {
+        return builder
+            .type(
+                named("java.net.URL")
+            ).transform(this)
+    }
+
+    override fun transform(builder: DynamicType.Builder<*>): DynamicType.Builder<*> {
+        return builder
+            .visit(Advice.to(OverrideUrlConnectionProxyAdvice::class.java)
+                .on(
+                    hasMethodName<MethodDescription>("openConnection")
+                    .and(takesArguments(1))
+                    .and(takesArgument(0, named("java.net.Proxy")))))
+    }
+}

test-app/src/main/java/tech/httptoolkit/testapp/cases/HttpUrlConnCase.java

@@ -3,8 +3,12 @@
 import java.io.IOException;
 import java.net.HttpURLConnection;
 import java.net.MalformedURLException;
+import java.net.Proxy;
 import java.net.URL;
 
+// We take over default HttpUrlConnections with no problem by setting config vars, but that doesn't apply in all
+// cases, e.g. if the target code manages its own proxy config, or disables it. We intercept here to forcibly
+// ensure that our proxy is _always_ used, regardless of the passed proxy configuration.
 public class HttpUrlConnCase extends ClientCase<URL> {
 
     @Override
@@ -14,7 +18,7 @@ public URL newClient(String url) throws MalformedURLException {
 
     @Override
     public int test(String url, URL urlInstance) throws IOException {
-        HttpURLConnection connection = (HttpURLConnection) urlInstance.openConnection();
+        HttpURLConnection connection = (HttpURLConnection) urlInstance.openConnection(Proxy.NO_PROXY);
         return connection.getResponseCode();
     }
 }