Merge branch 'master' into dependabot/gradle/com.squareup.okhttp3-okhttp-4.9.0

Author: ericbeland

browserup-proxy-core/build.gradle

@@ -58,7 +58,7 @@ uploadArchives {
 
 dependencies {
     api "io.netty:netty-codec:${nettyVersion}"
-    api('xyz.rogfam:littleproxy:2.0.0-beta-5') {
+    api('xyz.rogfam:littleproxy:2.0.1') {
         exclude(group: 'io.netty', module: 'netty-all')
     }
 
@@ -70,9 +70,9 @@ dependencies {
     implementation "com.fasterxml.jackson.core:jackson-core:${jacksonVersion}"
     implementation "com.fasterxml.jackson.core:jackson-databind:${jacksonVersion}"
     implementation "com.fasterxml.jackson.core:jackson-annotations:${jacksonVersion}"
-    implementation 'com.google.guava:guava:28.2-jre'
+    implementation 'com.google.guava:guava:30.0-jre'
     implementation 'com.jcraft:jzlib:1.1.3'
-    implementation 'dnsjava:dnsjava:3.1.0'
+    implementation 'dnsjava:dnsjava:3.3.1'
     implementation "io.netty:netty-all:${nettyVersion}"
     implementation "org.bouncycastle:bcpkix-jdk15on:${bcpVersion}"
     implementation "org.bouncycastle:bcprov-jdk15on:${bcpVersion}"
@@ -82,22 +82,21 @@ dependencies {
     implementation "org.slf4j:jcl-over-slf4j:${slf4jVersion}"
     implementation "org.slf4j:slf4j-api:${slf4jVersion}"
     implementation "com.squareup.okhttp3:okhttp:4.9.0"
-    implementation 'org.zeroturnaround:zt-exec:1.11'
-
-    testImplementation 'junit:junit:4.13'
-    testImplementation 'org.codehaus.groovy:groovy-all:2.5.8'
+    implementation 'org.zeroturnaround:zt-exec:1.12'
+    testImplementation 'junit:junit:4.13.1'
+    testImplementation 'org.codehaus.groovy:groovy-all:3.0.6'
     testImplementation "org.apache.logging.log4j:log4j-api:${log4jVersion}"
     testImplementation "org.apache.logging.log4j:log4j-core:${log4jVersion}"
     testImplementation "org.apache.logging.log4j:log4j-slf4j-impl:${log4jVersion}"
     testImplementation 'org.codehaus.groovy.modules.http-builder:http-builder:0.7.2'
     testImplementation "org.eclipse.jetty:jetty-server:${jettyVersion}"
     testImplementation "org.eclipse.jetty:jetty-servlet:${jettyVersion}"
     testImplementation "org.eclipse.jetty:jetty-servlets:${jettyVersion}"
-    testImplementation 'org.hamcrest:hamcrest:2.2'    
+    testImplementation 'org.hamcrest:hamcrest:2.2'
     testImplementation 'org.hamcrest:hamcrest-library:2.2'
     testImplementation 'com.github.tomakehurst:wiremock-jre8:2.26.3'
     testImplementation 'org.mockito:mockito-core:3.3.3'
     testImplementation 'org.seleniumhq.selenium:selenium-firefox-driver:3.141.59'
-    testImplementation 'org.apache.httpcomponents:httpclient:4.5.12'
+    testImplementation 'org.apache.httpcomponents:httpclient:4.5.13'
 }
 

browserup-proxy-core/src/main/java/com/browserup/bup/BrowserUpProxyServer.java

@@ -465,6 +465,21 @@ public SSLEngine newSslEngine() {
         proxyServer = bootstrap.start();
 
         addHarCaptureFilter();
+
+        addHttpFilterFactory(new HttpFiltersSourceAdapter() {
+            @Override
+            public HttpFilters filterRequest(HttpRequest originalRequest, ChannelHandlerContext ctx) {
+                return new BlacklistFilter(originalRequest, ctx, getBlacklist());
+            }
+        });
+
+        addHttpFilterFactory(new HttpFiltersSourceAdapter() {
+            @Override
+            public HttpFilters filterRequest(HttpRequest originalRequest, ChannelHandlerContext ctx) {
+                Whitelist currentWhitelist = whitelist.get();
+                return new WhitelistFilter(originalRequest, ctx, isWhitelistEnabled(), currentWhitelist.getStatusCode(), currentWhitelist.getPatterns());
+            }
+        });
     }
 
     @Override
@@ -1453,21 +1468,6 @@ public HttpFilters filterRequest(HttpRequest originalRequest, ChannelHandlerCont
             }
         });
 
-        addHttpFilterFactory(new HttpFiltersSourceAdapter() {
-            @Override
-            public HttpFilters filterRequest(HttpRequest originalRequest, ChannelHandlerContext ctx) {
-                return new BlacklistFilter(originalRequest, ctx, getBlacklist());
-            }
-        });
-
-        addHttpFilterFactory(new HttpFiltersSourceAdapter() {
-            @Override
-            public HttpFilters filterRequest(HttpRequest originalRequest, ChannelHandlerContext ctx) {
-                Whitelist currentWhitelist = whitelist.get();
-                return new WhitelistFilter(originalRequest, ctx, isWhitelistEnabled(), currentWhitelist.getStatusCode(), currentWhitelist.getPatterns());
-            }
-        });
-
         addHttpFilterFactory(new HttpFiltersSourceAdapter() {
             @Override
             public HttpFilters filterRequest(HttpRequest originalRequest, ChannelHandlerContext ctx) {

browserup-proxy-core/src/main/java/com/browserup/bup/filters/BlacklistFilter.java

@@ -6,13 +6,14 @@
 
 import io.netty.channel.ChannelHandlerContext;
 import io.netty.handler.codec.http.DefaultFullHttpResponse;
-import io.netty.handler.codec.http.HttpHeaders;
 import io.netty.handler.codec.http.HttpMethod;
 import io.netty.handler.codec.http.HttpObject;
 import io.netty.handler.codec.http.HttpRequest;
 import io.netty.handler.codec.http.HttpResponse;
 import io.netty.handler.codec.http.HttpResponseStatus;
 import com.browserup.bup.proxy.BlacklistEntry;
+import com.browserup.bup.util.HttpStatusClass;
+import io.netty.handler.codec.http.HttpUtil;
 
 import java.util.Collection;
 import java.util.Collections;
@@ -22,6 +23,7 @@
  * that the blacklist at the time of construction will contain the same values when the filter is actually invoked, if the entries are modified concurrently.
  */
 public class BlacklistFilter extends HttpsAwareFiltersAdapter {
+    public static final String BLOCKED_PHRASE = "Request blocked";
     private final Collection<BlacklistEntry> blacklistedUrls;
 
     public BlacklistFilter(HttpRequest originalRequest, ChannelHandlerContext ctx, Collection<BlacklistEntry> blacklistedUrls) {
@@ -39,18 +41,23 @@ public HttpResponse clientToProxyRequest(HttpObject httpObject) {
         if (httpObject instanceof HttpRequest) {
             HttpRequest httpRequest = (HttpRequest) httpObject;
 
-            String url = getFullUrl(httpRequest);
+            String url = getOriginalUrl();
 
             for (BlacklistEntry entry : blacklistedUrls) {
-                if (HttpMethod.CONNECT.equals(httpRequest.getMethod()) && entry.getHttpMethodPattern() == null) {
+                if (HttpMethod.CONNECT.equals(httpRequest.method()) && entry.getHttpMethodPattern() == null) {
                     // do not allow CONNECTs to be blacklisted unless a method pattern is explicitly specified
                     continue;
                 }
 
-                if (entry.matches(url, httpRequest.getMethod().name())) {
-                    HttpResponseStatus status = HttpResponseStatus.valueOf(entry.getStatusCode());
-                    HttpResponse resp = new DefaultFullHttpResponse(httpRequest.getProtocolVersion(), status);
-                    HttpHeaders.setContentLength(resp, 0L);
+                if (entry.matches(url, httpRequest.method().name())) {
+                    HttpResponseStatus status;
+                    if(HttpStatusClass.UNKNOWN.equals(HttpStatusClass.valueOf(entry.getStatusCode()))) {
+                        status = new HttpResponseStatus(entry.getStatusCode(), BLOCKED_PHRASE);
+                    } else {
+                        status = HttpResponseStatus.valueOf(entry.getStatusCode());
+                    }
+                    HttpResponse resp = new DefaultFullHttpResponse(httpRequest.protocolVersion(), status);
+                    HttpUtil.setContentLength(resp, 0L);
 
                     return resp;
                 }

browserup-proxy-core/src/main/java/com/browserup/bup/filters/HarCaptureFilter.java

@@ -47,7 +47,6 @@
 import java.util.EnumSet;
 import java.util.List;
 import java.util.Set;
-import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 
 import static com.browserup.bup.util.BrowserUpProxyUtil.getTotalElapsedTime;
@@ -120,12 +119,14 @@ public class HarCaptureFilter extends HttpsAwareFiltersAdapter {
      */
     private volatile HttpRequest capturedOriginalRequest;
 
+    private volatile boolean isResponse = false;
+
     /**
      * True if this filter instance processed a {@link #proxyToServerResolutionSucceeded(String, java.net.InetSocketAddress)} call, indicating
      * that the hostname was resolved and populated in the HAR (if this is not a CONNECT).
      */
     private volatile boolean addressResolved = false;
-
+    
     /**
      * Create a new instance of the HarCaptureFilter that will capture request and response information. If no har is specified in the
      * constructor, this filter will do nothing.
@@ -255,6 +256,7 @@ public HttpResponse clientToProxyRequest(HttpObject httpObject) {
 
     @Override
     public HttpObject serverToProxyResponse(HttpObject httpObject) {
+        isResponse = true;
         // if a ServerResponseCaptureFilter is configured, delegate to it to collect the server's response. if it is not
         // configured, we still need to capture basic information (timings, HTTP status, etc.), just not content.
         if (responseCaptureFilter != null) {
@@ -286,8 +288,20 @@ public HttpObject serverToProxyResponse(HttpObject httpObject) {
         return super.serverToProxyResponse(httpObject);
     }
 
+    @Override
+    public HttpObject proxyToClientResponse(HttpObject httpObject) {
+        // if a subsequent filter short-circuited the response, capture it here
+        if (!isResponse && httpObject instanceof HttpResponse) {
+            HttpResponse httpResponse = (HttpResponse) httpObject;
+            captureResponse(httpResponse);
+            harEntry.setTime(getTotalElapsedTime(harEntry.getTimings()));
+        }
+        return super.proxyToClientResponse(httpObject); 
+    }
+
     @Override
     public void serverToProxyResponseTimedOut() {
+        isResponse = true;
         // replace any existing HarResponse that was created if the server sent a partial response
         HarResponse response = HarCaptureUtil.createHarResponseForFailure();
         harEntry.setResponse(response);
@@ -678,6 +692,7 @@ public InetSocketAddress proxyToServerResolutionStarted(String resolvingServerHo
 
     @Override
     public void proxyToServerResolutionFailed(String hostAndPort) {
+        isResponse = true;
         HarResponse response = HarCaptureUtil.createHarResponseForFailure();
         this.harEntry.setResponse(response);
 
@@ -720,6 +735,7 @@ public void proxyToServerConnectionStarted() {
 
     @Override
     public void proxyToServerConnectionFailed() {
+        isResponse = true;
         HarResponse response = HarCaptureUtil.createHarResponseForFailure();
         this.harEntry.setResponse(response);
 
@@ -733,6 +749,7 @@ public void proxyToServerConnectionFailed() {
 
     @Override
     public void proxyToServerConnectionSucceeded(ChannelHandlerContext serverCtx) {
+        isResponse = true;
         long connectionSucceededTimeNanos = System.nanoTime();
 
         // make sure the previous timestamp was captured, to avoid setting an absurd value in the har (see serverToProxyResponseReceiving())

browserup-proxy-core/src/main/java/com/browserup/bup/filters/ResponseFilterAdapter.java

@@ -66,7 +66,7 @@ public void setModifiedHttpRequest(HttpRequest modifiedHttpRequest) {
      * and sets a maximum response buffer size of 2 MiB.
      */
     public static class FilterSource extends HttpFiltersSourceAdapter {
-        private static final int DEFAULT_MAXIMUM_RESPONSE_BUFFER_SIZE = 2097152;
+        private static final int DEFAULT_MAXIMUM_RESPONSE_BUFFER_SIZE = 10485760;
 
         private final ResponseFilter filter;
         private final int maximumResponseBufferSizeInBytes;

browserup-proxy-core/src/main/java/com/browserup/bup/filters/WhitelistFilter.java

@@ -4,9 +4,9 @@
 
 package com.browserup.bup.filters;
 
+import com.browserup.bup.util.HttpStatusClass;
 import io.netty.channel.ChannelHandlerContext;
 import io.netty.handler.codec.http.DefaultFullHttpResponse;
-import io.netty.handler.codec.http.HttpHeaders;
 import io.netty.handler.codec.http.HttpObject;
 import io.netty.handler.codec.http.HttpRequest;
 import io.netty.handler.codec.http.HttpResponse;
@@ -15,7 +15,6 @@
 import org.littleshoot.proxy.impl.ProxyUtils;
 
 import java.util.Collection;
-import java.util.Collections;
 import java.util.regex.Pattern;
 
 import static java.util.Collections.*;
@@ -58,12 +57,17 @@ public HttpResponse clientToProxyRequest(HttpObject httpObject) {
 
             boolean urlWhitelisted;
 
-            String url = getFullUrl(httpRequest);
+            String url = getOriginalUrl();
 
             urlWhitelisted = whitelistUrls.stream().anyMatch(pattern -> pattern.matcher(url).matches());
 
             if (!urlWhitelisted) {
-                HttpResponseStatus status = HttpResponseStatus.valueOf(whitelistResponseCode);
+                HttpResponseStatus status;
+                if(HttpStatusClass.UNKNOWN.equals(HttpStatusClass.valueOf(whitelistResponseCode))) {
+                    status = new HttpResponseStatus(whitelistResponseCode, BlacklistFilter.BLOCKED_PHRASE);
+                } else {
+                    status = HttpResponseStatus.valueOf(whitelistResponseCode);
+                }
                 HttpResponse resp = new DefaultFullHttpResponse(httpRequest.protocolVersion(), status);
                 HttpUtil.setContentLength(resp, 0L);
 

browserup-proxy-core/src/main/java/com/browserup/bup/mitmproxy/MitmProxyProcessManager.java

@@ -173,6 +173,8 @@ private void startProxy(int port, List<AbstractAddon> addons) {
       add("mitmdump");
       add("-p");
       add(String.valueOf(port));
+      add("--set");
+      add("flow_detail=3");
     }};
     if (trustAll) {
       command.add("--ssl-insecure");

browserup-proxy-core/src/main/java/com/browserup/bup/mitmproxy/management/HarCaptureManager.java

@@ -22,7 +22,6 @@ public class HarCaptureManager {
 
     private final AddonsManagerClient addonsManagerClient;
     private final MitmProxyProcessManager mitmProxyManager;
-    private Har lastHar = new Har();
     private EnumSet<CaptureType> lastCaptureTypes = EnumSet.noneOf(CaptureType.class);
 
     public HarCaptureManager(AddonsManagerClient addonsManagerClient, MitmProxyProcessManager mitmProxyManager) {
@@ -35,7 +34,7 @@ public Har getHar() {
     }
 
     public Har getHar(Boolean cleanHar) {
-        if (!mitmProxyManager.isRunning()) return lastHar;
+        if (!mitmProxyManager.isRunning()) return null;
 
         HarResponse response = addonsManagerClient.
                 getRequestToAddonsManager(
@@ -58,7 +57,7 @@ public Har newHar(String pageRef) {
     }
 
     public Har newHar(String pageRef, String pageTitle) {
-        if (!mitmProxyManager.isRunning()) return lastHar;
+        if (!mitmProxyManager.isRunning()) return null;
 
         HarResponse response = addonsManagerClient.
                 getRequestToAddonsManager(
@@ -73,7 +72,7 @@ public Har newHar(String pageRef, String pageTitle) {
     }
 
     public Har endHar() {
-        if (!mitmProxyManager.isRunning()) return lastHar;
+        if (!mitmProxyManager.isRunning()) return null;
 
         HarResponse response = addonsManagerClient.
                 getRequestToAddonsManager(
@@ -93,7 +92,7 @@ public Har newPage(String pageRef) {
     }
 
     public Har newPage(String pageRef, String pageTitle) {
-        if (!mitmProxyManager.isRunning()) return lastHar;
+        if (!mitmProxyManager.isRunning()) return null;
 
         HarResponse response = addonsManagerClient.
                 getRequestToAddonsManager(
@@ -127,7 +126,6 @@ private Har parseHar(String filePath) {
         } catch (IOException e) {
             throw new RuntimeException("Couldn't read HAR file: " + harFile.getAbsolutePath(), e);
         }
-        lastHar = har;
         return har;
     }
 

browserup-proxy-core/src/main/java/com/browserup/harreader/model/HarEntry.java

@@ -1,10 +1,6 @@
 package com.browserup.harreader.model;
 
-import com.fasterxml.jackson.annotation.JsonAnyGetter;
-import com.fasterxml.jackson.annotation.JsonAnySetter;
-import com.fasterxml.jackson.annotation.JsonFormat;
-import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
-import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.*;
 
 import java.util.Date;
 import java.util.HashMap;
@@ -29,6 +25,7 @@ public class HarEntry {
     private String serverIPAddress;
     private String connection;
     private String comment;
+    private String _url;
     private Map<String, Object> additional = new HashMap<>();
 
     /**
@@ -121,6 +118,19 @@ public void setTimings(HarTiming timings) {
         this.timings = timings;
     }
 
+    @JsonProperty("_url")
+    public String getUrl() {
+        if (_url == null) {
+            _url = "";
+        }
+        return _url;
+    }
+
+    @JsonProperty("_url")
+    public void setUrl(String url) {
+        this._url = url;
+    }
+
     /**
      * @return Server IP address (result of DNS resolution), null if not present.
      */

browserup-proxy-core/src/main/java/com/browserup/harreader/model/HarResponse.java

@@ -19,7 +19,7 @@ public class HarResponse {
 
     private HttpStatus status;
     private String statusText = "";
-    private String httpVersion = "HTTP/1.1";
+    private String httpVersion = "unknown";
     private List<HarCookie> cookies;
     private List<HarHeader> headers;
     private HarContent content;