Mitmdump binary management (#357)

Author: kiturutin

browserup-proxy-core/src/main/java/com/browserup/bup/mitmproxy/MitmProxyProcessManager.java

@@ -2,18 +2,17 @@
 
 import com.browserup.bup.mitmproxy.addons.*;
 import com.browserup.bup.mitmproxy.management.*;
+import org.apache.commons.lang3.SystemUtils;
 import org.awaitility.Awaitility;
 import org.awaitility.core.ConditionTimeoutException;
+import org.jetbrains.annotations.NotNull;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.zeroturnaround.exec.ProcessExecutor;
 import org.zeroturnaround.exec.StartedProcess;
 import org.zeroturnaround.exec.stream.LogOutputStream;
 import org.zeroturnaround.exec.stream.slf4j.Slf4jStream;
 
-import java.io.IOException;
-import java.io.PipedInputStream;
-import java.io.PipedOutputStream;
 import java.net.BindException;
 import java.net.InetSocketAddress;
 import java.util.ArrayList;
@@ -23,6 +22,9 @@
 
 public class MitmProxyProcessManager {
   private static final Logger LOGGER = LoggerFactory.getLogger(MitmProxyProcessManager.class);
+  private static final String MITMPROXY_BINARY_PATH_PROPERTY = "MITMPROXY_BINARY_PATH";
+  private static final String MITMPROXY_HOME_PATH = System.getProperty("user.home") + "/.browserup-mitmproxy";
+  private static final String MITMPROXY_DEFAULT_BINARY_PATH = MITMPROXY_HOME_PATH + "/" + getMitmproxyBinaryFileName();
 
   public enum MitmProxyLoggingLevel {
     error,
@@ -66,6 +68,10 @@ public enum MitmProxyLoggingLevel {
 
   private StringBuilder proxyLog = new StringBuilder();
 
+  private static String getMitmproxyBinaryFileName() {
+    return SystemUtils.IS_OS_WINDOWS ? "mitmdump.exe" : "mitmdump";
+  }
+
   public void start(int port) {
     start(port == 0 ? NetworkUtils.getFreePort() : port, defaultAddons());
   }
@@ -170,20 +176,7 @@ private void startProxyWithRetries(int port, List<AbstractAddon> addons, int ret
   }
 
   private void startProxy(int port, List<AbstractAddon> addons) {
-    List<String> command = new ArrayList<String>() {{
-      add("mitmdump");
-      add("-p");
-      add(String.valueOf(port));
-      add("--set");
-      add("flow_detail=3");
-    }};
-    if (trustAll) {
-      command.add("--ssl-insecure");
-    }
-
-    updateCommandWithUpstreamProxy(command);
-    updateCommandWithLogLevel(command);
-    updateCommandWithAddOns(addons, command);
+    List<String> command = createCommand(port, addons);
 
     LOGGER.info("Starting proxy using command: " + String.join(" ", command));
 
@@ -193,6 +186,10 @@ private void startProxy(int port, List<AbstractAddon> addons) {
     } catch (Exception ex) {
       throw new RuntimeException("Couldn't start mitmproxy process", ex);
     }
+    waitForReady();
+  }
+
+  private void waitForReady() {
     try {
       Awaitility.await()
               .atMost(5, TimeUnit.SECONDS)
@@ -202,6 +199,33 @@ private void startProxy(int port, List<AbstractAddon> addons) {
     }
   }
 
+  @NotNull
+  private ArrayList<String> createCommand(int port, List<AbstractAddon> addons) {
+    ArrayList<String> command = new ArrayList<String>() {{
+      add(getMitmproxyBinaryPath());
+      add("-p");
+      add(String.valueOf(port));
+      add("--set");
+      add("confdir=" + MITMPROXY_HOME_PATH);
+    }};
+    if (trustAll) {
+      command.add("--ssl-insecure");
+    }
+
+    updateCommandWithUpstreamProxy(command);
+    updateCommandWithLogLevel(command);
+    updateCommandWithAddOns(addons, command);
+    return command;
+  }
+
+  private String getMitmproxyBinaryPath() {
+    String mitmproxyBinaryPathProperty = System.getProperty(MITMPROXY_BINARY_PATH_PROPERTY);
+    if (mitmproxyBinaryPathProperty != null) {
+      return mitmproxyBinaryPathProperty + "/" + getMitmproxyBinaryFileName();
+    }
+    return MITMPROXY_DEFAULT_BINARY_PATH;
+  }
+
   private void handleHealthCheckFailure() {
     LOGGER.error("MitmProxy might not started properly, healthcheck failed for port: " + this.proxyPort);
     if (startedProcess == null) return;
@@ -251,8 +275,13 @@ private void updateCommandWithAddOns(List<AbstractAddon> addons, List<String> co
   }
 
   private void updateCommandWithLogLevel(List<String> command) {
+    MitmProxyLoggingLevel logLevel = getMitmProxyLoggingLevel();
     command.add("--set");
-    command.add("termlog_verbosity=" + getMitmProxyLoggingLevel());
+    command.add("termlog_verbosity=" + logLevel);
+    if (logLevel.equals(MitmProxyLoggingLevel.debug)) {
+      command.add("--set");
+      command.add("flow_detail=3");
+    }
   }
 
   private void updateCommandWithUpstreamProxy(List<String> command) {

browserup-proxy-dist/build.gradle

@@ -6,6 +6,10 @@ plugins {
     id 'application'
 }
 
+application {
+    applicationDefaultJvmArgs = ['-DMITMPROXY_BINARY_PATH=mitmproxy/']
+}
+
 uploadArchives {
     repositories {
         mavenDeployer {
@@ -83,6 +87,36 @@ dependencies {
 applicationName = 'browserup-proxy'
 mainClassName = 'com.browserup.bup.proxy.Main'
 
+def mitmproxyLinuxURL = 'https://mitmproxy-linux.s3.us-east-2.amazonaws.com/mitmdump'.toURL()
+def mitmproxyWindowsURL = 'https://mitmproxy-windows.s3.us-east-2.amazonaws.com/mitmdump.exe'.toURL()
+
+static void downloadFile(URL srcUrl, File dstFile) {
+    dstFile.withOutputStream { out ->
+        srcUrl.withInputStream { from ->
+            out << from
+        }
+    }
+}
+
+task downloadMitmproxyBinaries {
+    File mitmproxyDir = file("$buildDir/mitmproxy")
+    outputs.dir mitmproxyDir
+    doLast {
+        mitmproxyDir.mkdirs()
+        println("Downloading mitmproxy dependencies...")
+        def linuxBinaryFile = file("$buildDir/mitmproxy/mitmdump")
+        def windowsBinaryFile = file("$buildDir/mitmproxy/mitmdump.exe")
+        downloadFile(mitmproxyLinuxURL, linuxBinaryFile)
+        downloadFile(mitmproxyWindowsURL, windowsBinaryFile)
+        project.exec {
+            commandLine('chmod', '+x', linuxBinaryFile.absolutePath)
+            commandLine('chmod', '+x', windowsBinaryFile.absolutePath)
+        }
+        println("Downloaded mitmproxy dependencies.")
+        mitmproxyDir
+    }
+}
+
 distributions {
     main {
         baseName = 'browserup-proxy'
@@ -92,6 +126,9 @@ distributions {
             into ('ssl') {
                 from '../browserup-proxy-core/src/main/resources/sslSupport'
             }
+            from(downloadMitmproxyBinaries) {
+                into 'bin/mitmproxy'
+            }
         }
     }
 }