addons update

Author: kiturutin

browserup-proxy-core/src/main/resources/mitmproxy/har_dump.py

@@ -187,6 +187,9 @@ def __init__(self):
         self.response_receive_started_nanos = 0
         self.http_connect_timings = {}
 
+    def get_har_entry(self, flow):
+        return flow.request.har_entry
+
     def get_har(self, clean_har):
         if clean_har:
             return self.new_har(DEFAULT_PAGE_REF, DEFAULT_PAGE_TITLE)
@@ -511,9 +514,9 @@ def consume_http_connect_timing(self, client_conn):
     def populate_har_entry_with_default_response(self, flow):
         full_url = self.get_full_url(flow.request)
 
-        ctx.log.debug('Creating new har entry for request: {}'.format(full_url))
+        ctx.log.debug('Populating har entry for request: {}'.format(full_url))
 
-        har_entry = flow.metadata['har_entry']
+        har_entry = self.get_har_entry(flow)
 
         har_entry['pageref'] = self.get_current_page_ref()
         har_entry['startedDateTime'] = datetime.fromtimestamp(flow.request.timestamp_start, timezone.utc).isoformat()
@@ -536,6 +539,8 @@ def request(self, flow):
         if 'WhiteListFiltered' in flow.metadata or 'BlackListFiltered' in flow.metadata:
             return
 
+        har_entry = self.get_har_entry(flow)
+
         self.populate_har_entry_with_default_response(flow)
 
         req_url = 'none'
@@ -555,11 +560,10 @@ def request(self, flow):
         if HarCaptureTypes.RESPONSE_CONTENT in self.har_capture_types:
             self.capture_request_content(flow)
 
-        har_entry = flow.metadata['har_entry']
         har_entry['request']['bodySize'] = \
             len(flow.request.raw_content) if flow.request.raw_content else 0
 
-        connect_timing = self.consume_http_connect_timing(flow.client_conn)
+        connect_timing = self.consume_http_connect_timing(flow)
         if connect_timing is not None:
             har_entry['timings']['sslNanos'] = connect_timing['sslHandshakeTimeNanos']
             har_entry['timings']['connectNanos'] = connect_timing['connectTimeNanos']
@@ -589,7 +593,7 @@ def capture_request_content(self, flow):
         }
 
     def response(self, flow):
-        har_entry = flow.metadata['har_entry']
+        har_entry = self.get_har_entry(flow)
 
         ctx.log.debug('Incoming response for request to url: {}'.format(flow.request.url))
 
@@ -669,6 +673,8 @@ def response(self, flow):
             har_entry["serverIPAddress"] = str(
                 flow.server_conn.ip_address[0])
 
+        ctx.log.debug('Populated har entry for response: {}, entry: {}'.format(flow.request.url, str(har_entry)))
+
     def calculate_timings(self, connect_time, flow, ssl_time):
         timings_raw = {
             'sendNanos': flow.request.timestamp_end - flow.request.timestamp_start,

browserup-proxy-core/src/main/resources/mitmproxy/http_connect_capture.py

@@ -2,13 +2,19 @@
 import json
 import asyncio
 
+import typing
+
 from mitmproxy import ctx
+from mitmproxy import connections
 from mitmproxy.exceptions import TcpTimeout
 
 RESOLUTION_FAILED_ERROR_MESSAGE = "Unable to resolve host: "
 CONNECTION_FAILED_ERROR_MESSAGE = "Unable to connect to host"
 RESPONSE_TIMED_OUT_ERROR_MESSAGE = "Response timed out"
 
+# A list of server seen till now is maintained so we can avoid
+# using 'connect' time for entries that use an existing connection.
+SERVERS_SEEN: typing.Set[connections.ServerConnection] = set()
 
 class HttpConnectCaptureResource:
 
@@ -56,7 +62,7 @@ def generate_http_connect_timing(self):
     # TCP Callbacks
 
     def tcp_resolving_server_address_finished(self, flow):
-        if 'har_entry' not in flow.metadata:
+        if not hasattr(flow.request, 'har_entry'):
             return
         self.populate_dns_timings(flow)
         self.dns_resolution_finished_nanos = self.now_time_nanos()
@@ -119,7 +125,15 @@ def proxy_to_server_connection_succeeded(self, f):
         else:
             self.get_http_connect_timing()['sslHandshakeTimeNanos'] = 0
 
+    def init_har_entry(self, flow):
+        ctx.log.debug("Initializing har entry for flow request: {}".format(str(flow.request)))
+        setattr(flow.request, 'har_entry', self.har_dump_addon.generate_har_entry())
+        self.har_dump_addon.append_har_entry(flow.request.har_entry)
+
     def error(self, flow):
+        if not hasattr(flow.request, 'har_entry'):
+            self.init_har_entry(flow)
+
         req_host_port = flow.request.host
         if flow.request.port != 80:
             req_host_port = req_host_port + ':' + str(flow.request.port)
@@ -155,6 +169,25 @@ def populate_server_ip_address(self, flow, original_error):
             self.get_har_entry(flow)['serverIPAddress'] = str(
                 flow.server_conn.ip_address[0])
 
+    def populate_connect_and_ssl_timings(self, flow):
+        ssl_time = -1
+        connect_time = -1
+
+        if flow.server_conn and flow.server_conn not in SERVERS_SEEN:
+            connect_time = (flow.server_conn.timestamp_tcp_setup -
+                            flow.server_conn.timestamp_start)
+            connect_time = self.timestamp_to_nanos(connect_time)
+
+            if flow.server_conn.timestamp_tls_setup is not None:
+                ssl_time = (flow.server_conn.timestamp_tls_setup -
+                            flow.server_conn.timestamp_tcp_setup)
+                ssl_time = self.timestamp_to_nanos(ssl_time)
+
+            SERVERS_SEEN.add(flow.server_conn)
+        har_entry = self.get_har_entry(flow)
+        har_entry['timings']['sslNanos'] = ssl_time
+        har_entry['timings']['connectNanos'] = connect_time
+
     def get_resource(self):
         return HttpConnectCaptureResource(self)
 
@@ -179,6 +212,7 @@ def server_to_proxy_response_timed_out(self, flow, req_host_port, original_error
         self.create_har_entry_for_failed_connect(flow, msg)
         self.populate_timings_for_failed_connect(flow)
         self.populate_server_ip_address(flow, original_error)
+        self.populate_connect_and_ssl_timings(flow)
 
         current_time_nanos = self.now_time_nanos()
 
@@ -210,7 +244,7 @@ def calculate_total_elapsed_time(self, flow):
         return self.nano_to_ms(result)
 
     def get_har_entry(self, flow):
-        return flow.metadata['har_entry']
+        return flow.request.har_entry
 
     def get_http_connect_timing(self):
         if self.http_connect_timing is None:
@@ -231,6 +265,10 @@ def get_original_exception(flow_error):
     def now_time_nanos():
         return int(round(time.time() * 1000000000))
 
+    @staticmethod
+    def timestamp_to_nanos(timestamp):
+        return int(round(timestamp * 1000000000))
+
     @staticmethod
     def nano_to_ms(time_nano):
         return int(time_nano / 1000000)

browserup-proxy-core/src/main/resources/mitmproxy/init_flow.py

@@ -44,17 +44,18 @@ def __init__(self):
     def get_resource(self):
         return InitFlowResource(self)
 
-    def http_connect(self, flow):
-        if 'har_entry' not in flow.metadata:
-            self.init_har_entry(flow)
+    # def http_connect(self, flow):
+    #     if not hasattr(flow.request, 'har_entry'):
+    #         self.init_har_entry(flow)
 
     def request(self, flow):
-        if 'har_entry' not in flow.metadata:
+        if not hasattr(flow.request, 'har_entry'):
             self.init_har_entry(flow)
 
     def init_har_entry(self, flow):
-        flow.metadata['har_entry'] = self.har_dump_addon.generate_har_entry()
-        self.har_dump_addon.append_har_entry(flow.metadata['har_entry'])
+        ctx.log.debug("Initializing har entry for flow request: {}".format(str(flow.request)))
+        setattr(flow.request, 'har_entry', self.har_dump_addon.generate_har_entry())
+        self.har_dump_addon.append_har_entry(flow.request.har_entry)
 
 addons = [
     InitFlowAddOn()

browserup-proxy-core/src/test/groovy/com/browserup/bup/mitmproxy/NewHarTest.groovy

@@ -85,6 +85,7 @@ class NewHarTest extends MockServerTest {
         proxy.setDnsResolvingDelayMs(1000)
 
         proxy.start()
+
         int proxyPort = proxy.getPort()
 
         proxy.newHar()