fixing har entry scope issue

Author: kiturutin

browserup-proxy-core/src/main/resources/mitmproxy/har_dump.py

@@ -7,6 +7,8 @@
 from datetime import timezone
 import dateutil.parser
 
+import copy
+
 import asyncio
 
 from mitmproxy import ctx
@@ -175,7 +177,6 @@ class HarDumpAddOn:
     def __init__(self):
         self.num = 0
         self.har = None
-        self.har_entry = None
         self.har_page_count = 0
         self.har_capture_types = []
         self.current_har_page = None
@@ -507,32 +508,36 @@ def consume_http_connect_timing(self, client_conn):
             return self.http_connect_timings.pop(client_conn, None)
         return None
 
-    def create_har_entry_with_default_response(self, request):
-        full_url = self.get_full_url(request)
+    def populate_har_entry_with_default_response(self, flow):
+        full_url = self.get_full_url(flow.request)
 
         ctx.log.debug('Creating new har entry for request: {}'.format(full_url))
 
-        self.har_entry = self.generate_har_entry()
-        self.har_entry['pageref'] = self.get_current_page_ref()
-        self.har_entry['startedDateTime'] = datetime.fromtimestamp(
-            request.timestamp_start, timezone.utc).isoformat()
+        har_entry = flow.server_conn.currentHarEntry
+
+        har_entry['pageref'] = self.get_current_page_ref()
+        har_entry['startedDateTime'] = datetime.fromtimestamp(flow.request.timestamp_start, timezone.utc).isoformat()
         har_request = self.generate_har_entry_request()
-        har_request['method'] = request.method
+        har_request['method'] = flow.request.method
         har_request['url'] = full_url
-        har_request['httpVersion'] = request.http_version
-        har_request['queryString'] = self.name_value(request.query or {})
-        har_request['headersSize'] = len(str(request.headers))
+        har_request['httpVersion'] = flow.request.http_version
+        har_request['queryString'] = self.name_value(flow.request.query or {})
+        har_request['headersSize'] = len(str(flow.request.headers))
         har_response = self.generate_har_entry_response_for_failure()
 
-        self.har_entry['request'] = har_request
-        self.har_entry['response'] = har_response
+        har_entry['request'] = har_request
+        har_entry['response'] = har_response
 
-        self.har['log']['entries'].append(self.har_entry)
+    def append_har_entry(self, har_entry):
+        har = self.get_or_create_har(DEFAULT_PAGE_REF, DEFAULT_PAGE_TITLE, True)
+        har['log']['entries'].append(har_entry)
 
     def request(self, flow):
         if 'WhiteListFiltered' in flow.metadata or 'BlackListFiltered' in flow.metadata:
             return
 
+        self.populate_har_entry_with_default_response(flow)
+
         req_url = 'none'
         if flow.request is not None:
             req_url = flow.request.url
@@ -541,53 +546,52 @@ def request(self, flow):
 
         self.get_or_create_har(DEFAULT_PAGE_REF, DEFAULT_PAGE_TITLE, True)
 
-        self.create_har_entry_with_default_response(flow.request)
-
         if HarCaptureTypes.REQUEST_COOKIES in self.har_capture_types:
             self.capture_request_cookies(flow)
 
         if HarCaptureTypes.REQUEST_HEADERS in self.har_capture_types:
             self.capture_request_headers(flow)
 
         if HarCaptureTypes.RESPONSE_CONTENT in self.har_capture_types:
-            self.capture_request_content(flow.request)
+            self.capture_request_content(flow)
 
-        self.har_entry['request']['bodySize'] = \
+        har_entry = flow.server_conn.currentHarEntry
+        har_entry['request']['bodySize'] = \
             len(flow.request.raw_content) if flow.request.raw_content else 0
 
         connect_timing = self.consume_http_connect_timing(flow.client_conn)
         if connect_timing is not None:
-            self.har_entry['timings']['sslNanos'] = connect_timing['sslHandshakeTimeNanos']
-            self.har_entry['timings']['connectNanos'] = connect_timing['connectTimeNanos']
-            self.har_entry['timings']['blockedNanos'] = connect_timing['blockedTimeNanos']
-            self.har_entry['timings']['dnsNanos'] = connect_timing['dnsTimeNanos']
+            har_entry['timings']['sslNanos'] = connect_timing['sslHandshakeTimeNanos']
+            har_entry['timings']['connectNanos'] = connect_timing['connectTimeNanos']
+            har_entry['timings']['blockedNanos'] = connect_timing['blockedTimeNanos']
+            har_entry['timings']['dnsNanos'] = connect_timing['dnsTimeNanos']
 
     def capture_request_cookies(self, flow):
-        self.har_entry['request']['cookies'] = \
+        har_entry = flow.metadata['currentHarEntry']
+        har_entry['request']['cookies'] = \
             self.format_request_cookies(flow.request.cookies.fields)
 
     def capture_request_headers(self, flow):
-        self.har_entry['request']['headers'] = \
+        har_entry = flow.metadata['currentHarEntry']
+        har_entry['request']['headers'] = \
             self.name_value(flow.request.headers)
 
-    def capture_request_content(self, request):
+    def capture_request_content(self, flow):
+        har_entry = flow.metadata['currentHarEntry']
         params = [
             {"name": a, "value": b}
-            for a, b in request.urlencoded_form.items(multi=True)
+            for a, b in flow.request.urlencoded_form.items(multi=True)
         ]
-        self.har_entry["request"]["postData"] = {
-            "mimeType": request.headers.get("Content-Type", ""),
-            "text": request.get_text(strict=False),
+        har_entry["request"]["postData"] = {
+            "mimeType": flow.request.headers.get("Content-Type", ""),
+            "text": flow.request.get_text(strict=False),
             "params": params
         }
 
     def response(self, flow):
-        ctx.log.debug('Incoming response for request to url: {}'.format(flow.request.url))
+        har_entry = flow.server_conn.currentHarEntry
 
-        if self.har_entry is not None:
-            ctx.log.debug(
-                'Response handling for request: {} for current har entry with url: {}'
-                    .format(flow.request.url, self.har_entry['request']['url']))
+        ctx.log.debug('Incoming response for request to url: {}'.format(flow.request.url))
 
         if 'WhiteListFiltered' in flow.metadata or 'BlackListFiltered' in flow.metadata:
             ctx.log.debug('Black/White list filtered, return nothing.')
@@ -610,7 +614,7 @@ def response(self, flow):
             SERVERS_SEEN.add(flow.server_conn)
 
         timings = self.calculate_timings(connect_time, flow, ssl_time)
-        timings['dnsNanos'] = int(self.har_entry['timings']['dnsNanos'])
+        timings['dnsNanos'] = int(har_entry['timings']['dnsNanos'])
 
         full_time = sum(v for v in timings.values() if v > -1)
 
@@ -655,14 +659,14 @@ def response(self, flow):
         har_response["headersSize"] = len(str(flow.response.headers))
         har_response["bodySize"] = response_body_size
 
-        self.har_entry['response'] = har_response
-        self.har_entry['time'] = self.nano_to_ms(full_time)
-        self.har_entry['pageref'] = self.get_current_page_ref()
+        har_entry['response'] = har_response
+        har_entry['time'] = self.nano_to_ms(full_time)
+        har_entry['pageref'] = self.get_current_page_ref()
 
-        self.har_entry['timings'] = timings
+        har_entry['timings'] = timings
 
         if flow.server_conn.connected():
-            self.har_entry["serverIPAddress"] = str(
+            har_entry["serverIPAddress"] = str(
                 flow.server_conn.ip_address[0])
 
     def calculate_timings(self, connect_time, flow, ssl_time):

browserup-proxy-core/src/main/resources/mitmproxy/http_connect_capture.py

@@ -9,6 +9,7 @@
 CONNECTION_FAILED_ERROR_MESSAGE = "Unable to connect to host"
 RESPONSE_TIMED_OUT_ERROR_MESSAGE = "Response timed out"
 
+
 class HttpConnectCaptureResource:
 
     def addon_path(self):
@@ -54,21 +55,23 @@ def generate_http_connect_timing(self):
 
     # TCP Callbacks
 
-    def tcp_resolving_server_address_finished(self, sever_conn):
-        self.populate_dns_timings()
+    def tcp_resolving_server_address_finished(self, server_conn):
+        if not hasattr(server_conn, 'currentHarEntry'):
+            return
+        self.populate_dns_timings(server_conn)
         self.dns_resolution_finished_nanos = self.now_time_nanos()
 
         if self.dns_resolution_started_nanos > 0:
-            self.get_http_connect_timing()['dnsTimeNanos'] = self.dns_resolution_finished_nanos - self.dns_resolution_started_nanos
+            self.get_http_connect_timing()[
+                'dnsTimeNanos'] = self.dns_resolution_finished_nanos - self.dns_resolution_started_nanos
         else:
             self.get_http_connect_timing()['dnsTimeNanos'] = 0
 
-    def tcp_resolving_server_address_started(self, sever_conn):
+    def tcp_resolving_server_address_started(self, server_conn):
         self.dns_resolution_started_nanos = int(round(self.now_time_nanos()))
         self.connection_started_nanos = int(round(self.now_time_nanos()))
         self.proxy_to_server_resolution_started()
 
-
     # SSL Callbacks
     def ssl_handshake_started(self, flow):
         self.ssl_handshake_started_nanos = int(round(self.now_time_nanos()))
@@ -77,18 +80,19 @@ def ssl_handshake_started(self, flow):
 
     def http_connect(self, flow):
         self.http_connect_timing = self.get_http_connect_timing()
-        self.har_dump_addon.http_connect_timings[flow.client_conn] = self.http_connect_timing
+        self.har_dump_addon.http_connect_timings[
+            flow.client_conn] = self.http_connect_timing
 
     def http_proxy_to_server_request_started(self, flow):
         self.send_started_nanos = self.now_time_nanos()
 
     def http_proxy_to_server_request_finished(self, flow):
         self.send_finished_nanos = self.now_time_nanos()
         if self.send_started_nanos > 0:
-            self.get_har_entry()['timings'][
+            self.get_har_entry(flow.server_conn)['timings'][
                 'send'] = self.send_finished_nanos - self.send_started_nanos
         else:
-            self.get_har_entry()['timings']['send'] = 0
+            self.get_har_entry(flow.server_conn)['timings']['send'] = 0
 
     def http_server_to_proxy_response_receiving(self, flow):
         self.response_receive_started_nanos = self.now_time_nanos()
@@ -104,101 +108,99 @@ def proxy_to_server_connection_succeeded(self, f):
         self.connection_succeeded_time_nanos = self.now_time_nanos()
 
         if self.connection_started_nanos > 0:
-            self.get_http_connect_timing()['connectTimeNanos'] = self.connection_succeeded_time_nanos - self.connection_started_nanos
+            self.get_http_connect_timing()[
+                'connectTimeNanos'] = self.connection_succeeded_time_nanos - self.connection_started_nanos
         else:
             self.get_http_connect_timing()['connectTimeNanos'] = 0
 
         if self.ssl_handshake_started_nanos > 0:
-            self.get_http_connect_timing()['sslHandshakeTimeNanos'] = self.connection_succeeded_time_nanos - self.ssl_handshake_started_nanos
+            self.get_http_connect_timing()[
+                'sslHandshakeTimeNanos'] = self.connection_succeeded_time_nanos - self.ssl_handshake_started_nanos
         else:
             self.get_http_connect_timing()['sslHandshakeTimeNanos'] = 0
 
     def error(self, flow):
         req_host_port = flow.request.host
         if flow.request.port != 80:
             req_host_port = req_host_port + ':' + str(flow.request.port)
-        original_error = HttpConnectCaptureAddOn.get_original_exception(
-            flow.error)
+        original_error = HttpConnectCaptureAddOn.get_original_exception(flow.error)
+
+        self.har_dump_addon.populate_har_entry_with_default_response(flow)
 
         if 'Name or service not known' in str(original_error):
-            self.proxy_to_server_resolution_failed(flow, req_host_port,
-                                                   original_error)
+            self.proxy_to_server_resolution_failed(flow, req_host_port, original_error)
         elif isinstance(original_error, TcpTimeout):
-            self.server_to_proxy_response_timed_out(flow, req_host_port,
-                                                    original_error)
+            self.server_to_proxy_response_timed_out(flow, req_host_port, original_error)
         else:
             self.proxy_to_server_connection_failed(flow, original_error)
 
     # Populate data
 
-    def populate_dns_timings(self):
-        if self.dns_resolution_started_nanos > 0 and self.get_har_entry():
+    def populate_dns_timings(self, server_conn):
+        har_entry = self.get_har_entry(server_conn)
+        if self.dns_resolution_started_nanos > 0 and har_entry:
             time_now = self.now_time_nanos()
             dns_nanos = time_now - self.dns_resolution_started_nanos
-            self.get_har_entry()['timings']['dnsNanos'] = dns_nanos
+            har_entry['timings']['dnsNanos'] = dns_nanos
 
-    def populate_timings_for_failed_connect(self):
+    def populate_timings_for_failed_connect(self, flow):
+        har_entry = self.get_har_entry(flow.server_conn)
         if self.connection_started_nanos > 0:
             connect_nanos = self.now_time_nanos() - self.connection_started_nanos
-            self.get_har_entry()['timings']['connectNanos'] = connect_nanos
-        self.populate_dns_timings()
+            har_entry['timings']['connectNanos'] = connect_nanos
+        self.populate_dns_timings(flow.server_conn)
 
     def populate_server_ip_address(self, flow, original_error):
         if flow.server_conn is not None and flow.server_conn.ip_address is not None:
-            self.get_har_entry()['serverIPAddress'] = str(
+            self.get_har_entry(flow.server_conn)['serverIPAddress'] = str(
                 flow.server_conn.ip_address[0])
 
     def get_resource(self):
         return HttpConnectCaptureResource(self)
 
-    def proxy_to_server_resolution_failed(self, flow, req_host_port,
-                                          original_error):
+    def proxy_to_server_resolution_failed(self, flow, req_host_port, original_error):
         msg = RESOLUTION_FAILED_ERROR_MESSAGE + req_host_port
-        self.create_har_entry_for_failed_connect(flow.request, msg)
-        self.populate_dns_timings()
+        self.create_har_entry_for_failed_connect(flow, msg)
+        self.populate_dns_timings(flow.server_conn)
         self.populate_server_ip_address(flow, original_error)
 
-        self.get_har_entry()['time'] = self.calculate_total_elapsed_time()
+        self.get_har_entry(flow.server_conn)['time'] = self.calculate_total_elapsed_time(flow)
 
     def proxy_to_server_connection_failed(self, flow, original_error):
         msg = CONNECTION_FAILED_ERROR_MESSAGE
-        self.create_har_entry_for_failed_connect(flow.request, msg)
-        self.populate_timings_for_failed_connect()
+        self.create_har_entry_for_failed_connect(flow, msg)
+        self.populate_timings_for_failed_connect(flow)
         self.populate_server_ip_address(flow, original_error)
 
-        self.get_har_entry()['time'] = self.calculate_total_elapsed_time()
+        self.get_har_entry(flow.server_conn)['time'] = self.calculate_total_elapsed_time(flow)
 
-    def server_to_proxy_response_timed_out(self, flow, req_host_port,
-                                           original_error):
+    def server_to_proxy_response_timed_out(self, flow, req_host_port, original_error):
         msg = RESPONSE_TIMED_OUT_ERROR_MESSAGE
-        self.create_har_entry_for_failed_connect(flow.request, msg)
-        self.populate_timings_for_failed_connect()
+        self.create_har_entry_for_failed_connect(flow, msg)
+        self.populate_timings_for_failed_connect(flow)
         self.populate_server_ip_address(flow, original_error)
 
         current_time_nanos = self.now_time_nanos()
 
+        har_entry = self.get_har_entry(flow.server_conn)
+
         if self.send_started_nanos > 0 and self.send_finished_nanos == 0:
-            self.get_har_entry()['timings'][
-                'sendNanos'] = current_time_nanos - self.send_started_nanos
+            har_entry['timings']['sendNanos'] = current_time_nanos - self.send_started_nanos
 
         elif self.send_finished_nanos > 0 and self.response_receive_started_nanos == 0:
-            self.get_har_entry()['timings'][
-                'waitNanos'] = current_time_nanos - self.send_finished_nanos
+            har_entry['timings']['waitNanos'] = current_time_nanos - self.send_finished_nanos
 
         elif self.response_receive_started_nanos > 0:
-            self.get_har_entry()['timings'][
-                'receiveNanos'] = current_time_nanos - self.response_receive_started_nanos
-
-        self.get_har_entry()['time'] = self.calculate_total_elapsed_time()
+            har_entry['timings']['receiveNanos'] = current_time_nanos - self.response_receive_started_nanos
 
-    def create_har_entry_for_failed_connect(self, request, msg):
-        if not self.get_har_entry():
-            self.har_dump_addon.create_har_entry_with_default_response(request)
+        har_entry['time'] = self.calculate_total_elapsed_time(flow)
 
-        self.get_har_entry()['response']['_errorMessage'] = msg
+    def create_har_entry_for_failed_connect(self, flow, msg):
+        har_entry = self.get_har_entry(flow.server_conn)
+        har_entry['response']['_errorMessage'] = msg
 
-    def calculate_total_elapsed_time(self):
-        timings = self.get_har_entry()['timings']
+    def calculate_total_elapsed_time(self, flow):
+        timings = self.get_har_entry(flow.server_conn)['timings']
         result = (0 if timings.get('blockedNanos', -1) == -1 else timings['blockedNanos']) + \
                  (0 if timings.get('dnsNanos', -1) == -1 else timings['dnsNanos']) + \
                  (0 if timings.get('connectNanos', -1) == -1 else timings['connectNanos']) + \
@@ -207,8 +209,8 @@ def calculate_total_elapsed_time(self):
                  (0 if timings.get('receiveNanos', -1) == -1 else timings['receiveNanos'])
         return self.nano_to_ms(result)
 
-    def get_har_entry(self):
-        return self.har_dump_addon.har_entry
+    def get_har_entry(self, server_conn):
+        return server_conn.currentHarEntry
 
     def get_http_connect_timing(self):
         if self.http_connect_timing is None:

browserup-proxy-core/src/main/resources/mitmproxy/init_flow.py

@@ -44,8 +44,17 @@ def __init__(self):
     def get_resource(self):
         return InitFlowResource(self)
 
+    def http_connect(self, flow):
+        if not hasattr(flow.server_conn, 'currentHarEntry'):
+            self.init_har_entry(flow)
+
     def request(self, flow):
-        self.har_dump_addon.har_entry = None
+        if not hasattr(flow.server_conn, 'currentHarEntry'):
+            self.init_har_entry(flow)
+
+    def init_har_entry(self, flow):
+        setattr(flow.server_conn, 'currentHarEntry', self.har_dump_addon.generate_har_entry())
+        self.har_dump_addon.append_har_entry(flow.server_conn.currentHarEntry)
 
 addons = [
     InitFlowAddOn()