Added _url field to har entry

Author: kiturutin

browserup-proxy-core/src/main/java/com/browserup/harreader/model/HarEntry.java

@@ -1,10 +1,6 @@
 package com.browserup.harreader.model;
 
-import com.fasterxml.jackson.annotation.JsonAnyGetter;
-import com.fasterxml.jackson.annotation.JsonAnySetter;
-import com.fasterxml.jackson.annotation.JsonFormat;
-import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
-import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.*;
 
 import java.util.Date;
 import java.util.HashMap;
@@ -29,6 +25,7 @@ public class HarEntry {
     private String serverIPAddress;
     private String connection;
     private String comment;
+    private String _url;
     private Map<String, Object> additional = new HashMap<>();
 
     /**
@@ -121,6 +118,19 @@ public void setTimings(HarTiming timings) {
         this.timings = timings;
     }
 
+    @JsonProperty("_url")
+    public String getUrl() {
+        if (_url == null) {
+            _url = "";
+        }
+        return _url;
+    }
+
+    @JsonProperty("_url")
+    public void setUrl(String url) {
+        this._url = url;
+    }
+
     /**
      * @return Server IP address (result of DNS resolution), null if not present.
      */

browserup-proxy-core/src/main/resources/mitmproxy/har_dump.py

@@ -336,7 +336,7 @@ def generate_har_entry_response_for_failure(self):
         result['_errorMessage'] = "No response received"
         return result
 
-    def generate_har_entry(self):
+    def generate_har_entry(self, request_url):
         return {
             "pageref": "",
             "startedDateTime": "",
@@ -347,7 +347,8 @@ def generate_har_entry(self):
             "timings": self.generate_har_timings(),
             "serverIPAddress": "",
             "connection": "",
-            "comment": ""
+            "comment": "",
+            "_url": request_url
         }
 
     def get_or_create_har(self, page_ref, page_title, create_page=False):

browserup-proxy-core/src/main/resources/mitmproxy/http_connect_capture.py

@@ -127,7 +127,7 @@ def proxy_to_server_connection_succeeded(self, f):
 
     def init_har_entry(self, flow):
         ctx.log.debug("Initializing har entry for flow request: {}".format(str(flow.request)))
-        setattr(flow.request, 'har_entry', self.har_dump_addon.generate_har_entry())
+        setattr(flow.request, 'har_entry', self.har_dump_addon.generate_har_entry(flow.request.url))
         self.har_dump_addon.append_har_entry(flow.request.har_entry)
 
     def error(self, flow):

browserup-proxy-core/src/main/resources/mitmproxy/init_flow.py

@@ -54,7 +54,7 @@ def request(self, flow):
 
     def init_har_entry(self, flow):
         ctx.log.debug("Initializing har entry for flow request: {}".format(str(flow.request)))
-        setattr(flow.request, 'har_entry', self.har_dump_addon.generate_har_entry())
+        setattr(flow.request, 'har_entry', self.har_dump_addon.generate_har_entry(flow.request.url))
         self.har_dump_addon.append_har_entry(flow.request.har_entry)
 
 addons = [

browserup-proxy-core/src/test/groovy/com/browserup/bup/mitmproxy/HarValidationTest.groovy

@@ -8,12 +8,14 @@ import com.browserup.bup.MitmProxyServer
 import com.browserup.bup.proxy.test.util.MockServerTest
 import com.browserup.bup.proxy.test.util.NewProxyServerTestUtil
 import com.browserup.harreader.model.*
+import org.apache.commons.lang3.StringUtils
 import org.apache.http.client.methods.HttpGet
 import org.junit.After
 import org.junit.Test
 
 import static com.github.tomakehurst.wiremock.client.WireMock.*
 import static org.junit.Assert.assertNotNull
+import static org.junit.Assert.assertTrue
 
 class HarValidationTest extends MockServerTest {
     private MitmProxyServer proxy
@@ -25,6 +27,34 @@ class HarValidationTest extends MockServerTest {
         }
     }
 
+    @Test
+    void testHarEntryContainsUrlField() {
+        def stubUrl = "/testUrl.*"
+        stubFor(get(urlMatching(stubUrl)).willReturn(ok()))
+
+        proxy = new MitmProxyServer()
+        proxy.start()
+
+        proxy.newHar()
+
+        def requestUrl = "http://localhost:${mockServerPort}/testUrl"
+
+        NewProxyServerTestUtil.getNewHttpClient(proxy.port).withCloseable {
+            NewProxyServerTestUtil.toStringAndClose(it.execute(new HttpGet(requestUrl)).getEntity().getContent())
+        }
+
+        Thread.sleep(500)
+        def har = proxy.getHar()
+
+        assertNotNull("Expected not null log creator name", har.log.creator.name)
+        assertNotNull("Expected not null log creator version", har.log.creator.version)
+
+        har.log.entries.each {
+            assertTrue(StringUtils.isNotEmpty(it.url))
+        }
+        verify(1, getRequestedFor(urlMatching(stubUrl)))
+    }
+
     @Test
     void testDefaultValuesOfMockedHarResponse() {
         def stubUrl = "/testUrl.*"

browserup-proxy-core/src/test/groovy/com/browserup/bup/mitmproxy/NewHarTest.groovy

@@ -13,6 +13,7 @@ import com.browserup.bup.proxy.test.util.NewProxyServerTestUtil
 import com.browserup.harreader.model.*
 import com.github.tomakehurst.wiremock.client.WireMock
 import com.google.common.collect.Iterables
+import org.apache.commons.lang3.StringUtils
 import org.apache.http.client.config.RequestConfig
 import org.apache.http.client.methods.CloseableHttpResponse
 import org.apache.http.client.methods.HttpGet
@@ -390,6 +391,9 @@ class NewHarTest extends MockServerTest {
 
         assertEquals("Expected to not capture body content in HAR", "", content.text)
 
+        assertTrue("Expected HAR entries to have _url field",
+                har.log.entries.every { StringUtils.isNotEmpty(it.url) })
+
         verify(1, getRequestedFor(urlEqualTo(stubUrl)))
     }
 
@@ -467,6 +471,9 @@ class NewHarTest extends MockServerTest {
             assertNotNull("Expected to find HAR content", newContent)
 
             assertEquals("Expected to capture body content in HAR", "success", newContent.text)
+
+            assertTrue("Expected HAR entries to have _url field",
+                    populatedHar.log.entries.every { StringUtils.isNotEmpty(it.url) })
         }
     }
 
@@ -517,6 +524,8 @@ class NewHarTest extends MockServerTest {
 
         assertEquals("Expected HAR returned from newPage() not to contain second page", 1, harWithFirstPageOnly.log.pages.size())
         assertEquals("Expected id of HAR page to be 'first-page'", "first-page", harWithFirstPageOnly.log.pages.first().id)
+        assertTrue("Expected HAR entries to have _url field",
+                har.log.entries.every { StringUtils.isNotEmpty(it.url) })
     }
 
     @Test
@@ -546,7 +555,8 @@ class NewHarTest extends MockServerTest {
 
         String capturedUrl = har.log.entries[0].request.url
         assertEquals("URL captured in HAR did not match request URL", requestUrl, capturedUrl)
-
+        assertTrue("Expected HAR entries to have _url field",
+                har.log.entries.every { StringUtils.isNotEmpty(it.url) })
         verify(1, getRequestedFor(urlEqualTo(stubUrl)))
     }
 
@@ -582,7 +592,8 @@ class NewHarTest extends MockServerTest {
 
         assertEquals("Expected first query parameter name to be param1", "param1", har.log.entries[0].request.queryString[0].name)
         assertEquals("Expected first query parameter value to be value1", "value1", har.log.entries[0].request.queryString[0].value)
-
+        assertTrue("Expected HAR entries to have _url field",
+                har.log.entries.every { StringUtils.isNotEmpty(it.url) })
         verify(1, getRequestedFor(urlMatching(stubUrl)))
     }
 
@@ -620,7 +631,8 @@ class NewHarTest extends MockServerTest {
 
         assertEquals("Expected first query parameter name to be param1", "param1", har.log.entries[0].request.queryString[0].name)
         assertEquals("Expected first query parameter value to be value1", "value1", har.log.entries[0].request.queryString[0].value)
-
+        assertTrue("Expected HAR entries to have _url field",
+                har.log.entries.every { StringUtils.isNotEmpty(it.url) })
         verify(1, getRequestedFor(urlMatching(stubUrl)))
     }
 
@@ -662,7 +674,8 @@ class NewHarTest extends MockServerTest {
 
         assertEquals("Expected first query parameter name to be param1", "param1", har.log.entries[0].request.queryString[0].name)
         assertEquals("Expected first query parameter value to be value1", "value1", har.log.entries[0].request.queryString[0].value)
-
+        assertTrue("Expected HAR entries to have _url field",
+                har.log.entries.every { StringUtils.isNotEmpty(it.url) })
         verify(1, getRequestedFor(urlMatching(stubUrl)))
     }
 
@@ -711,6 +724,8 @@ class NewHarTest extends MockServerTest {
         assertEquals("Expected HAR timings to contain default values after DNS failure", 0L, harTimings.getSend(TimeUnit.NANOSECONDS))
         assertEquals("Expected HAR timings to contain default values after DNS failure", 0L, harTimings.getWait(TimeUnit.NANOSECONDS))
         assertEquals("Expected HAR timings to contain default values after DNS failure", 0L, harTimings.getReceive(TimeUnit.NANOSECONDS))
+        assertTrue("Expected HAR entries to have _url field",
+                har.log.entries.every { StringUtils.isNotEmpty(it.url) })
         assertNotNull(har.log.entries[0].time)
     }
 
@@ -760,6 +775,8 @@ class NewHarTest extends MockServerTest {
         assertEquals("Expected HAR timings to contain default values after DNS failure", 0L, harTimings.getSend(TimeUnit.NANOSECONDS))
         assertEquals("Expected HAR timings to contain default values after DNS failure", 0L, harTimings.getWait(TimeUnit.NANOSECONDS))
         assertEquals("Expected HAR timings to contain default values after DNS failure", 0L, harTimings.getReceive(TimeUnit.NANOSECONDS))
+        assertTrue("Expected HAR entries to have _url field",
+                har.log.entries.every { StringUtils.isNotEmpty(it.url) })
         assertNotNull(har.log.entries[0].time)
     }
 
@@ -808,6 +825,8 @@ class NewHarTest extends MockServerTest {
         assertEquals("Expected HAR timings to contain default values after connection failure", 0L, harTimings.getSend(TimeUnit.NANOSECONDS))
         assertEquals("Expected HAR timings to contain default values after connection failure", 0L, harTimings.getWait(TimeUnit.NANOSECONDS))
         assertEquals("Expected HAR timings to contain default values after connection failure", 0L, harTimings.getReceive(TimeUnit.NANOSECONDS))
+        assertTrue("Expected HAR entries to have _url field",
+                har.log.entries.every { StringUtils.isNotEmpty(it.url) })
         assertNotNull(har.log.entries[0].time)
     }
 
@@ -856,6 +875,8 @@ class NewHarTest extends MockServerTest {
         assertEquals("Expected HAR timings to contain default values after connection failure", 0L, harTimings.getSend(TimeUnit.NANOSECONDS))
         assertEquals("Expected HAR timings to contain default values after connection failure", 0L, harTimings.getWait(TimeUnit.NANOSECONDS))
         assertEquals("Expected HAR timings to contain default values after connection failure", 0L, harTimings.getReceive(TimeUnit.NANOSECONDS))
+        assertTrue("Expected HAR entries to have _url field",
+                har.log.entries.every { StringUtils.isNotEmpty(it.url) })
         assertTrue(har.log.entries[0].time > 0)
     }
 
@@ -914,6 +935,8 @@ class NewHarTest extends MockServerTest {
         assertThat("Expected wait time to be populated", harTimings.getWait(TimeUnit.NANOSECONDS), greaterThan(0L))
 
         assertEquals("Expected receive time to not be populated", 0L, harTimings.getReceive(TimeUnit.NANOSECONDS))
+        assertTrue("Expected HAR entries to have _url field",
+                har.log.entries.every { StringUtils.isNotEmpty(it.url) })
         assertTrue(har.log.entries[0].time > 0)
     }
 
@@ -973,6 +996,8 @@ class NewHarTest extends MockServerTest {
         assertThat("Expected wait time to be populated", harTimings.getWait(TimeUnit.NANOSECONDS), greaterThan(0L))
 
         assertEquals("Expected receive time to not be populated", 0L, harTimings.getReceive(TimeUnit.NANOSECONDS))
+        assertTrue("Expected HAR entries to have _url field",
+                har.log.entries.every { StringUtils.isNotEmpty(it.url) })
         assertTrue(har.log.entries[0].time > 0)
     }