Add didVisitURL delegate method (#816)

Trying to pave the way for closing
#790 with some
direction from @Lukasa.

I have no idea where is best to insert this new delegate method. I'm
currently doing it first thing in `receiveResponseHead0`, and not using
an EventLoopFuture for back pressure management. The state machine seems
pretty fragile and I don't want to leave too much of an imprint. Trying
to be a part of an EventLoopFuture chain seems really complicated and
would really leave a mark on the codebase, so I'm wondering if it's
possible to just warn the user "do not block"?

Anyway, just a jumping-off point and happy to take direction!

Author: gregcotten

Sources/AsyncHTTPClient/HTTPHandler.swift

@@ -668,7 +668,16 @@ public protocol HTTPClientResponseDelegate: AnyObject {
     ///     - task: Current request context.
     func didSendRequest(task: HTTPClient.Task<Response>)
 
-    /// Called when response head is received. Will be called once.
+    /// Called each time a response head is received (including redirects), and always called before ``HTTPClientResponseDelegate/didReceiveHead(task:_:)-9r4xd``.
+    /// You can use this method to keep an entire history of the request/response chain.
+    ///
+    /// - parameters:
+    ///     - task: Current request context.
+    ///     - request: The request that was sent.
+    ///     - head: Received response head.
+    func didVisitURL(task: HTTPClient.Task<Response>, _ request: HTTPClient.Request, _ head: HTTPResponseHead)
+
+    /// Called when the final response head is received (after redirects).
     /// You must return an `EventLoopFuture<Void>` that you complete when you have finished processing the body part.
     /// You can create an already succeeded future by calling `task.eventLoop.makeSucceededFuture(())`.
     ///
@@ -734,6 +743,11 @@ extension HTTPClientResponseDelegate {
     /// By default, this does nothing.
     public func didSendRequest(task: HTTPClient.Task<Response>) {}
 
+    /// Default implementation of ``HTTPClientResponseDelegate/didVisitURL(task:_:_:)-2el9y``.
+    ///
+    /// By default, this does nothing.
+    public func didVisitURL(task: HTTPClient.Task<Response>, _: HTTPClient.Request, _: HTTPResponseHead) {}
+
     /// Default implementation of ``HTTPClientResponseDelegate/didReceiveHead(task:_:)-9r4xd``.
     ///
     /// By default, this does nothing.

Sources/AsyncHTTPClient/RequestBag.swift

@@ -228,6 +228,8 @@ final class RequestBag<Delegate: HTTPClientResponseDelegate> {
     private func receiveResponseHead0(_ head: HTTPResponseHead) {
         self.task.eventLoop.assertInEventLoop()
 
+        self.delegate.didVisitURL(task: self.task, self.request, head)
+
         // runs most likely on channel eventLoop
         switch self.state.receiveResponseHead(head) {
         case .none:

Tests/AsyncHTTPClientTests/RequestBagTests.swift

@@ -127,6 +127,7 @@ final class RequestBagTests: XCTestCase {
         XCTAssertNoThrow(try executor.receiveEndOfStream())
         XCTAssertEqual(receivedBytes, bytesToSent, "We have sent all request bytes...")
 
+        XCTAssertTrue(delegate.history.isEmpty)
         XCTAssertNil(delegate.receivedHead, "Expected not to have a response head, before `receiveResponseHead`")
         let responseHead = HTTPResponseHead(
             version: .http1_1,
@@ -140,6 +141,10 @@ final class RequestBagTests: XCTestCase {
         XCTAssertEqual(responseHead, delegate.receivedHead)
         XCTAssertNoThrow(try XCTUnwrap(delegate.backpressurePromise).succeed(()))
         XCTAssertTrue(executor.signalledDemandForResponseBody)
+
+        XCTAssertEqual(delegate.history.map(\.request.url), [request.url])
+        XCTAssertEqual(delegate.history.map(\.response), [responseHead])
+
         executor.resetResponseStreamDemandSignal()
 
         // we will receive 20 chunks with each 10 byteBuffers and 32 bytes
@@ -747,13 +752,15 @@ final class RequestBagTests: XCTestCase {
         let executor = MockRequestExecutor(eventLoop: embeddedEventLoop)
         executor.runRequest(bag)
         XCTAssertFalse(executor.signalledDemandForResponseBody)
-        bag.receiveResponseHead(
-            .init(
-                version: .http1_1,
-                status: .permanentRedirect,
-                headers: ["content-length": "\(3 * 1024)", "location": "https://swift.org/sswg"]
-            )
+        XCTAssertTrue(delegate.history.isEmpty)
+        let responseHead = HTTPResponseHead(
+            version: .http1_1,
+            status: .permanentRedirect,
+            headers: ["content-length": "\(3 * 1024)", "location": "https://swift.org/sswg"]
         )
+        bag.receiveResponseHead(responseHead)
+        XCTAssertEqual(delegate.history.map(\.request.url), [request.url])
+        XCTAssertEqual(delegate.history.map(\.response), [responseHead])
         XCTAssertNil(delegate.backpressurePromise)
         XCTAssertTrue(executor.signalledDemandForResponseBody)
         executor.resetResponseStreamDemandSignal()
@@ -833,13 +840,15 @@ final class RequestBagTests: XCTestCase {
         let executor = MockRequestExecutor(eventLoop: embeddedEventLoop)
         executor.runRequest(bag)
         XCTAssertFalse(executor.signalledDemandForResponseBody)
-        bag.receiveResponseHead(
-            .init(
-                version: .http1_1,
-                status: .permanentRedirect,
-                headers: ["content-length": "\(4 * 1024)", "location": "https://swift.org/sswg"]
-            )
+        XCTAssertTrue(delegate.history.isEmpty)
+        let responseHead = HTTPResponseHead(
+            version: .http1_1,
+            status: .permanentRedirect,
+            headers: ["content-length": "\(4 * 1024)", "location": "https://swift.org/sswg"]
         )
+        bag.receiveResponseHead(responseHead)
+        XCTAssertEqual(delegate.history.map(\.request.url), [request.url])
+        XCTAssertEqual(delegate.history.map(\.response), [responseHead])
         XCTAssertNil(delegate.backpressurePromise)
         XCTAssertFalse(executor.signalledDemandForResponseBody)
         XCTAssertTrue(executor.isCancelled)
@@ -893,13 +902,15 @@ final class RequestBagTests: XCTestCase {
         let executor = MockRequestExecutor(eventLoop: embeddedEventLoop)
         executor.runRequest(bag)
         XCTAssertFalse(executor.signalledDemandForResponseBody)
-        bag.receiveResponseHead(
-            .init(
-                version: .http1_1,
-                status: .permanentRedirect,
-                headers: ["content-length": "\(3 * 1024)", "location": "https://swift.org/sswg"]
-            )
+        XCTAssertTrue(delegate.history.isEmpty)
+        let responseHead = HTTPResponseHead(
+            version: .http1_1,
+            status: .permanentRedirect,
+            headers: ["content-length": "\(3 * 1024)", "location": "https://swift.org/sswg"]
         )
+        bag.receiveResponseHead(responseHead)
+        XCTAssertEqual(delegate.history.map(\.request.url), [request.url])
+        XCTAssertEqual(delegate.history.map(\.response), [responseHead])
         XCTAssertNil(delegate.backpressurePromise)
         XCTAssertTrue(executor.signalledDemandForResponseBody)
         executor.resetResponseStreamDemandSignal()
@@ -1001,6 +1012,7 @@ class UploadCountingDelegate: HTTPClientResponseDelegate {
     private(set) var hitDidReceiveBodyPart = 0
     private(set) var hitDidReceiveError = 0
 
+    private(set) var history: [(request: HTTPClient.Request, response: HTTPResponseHead)] = []
     private(set) var receivedHead: HTTPResponseHead?
     private(set) var lastBodyPart: ByteBuffer?
     private(set) var backpressurePromise: EventLoopPromise<Void>?
@@ -1022,6 +1034,10 @@ class UploadCountingDelegate: HTTPClientResponseDelegate {
         self.hitDidSendRequest += 1
     }
 
+    func didVisitURL(task: HTTPClient.Task<Void>, _ request: HTTPClient.Request, _ head: HTTPResponseHead) {
+        self.history.append((request, head))
+    }
+
     func didReceiveHead(task: HTTPClient.Task<Void>, _ head: HTTPResponseHead) -> EventLoopFuture<Void> {
         self.receivedHead = head
         return self.createBackpressurePromise()