Fix request method override (#144)

Author: elacuesta

pylintrc

@@ -7,6 +7,7 @@ disable=
     missing-function-docstring,
     missing-module-docstring,
     too-few-public-methods,
+    too-many-arguments,
     too-many-instance-attributes,
     too-many-locals,
     unused-argument,

scrapy_playwright/handler.py

@@ -269,7 +269,8 @@ async def _download_request(self, request: Request, spider: Spider) -> Response:
             "**",
             self._make_request_handler(
                 method=request.method,
-                scrapy_headers=request.headers,
+                url=request.url,
+                headers=request.headers,
                 body=request.body,
                 encoding=request.encoding,
             ),
@@ -399,7 +400,12 @@ def close_browser_context_callback() -> None:
         return close_browser_context_callback
 
     def _make_request_handler(
-        self, method: str, scrapy_headers: Headers, body: Optional[bytes], encoding: str = "utf8"
+        self,
+        method: str,
+        url: str,
+        headers: Headers,
+        body: Optional[bytes],
+        encoding: str = "utf8",
     ) -> Callable:
         async def _request_handler(route: Route, playwright_request: PlaywrightRequest) -> None:
             """Override request headers, method and body."""
@@ -417,17 +423,18 @@ async def _request_handler(route: Route, playwright_request: PlaywrightRequest)
             else:
                 overrides["headers"] = final_headers = await _maybe_await(
                     self.process_request_headers(
-                        self.browser_type_name, playwright_request, scrapy_headers
+                        self.browser_type_name, playwright_request, headers
                     )
                 )
             # the request that reaches the callback should contain the final headers
-            scrapy_headers.clear()
-            scrapy_headers.update(final_headers)
+            headers.clear()
+            headers.update(final_headers)
             del final_headers
 
-            if playwright_request.is_navigation_request():
+            # if the request is triggered by scrapy, not playwright
+            if playwright_request.url == url:
                 overrides["method"] = method
-                if body is not None:
+                if body:
                     overrides["post_data"] = body.decode(encoding)
 
             try:
@@ -469,13 +476,14 @@ async def _log_request(request: PlaywrightRequest) -> None:
 def _make_response_logger(context_name: str) -> Callable:
     async def _log_request(response: PlaywrightResponse) -> None:
         referrer = await response.header_value("referer")
-        logger.debug(
-            "[Context=%s] Response: <%i %s> (referrer: %s)",
-            context_name,
-            response.status,
-            response.url,
-            referrer,
-        )
+        log_args = [context_name, response.status, response.url, referrer]
+        if 300 <= response.status < 400:
+            location = await response.header_value("location")
+            log_args.append(location)
+            msg = "[Context=%s] Response: <%i %s> (referrer: %s, location: %s)"
+        else:
+            msg = "[Context=%s] Response: <%i %s> (referrer: %s)"
+        logger.debug(msg, *log_args)
 
     return _log_request
 

tests/test_playwright_requests.py

@@ -125,10 +125,11 @@ async def test_route_continue_exception(self, logger):
         from unittest.mock import AsyncMock
 
         async with make_handler({"PLAYWRIGHT_BROWSER_TYPE": self.browser_type}) as handler:
-            req_handler = handler._make_request_handler("GET", Headers({}), body=None)
+            example_url = "https//example.org"
+            req_handler = handler._make_request_handler("GET", example_url, Headers({}), body=None)
             route = MagicMock()
             playwright_request = AsyncMock()
-            playwright_request.url = "https//example.org"
+            playwright_request.url = example_url
             playwright_request.is_navigation_request = MagicMock(return_value=True)
             playwright_request.all_headers.return_value = {}