Always override request headers (#98)

Author: elacuesta

scrapy_playwright/handler.py

@@ -390,15 +390,18 @@ async def _request_handler(route: Route, playwright_request: PlaywrightRequest)
 
             overrides: dict = {}
 
-            if self.process_request_headers is not None:
-                overrides["headers"] = await _maybe_await(
+            if self.process_request_headers is None:
+                final_headers = await playwright_request.all_headers()
+            else:
+                overrides["headers"] = final_headers = await _maybe_await(
                     self.process_request_headers(
                         self.browser_type_name, playwright_request, scrapy_headers
                     )
                 )
-                # the request that reaches the callback should contain the final headers
-                scrapy_headers.clear()
-                scrapy_headers.update(overrides["headers"])
+            # the request that reaches the callback should contain the final headers
+            scrapy_headers.clear()
+            scrapy_headers.update(final_headers)
+            del final_headers
 
             if playwright_request.is_navigation_request():
                 overrides["method"] = method

tests/test_headers.py

@@ -82,7 +82,10 @@ async def test_use_playwright_headers(self):
                 headers = json.loads(resp.css("pre::text").get())
                 headers = {key.lower(): value for key, value in headers.items()}
                 assert headers["user-agent"] == self.browser_type
+                assert req.headers["user-agent"].decode("utf-8") == self.browser_type
                 assert "asdf" not in headers
+                assert "asdf" not in req.headers
+                assert b"asdf" not in req.headers
 
     @pytest.mark.asyncio
     async def test_use_playwright_headers_deprecated(self):