chore: port record_har_* options (content, mode, url_filter) (#1382)

This is part 3/n of the 1.23 port.

Relates #1308, #1374, #1376.

Ports:

- [x] microsoft/playwright@fdcdd58 (feat(har): introduce urlFilter (#14693))
- [x] microsoft/playwright@c349c1d (feat: newContext.har (#14892))
- [x] microsoft/playwright@245c33a (feat(har): allow storing content as separate files (#14934))
- [x] microsoft/playwright@be64e9c (chore(har): attach resources for .zip hars (#14938))
- [x] microsoft/playwright@7bd7271 (feat(har): introduce the slim mode (#15053))

Author: rwoll

playwright/_impl/_browser.py

@@ -16,7 +16,7 @@
 import json
 from pathlib import Path
 from types import SimpleNamespace
-from typing import TYPE_CHECKING, Any, Dict, List, Union, cast
+from typing import TYPE_CHECKING, Any, Dict, List, Pattern, Union, cast
 
 from playwright._impl._api_structures import (
     Geolocation,
@@ -31,6 +31,8 @@
 from playwright._impl._helper import (
     ColorScheme,
     ForcedColors,
+    HarContentPolicy,
+    HarMode,
     ReducedMotion,
     ServiceWorkersPolicy,
     async_readfile,
@@ -40,6 +42,7 @@
 from playwright._impl._local_utils import LocalUtils
 from playwright._impl._network import serialize_headers
 from playwright._impl._page import Page
+from playwright._impl._str_utils import escape_regex_flags
 
 if TYPE_CHECKING:  # pragma: no cover
     from playwright._impl._browser_type import BrowserType
@@ -116,6 +119,9 @@ async def new_context(
         baseURL: str = None,
         strictSelectors: bool = None,
         serviceWorkers: ServiceWorkersPolicy = None,
+        recordHarUrlFilter: Union[Pattern, str] = None,
+        recordHarMode: HarMode = None,
+        recordHarContent: HarContentPolicy = None,
     ) -> BrowserContext:
         params = locals_to_params(locals())
         await normalize_context_params(self._connection._is_sync, params)
@@ -160,6 +166,9 @@ async def new_page(
         baseURL: str = None,
         strictSelectors: bool = None,
         serviceWorkers: ServiceWorkersPolicy = None,
+        recordHarUrlFilter: Union[Pattern, str] = None,
+        recordHarMode: HarMode = None,
+        recordHarContent: HarContentPolicy = None,
     ) -> Page:
         params = locals_to_params(locals())
         context = await self.new_context(**params)
@@ -217,9 +226,30 @@ async def normalize_context_params(is_sync: bool, params: Dict) -> None:
     if "recordHarPath" in params:
         recordHar: Dict[str, Any] = {"path": str(params["recordHarPath"])}
         params["recordHar"] = recordHar
+        if "recordHarUrlFilter" in params:
+            opt = params["recordHarUrlFilter"]
+            if isinstance(opt, str):
+                params["recordHar"]["urlGlob"] = opt
+            if isinstance(opt, Pattern):
+                params["recordHar"]["urlRegexSource"] = opt.pattern
+                params["recordHar"]["urlRegexFlags"] = escape_regex_flags(opt)
+            del params["recordHarUrlFilter"]
+        if "recordHarMode" in params:
+            params["recordHar"]["mode"] = params["recordHarMode"]
+            del params["recordHarMode"]
+
+        new_content_api = None
+        old_content_api = None
+        if "recordHarContent" in params:
+            new_content_api = params["recordHarContent"]
+            del params["recordHarContent"]
         if "recordHarOmitContent" in params:
-            params["recordHar"]["omitContent"] = params["recordHarOmitContent"]
+            old_content_api = params["recordHarOmitContent"]
             del params["recordHarOmitContent"]
+        content = new_content_api or ("omit" if old_content_api else None)
+        if content:
+            params["recordHar"]["content"] = content
+
         del params["recordHarPath"]
     if "recordVideoDir" in params:
         params["recordVideo"] = {"dir": str(params["recordVideoDir"])}

playwright/_impl/_browser_type.py

@@ -15,7 +15,7 @@
 import asyncio
 import pathlib
 from pathlib import Path
-from typing import TYPE_CHECKING, Dict, List, Optional, Union, cast
+from typing import TYPE_CHECKING, Dict, List, Optional, Pattern, Union, cast
 
 from playwright._impl._api_structures import (
     Geolocation,
@@ -36,6 +36,8 @@
     ColorScheme,
     Env,
     ForcedColors,
+    HarContentPolicy,
+    HarMode,
     ReducedMotion,
     ServiceWorkersPolicy,
     locals_to_params,
@@ -139,6 +141,9 @@ async def launch_persistent_context(
         baseURL: str = None,
         strictSelectors: bool = None,
         serviceWorkers: ServiceWorkersPolicy = None,
+        recordHarUrlFilter: Union[Pattern, str] = None,
+        recordHarMode: HarMode = None,
+        recordHarContent: HarContentPolicy = None,
     ) -> BrowserContext:
         userDataDir = str(Path(userDataDir))
         params = locals_to_params(locals())

playwright/_impl/_helper.py

@@ -65,6 +65,8 @@
 KeyboardModifier = Literal["Alt", "Control", "Meta", "Shift"]
 MouseButton = Literal["left", "middle", "right"]
 ServiceWorkersPolicy = Literal["allow", "block"]
+HarMode = Literal["full", "minimal"]
+HarContentPolicy = Literal["attach", "embed", "omit"]
 
 
 class ErrorPayload(TypedDict, total=False):

playwright/async_api/_generated.py

@@ -10647,7 +10647,10 @@ async def new_context(
         storage_state: typing.Union[StorageState, str, pathlib.Path] = None,
         base_url: str = None,
         strict_selectors: bool = None,
-        service_workers: Literal["allow", "block"] = None
+        service_workers: Literal["allow", "block"] = None,
+        record_har_url_filter: typing.Union[str, typing.Pattern] = None,
+        record_har_mode: Literal["full", "minimal"] = None,
+        record_har_content: Literal["attach", "embed", "omit"] = None
     ) -> "BrowserContext":
         """Browser.new_context
 
@@ -10756,6 +10759,14 @@ async def new_context(
             Whether to allow sites to register Service workers. Defaults to `'allow'`.
             - `'allow'`: [Service Workers](https://developer.mozilla.org/en-US/docs/Web/API/Service_Worker_API) can be registered.
             - `'block'`: Playwright will block all registration of Service Workers.
+        record_har_url_filter : Union[Pattern, str, NoneType]
+        record_har_mode : Union["full", "minimal", NoneType]
+            When set to `minimal`, only record information necessary for routing from HAR. This omits sizes, timing, page, cookies,
+            security and other types of HAR information that are not used when replaying from HAR. Defaults to `full`.
+        record_har_content : Union["attach", "embed", "omit", NoneType]
+            Optional setting to control resource content management. If `omit` is specified, content is not persisted. If `attach`
+            is specified, resources are persistet as separate files and all of these files are archived along with the HAR file.
+            Defaults to `embed`, which stores content inline the HAR file as per HAR specification.
 
         Returns
         -------
@@ -10795,6 +10806,9 @@ async def new_context(
                 baseURL=base_url,
                 strictSelectors=strict_selectors,
                 serviceWorkers=service_workers,
+                recordHarUrlFilter=record_har_url_filter,
+                recordHarMode=record_har_mode,
+                recordHarContent=record_har_content,
             )
         )
 
@@ -10831,7 +10845,10 @@ async def new_page(
         storage_state: typing.Union[StorageState, str, pathlib.Path] = None,
         base_url: str = None,
         strict_selectors: bool = None,
-        service_workers: Literal["allow", "block"] = None
+        service_workers: Literal["allow", "block"] = None,
+        record_har_url_filter: typing.Union[str, typing.Pattern] = None,
+        record_har_mode: Literal["full", "minimal"] = None,
+        record_har_content: Literal["attach", "embed", "omit"] = None
     ) -> "Page":
         """Browser.new_page
 
@@ -10935,6 +10952,14 @@ async def new_page(
             Whether to allow sites to register Service workers. Defaults to `'allow'`.
             - `'allow'`: [Service Workers](https://developer.mozilla.org/en-US/docs/Web/API/Service_Worker_API) can be registered.
             - `'block'`: Playwright will block all registration of Service Workers.
+        record_har_url_filter : Union[Pattern, str, NoneType]
+        record_har_mode : Union["full", "minimal", NoneType]
+            When set to `minimal`, only record information necessary for routing from HAR. This omits sizes, timing, page, cookies,
+            security and other types of HAR information that are not used when replaying from HAR. Defaults to `full`.
+        record_har_content : Union["attach", "embed", "omit", NoneType]
+            Optional setting to control resource content management. If `omit` is specified, content is not persisted. If `attach`
+            is specified, resources are persistet as separate files and all of these files are archived along with the HAR file.
+            Defaults to `embed`, which stores content inline the HAR file as per HAR specification.
 
         Returns
         -------
@@ -10974,6 +10999,9 @@ async def new_page(
                 baseURL=base_url,
                 strictSelectors=strict_selectors,
                 serviceWorkers=service_workers,
+                recordHarUrlFilter=record_har_url_filter,
+                recordHarMode=record_har_mode,
+                recordHarContent=record_har_content,
             )
         )
 
@@ -11269,7 +11297,10 @@ async def launch_persistent_context(
         record_video_size: ViewportSize = None,
         base_url: str = None,
         strict_selectors: bool = None,
-        service_workers: Literal["allow", "block"] = None
+        service_workers: Literal["allow", "block"] = None,
+        record_har_url_filter: typing.Union[str, typing.Pattern] = None,
+        record_har_mode: Literal["full", "minimal"] = None,
+        record_har_content: Literal["attach", "embed", "omit"] = None
     ) -> "BrowserContext":
         """BrowserType.launch_persistent_context
 
@@ -11413,6 +11444,14 @@ async def launch_persistent_context(
             Whether to allow sites to register Service workers. Defaults to `'allow'`.
             - `'allow'`: [Service Workers](https://developer.mozilla.org/en-US/docs/Web/API/Service_Worker_API) can be registered.
             - `'block'`: Playwright will block all registration of Service Workers.
+        record_har_url_filter : Union[Pattern, str, NoneType]
+        record_har_mode : Union["full", "minimal", NoneType]
+            When set to `minimal`, only record information necessary for routing from HAR. This omits sizes, timing, page, cookies,
+            security and other types of HAR information that are not used when replaying from HAR. Defaults to `full`.
+        record_har_content : Union["attach", "embed", "omit", NoneType]
+            Optional setting to control resource content management. If `omit` is specified, content is not persisted. If `attach`
+            is specified, resources are persistet as separate files and all of these files are archived along with the HAR file.
+            Defaults to `embed`, which stores content inline the HAR file as per HAR specification.
 
         Returns
         -------
@@ -11466,6 +11505,9 @@ async def launch_persistent_context(
                 baseURL=base_url,
                 strictSelectors=strict_selectors,
                 serviceWorkers=service_workers,
+                recordHarUrlFilter=record_har_url_filter,
+                recordHarMode=record_har_mode,
+                recordHarContent=record_har_content,
             )
         )
 

playwright/sync_api/_generated.py

@@ -10669,7 +10669,10 @@ def new_context(
         storage_state: typing.Union[StorageState, str, pathlib.Path] = None,
         base_url: str = None,
         strict_selectors: bool = None,
-        service_workers: Literal["allow", "block"] = None
+        service_workers: Literal["allow", "block"] = None,
+        record_har_url_filter: typing.Union[str, typing.Pattern] = None,
+        record_har_mode: Literal["full", "minimal"] = None,
+        record_har_content: Literal["attach", "embed", "omit"] = None
     ) -> "BrowserContext":
         """Browser.new_context
 
@@ -10778,6 +10781,14 @@ def new_context(
             Whether to allow sites to register Service workers. Defaults to `'allow'`.
             - `'allow'`: [Service Workers](https://developer.mozilla.org/en-US/docs/Web/API/Service_Worker_API) can be registered.
             - `'block'`: Playwright will block all registration of Service Workers.
+        record_har_url_filter : Union[Pattern, str, NoneType]
+        record_har_mode : Union["full", "minimal", NoneType]
+            When set to `minimal`, only record information necessary for routing from HAR. This omits sizes, timing, page, cookies,
+            security and other types of HAR information that are not used when replaying from HAR. Defaults to `full`.
+        record_har_content : Union["attach", "embed", "omit", NoneType]
+            Optional setting to control resource content management. If `omit` is specified, content is not persisted. If `attach`
+            is specified, resources are persistet as separate files and all of these files are archived along with the HAR file.
+            Defaults to `embed`, which stores content inline the HAR file as per HAR specification.
 
         Returns
         -------
@@ -10818,6 +10829,9 @@ def new_context(
                     baseURL=base_url,
                     strictSelectors=strict_selectors,
                     serviceWorkers=service_workers,
+                    recordHarUrlFilter=record_har_url_filter,
+                    recordHarMode=record_har_mode,
+                    recordHarContent=record_har_content,
                 )
             )
         )
@@ -10855,7 +10869,10 @@ def new_page(
         storage_state: typing.Union[StorageState, str, pathlib.Path] = None,
         base_url: str = None,
         strict_selectors: bool = None,
-        service_workers: Literal["allow", "block"] = None
+        service_workers: Literal["allow", "block"] = None,
+        record_har_url_filter: typing.Union[str, typing.Pattern] = None,
+        record_har_mode: Literal["full", "minimal"] = None,
+        record_har_content: Literal["attach", "embed", "omit"] = None
     ) -> "Page":
         """Browser.new_page
 
@@ -10959,6 +10976,14 @@ def new_page(
             Whether to allow sites to register Service workers. Defaults to `'allow'`.
             - `'allow'`: [Service Workers](https://developer.mozilla.org/en-US/docs/Web/API/Service_Worker_API) can be registered.
             - `'block'`: Playwright will block all registration of Service Workers.
+        record_har_url_filter : Union[Pattern, str, NoneType]
+        record_har_mode : Union["full", "minimal", NoneType]
+            When set to `minimal`, only record information necessary for routing from HAR. This omits sizes, timing, page, cookies,
+            security and other types of HAR information that are not used when replaying from HAR. Defaults to `full`.
+        record_har_content : Union["attach", "embed", "omit", NoneType]
+            Optional setting to control resource content management. If `omit` is specified, content is not persisted. If `attach`
+            is specified, resources are persistet as separate files and all of these files are archived along with the HAR file.
+            Defaults to `embed`, which stores content inline the HAR file as per HAR specification.
 
         Returns
         -------
@@ -10999,6 +11024,9 @@ def new_page(
                     baseURL=base_url,
                     strictSelectors=strict_selectors,
                     serviceWorkers=service_workers,
+                    recordHarUrlFilter=record_har_url_filter,
+                    recordHarMode=record_har_mode,
+                    recordHarContent=record_har_content,
                 )
             )
         )
@@ -11299,7 +11327,10 @@ def launch_persistent_context(
         record_video_size: ViewportSize = None,
         base_url: str = None,
         strict_selectors: bool = None,
-        service_workers: Literal["allow", "block"] = None
+        service_workers: Literal["allow", "block"] = None,
+        record_har_url_filter: typing.Union[str, typing.Pattern] = None,
+        record_har_mode: Literal["full", "minimal"] = None,
+        record_har_content: Literal["attach", "embed", "omit"] = None
     ) -> "BrowserContext":
         """BrowserType.launch_persistent_context
 
@@ -11443,6 +11474,14 @@ def launch_persistent_context(
             Whether to allow sites to register Service workers. Defaults to `'allow'`.
             - `'allow'`: [Service Workers](https://developer.mozilla.org/en-US/docs/Web/API/Service_Worker_API) can be registered.
             - `'block'`: Playwright will block all registration of Service Workers.
+        record_har_url_filter : Union[Pattern, str, NoneType]
+        record_har_mode : Union["full", "minimal", NoneType]
+            When set to `minimal`, only record information necessary for routing from HAR. This omits sizes, timing, page, cookies,
+            security and other types of HAR information that are not used when replaying from HAR. Defaults to `full`.
+        record_har_content : Union["attach", "embed", "omit", NoneType]
+            Optional setting to control resource content management. If `omit` is specified, content is not persisted. If `attach`
+            is specified, resources are persistet as separate files and all of these files are archived along with the HAR file.
+            Defaults to `embed`, which stores content inline the HAR file as per HAR specification.
 
         Returns
         -------
@@ -11497,6 +11536,9 @@ def launch_persistent_context(
                     baseURL=base_url,
                     strictSelectors=strict_selectors,
                     serviceWorkers=service_workers,
+                    recordHarUrlFilter=record_har_url_filter,
+                    recordHarMode=record_har_mode,
+                    recordHarContent=record_har_content,
                 )
             )
         )

scripts/expected_api_mismatch.txt

@@ -20,9 +20,6 @@ Method not implemented: Error.message
 Method not implemented: PlaywrightAssertions.expect
 
 # Pending 1.23 ports
-Parameter not implemented: BrowserType.launch_persistent_context(record_har_url_filter=)
 Method not implemented: BrowserContext.route_from_har
 Method not implemented: Route.fallback
-Parameter not implemented: Browser.new_page(record_har_url_filter=)
 Method not implemented: Page.route_from_har
-Parameter not implemented: Browser.new_context(record_har_url_filter=)

setup.py

@@ -30,7 +30,7 @@
     InWheel = None
 from wheel.bdist_wheel import bdist_wheel as BDistWheelCommand
 
-driver_version = "1.23.0-beta-1656026605000"
+driver_version = "1.23.0-beta-1656093125000"
 
 
 def extractall(zip: zipfile.ZipFile, path: str) -> None: