Pylint check

Author: elacuesta

.github/workflows/checks.yml

@@ -21,6 +21,9 @@ jobs:
         - python-version: 3.8
           env:
             TOXENV: typing
+        - python-version: 3.8
+          env:
+            TOXENV: pylint
 
     steps:
     - uses: actions/checkout@v2

pylintrc

@@ -0,0 +1,25 @@
+[MESSAGES CONTROL]
+disable=
+    attribute-defined-outside-init,
+    import-error,
+    invalid-name,
+    logging-fstring-interpolation,
+    missing-class-docstring,
+    missing-function-docstring,
+    missing-module-docstring,
+    too-few-public-methods,
+    too-many-instance-attributes,
+    too-many-locals,
+    unused-argument,
+    # tests
+    abstract-method,
+    bare-except,
+    duplicate-code,
+    import-outside-toplevel,
+    inconsistent-return-statements,
+    protected-access,
+
+
+[FORMAT]
+expected-line-ending-format=LF
+max-line-length=99

scrapy_playwright/handler.py

@@ -106,8 +106,8 @@ def __init__(self, crawler: Crawler) -> None:
             self.context_kwargs[name].update(kwargs)
         if "default" not in self.context_kwargs and default_context_kwargs:
             self.context_kwargs["default"] = default_context_kwargs
-        self.contexts: Dict[str, BrowserContext] = dict()
-        self.context_semaphores: Dict[str, asyncio.Semaphore] = dict()
+        self.contexts: Dict[str, BrowserContext] = {}
+        self.context_semaphores: Dict[str, asyncio.Semaphore] = {}
 
     @classmethod
     def from_crawler(cls: Type[PlaywrightHandler], crawler: Crawler) -> PlaywrightHandler:
@@ -132,8 +132,7 @@ async def _launch_browser(self) -> None:
         )
         self.contexts = dict(zip(self.context_kwargs.keys(), contexts))
         self.context_semaphores = {
-            name: asyncio.Semaphore(value=self.max_pages_per_context)
-            for name in self.contexts.keys()
+            name: asyncio.Semaphore(value=self.max_pages_per_context) for name in self.contexts
         }
 
     async def _create_browser_context(self, name: str, context_kwargs: dict) -> BrowserContext:

scrapy_playwright/headers.py

@@ -1,14 +1,13 @@
-from urllib.parse import urlparse
-
-from playwright.async_api import Request as PlaywrightRequest
-from scrapy.http.headers import Headers
-
-
 """
 This module includes functions to process request headers.
 Refer to the PLAYWRIGHT_PROCESS_REQUEST_HEADERS setting for more information.
 """
 
+from urllib.parse import urlparse
+
+from playwright.async_api import Request as PlaywrightRequest
+from scrapy.http.headers import Headers
+
 
 async def use_scrapy_headers(
     browser_type: str,
@@ -28,11 +27,11 @@ async def use_scrapy_headers(
             # otherwise this fails with playwright.helper.Error: NS_ERROR_NET_RESET
             headers["host"] = urlparse(playwright_request.url).netloc
         return headers
-    else:
-        # override user agent, for consistency with other requests
-        if headers.get("user-agent"):
-            playwright_request.headers["user-agent"] = headers["user-agent"]
-        return playwright_request.headers
+
+    # override user agent, for consistency with other requests
+    if headers.get("user-agent"):
+        playwright_request.headers["user-agent"] = headers["user-agent"]
+    return playwright_request.headers
 
 
 async def use_playwright_headers(

scrapy_playwright/page.py

@@ -11,6 +11,6 @@ def __init__(self, method: str, *args, **kwargs) -> None:
         self.result = None
 
     def __str__(self):
-        return "<%s for method '%s'>" % (self.__class__.__name__, self.method)
+        return f"<{self.__class__.__name__} for method '{self.method}'>"
 
     __repr__ = __str__

setup.py

@@ -3,7 +3,7 @@
 from scrapy_playwright import __version__
 
 
-with open("README.md", "r") as fh:
+with open("README.md", "r", encoding="utf-8") as fh:
     long_description = fh.read()
 
 

tests/mockserver.py

@@ -27,7 +27,7 @@ def __exit__(self, exc_type, exc_value, traceback):
         self.proc.communicate()
 
     def urljoin(self, url):
-        return urljoin("http://{}:{}".format(self.address, self.port), url)
+        return urljoin(f"http://{self.address}:{self.port}", url)
 
 
 class _RequestHandler(BaseHTTPRequestHandler):
@@ -70,7 +70,7 @@ def __exit__(self, exc_type, exc_value, traceback):
         self.thread.join()
 
     def urljoin(self, url: str) -> str:
-        return urljoin("http://{}:{}".format(self.address, self.port), url)
+        return urljoin(f"http://{self.address}:{self.port}", url)
 
 
 if __name__ == "__main__":

tests/test_playwright_requests.py

@@ -6,7 +6,11 @@
 from tempfile import NamedTemporaryFile
 
 import pytest
-from playwright.async_api import Dialog, Page as PlaywrightPage, TimeoutError
+from playwright.async_api import (
+    Dialog,
+    Page as PlaywrightPage,
+    TimeoutError as PlaywrightTimeoutError,
+)
 from scrapy import Spider, Request, FormRequest
 from scrapy.http.response.html import HtmlResponse
 
@@ -21,6 +25,7 @@ def get_mimetype(file):
         ["file", "--mime-type", "--brief", file.name],
         stdout=subprocess.PIPE,
         universal_newlines=True,
+        check=False,
     ).stdout.strip()
 
 
@@ -162,7 +167,7 @@ async def test_timeout(self):
         async with make_handler(settings_dict) as handler:
             with MockServer() as server:
                 req = Request(server.urljoin("/delay/2"), meta={"playwright": True})
-                with pytest.raises(TimeoutError):
+                with pytest.raises(PlaywrightTimeoutError):
                     await handler._download_request(req, Spider("foo"))
 
     @pytest.mark.asyncio
@@ -184,7 +189,7 @@ async def test_context_kwargs(self):
                         ],
                     },
                 )
-                with pytest.raises(TimeoutError):
+                with pytest.raises(PlaywrightTimeoutError):
                     await handler._download_request(req, Spider("foo"))
 
     @pytest.mark.asyncio
@@ -263,10 +268,12 @@ async def test_user_agent(self):
     @pytest.mark.asyncio
     async def test_use_playwright_headers(self):
         """Ignore Scrapy headers"""
+        from scrapy_playwright.headers import use_playwright_headers
+
         settings_dict = {
             "PLAYWRIGHT_BROWSER_TYPE": self.browser_type,
             "PLAYWRIGHT_CONTEXTS": {"default": {"user_agent": self.browser_type}},
-            "PLAYWRIGHT_PROCESS_REQUEST_HEADERS": "scrapy_playwright.headers.use_playwright_headers",  # noqa: E501
+            "PLAYWRIGHT_PROCESS_REQUEST_HEADERS": use_playwright_headers,
         }
         async with make_handler(settings_dict) as handler:
             with MockServer() as server:

tox.ini

@@ -1,5 +1,5 @@
 [tox]
-envlist = bandit,black,flake8,typing,py
+envlist = bandit,black,flake8,typing,pylint,py
 
 [testenv]
 deps =
@@ -21,30 +21,32 @@ setenv =
 basepython = python3
 
 [testenv:bandit]
-basepython = python3
 deps =
     bandit
 commands =
     bandit -r {posargs: scrapy_playwright setup.py examples}
 
 [testenv:black]
-basepython = python3.8
 deps =
     black==20.8b1
 commands =
     black --check {posargs: scrapy_playwright setup.py tests examples}
 
 [testenv:flake8]
-basepython = python3.8
 deps =
     flake8>=3.7.9
 commands =
     flake8 --exclude=.git,.tox,venv* {posargs: scrapy_playwright setup.py tests examples}
 
 [testenv:typing]
-basepython = python3.8
 deps =
     mypy==0.790
 commands =
     mypy --show-error-codes --ignore-missing-imports \
         --follow-imports=skip {posargs: scrapy_playwright setup.py tests examples}
+
+[testenv:pylint]
+deps =
+    pylint==2.12.2
+commands =
+    pylint {posargs: scrapy_playwright setup.py tests}