F6401 cannot-enumerate-pytest-fixtures (#16)

Author: reverbc

CHANGELOG.md

@@ -4,6 +4,7 @@
 ## Added
 - W6402 `useless-pytest-mark-decorator`: add warning for [using pytest.mark on fixtures](https://docs.pytest.org/en/stable/reference.html#marks) (thanks to @DKorytkin)
 - W6403 `deprecated-positional-argument-for-pytest-fixture`: add warning for [positional arguments to pytest.fixture()](https://docs.pytest.org/en/stable/deprecations.html#pytest-fixture-arguments-are-keyword-only) (thanks to @DKorytkin)
+- F6401 `cannot-enumerate-pytest-fixtures`: add fatal error when the plugin cannot enumerate and collect pytest fixtures for analysis (#27)
 
 ## [1.0.3] - 2021-03-13
 ## Fixed

README.md

@@ -128,6 +128,14 @@ def awesome_fixture():
     ...
 ```
 
+### F6401 `cannot-enumerate-pytest-fixtures`
+
+Raise when the plugin cannot enumerate and collect pytest fixtures for analysis
+
+```python
+import no_such_package  # <- pylint-pytest plugin cannot enumerate and collect pytest fixtures
+```
+
 ## Changelog
 
 See [CHANGELOG](CHANGELOG.md).

pylint_pytest/checkers/fixture.py

@@ -17,11 +17,16 @@
 
 class FixtureCollector:
     fixtures = {}
+    errors = set()
 
     def pytest_sessionfinish(self, session):
         # pylint: disable=protected-access
         self.fixtures = session._fixturemanager._arg2fixturedefs
 
+    def pytest_collectreport(self, report):
+        if report.failed:
+            self.errors.add(report)
+
 
 class FixtureChecker(BasePytestChecker):
     __implements__ = IAstroidChecker
@@ -44,6 +49,13 @@ class FixtureChecker(BasePytestChecker):
             'deprecated-positional-argument-for-pytest-fixture',
             'Pass scope as a kwarg, not positional arg, which is deprecated in future pytest.'
             'Take a look at: https://docs.pytest.org/en/stable/deprecations.html#pytest-fixture-arguments-are-keyword-only',
+        'F6401': (
+            (
+                'pylint-pytest plugin cannot enumerate and collect pytest fixtures. '
+                'Please run `pytest --fixtures --collect-only` and resolve any potential syntax error or package dependency issues'
+            ),
+            'cannot-enumerate-pytest-fixtures',
+            'Used when pylint-pytest has been unable to enumerate and collect pytest fixtures.',
         ),
     }
 
@@ -95,7 +107,7 @@ def visit_module(self, node):
                 # save and restore sys.path to prevent pytest.main from altering it
                 sys_path = sys.path.copy()
 
-                pytest.main(
+                ret = pytest.main(
                     [
                         node.file, '--fixtures', '--collect-only',
                         '--pythonwarnings=ignore:Module already imported:pytest.PytestWarning',
@@ -107,6 +119,9 @@ def visit_module(self, node):
                 sys.path = sys_path
 
                 FixtureChecker._pytest_fixtures = fixture_collector.fixtures
+
+                if ret != pytest.ExitCode.OK or fixture_collector.errors:
+                    self.add_message('cannot-enumerate-pytest-fixtures', node=node)
         finally:
             # restore output devices
             sys.stdout, sys.stderr = stdout, stderr

tests/input/cannot-enumerate-pytest-fixtures/import_corrupted_module.py

@@ -0,0 +1,5 @@
+from no_such_package import fixture
+
+
+def test_something(fixture):
+    pass

tests/input/cannot-enumerate-pytest-fixtures/no_such_package.py

@@ -0,0 +1,10 @@
+import pytest
+import this_is_invalid  # makes pytest fail
+
+
+@pytest.fixture
+def fixture():
+    pass
+
+def test_something(fixture):
+    pass

tests/test_cannot_enumerate_fixtures.py

@@ -0,0 +1,19 @@
+import pytest
+from pylint.checkers.variables import VariablesChecker
+from base_tester import BasePytestTester
+from pylint_pytest.checkers.fixture import FixtureChecker
+
+
+class TestCannotEnumerateFixtures(BasePytestTester):
+    CHECKER_CLASS = FixtureChecker
+    MSG_ID = 'cannot-enumerate-pytest-fixtures'
+
+    @pytest.mark.parametrize('enable_plugin', [True, False])
+    def test_no_such_package(self, enable_plugin):
+        self.run_linter(enable_plugin)
+        self.verify_messages(1 if enable_plugin else 0)
+
+    @pytest.mark.parametrize('enable_plugin', [True, False])
+    def test_import_corrupted_module(self, enable_plugin):
+        self.run_linter(enable_plugin)
+        self.verify_messages(1 if enable_plugin else 0)