pytestplugin/hooks: introduce pytest.mark.lg_xfail_feature

`@pytest.mark.lg_xfail_feature()` is meant to mark a test as "xfail" if
the target or global env contains the specified feature flag.

Signed-off-by: Bastian Krause <bst@pengutronix.de>

Author: Bastian-Krause

doc/usage.rst

@@ -730,6 +730,44 @@ Tests requiring multiple features are also possible:
    def test_camera(target):
       pass
 
+@pytest.mark.lg_xfail_feature()
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+labgrid supports :ref:`environment-configuration-feature-flags` in the
+:ref:`environment-configuration`.
+pytest supports the ``xfail`` marker, see
+`pytest.mark.xfail() <https://docs.pytest.org/en/stable/reference/reference.html#pytest-mark-xfail>`_.
+
+When having more specific features, tests can be marked as ``xfail`` for a
+particular feature.
+
+Imagine two targets have ``camera`` feature flags.
+One of them has the additional ``special-camera-2000`` feature flag.
+The other has the additional ``special-camera-3000`` feature flag.
+Due to a known bug on ``special-camera-3000``, the test is expected to
+fail.
+The test can be marked as ``xfail`` for that feature:
+
+.. code-block:: python
+
+   import pytest
+
+   @pytest.mark.lg_feature("camera"])
+   @pytest.mark.lg_xfail_feature(
+       "special-camera-3000",
+       reason="known bug xy on special-camera-3000",
+       raises=AssertionError,
+       strict=True,
+   )
+   def test_camera(target):
+      pass
+
+Features under the target and global ``features:`` keys are considered.
+
+``@pytest.mark.lg_xfail_feature(feature, **kwargs)``:
+  - ``feature`` (str) - Feature that should mark the test as ``xfail``, passed
+    as boolean ``condition=`` to ``pytest.mark.xfail()``.
+  - ``**kwargs`` - All kw-only args are passed to ``pytest.mark.xfail()``.
+
 Test Reports
 ~~~~~~~~~~~~
 

labgrid/pytestplugin/hooks.py

@@ -1,4 +1,5 @@
 import os
+import copy
 import logging
 import pytest
 
@@ -72,6 +73,9 @@ def pytest_configure(config):
 
     config.addinivalue_line("markers",
                             "lg_feature: skip tests on envs/targets without given labgrid feature flags")
+    config.addinivalue_line("markers",
+                            "lg_xfail_feature: mark tests xfail on envs/targets with given labgrid feature flag")
+
     lg_log = config.option.lg_log
     if lg_log:
         ConsoleLoggingReporter(lg_log)
@@ -120,6 +124,26 @@ def pytest_collection_modifyitems(config, items):
             reason = f'unsupported feature(s): {", ".join(missing_feature)}'
             item.add_marker(pytest.mark.skip(reason=reason))
 
+        # pytest.mark.lg_xfail_feature
+        lg_xfail_feature_signature = "pytest.mark.lg_xfail_feature(feature: str, *, **xfail_kwargs), xfail_kwargs as pytest.mark.xfail expects them"
+        for marker in item.iter_markers("lg_xfail_feature"):
+            if len(marker.args) != 1:
+                raise pytest.UsageError(f"Unexpected number of arguments for {lg_xfail_feature_signature}")
+            elif not isinstance(marker.args[0], str):
+                raise pytest.UsageError(f"Unsupported 'feature' argument type {type(marker.args[0])} for {lg_xfail_feature_signature}")
+            if "condition" in marker.kwargs:
+                raise pytest.UsageError(f"Unsupported 'condition' argument for {lg_xfail_feature_signature}")
+
+            kwargs = copy.copy(marker.kwargs)
+            reason = kwargs.pop("reason", marker.args[0])
+            item.add_marker(
+                pytest.mark.xfail(
+                    condition=marker.args[0] in have_feature,
+                    reason=reason,
+                    **kwargs,
+                )
+            )
+
 @pytest.hookimpl(tryfirst=True)
 def pytest_runtest_setup(item):
     """

tests/test_flags.py

@@ -172,3 +172,70 @@ def test(self, env):
         spawn.close()
         # pytest command line usage error leads to exit code 4
         assert spawn.exitstatus == 4
+
+def test_xfail_feature(tmpdir, env_feature_config):
+    conf = env_feature_config(["test"])
+    test = tmpdir.join("test.py")
+    test.write(
+"""
+import pytest
+
+@pytest.mark.lg_xfail_feature("test")
+def test(env):
+    assert False
+"""
+    )
+
+    with pexpect.spawn(f'pytest --lg-env {conf} {test}') as spawn:
+        spawn.expect("1 xfailed")
+        spawn.expect(pexpect.EOF)
+    assert spawn.exitstatus == 0
+
+def test_no_xfail_feature(tmpdir, env_feature_config):
+    conf = env_feature_config([])
+    test = tmpdir.join("test.py")
+    test.write(
+"""
+import pytest
+
+@pytest.mark.lg_xfail_feature("test")
+def test(env):
+    assert False
+"""
+    )
+
+    with pexpect.spawn(f'pytest --lg-env {conf} {test}') as spawn:
+        spawn.expect("1 failed")
+        spawn.expect(pexpect.EOF)
+    assert spawn.exitstatus == 1
+
+@pytest.mark.parametrize(
+    "marker_args_str,error",
+    [
+        ("", "Unexpected number of arguments"),
+        ("'too', 'many'", "Unexpected number of arguments"),
+        ("{'foo': 'bar'}", "Unsupported 'feature' argument type"),
+        ("'feature', condition='mycondition'", "Unsupported 'condition' argument"),
+    ],
+    ids=["no args", "too many args", "unsupported arg type", "unsupported condition"]
+)
+def test_lg_xfail_feature_unexpected_args(tmpdir, env_feature_config, marker_args_str, error):
+    # features do not matter here, simply generate a valid env config
+    conf = env_feature_config([])
+    test = tmpdir.join("test.py")
+    test.write(
+f"""
+import pytest
+
+@pytest.mark.lg_xfail_feature({marker_args_str})
+def test(self, env):
+    assert True
+"""
+    )
+
+    with pexpect.spawn(f'pytest --lg-env {conf} {test}') as spawn:
+        spawn.expect(error)
+        spawn.expect(pexpect.EOF)
+        spawn.close()
+        # pytest command line usage error leads to exit code 4
+        assert spawn.exitstatus == 4