Annotate warnings (#68)

* Annotate warnings

- Refactor workflow command generation

* Rename option to `--exclude-warning-annotations`

* Suppress windows relpath exception

Author: edgarrmondragon

.github/workflows/test.yml

@@ -17,6 +17,7 @@ env:
 jobs:
   test:
     strategy:
+      fail-fast: false
       matrix:
         os: [ubuntu-latest, windows-latest]
         python-version:

README.md

@@ -36,5 +36,9 @@ If your test is running in a Docker container, you have to install this plugin a
 
 If your tests are run from a subdirectory of the git repository, you have to set the `PYTEST_RUN_PATH` environment variable to the path of that directory relative to the repository root in order for GitHub to identify the files with errors correctly.
 
+### Warning annotations
+
+This plugin also supports warning annotations when used with Pytest 6.0+. To disable warning annotations, pass `--exclude-warning-annotations` to pytest.
+
 ## Screenshot
 [![Image from Gyazo](https://i.gyazo.com/b578304465dd1b755ceb0e04692a57d9.png)](https://gyazo.com/b578304465dd1b755ceb0e04692a57d9)

plugin_test.py

@@ -5,19 +5,17 @@
 import pytest
 from packaging import version
 
+PYTEST_VERSION = version.parse(pytest.__version__)
 pytest_plugins = "pytester"
 
 
-# result.stderr.no_fnmatch_line() is added to testdir on pytest 5.3.0
+# result.stderr.no_fnmatch_line() was added to testdir on pytest 5.3.0
 # https://docs.pytest.org/en/stable/changelog.html#pytest-5-3-0-2019-11-19
-def no_fnmatch_line(result, pattern):
-    if version.parse(pytest.__version__) >= version.parse("5.3.0"):
-        result.stderr.no_fnmatch_line(pattern + "*",)
-    else:
-        assert pattern not in result.stderr.str()
+def no_fnmatch_line(result: pytest.RunResult, pattern: str):
+    result.stderr.no_fnmatch_line(pattern + "*")
 
 
-def test_annotation_succeed_no_output(testdir):
+def test_annotation_succeed_no_output(testdir: pytest.Testdir):
     testdir.makepyfile(
         """
         import pytest
@@ -33,7 +31,7 @@ def test_success():
     no_fnmatch_line(result, "::error file=test_annotation_succeed_no_output.py")
 
 
-def test_annotation_pytest_error(testdir):
+def test_annotation_pytest_error(testdir: pytest.Testdir):
     testdir.makepyfile(
         """
         import pytest
@@ -55,7 +53,7 @@ def test_error():
     )
 
 
-def test_annotation_fail(testdir):
+def test_annotation_fail(testdir: pytest.Testdir):
     testdir.makepyfile(
         """
         import pytest
@@ -68,11 +66,13 @@ def test_fail():
     testdir.monkeypatch.setenv("GITHUB_ACTIONS", "true")
     result = testdir.runpytest_subprocess()
     result.stderr.fnmatch_lines(
-        ["::error file=test_annotation_fail.py,line=5::test_fail*assert 0*",]
+        [
+            "::error file=test_annotation_fail.py,line=5::test_fail*assert 0*",
+        ]
     )
 
 
-def test_annotation_exception(testdir):
+def test_annotation_exception(testdir: pytest.Testdir):
     testdir.makepyfile(
         """
         import pytest
@@ -86,11 +86,51 @@ def test_fail():
     testdir.monkeypatch.setenv("GITHUB_ACTIONS", "true")
     result = testdir.runpytest_subprocess()
     result.stderr.fnmatch_lines(
-        ["::error file=test_annotation_exception.py,line=5::test_fail*oops*",]
+        [
+            "::error file=test_annotation_exception.py,line=5::test_fail*oops*",
+        ]
+    )
+
+
+def test_annotation_warning(testdir: pytest.Testdir):
+    testdir.makepyfile(
+        """
+        import warnings
+        import pytest
+        pytest_plugins = 'pytest_github_actions_annotate_failures'
+
+        def test_warning():
+            warnings.warn('beware', Warning)
+            assert 1
+        """
     )
+    testdir.monkeypatch.setenv("GITHUB_ACTIONS", "true")
+    result = testdir.runpytest_subprocess()
+    result.stderr.fnmatch_lines(
+        [
+            "::warning file=test_annotation_warning.py,line=6::beware",
+        ]
+    )
+
+
+def test_annotation_exclude_warnings(testdir: pytest.Testdir):
+    testdir.makepyfile(
+        """
+        import warnings
+        import pytest
+        pytest_plugins = 'pytest_github_actions_annotate_failures'
+
+        def test_warning():
+            warnings.warn('beware', Warning)
+            assert 1
+        """
+    )
+    testdir.monkeypatch.setenv("GITHUB_ACTIONS", "true")
+    result = testdir.runpytest_subprocess("--exclude-warning-annotations")
+    assert not result.stderr.lines
 
 
-def test_annotation_third_party_exception(testdir):
+def test_annotation_third_party_exception(testdir: pytest.Testdir):
     testdir.makepyfile(
         my_module="""
         def fn():
@@ -111,11 +151,43 @@ def test_fail():
     testdir.monkeypatch.setenv("GITHUB_ACTIONS", "true")
     result = testdir.runpytest_subprocess()
     result.stderr.fnmatch_lines(
-        ["::error file=test_annotation_third_party_exception.py,line=6::test_fail*oops*",]
+        [
+            "::error file=test_annotation_third_party_exception.py,line=6::test_fail*oops*",
+        ]
+    )
+
+
+def test_annotation_third_party_warning(testdir: pytest.Testdir):
+    testdir.makepyfile(
+        my_module="""
+        import warnings
+
+        def fn():
+            warnings.warn('beware', Warning)
+        """
+    )
+
+    testdir.makepyfile(
+        """
+        import pytest
+        from my_module import fn
+        pytest_plugins = 'pytest_github_actions_annotate_failures'
+
+        def test_warning():
+            fn()
+        """
+    )
+    testdir.monkeypatch.setenv("GITHUB_ACTIONS", "true")
+    result = testdir.runpytest_subprocess()
+    result.stderr.fnmatch_lines(
+        # ["::warning file=test_annotation_third_party_warning.py,line=6::beware",]
+        [
+            "::warning file=my_module.py,line=4::beware",
+        ]
     )
 
 
-def test_annotation_fail_disabled_outside_workflow(testdir):
+def test_annotation_fail_disabled_outside_workflow(testdir: pytest.Testdir):
     testdir.makepyfile(
         """
         import pytest
@@ -132,7 +204,7 @@ def test_fail():
     )
 
 
-def test_annotation_fail_cwd(testdir):
+def test_annotation_fail_cwd(testdir: pytest.Testdir):
     testdir.makepyfile(
         """
         import pytest
@@ -148,11 +220,13 @@ def test_fail():
     testdir.makefile(".ini", pytest="[pytest]\ntestpaths=..")
     result = testdir.runpytest_subprocess("--rootdir=foo")
     result.stderr.fnmatch_lines(
-        ["::error file=test_annotation_fail_cwd.py,line=5::test_fail*assert 0*",]
+        [
+            "::error file=test_annotation_fail_cwd.py,line=5::test_fail*assert 0*",
+        ]
     )
 
 
-def test_annotation_fail_runpath(testdir):
+def test_annotation_fail_runpath(testdir: pytest.Testdir):
     testdir.makepyfile(
         """
         import pytest
@@ -166,11 +240,13 @@ def test_fail():
     testdir.monkeypatch.setenv("PYTEST_RUN_PATH", "some_path")
     result = testdir.runpytest_subprocess()
     result.stderr.fnmatch_lines(
-        ["::error file=some_path/test_annotation_fail_runpath.py,line=5::test_fail*assert 0*",]
+        [
+            "::error file=some_path/test_annotation_fail_runpath.py,line=5::test_fail*assert 0*",
+        ]
     )
 
 
-def test_annotation_long(testdir):
+def test_annotation_long(testdir: pytest.Testdir):
     testdir.makepyfile(
         """
         import pytest
@@ -202,7 +278,7 @@ def test_fail():
     no_fnmatch_line(result, "::*assert x += 1*")
 
 
-def test_class_method(testdir):
+def test_class_method(testdir: pytest.Testdir):
     testdir.makepyfile(
         """
         import pytest
@@ -224,7 +300,7 @@ def test_method(self):
     no_fnmatch_line(result, "::*x = 1*")
 
 
-def test_annotation_param(testdir):
+def test_annotation_param(testdir: pytest.Testdir):
     testdir.makepyfile(
         """
         import pytest

pytest_github_actions_annotate_failures/plugin.py

@@ -1,13 +1,14 @@
 
 from __future__ import annotations
 
+import contextlib
 import os
 import sys
-from collections import OrderedDict
 from typing import TYPE_CHECKING
 
 import pytest
 from _pytest._code.code import ExceptionRepr
+from packaging import version
 
 if TYPE_CHECKING:
     from _pytest.nodes import Item
@@ -23,6 +24,9 @@
 # https://github.com/pytest-dev/pytest/blob/master/src/_pytest/terminal.py
 
 
+PYTEST_VERSION = version.parse(pytest.__version__)
+
+
 @pytest.hookimpl(tryfirst=True, hookwrapper=True)
 def pytest_runtest_makereport(item: Item, call):  # noqa: ARG001
     # execute all other hooks to obtain the report object
@@ -79,25 +83,91 @@ def pytest_runtest_makereport(item: Item, call):  # noqa: ARG001
         elif isinstance(report.longrepr, str):
             longrepr += "\n\n" + report.longrepr
 
-        print(
-            _error_workflow_command(filesystempath, lineno, longrepr), file=sys.stderr
+        workflow_command = _build_workflow_command(
+            "error",
+            filesystempath,
+            lineno,
+            message=longrepr,
         )
+        print(workflow_command, file=sys.stderr)
 
 
-def _error_workflow_command(filesystempath, lineno, longrepr):
-    # Build collection of arguments. Ordering is strict for easy testing
-    details_dict = OrderedDict()
-    details_dict["file"] = filesystempath
-    if lineno is not None:
-        details_dict["line"] = lineno
-
-    details = ",".join(f"{k}={v}" for k, v in details_dict.items())
+class _AnnotateWarnings:
+    def pytest_warning_recorded(self, warning_message, when, nodeid, location):  # noqa: ARG002
+        # enable only in a workflow of GitHub Actions
+        # ref: https://help.github.com/en/actions/configuring-and-managing-workflows/using-environment-variables#default-environment-variables
+        if os.environ.get("GITHUB_ACTIONS") != "true":
+            return
 
-    if longrepr is None:
-        return f"\n::error {details}"
+        filesystempath = warning_message.filename
+        workspace = os.environ.get("GITHUB_WORKFLOW")
 
-    longrepr = _escape(longrepr)
-    return f"\n::error {details}::{longrepr}"
+        if workspace:
+            try:
+                rel_path = os.path.relpath(filesystempath, workspace)
+            except ValueError:
+                # os.path.relpath() will raise ValueError on Windows
+                # when full_path and workspace have different mount points.
+                rel_path = filesystempath
+            if not rel_path.startswith(".."):
+                filesystempath = rel_path
+        else:
+            with contextlib.suppress(ValueError):
+                filesystempath = os.path.relpath(filesystempath)
+
+        workflow_command = _build_workflow_command(
+            "warning",
+            filesystempath,
+            warning_message.lineno,
+            message=warning_message.message.args[0],
+        )
+        print(workflow_command, file=sys.stderr)
+
+
+def pytest_addoption(parser):
+    group = parser.getgroup("pytest_github_actions_annotate_failures")
+    group.addoption(
+        "--exclude-warning-annotations",
+        action="store_true",
+        default=False,
+        help="Annotate failures in GitHub Actions.",
+    )
+
+def pytest_configure(config):
+    if not config.option.exclude_warning_annotations:
+        config.pluginmanager.register(_AnnotateWarnings(), "annotate_warnings")
+
+
+def _build_workflow_command(
+    command_name,
+    file,
+    line,
+    end_line=None,
+    column=None,
+    end_column=None,
+    title=None,
+    message=None,
+):
+    """Build a command to annotate a workflow."""
+    result = f"::{command_name} "
+
+    entries = [
+        ("file", file),
+        ("line", line),
+        ("endLine", end_line),
+        ("col", column),
+        ("endColumn", end_column),
+        ("title", title),
+    ]
+
+    result = result + ",".join(
+        f"{k}={v}" for k, v in entries if v is not None
+    )
+
+    if message is not None:
+        result = result + "::" + _escape(message)
+
+    return result
 
 
 def _escape(s):