Add cpp_harness_collect feature (#90)

Close #89

Author: tujh-rf

CHANGELOG.md

@@ -1,10 +1,13 @@
+# 2.2.2 (UNRELEASED)
+
+- New `cpp_harness_collect` configuration option allows users to add prefix arguments to run C++ test executables, allowing to use additional tools like `wine` of `qemu` for a test runners in cross-platform development.
+
 # 2.2.1 (2022-09-23)
 
 - Fixed a problem while handling errors using``--gtest_filter``. The recommendation is to use pytest's own
   filtering facilities (like `-k`) instead of passing filtering arguments to the underlying framework
   ([#84](https://github.com/pytest-dev/pytest-cpp/issues/84)).
 
-
 # 2.2.0 (2022-08-22)
 
 - Dropped support for Python 3.6.

README.rst

@@ -129,6 +129,28 @@ wrapping the test binary, like valgrind and memcheck:
     cpp_harness = valgrind --tool=memcheck
 
 
+cpp_harness_collect
+^^^^^^^^^^^^^^^^^^^
+
+This option allows the usage of tools or emulators (like wine or qemu) that are used by invoking them
+on the console wrapping the test binary during a test collection.
+
+Might be used in the combination with ``cpp_harness`` to run a binary in emulators, like wine or qemu
+in cross-compilation targets.
+
+.. code-block:: ini
+
+    [pytest]
+    cpp_harness_collect = qemu-x86_64 -L libs/
+
+or
+
+.. code-block:: ini
+
+    [pytest]
+    cpp_harness_collect = qemu-x86_64 -L libs/
+    cpp_harness = qemu-x86_64 -L libs/
+
 Changelog
 =========
 

src/pytest_cpp/boost.py

@@ -10,6 +10,7 @@
 from pytest_cpp.error import CppTestFailure
 from pytest_cpp.error import Markup
 from pytest_cpp.facade_abc import AbstractFacade
+from pytest_cpp.helpers import make_cmdline
 
 
 class BoostTestFacade(AbstractFacade):
@@ -18,10 +19,15 @@ class BoostTestFacade(AbstractFacade):
     """
 
     @classmethod
-    def is_test_suite(cls, executable: str) -> bool:
+    def is_test_suite(
+        cls,
+        executable: str,
+        harness_collect: Sequence[str] = (),
+    ) -> bool:
+        args = make_cmdline(harness_collect, executable, ["--help"])
         try:
             output = subprocess.check_output(
-                [executable, "--help"],
+                args,
                 stderr=subprocess.STDOUT,
                 universal_newlines=True,
             )
@@ -30,7 +36,12 @@ def is_test_suite(cls, executable: str) -> bool:
         else:
             return "--output_format" in output and "log_format" in output
 
-    def list_tests(self, executable: str) -> list[str]:
+    def list_tests(
+        self,
+        executable: str,
+        harness_collect: Sequence[str] = (),
+    ) -> list[str]:
+        del harness_collect
         # unfortunately boost doesn't provide us with a way to list the tests
         # inside the executable, so the test_id is a dummy placeholder :(
         return [os.path.basename(os.path.splitext(executable)[0])]
@@ -50,15 +61,17 @@ def read_file(name: str) -> str:
                 return ""
 
         with tempfile.TemporaryDirectory(prefix="pytest-cpp") as temp_dir:
-            log_xml = os.path.join(temp_dir, "log.xml")
-            report_xml = os.path.join(temp_dir, "report.xml")
-            args = list(harness) + [
-                executable,
-                "--output_format=XML",
-                "--log_sink=%s" % log_xml,
-                "--report_sink=%s" % report_xml,
-            ]
+            # On Windows, ValueError is raised when path and start are on different drives.
+            # In this case failing back to the absolute path.
+            try:
+                log_xml = os.path.join(os.path.relpath(temp_dir), "log.xml")
+                report_xml = os.path.join(os.path.relpath(temp_dir), "report.xml")
+            except ValueError:
+                log_xml = os.path.join(temp_dir, "log.xml")
+                report_xml = os.path.join(temp_dir, "report.xml")
+            args = make_cmdline(harness, executable, ["--output_format=XML", f"--log_sink={log_xml}", f"--report_sink={report_xml}"])
             args.extend(test_args)
+
             p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
             raw_stdout, _ = p.communicate()
             stdout = raw_stdout.decode("utf-8") if raw_stdout else ""

src/pytest_cpp/catch2.py

@@ -11,6 +11,7 @@
 from pytest_cpp.error import CppTestFailure
 from pytest_cpp.error import Markup
 from pytest_cpp.facade_abc import AbstractFacade
+from pytest_cpp.helpers import make_cmdline
 
 
 class Catch2Facade(AbstractFacade):
@@ -19,10 +20,15 @@ class Catch2Facade(AbstractFacade):
     """
 
     @classmethod
-    def is_test_suite(cls, executable: str) -> bool:
+    def is_test_suite(
+        cls,
+        executable: str,
+        harness_collect: Sequence[str] = (),
+    ) -> bool:
+        args = make_cmdline(harness_collect, executable, ["--help"])
         try:
             output = subprocess.check_output(
-                [executable, "--help"],
+                args,
                 stderr=subprocess.STDOUT,
                 universal_newlines=True,
             )
@@ -31,7 +37,11 @@ def is_test_suite(cls, executable: str) -> bool:
         else:
             return "--list-test-names-only" in output
 
-    def list_tests(self, executable: str) -> list[str]:
+    def list_tests(
+        self,
+        executable: str,
+        harness_collect: Sequence[str] = (),
+    ) -> list[str]:
         """
         Executes test with "--list-test-names-only" and gets list of tests
         parsing output like this:
@@ -41,9 +51,10 @@ def list_tests(self, executable: str) -> list[str]:
         2: Factorials of 1 and higher are computed (pass)
         """
         # This will return an exit code with the number of tests available
+        args = make_cmdline(harness_collect, executable, ["--list-test-names-only"])
         try:
             output = subprocess.check_output(
-                [executable, "--list-test-names-only"],
+                args,
                 stderr=subprocess.STDOUT,
                 universal_newlines=True,
             )
@@ -61,15 +72,16 @@ def run_test(
         test_args: Sequence[str] = (),
         harness: Sequence[str] = (),
     ) -> tuple[Sequence[Catch2Failure] | None, str]:
-        with tempfile.TemporaryDirectory(prefix="pytest-cpp") as tmp_dir:
-            xml_filename = os.path.join(tmp_dir, "cpp-report.xml")
-            args = list(harness) + [
-                executable,
-                test_id,
-                "--success",
-                "--reporter=xml",
-                "--out %s" % xml_filename,
-            ]
+        with tempfile.TemporaryDirectory(prefix="pytest-cpp") as temp_dir:
+            """
+            On Windows, ValueError is raised when path and start are on different drives.
+            In this case failing back to the absolute path.
+            """
+            try:
+                xml_filename = os.path.join(os.path.relpath(temp_dir), "cpp-report.xml")
+            except ValueError:
+                xml_filename = os.path.join(temp_dir, "cpp-report.xml")
+            args = make_cmdline(harness, executable, [test_id, "--success", "--reporter=xml", f"--out {xml_filename}"])
             args.extend(test_args)
 
             try:

src/pytest_cpp/facade_abc.py

@@ -10,11 +10,19 @@
 class AbstractFacade(ABC):
     @classmethod
     @abstractmethod
-    def is_test_suite(cls, executable: str) -> bool:
+    def is_test_suite(
+        cls,
+        executable: str,
+        harness_collect: Sequence[str] = (),
+    ) -> bool:
         """Return True if the given path to an executable contains tests for this framework."""
 
     @abstractmethod
-    def list_tests(self, executable: str) -> list[str]:
+    def list_tests(
+        self,
+        executable: str,
+        harness_collect: Sequence[str] = (),
+    ) -> list[str]:
         """Return a list of test ids found in the given executable."""
 
     @abstractmethod

src/pytest_cpp/google.py

@@ -11,6 +11,7 @@
 from pytest_cpp.error import CppTestFailure
 from pytest_cpp.error import Markup
 from pytest_cpp.facade_abc import AbstractFacade
+from pytest_cpp.helpers import make_cmdline
 
 
 class GoogleTestFacade(AbstractFacade):
@@ -19,10 +20,15 @@ class GoogleTestFacade(AbstractFacade):
     """
 
     @classmethod
-    def is_test_suite(cls, executable: str) -> bool:
+    def is_test_suite(
+        cls,
+        executable: str,
+        harness_collect: Sequence[str] = (),
+    ) -> bool:
+        args = make_cmdline(harness_collect, executable, ["--help"])
         try:
             output = subprocess.check_output(
-                [executable, "--help"],
+                args,
                 stderr=subprocess.STDOUT,
                 universal_newlines=True,
             )
@@ -31,7 +37,11 @@ def is_test_suite(cls, executable: str) -> bool:
         else:
             return "--gtest_list_tests" in output
 
-    def list_tests(self, executable: str) -> list[str]:
+    def list_tests(
+        self,
+        executable: str,
+        harness_collect: Sequence[str] = (),
+    ) -> list[str]:
         """
         Executes google-test with "--gtest_list_tests" and gets list of tests
         parsing output like this:
@@ -41,8 +51,9 @@ def list_tests(self, executable: str) -> list[str]:
           ReturnsTrueForPrimes
           CanGetNextPrime
         """
+        args = make_cmdline(harness_collect, executable, ["--gtest_list_tests"])
         output = subprocess.check_output(
-            [executable, "--gtest_list_tests"],
+            args,
             stderr=subprocess.STDOUT,
             universal_newlines=True,
         )
@@ -71,13 +82,14 @@ def run_test(
         test_args: Sequence[str] = (),
         harness: Sequence[str] = (),
     ) -> tuple[list[GoogleTestFailure] | None, str]:
-        with tempfile.TemporaryDirectory(prefix="pytest-cpp") as tmp_dir:
-            xml_filename = os.path.join(tmp_dir, "cpp-report.xml")
-            args = list(harness) + [
-                executable,
-                "--gtest_filter=" + test_id,
-                "--gtest_output=xml:%s" % xml_filename,
-            ]
+        with tempfile.TemporaryDirectory(prefix="pytest-cpp") as temp_dir:
+            # On Windows, ValueError is raised when path and start are on different drives.
+            # In this case failing back to the absolute path.
+            try:
+                xml_filename = os.path.join(os.path.relpath(temp_dir), "cpp-report.xml")
+            except ValueError:
+                xml_filename = os.path.join(temp_dir, "cpp-report.xml")
+            args = make_cmdline(harness, executable, [f"--gtest_filter={test_id}", f"--gtest_output=xml:{xml_filename}"])
             args.extend(test_args)
 
             try:

src/pytest_cpp/helpers.py

@@ -0,0 +1,6 @@
+
+from typing import Sequence
+
+
+def make_cmdline(harness: Sequence[str], executable: str, arg: Sequence[str] = ()) ->Sequence[str]:
+    return [*harness, executable, *arg]

src/pytest_cpp/plugin.py

@@ -67,8 +67,12 @@ def pytest_collect_file(
     ):
         return None
 
+    harness_collect = parent.config.getini("cpp_harness_collect")
     for facade_class in FACADES:
-        if facade_class.is_test_suite(str(file_path)):
+        if facade_class.is_test_suite(
+                str(file_path),
+                harness_collect=harness_collect
+            ):
             return CppFile.from_parent(
                 path=file_path,
                 parent=parent,
@@ -102,7 +106,13 @@ def pytest_addoption(parser: pytest.Parser) -> None:
         "cpp_harness",
         type="args",
         default=(),
-        help="command that wraps the cpp binary",
+        help="command that wraps the cpp binary when running tests",
+    )
+    parser.addini(
+        "cpp_harness_collect",
+        type="args",
+        default=(),
+        help="command that wraps the cpp binary when collecting tests",
     )
     parser.addini(
         "cpp_verbose",
@@ -144,7 +154,11 @@ def from_parent(  # type:ignore[override]
         )
 
     def collect(self) -> Iterator[CppItem]:
-        for test_id in self.facade.list_tests(str(self.fspath)):
+        harness_collect=self.config.getini("cpp_harness_collect")
+        for test_id in self.facade.list_tests(
+                str(self.fspath),
+                harness_collect=harness_collect,
+            ):
             yield CppItem.from_parent(
                 parent=self,
                 name=test_id,

tests/test_pytest_cpp.py

@@ -11,6 +11,7 @@
 from pytest_cpp.error import CppFailureRepr
 from pytest_cpp.error import CppTestFailure
 from pytest_cpp.google import GoogleTestFacade
+from pytest_cpp.helpers import make_cmdline
 
 
 def assert_outcomes(result, expected_outcomes):
@@ -91,6 +92,21 @@ def test_success(facade, name, test_id, exes):
     assert facade.run_test(exes.get(name), test_id)[0] is None
 
 
+def test_cmdline_builder_happy_flow():
+    arg_string = make_cmdline( ["wine"], "gtest", ["--help"] )
+    assert arg_string == ["wine", "gtest", "--help"]
+
+
+def test_cmdline_builder_with_empty_harness():
+    arg_string = make_cmdline(list(), "boost_test", ["--output_format=XML", "--log_sink=dummy.log"])
+    assert arg_string == ["boost_test", "--output_format=XML", "--log_sink=dummy.log"]
+
+
+def test_cmdline_builder_with_empty_args():
+    arg_string = make_cmdline( ["wine"], "gtest")
+    assert arg_string == ["wine", "gtest"]
+
+
 def test_google_failure(exes):
     facade = GoogleTestFacade()
     failures, _ = facade.run_test(exes.get("gtest"), "FooTest.test_failure")