Get the Full Test Suite Running on PRs for Windows (#174)

(fixes #168)

Author: ericsnowcurrently

.github/workflows/main.yml

@@ -45,4 +45,4 @@ jobs:
     - name: Display Python version
       run: python -c "import sys; print(sys.version)"
     - name: Run Tests
-      run: python runtests.py
+      run: python -u -m pyperformance.tests

pyperformance/commands.py

@@ -45,9 +45,9 @@ def cmd_venv_create(options, root, python, benchmarks):
     venv.ensure_pip()
     try:
         venv.install_pyperformance()
+        venv.ensure_reqs(requirements)
     except _venv.RequirementsInstallationFailedError:
         sys.exit(1)
-    venv.ensure_reqs(requirements, exitonerror=True)
     print("The virtual environment %s has been created" % root)
 
 
@@ -67,10 +67,10 @@ def cmd_venv_recreate(options, root, python, benchmarks):
             )
             venv.ensure_pip()
             try:
-                venv.install_pyperformance()
+                venv.ensure_reqs(requirements)
+                venv.ensure_reqs(requirements)
             except _venv.RequirementsInstallationFailedError:
                 sys.exit(1)
-            venv.ensure_reqs(requirements, exitonerror=True)
         else:
             print("The virtual environment %s already exists" % root)
             _utils.safe_rmtree(root)
@@ -84,9 +84,9 @@ def cmd_venv_recreate(options, root, python, benchmarks):
             venv.ensure_pip()
             try:
                 venv.install_pyperformance()
+                venv.ensure_reqs(requirements)
             except _venv.RequirementsInstallationFailedError:
                 sys.exit(1)
-            venv.ensure_reqs(requirements, exitonerror=True)
             print("The virtual environment %s has been recreated" % root)
     else:
         venv = VenvForBenchmarks.create(
@@ -97,9 +97,9 @@ def cmd_venv_recreate(options, root, python, benchmarks):
         venv.ensure_pip()
         try:
             venv.install_pyperformance()
+            venv.ensure_reqs(requirements)
         except _venv.RequirementsInstallationFailedError:
             sys.exit(1)
-        venv.ensure_reqs(requirements, exitonerror=True)
         print("The virtual environment %s has been created" % root)
 
 
@@ -225,9 +225,13 @@ def cmd_show(options):
 
 
 def cmd_compare(options):
-    from .compare import compare_results, write_csv
+    from .compare import compare_results, write_csv, VersionMismatchError
 
-    results = compare_results(options)
+    try:
+        results = compare_results(options)
+    except VersionMismatchError as exc:
+        print(f'ERROR: {exc}')
+        sys.exit(1)
 
     if options.csv:
         write_csv(results, options.csv)

pyperformance/compare.py

@@ -10,6 +10,16 @@
 NO_VERSION = "<not set>"
 
 
+class VersionMismatchError(Exception):
+
+    def __init__(self, version1, version2):
+        super().__init__(
+            f"Performance versions are different ({version1} != {version2})",
+        )
+        self.version1 = version1
+        self.version2 = version2
+
+
 def format_result(bench):
     mean = bench.mean()
     if bench.get_nvalue() >= 2:
@@ -372,10 +382,7 @@ def compare_results(options):
     version1 = base_suite.get_metadata().get('performance_version', NO_VERSION)
     version2 = changed_suite.get_metadata().get('performance_version', NO_VERSION)
     if version1 != version2 or (version1 == version2 == NO_VERSION):
-        print()
-        print("ERROR: Performance versions are different: %s != %s"
-              % (version1, version2))
-        sys.exit(1)
+        raise VersionMismatchError(version1, version2)
 
     return results
 

pyperformance/tests/__init__.py

@@ -7,6 +7,7 @@
 import subprocess
 import sys
 import tempfile
+import unittest
 
 
 TESTS_ROOT = os.path.realpath(os.path.dirname(__file__))
@@ -99,7 +100,35 @@ def cleanup(self):
 
 
 #############################
-# fixtures and mixins
+# testing fixtures, mixins, and helpers
+
+def apply_to_test_methods(cls, decorator):
+    for name, func in vars(cls).items():
+        if not name.startswith('test_'):
+            continue
+        func = decorator(func)
+        setattr(cls, name, func)
+
+
+def mark(label, func=None):
+    """Mark the function/class with the given label.
+
+    This may be used as a decorator.
+    """
+    if func is None:
+        def decorator(func):
+            return mark(label, func)
+        return decorator
+    if isinstance(func, type):
+        cls = func
+        apply_to_test_methods(cls, mark(label))
+        return cls
+    try:
+        func._pyperformance_test_labels.append(label)
+    except AttributeError:
+        func._pyperformance_test_labels = [label]
+    return func
+
 
 class Compat:
     """A mixin that lets older Pythons use newer unittest features."""
@@ -124,6 +153,18 @@ def addClassCleanup(cls, cleanup):
 #############################
 # functional tests
 
+CPYTHON_ONLY = unittest.skipIf(
+    sys.implementation.name != 'cpython',
+    'CPython-only',
+)
+NON_WINDOWS_ONLY = unittest.skipIf(os.name == 'nt', 'skipping Windows')
+
+# XXX Provide a way to run slow tests.
+SLOW = (lambda f:
+            unittest.skip('way too slow')(
+                mark('slow', f)))
+
+
 class Functional(Compat):
     """A mixin for functional tests.
 

pyperformance/tests/test_commands.py

@@ -9,12 +9,6 @@
 from pyperformance import tests
 
 
-CPYTHON_ONLY = unittest.skipIf(
-    sys.implementation.name != 'cpython',
-    'CPython-only',
-)
-
-
 class FullStackTests(tests.Functional, unittest.TestCase):
 
     maxDiff = 80 * 100
@@ -108,56 +102,41 @@ def div():
             print('---')
             print()
 
+        def expect_success(*args):
+            text = self.run_pyperformance(
+                *args,
+                capture=None,
+            )
+
+        def expect_failure(*args):
+            text = self.run_pyperformance(
+                *args,
+                capture=None,
+                exitcode=1,
+            )
+
         # It doesn't exist yet.
-        self.run_pyperformance(
-            'venv', 'show', '--venv', root,
-            capture=None,
-        )
+        expect_success('venv', 'show', '--venv', root)
         div()
         # It gets created.
-        self.run_pyperformance(
-            'venv', 'create', '--venv', root,
-            capture=None,
-        )
+        expect_success('venv', 'create', '--venv', root)
         div()
-        self.run_pyperformance(
-            'venv', 'show', '--venv', root,
-            capture=None,
-        )
+        expect_success('venv', 'show', '--venv', root)
         div()
         # It alraedy exists.
-        self.run_pyperformance(
-            'venv', 'create', '--venv', root,
-            capture=None,
-            exitcode=1,
-        )
+        expect_failure('venv', 'create', '--venv', root)
         div()
-        self.run_pyperformance(
-            'venv', 'show', '--venv', root,
-            capture=None,
-        )
+        expect_success('venv', 'show', '--venv', root)
         div()
         # It gets re-created.
-        self.run_pyperformance(
-            'venv', 'recreate', '--venv', root,
-            capture=None,
-        )
+        expect_success('venv', 'recreate', '--venv', root)
         div()
-        self.run_pyperformance(
-            'venv', 'show', '--venv', root,
-            capture=None,
-        )
+        expect_success('venv', 'show', '--venv', root)
         div()
         # It get deleted.
-        self.run_pyperformance(
-            'venv', 'remove', '--venv', root,
-            capture=None,
-        )
+        expect_success('venv', 'remove', '--venv', root)
         div()
-        self.run_pyperformance(
-            'venv', 'show', '--venv', root,
-            capture=None,
-        )
+        expect_success('venv', 'show', '--venv', root)
 
     ###################################
     # run
@@ -170,7 +149,7 @@ def test_run_and_show(self):
         # --debug-single-value: benchmark results don't matter, we only
         # check that running benchmarks don't fail.
         # XXX Capture and check the output.
-        self.run_pyperformance(
+        text = self.run_pyperformance(
             'run',
             '-b', 'all',
             '--debug-single-value',
@@ -259,8 +238,9 @@ def create_compile_config(self, *revisions,
             outfile.write(text)
         return cfgfile
 
-    @CPYTHON_ONLY
-    @unittest.skip('way too slow')
+    @tests.CPYTHON_ONLY
+    @tests.NON_WINDOWS_ONLY
+    @tests.SLOW
     def test_compile(self):
         cfgfile = self.create_compile_config()
         revision = 'a58ebcc701dd'  # tag: v3.10.2
@@ -271,8 +251,9 @@ def test_compile(self):
             capture=None,
         )
 
-    @CPYTHON_ONLY
-    @unittest.skip('way too slow')
+    @tests.CPYTHON_ONLY
+    @tests.NON_WINDOWS_ONLY
+    @tests.SLOW
     def test_compile_all(self):
         rev1 = '2cd268a3a934'  # tag: v3.10.1
         rev2 = 'a58ebcc701dd'  # tag: v3.10.2
@@ -284,7 +265,8 @@ def test_compile_all(self):
             capture=None,
         )
 
-    @CPYTHON_ONLY
+    @tests.CPYTHON_ONLY
+    @tests.NON_WINDOWS_ONLY
     @unittest.expectedFailure
     def test_upload(self):
         url = '<bogus>'
@@ -387,8 +369,7 @@ def test_compare_wrong_version(self):
             Skipped 1 benchmarks only in py36.json: telco
 
             Skipped 1 benchmarks only in py3_performance03.json: call_simple
-
-            ERROR: Performance versions are different: 1.0.1 != 0.3
+            ERROR: Performance versions are different (1.0.1 != 0.3)
             ''').lstrip())
 
     def test_compare_single_value(self):