CI - Parallelize smoketests

.github/docker-compose.yml

@@ -6,9 +6,9 @@ services:
       context: ../
       dockerfile: .github/Dockerfile
     ports:
-      - "3000:3000"
+      - "${STDB_PORT:-3000}:3000"
       # Postgres
-      - "5432:5432"
+      - "${STDB_PG_PORT:-5432}:5432"
     entrypoint: spacetime start --pg-port 5432
     privileged: true
     environment:

.github/workflows/ci.yml

@@ -36,6 +36,8 @@ jobs:
     container: ${{ matrix.container }}
     env:
       CARGO_TARGET_DIR: ${{ github.workspace }}/target
+      # Note: clear_database and replication only work in private
+      SMOKETEST_ARGS: ${{ matrix.smoketest_args }} -x clear_database replication teams
     steps:
       - name: Find Git ref
         env:
@@ -87,35 +89,29 @@ jobs:
         if: runner.os == 'Linux'
         run: /usr/local/bin/start-docker.sh
 
-      - name: Build and start database (Linux)
-        if: runner.os == 'Linux'
-        run: |
-          # Our .dockerignore omits `target`, which our CI Dockerfile needs.
-          rm .dockerignore
-          docker compose -f .github/docker-compose.yml up -d
-      - name: Build and start database (Windows)
+      # the sdk-manifests on windows-latest are messed up, so we need to update them
+      - name: Fix sdk-manifests
         if: runner.os == 'Windows'
+        working-directory: modules
+        # Powershell doesn't early-exit properly from a multi-line command if one fails
+        shell: bash
         run: |
-          # Fail properly if any individual command fails
-          $ErrorActionPreference = 'Stop'
-          $PSNativeCommandUseErrorActionPreference = $true
-
-          Start-Process target/debug/spacetimedb-cli.exe -ArgumentList 'start --pg-port 5432'
-          cd modules
-          # the sdk-manifests on windows-latest are messed up, so we need to update them
           dotnet workload config --update-mode manifests
           dotnet workload update
       - uses: actions/setup-python@v5
         with: { python-version: '3.12' }
         if: runner.os == 'Windows'
       - name: Install python deps
         run: python -m pip install -r smoketests/requirements.txt
-      - name: Run smoketests
-        # Note: clear_database and replication only work in private
-        run: cargo ci smoketests -- ${{ matrix.smoketest_args }} -x clear_database replication teams
-      - name: Stop containers (Linux)
-        if: always() && runner.os == 'Linux'
-        run: docker compose -f .github/docker-compose.yml down
+      - name: Run smoketests (Linux)
+        if: runner.os == 'Linux'
+        run: |
+          # Our .dockerignore omits `target`, which our CI Dockerfile needs.
+          rm .dockerignore
+          cargo ci smoketests --docker .github/docker-compose.yml --parallel -- ${SMOKETEST_ARGS}
+      - name: Run smoketests (Windows)
+        if: runner.os == 'Windows'
+        run: cargo ci smoketests --parallel -- ${SMOKETEST_ARGS}
 
   test:
     name: Test Suite

docker-compose.yml

@@ -24,11 +24,11 @@ services:
       - key_files:/etc/spacetimedb
       - /stdb
     ports:
-      - "3000:3000"
+      - "${STDB_PORT:-3000}:3000"
       # Postgres
-      - "5432:5432"
+      - "${STDB_PG_PORT:-5432}:5432"
       # Tracy
-      - "8086:8086"
+      - "${STDB_TRACY_PORT:-8086}:8086"
     entrypoint: cargo watch -i flamegraphs -i log.conf --why -C crates/standalone -x 'run start --data-dir=/stdb/data --jwt-pub-key-path=/etc/spacetimedb/id_ecdsa.pub --jwt-priv-key-path=/etc/spacetimedb/id_ecdsa --pg-port 5432'
     privileged: true
     environment:

smoketests/__init__.py

@@ -80,6 +80,7 @@ def requires_anonymous_login(item):
     return item
 
 def requires_local_server(item):
+    setattr(item, "_requires_local_server", True)
     if REMOTE_SERVER:
         return unittest.skip("running against a remote server")(item)
     return item

smoketests/__main__.py

@@ -77,16 +77,15 @@ def main():
     parser.add_argument("--no-docker-logs", action="store_true")
     parser.add_argument("--skip-dotnet", action="store_true", help="ignore tests which require dotnet")
     parser.add_argument("--show-all-output", action="store_true", help="show all stdout/stderr from the tests as they're running")
-    parser.add_argument("--parallel", action="store_true", help="run test classes in parallel")
-    parser.add_argument("-j", dest='jobs', help="Set number of jobs for parallel test runs. Default is `nproc`", type=int, default=0)
     parser.add_argument('-k', dest='testNamePatterns',
                         action='append', type=_convert_select_pattern,
                         help='Only run tests which match the given substring')
     parser.add_argument("-x", dest="exclude", nargs="*", default=[])
     parser.add_argument("--no-build-cli", action="store_true", help="don't cargo build the cli")
-    parser.add_argument("--list", action="store_true", help="list the tests that would be run, but don't run them")
+    parser.add_argument("--list", nargs="?", const="text", choices=("text", "json"), default=None, help="list the tests that would be run (optionally as 'text' or 'json'), but don't run them")
     parser.add_argument("--remote-server", action="store", help="Run against a remote server")
     parser.add_argument("--spacetime-login", action="store_true", help="Use `spacetime login` for these tests (and disable tests that don't work with that)")
+    parser.add_argument("--local-only", action="store_true", help="Only run tests that require a local server")
     args = parser.parse_args()
 
     if args.docker:
@@ -116,22 +115,58 @@ def main():
     loader.testNamePatterns = args.testNamePatterns
 
     tests = loader.loadTestsFromNames(testlist)
-    if args.list:
+
+    if args.local_only:
+        def _is_local_only(test_case):
+            method_name = getattr(test_case, "_testMethodName", None)
+            if method_name is not None and hasattr(test_case, method_name):
+                method = getattr(test_case, method_name)
+                if getattr(method, "_requires_local_server", False):
+                    return True
+            # Also allow class-level decoration
+            if getattr(test_case.__class__, "_requires_local_server", False):
+                return True
+            return False
+
+        filtered = unittest.TestSuite()
+        for t in _iter_all_tests(tests):
+            if _is_local_only(t):
+                filtered.addTest(t)
+        tests = filtered
+
+    if args.list is not None:
         failed_cls = getattr(unittest.loader, "_FailedTest", None)
         any_failed = False
+        test_names = []
+        failed_tests = []
         for test in _iter_all_tests(tests):
             name = test.id()
             if isinstance(test, failed_cls):
                 any_failed = True
-                print('')
-                print("Failed to construct %s:" % test.id())
                 exc = getattr(test, "_exception", None)
-                if exc is not None:
-                    tb = ''.join(traceback.format_exception(exc))
-                    print(tb.rstrip())
-                print('')
+                tb = ''.join(traceback.format_exception(exc)) if exc is not None else None
+                failed_tests.append({
+                    "test_id": name,
+                    "error": tb.rstrip() if tb is not None else None,
+                })
+                if args.list == "text":
+                    print('')
+                    print("Failed to construct %s:" % name)
+                    if tb is not None:
+                        print(tb.rstrip())
+                    print('')
             else:
-                print(f"{name}")
+                test_names.append(name)
+                if args.list == "text":
+                    print(f"{name}")
+
+        if args.list == "json":
+            output = {
+                "tests": test_names,
+                "errors": failed_tests,
+            }
+            print(json.dumps(output))
+
         exit(1 if any_failed else 0)
 
     if not args.no_build_cli:
@@ -176,14 +211,9 @@ def main():
     buffer = not args.show_all_output
     verbosity = 2
 
-    if args.parallel:
-        print("parallel test running is under construction, this will probably not work correctly")
-        from . import unittest_parallel
-        unittest_parallel.main(buffer=buffer, verbose=verbosity, level="class", discovered_tests=tests, jobs=args.jobs)
-    else:
-        result = unittest.TextTestRunner(buffer=buffer, verbosity=verbosity).run(tests)
-        if not result.wasSuccessful():
-            parser.exit(status=1)
+    result = unittest.TextTestRunner(buffer=buffer, verbosity=verbosity).run(tests)
+    if not result.wasSuccessful():
+        parser.exit(status=1)
 
 
 if __name__ == '__main__':

tools/ci/Cargo.toml

@@ -10,3 +10,4 @@ chrono = { workspace = true, features=["clock"] }
 clap.workspace = true
 regex.workspace = true
 duct.workspace = true
+serde_json.workspace = true