From 7789f2f41cc180b36b1d5dbea3ee09ae620fa985 Mon Sep 17 00:00:00 2001 From: Zeke Foppa Date: Wed, 19 Nov 2025 13:04:59 -0800 Subject: [PATCH 01/41] [bfops/fix-list-tests]: Fix running smoketests --list --- smoketests/__main__.py | 67 +++++++++++++++++++++--------------------- 1 file changed, 33 insertions(+), 34 deletions(-) diff --git a/smoketests/__main__.py b/smoketests/__main__.py index 6161dc5a7aa..8f19238adc8 100644 --- a/smoketests/__main__.py +++ b/smoketests/__main__.py @@ -79,6 +79,39 @@ def main(): parser.add_argument("--spacetime-login", action="store_true", help="Use `spacetime login` for these tests (and disable tests that don't work with that)") args = parser.parse_args() + if args.docker: + # have docker logs print concurrently with the test output + if args.compose_file: + smoketests.COMPOSE_FILE = args.compose_file + if not args.no_docker_logs: + if args.compose_file: + subprocess.Popen(["docker", "compose", "-f", args.compose_file, "logs", "-f"]) + else: + docker_container = check_docker() + subprocess.Popen(["docker", "logs", "-f", docker_container]) + smoketests.HAVE_DOCKER = True + + if not args.skip_dotnet: + smoketests.HAVE_DOTNET = check_dotnet() + if not smoketests.HAVE_DOTNET: + print("no suitable dotnet installation found") + exit(1) + + add_prefix = lambda testlist: [TESTPREFIX + test for test in testlist] + import fnmatch + excludelist = add_prefix(args.exclude) + testlist = add_prefix(args.test) + + loader = ExclusionaryTestLoader(excludelist) + loader.testNamePatterns = args.testNamePatterns + + tests = loader.loadTestsFromNames(testlist) + if args.list: + print("Selected tests:\n") + for test in itertools.chain(*itertools.chain(*tests)): + print(f"{test}") + exit(0) + if not args.no_build_cli: logging.info("Compiling spacetime cli...") smoketests.run_cmd("cargo", "build", cwd=TEST_DIR.parent, capture_stderr=False) @@ -100,18 +133,6 @@ def main(): os.environ["SPACETIME_SKIP_CLIPPY"] = "1" - if args.docker: - # have docker logs print concurrently with the test output - if args.compose_file: - smoketests.COMPOSE_FILE = args.compose_file - if not args.no_docker_logs: - if args.compose_file: - subprocess.Popen(["docker", "compose", "-f", args.compose_file, "logs", "-f"]) - else: - docker_container = check_docker() - subprocess.Popen(["docker", "logs", "-f", docker_container]) - smoketests.HAVE_DOCKER = True - with tempfile.NamedTemporaryFile(mode="w+b", suffix=".toml", buffering=0, delete_on_close=False) as config_file: with BASE_STDB_CONFIG_PATH.open("rb") as src, config_file.file as dst: shutil.copyfileobj(src, dst) @@ -130,28 +151,6 @@ def main(): smoketests.STDB_CONFIG = Path(config_file.name).read_text() build_template_target() - - if not args.skip_dotnet: - smoketests.HAVE_DOTNET = check_dotnet() - if not smoketests.HAVE_DOTNET: - print("no suitable dotnet installation found") - exit(1) - - add_prefix = lambda testlist: [TESTPREFIX + test for test in testlist] - import fnmatch - excludelist = add_prefix(args.exclude) - testlist = add_prefix(args.test) - - loader = ExclusionaryTestLoader(excludelist) - loader.testNamePatterns = args.testNamePatterns - - tests = loader.loadTestsFromNames(testlist) - if args.list: - print("Selected tests:\n") - for test in itertools.chain(*itertools.chain(*tests)): - print(f"{test}") - exit(0) - buffer = not args.show_all_output verbosity = 2 From 8bd2b44974d9290378fce97730936039b0329da6 Mon Sep 17 00:00:00 2001 From: Zeke Foppa Date: Wed, 19 Nov 2025 13:24:58 -0800 Subject: [PATCH 02/41] [bfops/share-python-deps]: Add smoketests/requirements.txt --- .github/workflows/ci.yml | 6 ++++-- smoketests/requirements.txt | 3 +++ 2 files changed, 7 insertions(+), 2 deletions(-) create mode 100644 smoketests/requirements.txt diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4f18f1209e8..1d4409bb2c4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -109,10 +109,12 @@ jobs: with: { python-version: '3.12' } if: runner.os == 'Windows' - name: Install python deps - run: python -m pip install psycopg2-binary xmltodict + run: | + python -m venv venv + venv/pip install -r smoketests/requirements.txt - name: Run smoketests # Note: clear_database and replication only work in private - run: python -m smoketests ${{ matrix.smoketest_args }} -x clear_database replication teams + run: venv/bin/python -m smoketests ${{ matrix.smoketest_args }} -x clear_database replication teams - name: Stop containers (Linux) if: always() && runner.os == 'Linux' run: docker compose -f .github/docker-compose.yml down diff --git a/smoketests/requirements.txt b/smoketests/requirements.txt new file mode 100644 index 00000000000..697856f84f6 --- /dev/null +++ b/smoketests/requirements.txt @@ -0,0 +1,3 @@ +psycopg2-binary +toml +xmltodict From a4c3ca8fa19e5c4993094daf8f5da5787fb2aa29 Mon Sep 17 00:00:00 2001 From: Zeke Foppa Date: Wed, 19 Nov 2025 13:36:34 -0800 Subject: [PATCH 03/41] [bfops/share-python-deps]: fix --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1d4409bb2c4..b0c2497c258 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -111,7 +111,7 @@ jobs: - name: Install python deps run: | python -m venv venv - venv/pip install -r smoketests/requirements.txt + venv/bin/pip install -r smoketests/requirements.txt - name: Run smoketests # Note: clear_database and replication only work in private run: venv/bin/python -m smoketests ${{ matrix.smoketest_args }} -x clear_database replication teams From db51dc7c14de10b47c15c0465cc281e952f63e37 Mon Sep 17 00:00:00 2001 From: Zeke Foppa Date: Wed, 19 Nov 2025 13:38:45 -0800 Subject: [PATCH 04/41] [bfops/fix-list-tests]: restore --- smoketests/__main__.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/smoketests/__main__.py b/smoketests/__main__.py index 8f19238adc8..53b7ed845c5 100644 --- a/smoketests/__main__.py +++ b/smoketests/__main__.py @@ -57,6 +57,15 @@ def _convert_select_pattern(pattern): TESTPREFIX = "smoketests.tests." + +def _iter_all_tests(suite_or_case): + """Yield all individual tests from possibly nested TestSuite structures.""" + if isinstance(suite_or_case, unittest.TestSuite): + for t in suite_or_case: + yield from _iter_all_tests(t) + else: + yield suite_or_case + def main(): tests = [fname.removesuffix(".py") for fname in os.listdir(TEST_DIR / "tests") if fname.endswith(".py") and fname != "__init__.py"] @@ -108,8 +117,9 @@ def main(): tests = loader.loadTestsFromNames(testlist) if args.list: print("Selected tests:\n") - for test in itertools.chain(*itertools.chain(*tests)): - print(f"{test}") + for test in _iter_all_tests(tests): + name = test.id() + print(f"{name}") exit(0) if not args.no_build_cli: From fae85d5902e6198d0c2703d86b4ac6fa6971e313 Mon Sep 17 00:00:00 2001 From: Zeke Foppa Date: Wed, 19 Nov 2025 13:51:15 -0800 Subject: [PATCH 05/41] [bfops/fix-list-tests]: print any failed --- smoketests/__main__.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/smoketests/__main__.py b/smoketests/__main__.py index 53b7ed845c5..cc3b0d004b6 100644 --- a/smoketests/__main__.py +++ b/smoketests/__main__.py @@ -15,6 +15,7 @@ import tempfile from pathlib import Path import shutil +import traceback def check_docker(): docker_ps = smoketests.run_cmd("docker", "ps", "--format=json") @@ -116,11 +117,22 @@ def main(): tests = loader.loadTestsFromNames(testlist) if args.list: - print("Selected tests:\n") + failed_cls = getattr(unittest.loader, "_FailedTest", None) + any_failed = False for test in _iter_all_tests(tests): name = test.id() - print(f"{name}") - exit(0) + if isinstance(test, failed_cls): + any_failed = True + print('') + print("Failed to construct %s:" % test.id()) + exc = getattr(test, "_exception", None) + if exc is not None: + tb = ''.join(traceback.format_exception(exc)) + print(tb.rstrip()) + print('') + else: + print(f"{name}") + exit(1 if any_failed else 0) if not args.no_build_cli: logging.info("Compiling spacetime cli...") From e7f9f9f95897d24d8ee128b9aa333928d030fe80 Mon Sep 17 00:00:00 2001 From: Zeke Foppa Date: Wed, 19 Nov 2025 13:55:03 -0800 Subject: [PATCH 06/41] [bfops/share-python-deps]: review --- .gitignore | 3 +++ smoketests/README.md | 11 +++++++++++ 2 files changed, 14 insertions(+) create mode 100644 smoketests/README.md diff --git a/.gitignore b/.gitignore index 32bf1df4385..5189e22e2c1 100644 --- a/.gitignore +++ b/.gitignore @@ -226,3 +226,6 @@ new.json # Symlinked output from `nix build` result + +# Python venv directories +venv/ diff --git a/smoketests/README.md b/smoketests/README.md new file mode 100644 index 00000000000..ec53333efad --- /dev/null +++ b/smoketests/README.md @@ -0,0 +1,11 @@ +To use the smoketests, you first need to install the dependencies: + +``` +python -m venv smoketests/venv +smoketests/venv/bin/pip install -r smoketests/requirements.txt +``` + +Then, run the smoketests like so: +``` +smoketests/venv/bin/python -m smoketests +``` From 9aca3b7477a5fc36d7e5f1dad70ac9df2cafcaba Mon Sep 17 00:00:00 2001 From: Zeke Foppa Date: Wed, 19 Nov 2025 13:57:13 -0800 Subject: [PATCH 07/41] [bfops/share-python-deps]: review --- .github/workflows/ci.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b0c2497c258..0e01f47e6a2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -109,12 +109,10 @@ jobs: with: { python-version: '3.12' } if: runner.os == 'Windows' - name: Install python deps - run: | - python -m venv venv - venv/bin/pip install -r smoketests/requirements.txt + run: python -m pip install -r smoketests/requirements.txt - name: Run smoketests # Note: clear_database and replication only work in private - run: venv/bin/python -m smoketests ${{ matrix.smoketest_args }} -x clear_database replication teams + run: python -m smoketests ${{ matrix.smoketest_args }} -x clear_database replication teams - name: Stop containers (Linux) if: always() && runner.os == 'Linux' run: docker compose -f .github/docker-compose.yml down From 5f1c35dc6fc2181912b875faefa03c00384605ac Mon Sep 17 00:00:00 2001 From: Zeke Foppa Date: Wed, 19 Nov 2025 14:08:53 -0800 Subject: [PATCH 08/41] [bfops/parallel-smoketests]: empty From 2dc780c4f1395f18a0d0d4ef6fc6f6128dc55203 Mon Sep 17 00:00:00 2001 From: Zeke Foppa Date: Thu, 20 Nov 2025 11:11:12 -0800 Subject: [PATCH 09/41] [bfops/parallel-smoketests]: remove python parallel flags --- smoketests/__main__.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/smoketests/__main__.py b/smoketests/__main__.py index cc3b0d004b6..7d435fc9f89 100644 --- a/smoketests/__main__.py +++ b/smoketests/__main__.py @@ -77,8 +77,6 @@ def main(): parser.add_argument("--no-docker-logs", action="store_true") parser.add_argument("--skip-dotnet", action="store_true", help="ignore tests which require dotnet") parser.add_argument("--show-all-output", action="store_true", help="show all stdout/stderr from the tests as they're running") - parser.add_argument("--parallel", action="store_true", help="run test classes in parallel") - parser.add_argument("-j", dest='jobs', help="Set number of jobs for parallel test runs. Default is `nproc`", type=int, default=0) parser.add_argument('-k', dest='testNamePatterns', action='append', type=_convert_select_pattern, help='Only run tests which match the given substring') @@ -176,14 +174,9 @@ def main(): buffer = not args.show_all_output verbosity = 2 - if args.parallel: - print("parallel test running is under construction, this will probably not work correctly") - from . import unittest_parallel - unittest_parallel.main(buffer=buffer, verbose=verbosity, level="class", discovered_tests=tests, jobs=args.jobs) - else: - result = unittest.TextTestRunner(buffer=buffer, verbosity=verbosity).run(tests) - if not result.wasSuccessful(): - parser.exit(status=1) + result = unittest.TextTestRunner(buffer=buffer, verbosity=verbosity).run(tests) + if not result.wasSuccessful(): + parser.exit(status=1) if __name__ == '__main__': From 7fe19dda37eda7f871190ad193da049af055d115 Mon Sep 17 00:00:00 2001 From: Zeke Foppa Date: Thu, 20 Nov 2025 12:54:14 -0800 Subject: [PATCH 10/41] [bfops/parallel-smoketests]: wip --- tools/ci/src/main.rs | 60 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 58 insertions(+), 2 deletions(-) diff --git a/tools/ci/src/main.rs b/tools/ci/src/main.rs index 92f9212f3c0..36a2b5aeeb3 100644 --- a/tools/ci/src/main.rs +++ b/tools/ci/src/main.rs @@ -51,6 +51,12 @@ enum CiCmd { /// /// Executes the smoketests suite with some default exclusions. Smoketests { + #[arg( + long, + default_value_t = false, + long_help = "Start SpacetimeDB in docker (Linux) instead of bare. Also forwarded to the smoketests runner." + )] + docker: bool, #[arg( trailing_var_arg = true, long_help = "Additional arguments to pass to the smoketests runner. These are usually set by the CI environment, such as `-- --docker`" @@ -168,9 +174,59 @@ fn main() -> Result<()> { run!("cargo run -p spacetimedb-cli -- build --project-path modules/module-test")?; } - Some(CiCmd::Smoketests { args }) => { + Some(CiCmd::Smoketests { docker, args }) => { + // Prepare arguments for the smoketests runner + let mut st_args = args.clone(); + if docker { + st_args.push("--docker".to_string()); + } + + // Start SpacetimeDB depending on platform and docker flag + if docker { + // Linux docker compose flow + if cfg!(target_os = "linux") { + // Our .dockerignore omits `target`, which our CI Dockerfile needs. + run!("rm -f .dockerignore")?; + run!("docker compose -f .github/docker-compose.yml up -d")?; + } + } else { + // Bare mode + if cfg!(target_os = "windows") { + // Start spacetimedb-cli in the background with a fixed pg port + // Use Git Bash compatible backgrounding + run!(r#"target/debug/spacetimedb-cli.exe start --pg-port 5432 &"#)?; + // Ensure dotnet workloads are present for module builds on Windows + run!( + r#"(cd modules && dotnet workload config --update-mode manifests && dotnet workload update)"# + )?; + } else { + run!("target/debug/spacetimedb-cli start --pg-port 5432 &")?; + } + } + + // Always exclude some tests by default (mirrors CI workflow) // Note: clear_database and replication only work in private - run!(&format!("python -m smoketests {}", args.join(" ")))?; + let default_excludes = "-x clear_database replication teams"; + let joined = st_args.join(" "); + let cmdline = if joined.is_empty() { + format!("python -m smoketests {}", default_excludes) + } else { + format!("python -m smoketests {} {}", joined, default_excludes) + }; + + let mut test_err: Option = None; + if let Err(e) = run!(&cmdline) { + test_err = Some(e); + } + + // Teardown if we started docker + if docker && cfg!(target_os = "linux") { + let _ = run!("docker compose -f .github/docker-compose.yml down"); + } + + if let Some(e) = test_err { + return Err(e); + } } Some(CiCmd::UpdateFlow { From deb123a3bb0fc74f9e3df2e90ec7a8df51e67f69 Mon Sep 17 00:00:00 2001 From: Zeke Foppa Date: Thu, 20 Nov 2025 13:02:09 -0800 Subject: [PATCH 11/41] [bfops/parallel-smoketests]: review --- tools/ci/src/main.rs | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/tools/ci/src/main.rs b/tools/ci/src/main.rs index 36a2b5aeeb3..76a8d1cdd3f 100644 --- a/tools/ci/src/main.rs +++ b/tools/ci/src/main.rs @@ -54,7 +54,7 @@ enum CiCmd { #[arg( long, default_value_t = false, - long_help = "Start SpacetimeDB in docker (Linux) instead of bare. Also forwarded to the smoketests runner." + long_help = "Start SpacetimeDB in docker (Linux) instead of bare." )] docker: bool, #[arg( @@ -175,20 +175,11 @@ fn main() -> Result<()> { } Some(CiCmd::Smoketests { docker, args }) => { - // Prepare arguments for the smoketests runner - let mut st_args = args.clone(); - if docker { - st_args.push("--docker".to_string()); - } - // Start SpacetimeDB depending on platform and docker flag if docker { - // Linux docker compose flow - if cfg!(target_os = "linux") { - // Our .dockerignore omits `target`, which our CI Dockerfile needs. - run!("rm -f .dockerignore")?; - run!("docker compose -f .github/docker-compose.yml up -d")?; - } + // Our .dockerignore omits `target`, which our CI Dockerfile needs. + run!("rm -f .dockerignore")?; + run!("docker compose -f .github/docker-compose.yml up -d")?; } else { // Bare mode if cfg!(target_os = "windows") { From 282ece6ff597ae4a3ecc5c75de759a80dcccb48f Mon Sep 17 00:00:00 2001 From: Zeke Foppa Date: Thu, 20 Nov 2025 13:15:46 -0800 Subject: [PATCH 12/41] [bfops/parallel-smoketests]: Start server as part of running smoketests --- .github/workflows/ci.yml | 30 +++++++++++++----------------- tools/ci/src/main.rs | 13 +------------ 2 files changed, 14 insertions(+), 29 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 870d61b9613..d15587c37ce 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,6 +36,8 @@ jobs: container: ${{ matrix.container }} env: CARGO_TARGET_DIR: ${{ github.workspace }}/target + # Note: clear_database and replication only work in private + SMOKETEST_ARGS: ${{ matrix.smoketest_args }} -x clear_database replication teams steps: - name: Find Git ref env: @@ -87,22 +89,13 @@ jobs: if: runner.os == 'Linux' run: /usr/local/bin/start-docker.sh - - name: Build and start database (Linux) - if: runner.os == 'Linux' - run: | - # Our .dockerignore omits `target`, which our CI Dockerfile needs. - rm .dockerignore - docker compose -f .github/docker-compose.yml up -d - - name: Build and start database (Windows) + # the sdk-manifests on windows-latest are messed up, so we need to update them + - name: Fix sdk-manifests if: runner.os == 'Windows' + working-directory: modules + # Powershell doesn't early-exit properly from a multi-line command if one fails + shell: bash run: | - # Fail properly if any individual command fails - $ErrorActionPreference = 'Stop' - $PSNativeCommandUseErrorActionPreference = $true - - Start-Process target/debug/spacetimedb-cli.exe -ArgumentList 'start --pg-port 5432' - cd modules - # the sdk-manifests on windows-latest are messed up, so we need to update them dotnet workload config --update-mode manifests dotnet workload update - uses: actions/setup-python@v5 @@ -110,9 +103,12 @@ jobs: if: runner.os == 'Windows' - name: Install python deps run: python -m pip install -r smoketests/requirements.txt - - name: Run smoketests - # Note: clear_database and replication only work in private - run: cargo ci smoketests -- ${{ matrix.smoketest_args }} -x clear_database replication teams + - name: Run smoketests (Linux) + if: runner.os == 'Linux' + run: cargo ci smoketests --docker -- ${SMOKETEST_ARGS} + - name: Run smoketests (Windows) + if: runner.os == 'Windows' + run: cargo ci smoketests -- ${SMOKETEST_ARGS} - name: Stop containers (Linux) if: always() && runner.os == 'Linux' run: docker compose -f .github/docker-compose.yml down diff --git a/tools/ci/src/main.rs b/tools/ci/src/main.rs index 76a8d1cdd3f..2263fdf5d38 100644 --- a/tools/ci/src/main.rs +++ b/tools/ci/src/main.rs @@ -181,18 +181,7 @@ fn main() -> Result<()> { run!("rm -f .dockerignore")?; run!("docker compose -f .github/docker-compose.yml up -d")?; } else { - // Bare mode - if cfg!(target_os = "windows") { - // Start spacetimedb-cli in the background with a fixed pg port - // Use Git Bash compatible backgrounding - run!(r#"target/debug/spacetimedb-cli.exe start --pg-port 5432 &"#)?; - // Ensure dotnet workloads are present for module builds on Windows - run!( - r#"(cd modules && dotnet workload config --update-mode manifests && dotnet workload update)"# - )?; - } else { - run!("target/debug/spacetimedb-cli start --pg-port 5432 &")?; - } + run!(r#"cargo run -p spacetimedb-cli -- start --pg-port 5432 &"#)?; } // Always exclude some tests by default (mirrors CI workflow) From f095a1dfa643bf8fad744d34ff7105ab6c5ffcff Mon Sep 17 00:00:00 2001 From: Zeke Foppa Date: Thu, 20 Nov 2025 13:50:07 -0800 Subject: [PATCH 13/41] [bfops/parallel-smoketests]: WIP --- .github/workflows/ci.yml | 3 -- tools/ci/src/main.rs | 108 +++++++++++++++++++++++++++------------ 2 files changed, 76 insertions(+), 35 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d15587c37ce..e9b7897f863 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -109,9 +109,6 @@ jobs: - name: Run smoketests (Windows) if: runner.os == 'Windows' run: cargo ci smoketests -- ${SMOKETEST_ARGS} - - name: Stop containers (Linux) - if: always() && runner.os == 'Linux' - run: docker compose -f .github/docker-compose.yml down test: name: Test Suite diff --git a/tools/ci/src/main.rs b/tools/ci/src/main.rs index 2263fdf5d38..b245c9b88fa 100644 --- a/tools/ci/src/main.rs +++ b/tools/ci/src/main.rs @@ -1,5 +1,5 @@ use anyhow::{bail, Result}; -use clap::{CommandFactory, Parser, Subcommand}; +use clap::{CommandFactory, Parser, Subcommand, ValueEnum}; use duct::cmd; use std::collections::HashMap; use std::path::Path; @@ -31,6 +31,16 @@ struct Cli { skip: Vec, } +#[derive(Copy, Clone, Debug, Eq, PartialEq, ValueEnum)] +enum StartServer { + /// Do not start a server; assume one is already running or remote + No, + /// Start a local server using the spacetimedb CLI + Bare, + /// Start services using docker compose + Docker, +} + #[derive(Subcommand)] enum CiCmd { /// Runs tests @@ -52,11 +62,12 @@ enum CiCmd { /// Executes the smoketests suite with some default exclusions. Smoketests { #[arg( - long, - default_value_t = false, - long_help = "Start SpacetimeDB in docker (Linux) instead of bare." + long = "start-server", + value_enum, + default_value_t = StartServer::Bare, + long_help = "How to start SpacetimeDB before running smoketests: no | bare | docker" )] - docker: bool, + start_server: StartServer, #[arg( trailing_var_arg = true, long_help = "Additional arguments to pass to the smoketests runner. These are usually set by the CI environment, such as `-- --docker`" @@ -174,39 +185,72 @@ fn main() -> Result<()> { run!("cargo run -p spacetimedb-cli -- build --project-path modules/module-test")?; } - Some(CiCmd::Smoketests { docker, args }) => { - // Start SpacetimeDB depending on platform and docker flag - if docker { - // Our .dockerignore omits `target`, which our CI Dockerfile needs. - run!("rm -f .dockerignore")?; - run!("docker compose -f .github/docker-compose.yml up -d")?; - } else { - run!(r#"cargo run -p spacetimedb-cli -- start --pg-port 5432 &"#)?; + Some(CiCmd::Smoketests { start_server, args }) => { + let mut started_pid: Option = None; + match start_server { + StartServer::Docker => { + // This means we ignore `.dockerignore`, beacuse it omits `target`, which our CI Dockerfile needs. + run!("cd .github && docker compose -f docker-compose.yml up -d")?; + } + StartServer::Bare => { + let pid_str; + if cfg!(target_os = "windows") { + pid_str = cmd!( + "powershell", + "-NoProfile", + "-Command", + "$p = Start-Process cargo -ArgumentList 'run -p spacetimedb-cli -- start --pg-port 5432' -PassThru; $p.Id" + ) + .read() + .unwrap_or_default(); + } else { + pid_str = cmd!( + "bash", + "-lc", + "cargo run -p spacetimedb-cli -- start --pg-port 5432 & echo $!" + ) + .read() + .unwrap_or_default(); + } + started_pid = Some( + pid_str + .trim() + .parse::() + .expect("Failed to get PID of started process"), + ) + } + StartServer::No => {} } - // Always exclude some tests by default (mirrors CI workflow) - // Note: clear_database and replication only work in private - let default_excludes = "-x clear_database replication teams"; - let joined = st_args.join(" "); - let cmdline = if joined.is_empty() { - format!("python -m smoketests {}", default_excludes) - } else { - format!("python -m smoketests {} {}", joined, default_excludes) - }; + let py3_available = cmd!("bash", "-lc", "command -v python3 >/dev/null 2>&1") + .run() + .map(|s| s.status.success()) + .unwrap_or(false); + let python = if py3_available { "python3" } else { "python" }; - let mut test_err: Option = None; - if let Err(e) = run!(&cmdline) { - test_err = Some(e); - } + let test_result = run!(&format!("{python} -m smoketests {}", args.join(" "))); - // Teardown if we started docker - if docker && cfg!(target_os = "linux") { - let _ = run!("docker compose -f .github/docker-compose.yml down"); + match start_server { + StartServer::Docker => { + let _ = run!("docker compose -f .github/docker-compose.yml down"); + } + StartServer::Bare => { + let pid = if let Some(pid) = started_pid { + pid + } else { + // We constructed a `Some` above if `StartServer::Bare`. + unreachable!(); + }; + if cfg!(target_os = "windows") { + let _ = run!(&format!("powershell -NoProfile -Command \"Stop-Process -Id {} -Force -ErrorAction SilentlyContinue\"", pid)); + } else { + let _ = run!(&format!("bash -lc 'kill {} 2>/dev/null || true'", pid)); + } + } + StartServer::No => {} } - if let Some(e) = test_err { - return Err(e); - } + test_result?; } Some(CiCmd::UpdateFlow { From 336337acd70206e3df5ad51172b25329af887f9d Mon Sep 17 00:00:00 2001 From: Zeke Foppa Date: Thu, 20 Nov 2025 14:25:00 -0800 Subject: [PATCH 14/41] [bfops/parallel-smoketests]: review --- tools/ci/src/main.rs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/tools/ci/src/main.rs b/tools/ci/src/main.rs index b245c9b88fa..2895925e33f 100644 --- a/tools/ci/src/main.rs +++ b/tools/ci/src/main.rs @@ -187,12 +187,17 @@ fn main() -> Result<()> { Some(CiCmd::Smoketests { start_server, args }) => { let mut started_pid: Option = None; + println!("Starting server.."); match start_server { StartServer::Docker => { // This means we ignore `.dockerignore`, beacuse it omits `target`, which our CI Dockerfile needs. run!("cd .github && docker compose -f docker-compose.yml up -d")?; } StartServer::Bare => { + // Pre-build so that `cargo run -p spacetimedb-cli` will immediately start. Otherwise we risk starting the tests + // before the server is up. + run!("cargo build -p spacetimedb-cli -p spacetimedb-standalone")?; + let pid_str; if cfg!(target_os = "windows") { pid_str = cmd!( @@ -207,7 +212,7 @@ fn main() -> Result<()> { pid_str = cmd!( "bash", "-lc", - "cargo run -p spacetimedb-cli -- start --pg-port 5432 & echo $!" + "nohup cargo run -p spacetimedb-cli -- start --pg-port 5432 >/dev/null 2>&1 & echo $!" ) .read() .unwrap_or_default(); @@ -228,8 +233,11 @@ fn main() -> Result<()> { .unwrap_or(false); let python = if py3_available { "python3" } else { "python" }; + println!("Running smoketests.."); let test_result = run!(&format!("{python} -m smoketests {}", args.join(" "))); + println!("Shutting down server.."); + // TODO: Make an effort to run the wind-down behavior if we ctrl-C this process match start_server { StartServer::Docker => { let _ = run!("docker compose -f .github/docker-compose.yml down"); @@ -244,7 +252,7 @@ fn main() -> Result<()> { if cfg!(target_os = "windows") { let _ = run!(&format!("powershell -NoProfile -Command \"Stop-Process -Id {} -Force -ErrorAction SilentlyContinue\"", pid)); } else { - let _ = run!(&format!("bash -lc 'kill {} 2>/dev/null || true'", pid)); + let _ = run!(&format!("bash -lc 'kill {} 2>/dev/null'", pid)); } } StartServer::No => {} From 38859ab0feca9dfda6e60d39a83213b8f05f11f7 Mon Sep 17 00:00:00 2001 From: Zeke Foppa Date: Thu, 20 Nov 2025 14:26:46 -0800 Subject: [PATCH 15/41] [bfops/parallel-smoketests]: review --- tools/ci/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/ci/src/main.rs b/tools/ci/src/main.rs index 2895925e33f..4af9e0db4ed 100644 --- a/tools/ci/src/main.rs +++ b/tools/ci/src/main.rs @@ -252,7 +252,7 @@ fn main() -> Result<()> { if cfg!(target_os = "windows") { let _ = run!(&format!("powershell -NoProfile -Command \"Stop-Process -Id {} -Force -ErrorAction SilentlyContinue\"", pid)); } else { - let _ = run!(&format!("bash -lc 'kill {} 2>/dev/null'", pid)); + let _ = run!(&format!("kill {}", pid)); } } StartServer::No => {} From 8ae19715439dfa91b07d42c6c32d86faafa9a921 Mon Sep 17 00:00:00 2001 From: Zeke Foppa Date: Thu, 20 Nov 2025 14:41:21 -0800 Subject: [PATCH 16/41] [bfops/parallel-smoketests]: fix workflow file --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e9b7897f863..bc11d0d6f5b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -105,7 +105,7 @@ jobs: run: python -m pip install -r smoketests/requirements.txt - name: Run smoketests (Linux) if: runner.os == 'Linux' - run: cargo ci smoketests --docker -- ${SMOKETEST_ARGS} + run: cargo ci smoketests --start-server docker -- ${SMOKETEST_ARGS} - name: Run smoketests (Windows) if: runner.os == 'Windows' run: cargo ci smoketests -- ${SMOKETEST_ARGS} From 9bbe781bd7ff18f7359ac0bad36504dd0f952231 Mon Sep 17 00:00:00 2001 From: Zeke Foppa Date: Thu, 20 Nov 2025 14:48:07 -0800 Subject: [PATCH 17/41] [bfops/parallel-smoketests]: maybe fix? --- .github/Dockerfile | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/Dockerfile b/.github/Dockerfile index 00d20a86386..40ad7c86d8b 100644 --- a/.github/Dockerfile +++ b/.github/Dockerfile @@ -1,6 +1,11 @@ # Minimal Dockerfile that just wraps pre-built binaries, so we can test the server inside docker + +FROM rust:1.90.0 as not-ignored +COPY ./target/debug/spacetimedb-standalone ./target/debug/spacetimedb-cli +COPY ./crates/standalone/config.toml config.toml + FROM rust:1.90.0 RUN mkdir -p /stdb/data -COPY ./target/debug/spacetimedb-standalone ./target/debug/spacetimedb-cli /usr/local/bin/ -COPY ./crates/standalone/config.toml /stdb/data/config.toml +COPY --from=not-ignored ./spacetimedb-standalone ./spacetimedb-cli /usr/local/bin/ +COPY --from=not-ignored ./config.toml /stdb/data/config.toml RUN ln -s /usr/local/bin/spacetimedb-cli /usr/local/bin/spacetime From 393776811062cc59ae2b70868042e2fced0ff201 Mon Sep 17 00:00:00 2001 From: Zeke Foppa Date: Thu, 20 Nov 2025 15:17:04 -0800 Subject: [PATCH 18/41] [bfops/parallel-smoketests]: fix copy --- .github/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/Dockerfile b/.github/Dockerfile index 40ad7c86d8b..56dc4cda77e 100644 --- a/.github/Dockerfile +++ b/.github/Dockerfile @@ -1,8 +1,8 @@ # Minimal Dockerfile that just wraps pre-built binaries, so we can test the server inside docker FROM rust:1.90.0 as not-ignored -COPY ./target/debug/spacetimedb-standalone ./target/debug/spacetimedb-cli -COPY ./crates/standalone/config.toml config.toml +COPY ./target/debug/spacetimedb-standalone ./target/debug/spacetimedb-cli . +COPY ./crates/standalone/config.toml . FROM rust:1.90.0 RUN mkdir -p /stdb/data From f7e7893fa8fed87f5ec18371540a1d3fa175585c Mon Sep 17 00:00:00 2001 From: Zeke Foppa Date: Thu, 20 Nov 2025 15:18:13 -0800 Subject: [PATCH 19/41] [bfops/parallel-smoketests]: fix --- .github/workflows/ci.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bc11d0d6f5b..e5e1faf1b22 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -105,7 +105,10 @@ jobs: run: python -m pip install -r smoketests/requirements.txt - name: Run smoketests (Linux) if: runner.os == 'Linux' - run: cargo ci smoketests --start-server docker -- ${SMOKETEST_ARGS} + run: | + # .dockerignore ignores target, which our CI dockerfile uses + rm .dockerignore + cargo ci smoketests --docker .github/docker-compose.yml -- ${SMOKETEST_ARGS} - name: Run smoketests (Windows) if: runner.os == 'Windows' run: cargo ci smoketests -- ${SMOKETEST_ARGS} From 680042e3e1001366322a3560335d24ffc09fd2b3 Mon Sep 17 00:00:00 2001 From: Zeke Foppa Date: Fri, 21 Nov 2025 11:51:36 -0800 Subject: [PATCH 20/41] [bfops/parallel-smoketests]: review --- tools/ci/src/main.rs | 75 ++++++++++++++++++++++++++------------------ 1 file changed, 45 insertions(+), 30 deletions(-) diff --git a/tools/ci/src/main.rs b/tools/ci/src/main.rs index c4d5414dd22..20f9f35f45b 100644 --- a/tools/ci/src/main.rs +++ b/tools/ci/src/main.rs @@ -1,6 +1,7 @@ use anyhow::{bail, Result}; -use clap::{CommandFactory, Parser, Subcommand, ValueEnum}; +use clap::{CommandFactory, Parser, Subcommand}; use duct::cmd; +use log::warn; use std::collections::HashMap; use std::path::Path; use std::{env, fs}; @@ -31,16 +32,6 @@ struct Cli { skip: Vec, } -#[derive(Copy, Clone, Debug, Eq, PartialEq, ValueEnum)] -enum StartServer { - /// Do not start a server; assume one is already running or remote - No, - /// Start a local server using the spacetimedb CLI - Bare, - /// Start services using docker compose - Docker, -} - #[derive(Subcommand)] enum CiCmd { /// Runs tests @@ -63,11 +54,18 @@ enum CiCmd { Smoketests { #[arg( long = "start-server", - value_enum, - default_value_t = StartServer::Bare, - long_help = "How to start SpacetimeDB before running smoketests: no | bare | docker" + default_value_t = true, + long_help = "Whether to start a local SpacetimeDB server before running smoketests" + )] + start_server: bool, + #[arg( + long = "docker", + value_name = "COMPOSE_FILE", + num_args(0..=1), + default_missing_value = "docker-compose.yml", + long_help = "Use docker for smoketests, specifying a docker compose file. If no value is provided, docker-compose.yml is used by default. This cannot be combined with --start-server." )] - start_server: StartServer, + docker: Option, #[arg( trailing_var_arg = true, long_help = "Additional arguments to pass to the smoketests runner. These are usually set by the CI environment, such as `-- --docker`" @@ -185,19 +183,27 @@ fn main() -> Result<()> { bash!("cargo run -p spacetimedb-cli -- build --project-path modules/module-test")?; } - Some(CiCmd::Smoketests { start_server, args }) => { + Some(CiCmd::Smoketests { + start_server, + docker, + args, + }) => { let mut started_pid: Option = None; - println!("Starting server.."); - match start_server { - StartServer::Docker => { + match (start_server, docker.as_ref()) { + (start_server, Some(compose_file)) => { + if !start_server { + warn!("--docker implies --start-server=true"); + } + println!("Starting server.."); // This means we ignore `.dockerignore`, beacuse it omits `target`, which our CI Dockerfile needs. - bash!("cd .github && docker compose -f docker-compose.yml up -d")?; + bash!(&format!("docker compose -f {compose_file} up -d"))?; } - StartServer::Bare => { + (true, None) => { // Pre-build so that `cargo run -p spacetimedb-cli` will immediately start. Otherwise we risk starting the tests // before the server is up. bash!("cargo build -p spacetimedb-cli -p spacetimedb-standalone")?; + println!("Starting server.."); let pid_str; if cfg!(target_os = "windows") { pid_str = cmd!( @@ -224,9 +230,10 @@ fn main() -> Result<()> { .expect("Failed to get PID of started process"), ) } - StartServer::No => {} + (false, None) => {} } + // TODO: does this work on windows? let py3_available = cmd!("bash", "-lc", "command -v python3 >/dev/null 2>&1") .run() .map(|s| s.status.success()) @@ -234,19 +241,27 @@ fn main() -> Result<()> { let python = if py3_available { "python3" } else { "python" }; println!("Running smoketests.."); - let test_result = bash!(&format!("{python} -m smoketests {}", args.join(" "))); + let mut smoketests_args = args.clone(); + if let Some(compose_file) = docker.as_ref() { + // Note that we do not assume that the user wants to pass --docker to the tests. We leave them the power to + // run the server in docker while still retaining full control over what tests they want. + smoketests_args.push("--compose-file".to_string()); + smoketests_args.push(compose_file.to_string()); + } + let test_result = bash!(&format!("{python} -m smoketests {}", smoketests_args.join(" "))); - println!("Shutting down server.."); // TODO: Make an effort to run the wind-down behavior if we ctrl-C this process - match start_server { - StartServer::Docker => { - let _ = bash!("docker compose -f .github/docker-compose.yml down"); + match (start_server, docker.as_ref()) { + (_, Some(compose_file)) => { + println!("Shutting down server.."); + let _ = bash!(&format!("docker compose -f {compose_file} down")); } - StartServer::Bare => { + (true, None) => { + println!("Shutting down server.."); let pid = if let Some(pid) = started_pid { pid } else { - // We constructed a `Some` above if `StartServer::Bare`. + // We constructed a `Some` above in this case unreachable!(); }; if cfg!(target_os = "windows") { @@ -255,7 +270,7 @@ fn main() -> Result<()> { let _ = bash!(&format!("kill {}", pid)); } } - StartServer::No => {} + (false, None) => {} } test_result?; From 60de57d99c0cbd5901520a35c137b7bd08e8b76d Mon Sep 17 00:00:00 2001 From: Zeke Foppa Date: Fri, 21 Nov 2025 11:54:02 -0800 Subject: [PATCH 21/41] [bfops/ci-start-server]: revert --- smoketests/__main__.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/smoketests/__main__.py b/smoketests/__main__.py index 7d435fc9f89..cc3b0d004b6 100644 --- a/smoketests/__main__.py +++ b/smoketests/__main__.py @@ -77,6 +77,8 @@ def main(): parser.add_argument("--no-docker-logs", action="store_true") parser.add_argument("--skip-dotnet", action="store_true", help="ignore tests which require dotnet") parser.add_argument("--show-all-output", action="store_true", help="show all stdout/stderr from the tests as they're running") + parser.add_argument("--parallel", action="store_true", help="run test classes in parallel") + parser.add_argument("-j", dest='jobs', help="Set number of jobs for parallel test runs. Default is `nproc`", type=int, default=0) parser.add_argument('-k', dest='testNamePatterns', action='append', type=_convert_select_pattern, help='Only run tests which match the given substring') @@ -174,9 +176,14 @@ def main(): buffer = not args.show_all_output verbosity = 2 - result = unittest.TextTestRunner(buffer=buffer, verbosity=verbosity).run(tests) - if not result.wasSuccessful(): - parser.exit(status=1) + if args.parallel: + print("parallel test running is under construction, this will probably not work correctly") + from . import unittest_parallel + unittest_parallel.main(buffer=buffer, verbose=verbosity, level="class", discovered_tests=tests, jobs=args.jobs) + else: + result = unittest.TextTestRunner(buffer=buffer, verbosity=verbosity).run(tests) + if not result.wasSuccessful(): + parser.exit(status=1) if __name__ == '__main__': From e7443f4ad1bdb6156d52d99e997449b7e2859b0e Mon Sep 17 00:00:00 2001 From: Zeke Foppa Date: Fri, 21 Nov 2025 11:55:37 -0800 Subject: [PATCH 22/41] [bfops/ci-start-server]: review --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e5e1faf1b22..b1928b0a600 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -106,7 +106,7 @@ jobs: - name: Run smoketests (Linux) if: runner.os == 'Linux' run: | - # .dockerignore ignores target, which our CI dockerfile uses + # Our .dockerignore omits `target`, which our CI Dockerfile needs. rm .dockerignore cargo ci smoketests --docker .github/docker-compose.yml -- ${SMOKETEST_ARGS} - name: Run smoketests (Windows) From 53279560841ddded27c07d1d56afa96f5c21e74d Mon Sep 17 00:00:00 2001 From: Zeke Foppa Date: Fri, 21 Nov 2025 11:56:42 -0800 Subject: [PATCH 23/41] [bfops/ci-start-server]: revert --- .github/Dockerfile | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/.github/Dockerfile b/.github/Dockerfile index 56dc4cda77e..00d20a86386 100644 --- a/.github/Dockerfile +++ b/.github/Dockerfile @@ -1,11 +1,6 @@ # Minimal Dockerfile that just wraps pre-built binaries, so we can test the server inside docker - -FROM rust:1.90.0 as not-ignored -COPY ./target/debug/spacetimedb-standalone ./target/debug/spacetimedb-cli . -COPY ./crates/standalone/config.toml . - FROM rust:1.90.0 RUN mkdir -p /stdb/data -COPY --from=not-ignored ./spacetimedb-standalone ./spacetimedb-cli /usr/local/bin/ -COPY --from=not-ignored ./config.toml /stdb/data/config.toml +COPY ./target/debug/spacetimedb-standalone ./target/debug/spacetimedb-cli /usr/local/bin/ +COPY ./crates/standalone/config.toml /stdb/data/config.toml RUN ln -s /usr/local/bin/spacetimedb-cli /usr/local/bin/spacetime From 9afb403f3e54b9f3f6709b113a4526b6e268a17a Mon Sep 17 00:00:00 2001 From: Zeke Foppa Date: Fri, 21 Nov 2025 12:26:55 -0800 Subject: [PATCH 24/41] [bfops/parallel-smoketests]: Revert "[bfops/ci-start-server]: revert" This reverts commit 60de57d99c0cbd5901520a35c137b7bd08e8b76d. --- smoketests/__main__.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/smoketests/__main__.py b/smoketests/__main__.py index cc3b0d004b6..7d435fc9f89 100644 --- a/smoketests/__main__.py +++ b/smoketests/__main__.py @@ -77,8 +77,6 @@ def main(): parser.add_argument("--no-docker-logs", action="store_true") parser.add_argument("--skip-dotnet", action="store_true", help="ignore tests which require dotnet") parser.add_argument("--show-all-output", action="store_true", help="show all stdout/stderr from the tests as they're running") - parser.add_argument("--parallel", action="store_true", help="run test classes in parallel") - parser.add_argument("-j", dest='jobs', help="Set number of jobs for parallel test runs. Default is `nproc`", type=int, default=0) parser.add_argument('-k', dest='testNamePatterns', action='append', type=_convert_select_pattern, help='Only run tests which match the given substring') @@ -176,14 +174,9 @@ def main(): buffer = not args.show_all_output verbosity = 2 - if args.parallel: - print("parallel test running is under construction, this will probably not work correctly") - from . import unittest_parallel - unittest_parallel.main(buffer=buffer, verbose=verbosity, level="class", discovered_tests=tests, jobs=args.jobs) - else: - result = unittest.TextTestRunner(buffer=buffer, verbosity=verbosity).run(tests) - if not result.wasSuccessful(): - parser.exit(status=1) + result = unittest.TextTestRunner(buffer=buffer, verbosity=verbosity).run(tests) + if not result.wasSuccessful(): + parser.exit(status=1) if __name__ == '__main__': From 961d8ba04e1a68f9a5727ce4086b91f0f15dd909 Mon Sep 17 00:00:00 2001 From: Zeke Foppa Date: Fri, 21 Nov 2025 13:58:21 -0800 Subject: [PATCH 25/41] [bfops/parallel-smoketests]: WIP --- tools/ci/src/main.rs | 300 ++++++++++++++++++++++++++++++++----------- 1 file changed, 225 insertions(+), 75 deletions(-) diff --git a/tools/ci/src/main.rs b/tools/ci/src/main.rs index 20f9f35f45b..2480e248f0a 100644 --- a/tools/ci/src/main.rs +++ b/tools/ci/src/main.rs @@ -1,9 +1,11 @@ -use anyhow::{bail, Result}; +use anyhow::{bail, Context, Result}; use clap::{CommandFactory, Parser, Subcommand}; use duct::cmd; use log::warn; use std::collections::HashMap; +use std::net::TcpListener; use std::path::Path; +use std::thread; use std::{env, fs}; const README_PATH: &str = "tools/ci/README.md"; @@ -66,6 +68,12 @@ enum CiCmd { long_help = "Use docker for smoketests, specifying a docker compose file. If no value is provided, docker-compose.yml is used by default. This cannot be combined with --start-server." )] docker: Option, + #[arg( + long = "parallel", + default_value_t = false, + long_help = "Run smoketest suites in parallel, one process per top-level suite" + )] + parallel: bool, #[arg( trailing_var_arg = true, long_help = "Additional arguments to pass to the smoketests runner. These are usually set by the CI environment, such as `-- --docker`" @@ -135,6 +143,168 @@ fn run_all_clap_subcommands(skips: &[String]) -> Result<()> { Ok(()) } +fn prebuild_bare_server() -> Result<()> { + // Pre-build so that `cargo run -p spacetimedb-cli` will immediately start. Otherwise we risk starting the tests + // before the server is up. + bash!("cargo build -p spacetimedb-cli -p spacetimedb-standalone") +} + +fn run_smoketests_inner( + python: &str, + suite_name: Option<&str>, + start_server: bool, + docker: &Option, + per_suite: bool, + args: &[String], +) -> (String, bool) { + let mut output = String::new(); + let mut ok = true; + + // Server setup + let mut pid: Option = None; + let mut port: Option = None; + if let Some(compose_file) = docker.as_ref() { + output.push_str("Starting server..\n"); + // This means we ignore `.dockerignore`, beacuse it omits `target`, which our CI Dockerfile needs. + let cmdline = format!("docker compose -f {} up -d", compose_file); + let res = cmd!("bash", "-lc", &cmdline).stderr_to_stdout().run(); + if let Err(e) = res { + output.push_str(&format!("Failed to start docker: {}\n", e)); + return (output, false); + } + } else if start_server { + // Bare server: shared (port 5432) in serial mode, or per-suite port + --remote-server in parallel mode. + if !per_suite { + if let Err(e) = prebuild_bare_server() { + output.push_str(&format!("Failed to prebuild server: {}\n", e)); + return (output, false); + } + output.push_str("Starting server..\n"); + let cmdline = "nohup cargo run -p spacetimedb-cli -- start --pg-port 5432 >/dev/null 2>&1 & echo $!"; + let pid_str = cmd!("bash", "-lc", cmdline).read().unwrap_or_default(); + match pid_str.trim().parse::() { + Ok(p) => pid = Some(p), + Err(e) => { + output.push_str(&format!("Failed to parse server PID from '{}': {}\n", pid_str, e)); + return (output, false); + } + } + } else { + let free_port = match find_free_port() { + Ok(p) => p, + Err(e) => { + output.push_str(&format!("Failed to find free port: {}\n", e)); + return (output, false); + } + }; + port = Some(free_port); + output.push_str(&format!("Starting local server on port {}..\n", free_port)); + let cmdline = format!( + "nohup cargo run -p spacetimedb-cli -- start --pg-port {} >/dev/null 2>&1 & echo $!", + free_port + ); + let pid_str = cmd!("bash", "-lc", &cmdline).read().unwrap_or_default(); + match pid_str.trim().parse::() { + Ok(p) => pid = Some(p), + Err(e) => { + output.push_str(&format!("Failed to parse server PID from '{}': {}\n", pid_str, e)); + return (output, false); + } + } + } + } + + // Build smoketests args + let mut smoketests_args = Vec::new(); + if let Some(name) = suite_name { + smoketests_args.push(name.to_string()); + } + smoketests_args.extend(args.iter().cloned()); + + if let Some(compose_file) = docker.as_ref() { + // Note that we do not assume that the user wants to pass --docker to the tests. We leave them the power to + // run the server in docker while still retaining full control over what tests they want. + smoketests_args.push("--compose-file".to_string()); + smoketests_args.push(compose_file.to_string()); + } + + if start_server && docker.is_none() && per_suite { + if let Some(p) = port { + smoketests_args.push("--remote-server".to_string()); + smoketests_args.push(format!("http://127.0.0.1:{}", p)); + } + } + + output.push_str("Running smoketests..\n"); + let cmdline = format!("{} -m smoketests {}", python, smoketests_args.join(" ")); + let res = cmd!("bash", "-lc", &cmdline).stderr_to_stdout().read(); + match res { + Ok(out) => { + output.push_str(&out); + } + Err(e) => { + output.push_str(&format!("smoketests failed: {}\n", e)); + ok = false; + } + } + + // Shutdown + if let Some(compose_file) = docker.as_ref() { + output.push_str("Shutting down server..\n"); + let down_cmd = format!("docker compose -f {} down", compose_file); + let _ = cmd!("bash", "-lc", &down_cmd).run(); + } + + if let Some(p) = pid { + output.push_str("Shutting down server..\n"); + let kill_cmd = format!("kill {}", p); + let _ = cmd!("bash", "-lc", &kill_cmd).run(); + } + + (output, ok) +} + +fn run_smoketests_serial(python: &str, start_server: bool, docker: &Option, args: &[String]) -> Result<()> { + let (output, ok) = run_smoketests_inner(python, None, start_server, docker, false, args); + print!("{}", output); + if !ok { + bail!("smoketests failed"); + } + Ok(()) +} + +fn find_free_port() -> Result { + let listener = TcpListener::bind("127.0.0.1:0").context("failed to bind to an ephemeral port")?; + let port = listener + .local_addr() + .context("failed to read local address for ephemeral port")? + .port(); + drop(listener); + Ok(port) +} + +fn list_smoketest_suites() -> Result> { + let mut suites = Vec::new(); + for entry in fs::read_dir("smoketests/tests").context("failed to read smoketests/tests directory")? { + let entry = entry?; + let path = entry.path(); + if let Some(name) = path.file_name().and_then(|n| n.to_str()) { + if name == "__init__.py" { + continue; + } + if let Some(ext) = path.extension().and_then(|e| e.to_str()) { + if ext == "py" { + if let Some(stem) = path.file_stem().and_then(|s| s.to_str()) { + suites.push(stem.to_string()); + } + } + } + } + } + suites.sort(); + Ok(suites) +} + fn run_bash(cmdline: &str, additional_env: &[(&str, &str)]) -> Result<()> { let mut env = env::vars().collect::>(); env.extend(additional_env.iter().map(|(k, v)| (k.to_string(), v.to_string()))); @@ -186,53 +356,9 @@ fn main() -> Result<()> { Some(CiCmd::Smoketests { start_server, docker, + parallel, args, }) => { - let mut started_pid: Option = None; - match (start_server, docker.as_ref()) { - (start_server, Some(compose_file)) => { - if !start_server { - warn!("--docker implies --start-server=true"); - } - println!("Starting server.."); - // This means we ignore `.dockerignore`, beacuse it omits `target`, which our CI Dockerfile needs. - bash!(&format!("docker compose -f {compose_file} up -d"))?; - } - (true, None) => { - // Pre-build so that `cargo run -p spacetimedb-cli` will immediately start. Otherwise we risk starting the tests - // before the server is up. - bash!("cargo build -p spacetimedb-cli -p spacetimedb-standalone")?; - - println!("Starting server.."); - let pid_str; - if cfg!(target_os = "windows") { - pid_str = cmd!( - "powershell", - "-NoProfile", - "-Command", - "$p = Start-Process cargo -ArgumentList 'run -p spacetimedb-cli -- start --pg-port 5432' -PassThru; $p.Id" - ) - .read() - .unwrap_or_default(); - } else { - pid_str = cmd!( - "bash", - "-lc", - "nohup cargo run -p spacetimedb-cli -- start --pg-port 5432 >/dev/null 2>&1 & echo $!" - ) - .read() - .unwrap_or_default(); - } - started_pid = Some( - pid_str - .trim() - .parse::() - .expect("Failed to get PID of started process"), - ) - } - (false, None) => {} - } - // TODO: does this work on windows? let py3_available = cmd!("bash", "-lc", "command -v python3 >/dev/null 2>&1") .run() @@ -240,40 +366,64 @@ fn main() -> Result<()> { .unwrap_or(false); let python = if py3_available { "python3" } else { "python" }; - println!("Running smoketests.."); - let mut smoketests_args = args.clone(); - if let Some(compose_file) = docker.as_ref() { - // Note that we do not assume that the user wants to pass --docker to the tests. We leave them the power to - // run the server in docker while still retaining full control over what tests they want. - smoketests_args.push("--compose-file".to_string()); - smoketests_args.push(compose_file.to_string()); - } - let test_result = bash!(&format!("{python} -m smoketests {}", smoketests_args.join(" "))); + if !parallel { + run_smoketests_serial(python, start_server, &docker, &args)?; + } else { + // Parallel mode: run each top-level smoketest suite in its own process. + let suites = list_smoketest_suites()?; + if suites.is_empty() { + bail!("No smoketest suites found in smoketests/tests"); + } + + let mut handles = Vec::new(); + for suite in suites { + let suite_name = suite.clone(); + let docker = docker.clone(); + let args = args.clone(); + let python = python.to_string(); + let start_server = start_server; - // TODO: Make an effort to run the wind-down behavior if we ctrl-C this process - match (start_server, docker.as_ref()) { - (_, Some(compose_file)) => { - println!("Shutting down server.."); - let _ = bash!(&format!("docker compose -f {compose_file} down")); + let handle = thread::spawn(move || { + let (output, ok) = + run_smoketests_inner(&python, Some(&suite_name), start_server, &docker, true, &args); + (suite_name, output, ok) + }); + + handles.push(handle); } - (true, None) => { - println!("Shutting down server.."); - let pid = if let Some(pid) = started_pid { - pid - } else { - // We constructed a `Some` above in this case - unreachable!(); - }; - if cfg!(target_os = "windows") { - let _ = bash!(&format!("powershell -NoProfile -Command \"Stop-Process -Id {} -Force -ErrorAction SilentlyContinue\"", pid)); - } else { - let _ = bash!(&format!("kill {}", pid)); + + let mut all_ok = true; + let mut results = Vec::new(); + for handle in handles { + match handle.join() { + Ok((suite, output, ok)) => { + results.push((suite, output, ok)); + } + Err(_) => { + results.push(("".to_string(), "thread panicked".to_string(), false)); + } + } + } + + // Print outputs in a stable order. + results.sort_by(|a, b| a.0.cmp(&b.0)); + for (suite, output, ok) in &results { + println!("===== smoketests suite: {} =====", suite); + print!("{}", output); + println!( + "===== end suite: {} (status: {}) =====", + suite, + if *ok { "ok" } else { "FAILED" } + ); + if !ok { + all_ok = false; } } - (false, None) => {} - } - test_result?; + if !all_ok { + bail!("One or more smoketest suites failed"); + } + } } Some(CiCmd::UpdateFlow { From 3516f6908f9804d27a5d8a5ace4eec1af68d73a4 Mon Sep 17 00:00:00 2001 From: Zeke Foppa Date: Fri, 21 Nov 2025 14:02:00 -0800 Subject: [PATCH 26/41] [bfops/parallel-smoketests]: WIP --- tools/ci/src/main.rs | 313 ++++++++++++------------------------------- 1 file changed, 85 insertions(+), 228 deletions(-) diff --git a/tools/ci/src/main.rs b/tools/ci/src/main.rs index 2480e248f0a..a9d2c171c1c 100644 --- a/tools/ci/src/main.rs +++ b/tools/ci/src/main.rs @@ -1,11 +1,9 @@ -use anyhow::{bail, Context, Result}; +use anyhow::{bail, Result}; use clap::{CommandFactory, Parser, Subcommand}; use duct::cmd; use log::warn; use std::collections::HashMap; -use std::net::TcpListener; use std::path::Path; -use std::thread; use std::{env, fs}; const README_PATH: &str = "tools/ci/README.md"; @@ -68,12 +66,6 @@ enum CiCmd { long_help = "Use docker for smoketests, specifying a docker compose file. If no value is provided, docker-compose.yml is used by default. This cannot be combined with --start-server." )] docker: Option, - #[arg( - long = "parallel", - default_value_t = false, - long_help = "Run smoketest suites in parallel, one process per top-level suite" - )] - parallel: bool, #[arg( trailing_var_arg = true, long_help = "Additional arguments to pass to the smoketests runner. These are usually set by the CI environment, such as `-- --docker`" @@ -143,179 +135,109 @@ fn run_all_clap_subcommands(skips: &[String]) -> Result<()> { Ok(()) } -fn prebuild_bare_server() -> Result<()> { - // Pre-build so that `cargo run -p spacetimedb-cli` will immediately start. Otherwise we risk starting the tests - // before the server is up. - bash!("cargo build -p spacetimedb-cli -p spacetimedb-standalone") +fn run_bash(cmdline: &str, additional_env: &[(&str, &str)]) -> Result<()> { + let mut env = env::vars().collect::>(); + env.extend(additional_env.iter().map(|(k, v)| (k.to_string(), v.to_string()))); + log::debug!("$ {cmdline}"); + let status = cmd!("bash", "-lc", cmdline).full_env(env).run()?; + if !status.status.success() { + let e = anyhow::anyhow!("command failed: {cmdline}"); + log::error!("{e}"); + return Err(e); + } + Ok(()) } -fn run_smoketests_inner( - python: &str, - suite_name: Option<&str>, - start_server: bool, - docker: &Option, - per_suite: bool, - args: &[String], -) -> (String, bool) { - let mut output = String::new(); - let mut ok = true; - - // Server setup - let mut pid: Option = None; - let mut port: Option = None; - if let Some(compose_file) = docker.as_ref() { - output.push_str("Starting server..\n"); - // This means we ignore `.dockerignore`, beacuse it omits `target`, which our CI Dockerfile needs. - let cmdline = format!("docker compose -f {} up -d", compose_file); - let res = cmd!("bash", "-lc", &cmdline).stderr_to_stdout().run(); - if let Err(e) = res { - output.push_str(&format!("Failed to start docker: {}\n", e)); - return (output, false); - } - } else if start_server { - // Bare server: shared (port 5432) in serial mode, or per-suite port + --remote-server in parallel mode. - if !per_suite { - if let Err(e) = prebuild_bare_server() { - output.push_str(&format!("Failed to prebuild server: {}\n", e)); - return (output, false); - } - output.push_str("Starting server..\n"); - let cmdline = "nohup cargo run -p spacetimedb-cli -- start --pg-port 5432 >/dev/null 2>&1 & echo $!"; - let pid_str = cmd!("bash", "-lc", cmdline).read().unwrap_or_default(); - match pid_str.trim().parse::() { - Ok(p) => pid = Some(p), - Err(e) => { - output.push_str(&format!("Failed to parse server PID from '{}': {}\n", pid_str, e)); - return (output, false); - } +fn run_smoketests_batch(start_server: bool, docker: &Option, args: &[String]) -> Result<()> { + let mut started_pid: Option = None; + match (start_server, docker.as_ref()) { + (start_server, Some(compose_file)) => { + if !start_server { + warn!("--docker implies --start-server=true"); } - } else { - let free_port = match find_free_port() { - Ok(p) => p, - Err(e) => { - output.push_str(&format!("Failed to find free port: {}\n", e)); - return (output, false); - } - }; - port = Some(free_port); - output.push_str(&format!("Starting local server on port {}..\n", free_port)); - let cmdline = format!( - "nohup cargo run -p spacetimedb-cli -- start --pg-port {} >/dev/null 2>&1 & echo $!", - free_port - ); - let pid_str = cmd!("bash", "-lc", &cmdline).read().unwrap_or_default(); - match pid_str.trim().parse::() { - Ok(p) => pid = Some(p), - Err(e) => { - output.push_str(&format!("Failed to parse server PID from '{}': {}\n", pid_str, e)); - return (output, false); - } + println!("Starting server.."); + // This means we ignore `.dockerignore`, beacuse it omits `target`, which our CI Dockerfile needs. + bash!(&format!("docker compose -f {compose_file} up -d"))?; + } + (true, None) => { + // Pre-build so that `cargo run -p spacetimedb-cli` will immediately start. Otherwise we risk starting the tests + // before the server is up. + bash!("cargo build -p spacetimedb-cli -p spacetimedb-standalone")?; + + println!("Starting server.."); + let pid_str; + if cfg!(target_os = "windows") { + pid_str = cmd!( + "powershell", + "-NoProfile", + "-Command", + "$p = Start-Process cargo -ArgumentList 'run -p spacetimedb-cli -- start --pg-port 5432' -PassThru; $p.Id" + ) + .read() + .unwrap_or_default(); + } else { + pid_str = cmd!( + "bash", + "-lc", + "nohup cargo run -p spacetimedb-cli -- start --pg-port 5432 >/dev/null 2>&1 & echo $!" + ) + .read() + .unwrap_or_default(); } + started_pid = Some( + pid_str + .trim() + .parse::() + .expect("Failed to get PID of started process"), + ) } + (false, None) => {} } - // Build smoketests args - let mut smoketests_args = Vec::new(); - if let Some(name) = suite_name { - smoketests_args.push(name.to_string()); - } - smoketests_args.extend(args.iter().cloned()); + // TODO: does this work on windows? + let py3_available = cmd!("bash", "-lc", "command -v python3 >/dev/null 2>&1") + .run() + .map(|s| s.status.success()) + .unwrap_or(false); + let python = if py3_available { "python3" } else { "python" }; + println!("Running smoketests.."); + let mut smoketests_args = args.to_vec(); if let Some(compose_file) = docker.as_ref() { // Note that we do not assume that the user wants to pass --docker to the tests. We leave them the power to // run the server in docker while still retaining full control over what tests they want. smoketests_args.push("--compose-file".to_string()); smoketests_args.push(compose_file.to_string()); } + let test_result = bash!(&format!("{python} -m smoketests {}", smoketests_args.join(" "))); - if start_server && docker.is_none() && per_suite { - if let Some(p) = port { - smoketests_args.push("--remote-server".to_string()); - smoketests_args.push(format!("http://127.0.0.1:{}", p)); - } - } - - output.push_str("Running smoketests..\n"); - let cmdline = format!("{} -m smoketests {}", python, smoketests_args.join(" ")); - let res = cmd!("bash", "-lc", &cmdline).stderr_to_stdout().read(); - match res { - Ok(out) => { - output.push_str(&out); - } - Err(e) => { - output.push_str(&format!("smoketests failed: {}\n", e)); - ok = false; + // TODO: Make an effort to run the wind-down behavior if we ctrl-C this process + match (start_server, docker.as_ref()) { + (_, Some(compose_file)) => { + println!("Shutting down server.."); + let _ = bash!(&format!("docker compose -f {compose_file} down")); } - } - - // Shutdown - if let Some(compose_file) = docker.as_ref() { - output.push_str("Shutting down server..\n"); - let down_cmd = format!("docker compose -f {} down", compose_file); - let _ = cmd!("bash", "-lc", &down_cmd).run(); - } - - if let Some(p) = pid { - output.push_str("Shutting down server..\n"); - let kill_cmd = format!("kill {}", p); - let _ = cmd!("bash", "-lc", &kill_cmd).run(); - } - - (output, ok) -} - -fn run_smoketests_serial(python: &str, start_server: bool, docker: &Option, args: &[String]) -> Result<()> { - let (output, ok) = run_smoketests_inner(python, None, start_server, docker, false, args); - print!("{}", output); - if !ok { - bail!("smoketests failed"); - } - Ok(()) -} - -fn find_free_port() -> Result { - let listener = TcpListener::bind("127.0.0.1:0").context("failed to bind to an ephemeral port")?; - let port = listener - .local_addr() - .context("failed to read local address for ephemeral port")? - .port(); - drop(listener); - Ok(port) -} - -fn list_smoketest_suites() -> Result> { - let mut suites = Vec::new(); - for entry in fs::read_dir("smoketests/tests").context("failed to read smoketests/tests directory")? { - let entry = entry?; - let path = entry.path(); - if let Some(name) = path.file_name().and_then(|n| n.to_str()) { - if name == "__init__.py" { - continue; - } - if let Some(ext) = path.extension().and_then(|e| e.to_str()) { - if ext == "py" { - if let Some(stem) = path.file_stem().and_then(|s| s.to_str()) { - suites.push(stem.to_string()); - } - } + (true, None) => { + println!("Shutting down server.."); + let pid = if let Some(pid) = started_pid { + pid + } else { + // We constructed a `Some` above in this case + unreachable!(); + }; + if cfg!(target_os = "windows") { + let _ = bash!(&format!( + "powershell -NoProfile -Command \"Stop-Process -Id {} -Force -ErrorAction SilentlyContinue\"", + pid + )); + } else { + let _ = bash!(&format!("kill {}", pid)); } } + (false, None) => {} } - suites.sort(); - Ok(suites) -} -fn run_bash(cmdline: &str, additional_env: &[(&str, &str)]) -> Result<()> { - let mut env = env::vars().collect::>(); - env.extend(additional_env.iter().map(|(k, v)| (k.to_string(), v.to_string()))); - log::debug!("$ {cmdline}"); - let status = cmd!("bash", "-lc", cmdline).full_env(env).run()?; - if !status.status.success() { - let e = anyhow::anyhow!("command failed: {cmdline}"); - log::error!("{e}"); - return Err(e); - } - Ok(()) + test_result } fn main() -> Result<()> { @@ -356,74 +278,9 @@ fn main() -> Result<()> { Some(CiCmd::Smoketests { start_server, docker, - parallel, args, }) => { - // TODO: does this work on windows? - let py3_available = cmd!("bash", "-lc", "command -v python3 >/dev/null 2>&1") - .run() - .map(|s| s.status.success()) - .unwrap_or(false); - let python = if py3_available { "python3" } else { "python" }; - - if !parallel { - run_smoketests_serial(python, start_server, &docker, &args)?; - } else { - // Parallel mode: run each top-level smoketest suite in its own process. - let suites = list_smoketest_suites()?; - if suites.is_empty() { - bail!("No smoketest suites found in smoketests/tests"); - } - - let mut handles = Vec::new(); - for suite in suites { - let suite_name = suite.clone(); - let docker = docker.clone(); - let args = args.clone(); - let python = python.to_string(); - let start_server = start_server; - - let handle = thread::spawn(move || { - let (output, ok) = - run_smoketests_inner(&python, Some(&suite_name), start_server, &docker, true, &args); - (suite_name, output, ok) - }); - - handles.push(handle); - } - - let mut all_ok = true; - let mut results = Vec::new(); - for handle in handles { - match handle.join() { - Ok((suite, output, ok)) => { - results.push((suite, output, ok)); - } - Err(_) => { - results.push(("".to_string(), "thread panicked".to_string(), false)); - } - } - } - - // Print outputs in a stable order. - results.sort_by(|a, b| a.0.cmp(&b.0)); - for (suite, output, ok) in &results { - println!("===== smoketests suite: {} =====", suite); - print!("{}", output); - println!( - "===== end suite: {} (status: {}) =====", - suite, - if *ok { "ok" } else { "FAILED" } - ); - if !ok { - all_ok = false; - } - } - - if !all_ok { - bail!("One or more smoketest suites failed"); - } - } + run_smoketests_batch(start_server, &docker, &args)?; } Some(CiCmd::UpdateFlow { From a0ccb9ebf41441ed9d494b677a5906be6e9784ee Mon Sep 17 00:00:00 2001 From: Zeke Foppa Date: Fri, 21 Nov 2025 14:39:09 -0800 Subject: [PATCH 27/41] [bfops/parallel-smoketests]: WIP --- .github/docker-compose.yml | 4 +-- docker-compose.yml | 6 ++-- tools/ci/src/main.rs | 65 +++++++++++++++++++++++++++++++------- 3 files changed, 59 insertions(+), 16 deletions(-) diff --git a/.github/docker-compose.yml b/.github/docker-compose.yml index 400c044c4ed..76e3eb251d2 100644 --- a/.github/docker-compose.yml +++ b/.github/docker-compose.yml @@ -6,9 +6,9 @@ services: context: ../ dockerfile: .github/Dockerfile ports: - - "3000:3000" + - "${STDB_PORT:-3000}:3000" # Postgres - - "5432:5432" + - "${STDB_PG_PORT:-5432}:5432" entrypoint: spacetime start --pg-port 5432 privileged: true environment: diff --git a/docker-compose.yml b/docker-compose.yml index 718281892dd..0b00256a808 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -24,11 +24,11 @@ services: - key_files:/etc/spacetimedb - /stdb ports: - - "3000:3000" + - "${STDB_PORT:-3000}:3000" # Postgres - - "5432:5432" + - "${STDB_PG_PORT:-5432}:5432" # Tracy - - "8086:8086" + - "${STDB_TRACY_PORT:-8086}:8086" entrypoint: cargo watch -i flamegraphs -i log.conf --why -C crates/standalone -x 'run start --data-dir=/stdb/data --jwt-pub-key-path=/etc/spacetimedb/id_ecdsa.pub --jwt-priv-key-path=/etc/spacetimedb/id_ecdsa --pg-port 5432' privileged: true environment: diff --git a/tools/ci/src/main.rs b/tools/ci/src/main.rs index a9d2c171c1c..9e93d996902 100644 --- a/tools/ci/src/main.rs +++ b/tools/ci/src/main.rs @@ -1,8 +1,9 @@ -use anyhow::{bail, Result}; +use anyhow::{bail, Context, Result}; use clap::{CommandFactory, Parser, Subcommand}; use duct::cmd; use log::warn; use std::collections::HashMap; +use std::net::TcpListener; use std::path::Path; use std::{env, fs}; @@ -148,30 +149,57 @@ fn run_bash(cmdline: &str, additional_env: &[(&str, &str)]) -> Result<()> { Ok(()) } -fn run_smoketests_batch(start_server: bool, docker: &Option, args: &[String]) -> Result<()> { +#[derive(Debug, Clone)] +pub enum StartServer { + /// Do not start a server. + No, + + /// Start a server normally. + Yes { random_port: bool }, + + /// Start a server using Docker Compose. + Docker { random_port: bool, compose_file: PathBuf }, +} + +fn find_free_port() -> Result { + let listener = TcpListener::bind("127.0.0.1:0").context("failed to bind to an ephemeral port")?; + let port = listener + .local_addr() + .context("failed to read local address for ephemeral port")? + .port(); + drop(listener); + Ok(port) +} + +fn run_smoketests_batch(start_server: StartServer, args: &[String]) -> Result<()> { let mut started_pid: Option = None; match (start_server, docker.as_ref()) { (start_server, Some(compose_file)) => { - if !start_server { - warn!("--docker implies --start-server=true"); - } println!("Starting server.."); - // This means we ignore `.dockerignore`, beacuse it omits `target`, which our CI Dockerfile needs. - bash!(&format!("docker compose -f {compose_file} up -d"))?; + let server_port = find_free_port()?; + let pg_port = find_free_port()?; + let tracy_port = find_free_port()?; + bash!(&format!( + "STDB_PORT={server_port} STDB_PG_PORT={pg_port} STDB_TRACY_PORT={tracy_port} docker compose -f {compose_file} up -d" + ))?; } (true, None) => { // Pre-build so that `cargo run -p spacetimedb-cli` will immediately start. Otherwise we risk starting the tests // before the server is up. bash!("cargo build -p spacetimedb-cli -p spacetimedb-standalone")?; - println!("Starting server.."); + // TODO: Make sure that this isn't brittle / multiple parallel batches don't grab the same port + let server_port = find_free_port()?; + println!("Starting server on port {server_port}.."); let pid_str; if cfg!(target_os = "windows") { pid_str = cmd!( "powershell", "-NoProfile", "-Command", - "$p = Start-Process cargo -ArgumentList 'run -p spacetimedb-cli -- start --pg-port 5432' -PassThru; $p.Id" + &format!( + "$p = Start-Process cargo -ArgumentList 'run -p spacetimedb-cli -- start --listen-addr 0.0.0.0:{server_port} --pg-port 5432' -PassThru; $p.Id" + ) ) .read() .unwrap_or_default(); @@ -179,7 +207,9 @@ fn run_smoketests_batch(start_server: bool, docker: &Option, args: &[Str pid_str = cmd!( "bash", "-lc", - "nohup cargo run -p spacetimedb-cli -- start --pg-port 5432 >/dev/null 2>&1 & echo $!" + &format!( + "nohup cargo run -p spacetimedb-cli -- start --listen-addr 0.0.0.0:{server_port} --pg-port 5432 >/dev/null 2>&1 & echo $!" + ) ) .read() .unwrap_or_default(); @@ -280,7 +310,20 @@ fn main() -> Result<()> { docker, args, }) => { - run_smoketests_batch(start_server, &docker, &args)?; + let start_server = match (start_server, docker.as_ref()) { + (start_server, Some(compose_file)) => { + if !start_server { + warn!("--docker implies --start-server=true"); + } + StartServer::Docker { + random_port: false, + compose_file: compose_file.into(), + } + } + (true, None) => StartServer::Yes { random_port: false }, + (false, None) => StartServer::No, + }; + run_smoketests_batch(start_server, &args)?; } Some(CiCmd::UpdateFlow { From cd2db36ce568f0c8f62c4ac248b6a67be1696118 Mon Sep 17 00:00:00 2001 From: Zeke Foppa Date: Fri, 21 Nov 2025 15:02:26 -0800 Subject: [PATCH 28/41] [bfops/parallel-smoketests]: builds with random port selection / project names --- tools/ci/src/main.rs | 118 ++++++++++++++++++++++++------------------- 1 file changed, 66 insertions(+), 52 deletions(-) diff --git a/tools/ci/src/main.rs b/tools/ci/src/main.rs index 9e93d996902..b2e29377c4b 100644 --- a/tools/ci/src/main.rs +++ b/tools/ci/src/main.rs @@ -4,7 +4,7 @@ use duct::cmd; use log::warn; use std::collections::HashMap; use std::net::TcpListener; -use std::path::Path; +use std::path::{Path, PathBuf}; use std::{env, fs}; const README_PATH: &str = "tools/ci/README.md"; @@ -151,14 +151,16 @@ fn run_bash(cmdline: &str, additional_env: &[(&str, &str)]) -> Result<()> { #[derive(Debug, Clone)] pub enum StartServer { - /// Do not start a server. No, - - /// Start a server normally. Yes { random_port: bool }, + Docker { compose_file: PathBuf, random_port: bool }, +} - /// Start a server using Docker Compose. - Docker { random_port: bool, compose_file: PathBuf }, +#[derive(Debug, Clone)] +pub enum ServerState { + None, + Yes { pid: i32 }, + Docker { compose_file: PathBuf, project: String }, } fn find_free_port() -> Result { @@ -171,58 +173,75 @@ fn find_free_port() -> Result { Ok(port) } -fn run_smoketests_batch(start_server: StartServer, args: &[String]) -> Result<()> { - let mut started_pid: Option = None; - match (start_server, docker.as_ref()) { - (start_server, Some(compose_file)) => { +fn run_smoketests_batch(server_mode: StartServer, args: &[String]) -> Result<()> { + let server_state = match server_mode { + StartServer::No => ServerState::None, + StartServer::Docker { + compose_file, + random_port, + } => { println!("Starting server.."); - let server_port = find_free_port()?; - let pg_port = find_free_port()?; - let tracy_port = find_free_port()?; + let env_string; + let project; + if random_port { + let server_port = find_free_port()?; + let pg_port = find_free_port()?; + let tracy_port = find_free_port()?; + env_string = format!("STDB_PORT={server_port} STDB_PG_PORT={pg_port} STDB_TRACY_PORT={tracy_port}"); + project = format!("spacetimedb-smoketests-{server_port}"); + } else { + env_string = String::new(); + project = "spacetimedb-smoketests".to_string(); + }; + let compose_str = compose_file.to_string_lossy(); bash!(&format!( - "STDB_PORT={server_port} STDB_PG_PORT={pg_port} STDB_TRACY_PORT={tracy_port} docker compose -f {compose_file} up -d" + "{env_string} docker compose -f {compose_str} --project {project} up -d" ))?; + ServerState::Docker { compose_file, project } } - (true, None) => { + StartServer::Yes { random_port } => { // Pre-build so that `cargo run -p spacetimedb-cli` will immediately start. Otherwise we risk starting the tests // before the server is up. bash!("cargo build -p spacetimedb-cli -p spacetimedb-standalone")?; // TODO: Make sure that this isn't brittle / multiple parallel batches don't grab the same port - let server_port = find_free_port()?; - println!("Starting server on port {server_port}.."); + let arg_string = if random_port { + let server_port = find_free_port()?; + let pg_port = find_free_port()?; + &format!("--listen-addr 0.0.0.0:{server_port} --pg-port {pg_port}") + } else { + "--pg-port 5432" + }; + println!("Starting server.."); let pid_str; if cfg!(target_os = "windows") { pid_str = cmd!( - "powershell", - "-NoProfile", - "-Command", - &format!( - "$p = Start-Process cargo -ArgumentList 'run -p spacetimedb-cli -- start --listen-addr 0.0.0.0:{server_port} --pg-port 5432' -PassThru; $p.Id" + "powershell", + "-NoProfile", + "-Command", + &format!( + "$p = Start-Process cargo -ArgumentList 'run -p spacetimedb-cli -- start {arg_string}' -PassThru; $p.Id" + ) ) - ) - .read() - .unwrap_or_default(); + .read() + .unwrap_or_default(); } else { pid_str = cmd!( "bash", "-lc", - &format!( - "nohup cargo run -p spacetimedb-cli -- start --listen-addr 0.0.0.0:{server_port} --pg-port 5432 >/dev/null 2>&1 & echo $!" - ) + &format!("nohup cargo run -p spacetimedb-cli -- start {arg_string} >/dev/null 2>&1 & echo $!") ) .read() .unwrap_or_default(); } - started_pid = Some( - pid_str + ServerState::Yes { + pid: pid_str .trim() .parse::() .expect("Failed to get PID of started process"), - ) + } } - (false, None) => {} - } + }; // TODO: does this work on windows? let py3_available = cmd!("bash", "-lc", "command -v python3 >/dev/null 2>&1") @@ -232,29 +251,18 @@ fn run_smoketests_batch(start_server: StartServer, args: &[String]) -> Result<() let python = if py3_available { "python3" } else { "python" }; println!("Running smoketests.."); - let mut smoketests_args = args.to_vec(); - if let Some(compose_file) = docker.as_ref() { - // Note that we do not assume that the user wants to pass --docker to the tests. We leave them the power to - // run the server in docker while still retaining full control over what tests they want. - smoketests_args.push("--compose-file".to_string()); - smoketests_args.push(compose_file.to_string()); - } - let test_result = bash!(&format!("{python} -m smoketests {}", smoketests_args.join(" "))); + let test_result = bash!(&format!("{python} -m smoketests {}", args.join(" "))); // TODO: Make an effort to run the wind-down behavior if we ctrl-C this process - match (start_server, docker.as_ref()) { - (_, Some(compose_file)) => { + match server_state { + ServerState::None => {} + ServerState::Docker { compose_file, project } => { println!("Shutting down server.."); - let _ = bash!(&format!("docker compose -f {compose_file} down")); + let compose_str = compose_file.to_string_lossy(); + let _ = bash!(&format!("docker compose -f {compose_str} --project {project} down")); } - (true, None) => { + ServerState::Yes { pid } => { println!("Shutting down server.."); - let pid = if let Some(pid) = started_pid { - pid - } else { - // We constructed a `Some` above in this case - unreachable!(); - }; if cfg!(target_os = "windows") { let _ = bash!(&format!( "powershell -NoProfile -Command \"Stop-Process -Id {} -Force -ErrorAction SilentlyContinue\"", @@ -264,7 +272,6 @@ fn run_smoketests_batch(start_server: StartServer, args: &[String]) -> Result<() let _ = bash!(&format!("kill {}", pid)); } } - (false, None) => {} } test_result @@ -323,6 +330,13 @@ fn main() -> Result<()> { (true, None) => StartServer::Yes { random_port: false }, (false, None) => StartServer::No, }; + let mut args = args.to_vec(); + if let Some(compose_file) = docker.as_ref() { + // Note that we do not assume that the user wants to pass --docker to the tests. We leave them the power to + // run the server in docker while still retaining full control over what tests they want. + args.push("--compose-file".to_string()); + args.push(compose_file.to_string()); + } run_smoketests_batch(start_server, &args)?; } From 045753431992a67c7ab8503eab66a60740186189 Mon Sep 17 00:00:00 2001 From: Zeke Foppa Date: Fri, 21 Nov 2025 15:05:31 -0800 Subject: [PATCH 29/41] [bfops/parallel-smoketests]: add --local-only --- smoketests/__init__.py | 1 + smoketests/__main__.py | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/smoketests/__init__.py b/smoketests/__init__.py index 93e179e30dd..cd50ef11575 100644 --- a/smoketests/__init__.py +++ b/smoketests/__init__.py @@ -80,6 +80,7 @@ def requires_anonymous_login(item): return item def requires_local_server(item): + setattr(item, "_requires_local_server", True) if REMOTE_SERVER: return unittest.skip("running against a remote server")(item) return item diff --git a/smoketests/__main__.py b/smoketests/__main__.py index 7d435fc9f89..c963744f230 100644 --- a/smoketests/__main__.py +++ b/smoketests/__main__.py @@ -85,6 +85,7 @@ def main(): parser.add_argument("--list", action="store_true", help="list the tests that would be run, but don't run them") parser.add_argument("--remote-server", action="store", help="Run against a remote server") parser.add_argument("--spacetime-login", action="store_true", help="Use `spacetime login` for these tests (and disable tests that don't work with that)") + parser.add_argument("--local-only", action="store_true", help="Only run tests that require a local server") args = parser.parse_args() if args.docker: @@ -114,6 +115,25 @@ def main(): loader.testNamePatterns = args.testNamePatterns tests = loader.loadTestsFromNames(testlist) + + if args.local_only: + def _is_local_only(test_case): + method_name = getattr(test_case, "_testMethodName", None) + if method_name is not None and hasattr(test_case, method_name): + method = getattr(test_case, method_name) + if getattr(method, "_requires_local_server", False): + return True + # Also allow class-level decoration + if getattr(test_case.__class__, "_requires_local_server", False): + return True + return False + + filtered = unittest.TestSuite() + for t in _iter_all_tests(tests): + if _is_local_only(t): + filtered.addTest(t) + tests = filtered + if args.list: failed_cls = getattr(unittest.loader, "_FailedTest", None) any_failed = False From 9eef7cbeca40c70fbb5d1224a3cd9b7a178dde38 Mon Sep 17 00:00:00 2001 From: Zeke Foppa Date: Mon, 24 Nov 2025 12:07:00 -0800 Subject: [PATCH 30/41] [bfops/parallel-smoketests]: add smoketests --list=json --- smoketests/__main__.py | 35 ++++++++++++++++++++++++++--------- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/smoketests/__main__.py b/smoketests/__main__.py index c963744f230..b4f2160352f 100644 --- a/smoketests/__main__.py +++ b/smoketests/__main__.py @@ -82,7 +82,7 @@ def main(): help='Only run tests which match the given substring') parser.add_argument("-x", dest="exclude", nargs="*", default=[]) parser.add_argument("--no-build-cli", action="store_true", help="don't cargo build the cli") - parser.add_argument("--list", action="store_true", help="list the tests that would be run, but don't run them") + parser.add_argument("--list", nargs="?", const="text", choices=("text", "json"), default=None, help="list the tests that would be run (optionally as 'text' or 'json'), but don't run them") parser.add_argument("--remote-server", action="store", help="Run against a remote server") parser.add_argument("--spacetime-login", action="store_true", help="Use `spacetime login` for these tests (and disable tests that don't work with that)") parser.add_argument("--local-only", action="store_true", help="Only run tests that require a local server") @@ -134,22 +134,39 @@ def _is_local_only(test_case): filtered.addTest(t) tests = filtered - if args.list: + if args.list is not None: failed_cls = getattr(unittest.loader, "_FailedTest", None) any_failed = False + test_names = [] + failed_tests = [] for test in _iter_all_tests(tests): name = test.id() if isinstance(test, failed_cls): any_failed = True - print('') - print("Failed to construct %s:" % test.id()) exc = getattr(test, "_exception", None) - if exc is not None: - tb = ''.join(traceback.format_exception(exc)) - print(tb.rstrip()) - print('') + tb = ''.join(traceback.format_exception(exc)) if exc is not None else None + failed_tests.append({ + "test_id": name, + "error": tb.rstrip() if tb is not None else None, + }) + if args.list == "text": + print('') + print("Failed to construct %s:" % name) + if tb is not None: + print(tb.rstrip()) + print('') else: - print(f"{name}") + test_names.append(name) + if args.list == "text": + print(f"{name}") + + if args.list == "json": + output = { + "tests": test_names, + "errors": failed_tests, + } + print(json.dumps(output)) + exit(1 if any_failed else 0) if not args.no_build_cli: From a5149212cd578818888f1ee4693885d359ade9af Mon Sep 17 00:00:00 2001 From: Zeke Foppa Date: Mon, 24 Nov 2025 12:53:44 -0800 Subject: [PATCH 31/41] [bfops/parallel-smoketests]: WIP parallel logic --- Cargo.lock | 1 + tools/ci/Cargo.toml | 1 + tools/ci/src/main.rs | 99 ++++++++++++++++++++++++++++++++++++++------ 3 files changed, 89 insertions(+), 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 66c824d694f..09db7759ff2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -874,6 +874,7 @@ dependencies = [ "duct", "log", "regex", + "serde_json", ] [[package]] diff --git a/tools/ci/Cargo.toml b/tools/ci/Cargo.toml index e907526a42c..19a40d0afe1 100644 --- a/tools/ci/Cargo.toml +++ b/tools/ci/Cargo.toml @@ -10,3 +10,4 @@ chrono = { workspace = true, features=["clock"] } clap.workspace = true regex.workspace = true duct.workspace = true +serde_json.workspace = true diff --git a/tools/ci/src/main.rs b/tools/ci/src/main.rs index b2e29377c4b..c4016b299a5 100644 --- a/tools/ci/src/main.rs +++ b/tools/ci/src/main.rs @@ -2,7 +2,8 @@ use anyhow::{bail, Context, Result}; use clap::{CommandFactory, Parser, Subcommand}; use duct::cmd; use log::warn; -use std::collections::HashMap; +use serde_json; +use std::collections::{HashMap, HashSet}; use std::net::TcpListener; use std::path::{Path, PathBuf}; use std::{env, fs}; @@ -67,6 +68,12 @@ enum CiCmd { long_help = "Use docker for smoketests, specifying a docker compose file. If no value is provided, docker-compose.yml is used by default. This cannot be combined with --start-server." )] docker: Option, + #[arg( + long = "parallel", + default_value_t = false, + long_help = "Run smoketests in parallel batches grouped by test suite" + )] + parallel: bool, #[arg( trailing_var_arg = true, long_help = "Additional arguments to pass to the smoketests runner. These are usually set by the CI environment, such as `-- --docker`" @@ -173,7 +180,7 @@ fn find_free_port() -> Result { Ok(port) } -fn run_smoketests_batch(server_mode: StartServer, args: &[String]) -> Result<()> { +fn run_smoketests_batch(server_mode: StartServer, args: &[String], python: &str) -> Result<()> { let server_state = match server_mode { StartServer::No => ServerState::None, StartServer::Docker { @@ -243,13 +250,6 @@ fn run_smoketests_batch(server_mode: StartServer, args: &[String]) -> Result<()> } }; - // TODO: does this work on windows? - let py3_available = cmd!("bash", "-lc", "command -v python3 >/dev/null 2>&1") - .run() - .map(|s| s.status.success()) - .unwrap_or(false); - let python = if py3_available { "python3" } else { "python" }; - println!("Running smoketests.."); let test_result = bash!(&format!("{python} -m smoketests {}", args.join(" "))); @@ -315,6 +315,7 @@ fn main() -> Result<()> { Some(CiCmd::Smoketests { start_server, docker, + parallel, args, }) => { let start_server = match (start_server, docker.as_ref()) { @@ -323,11 +324,11 @@ fn main() -> Result<()> { warn!("--docker implies --start-server=true"); } StartServer::Docker { - random_port: false, + random_port: parallel, compose_file: compose_file.into(), } } - (true, None) => StartServer::Yes { random_port: false }, + (true, None) => StartServer::Yes { random_port: parallel }, (false, None) => StartServer::No, }; let mut args = args.to_vec(); @@ -337,7 +338,81 @@ fn main() -> Result<()> { args.push("--compose-file".to_string()); args.push(compose_file.to_string()); } - run_smoketests_batch(start_server, &args)?; + + // TODO: does this work on windows? + let py3_available = cmd!("bash", "-lc", "command -v python3 >/dev/null 2>&1") + .run() + .map(|s| s.status.success()) + .unwrap_or(false); + let python = if py3_available { "python3" } else { "python" }; + + if parallel { + println!("Listing smoketests for parallel execution.."); + + let mut list_args: Vec = args.to_vec(); + list_args.push("--list=json".to_string()); + let list_cmdline = format!("{python} -m smoketests {}", list_args.join(" ")); + + // TODO: do actually check the return code here. and make --list=json not return non-zero if there are errors. + let list_output = cmd!("bash", "-lc", list_cmdline) + .stderr_to_stdout() + .unchecked() + .read()?; + + let parsed: serde_json::Value = serde_json::from_str(&list_output)?; + let tests = parsed.get("tests").and_then(|v| v.as_array()).cloned().unwrap(); + let errors = parsed + .get("errors") + .and_then(|v| v.as_array()) + .cloned() + .unwrap_or_default(); + + if !errors.is_empty() { + println!("Errors while constructing smoketests:"); + for err in &errors { + let test_id = err.get("test_id").and_then(|v| v.as_str()).unwrap(); + let msg = err.get("error").and_then(|v| v.as_str()).unwrap(); + println!("{test_id}"); + println!("{msg}"); + } + // If there were errors constructing tests, treat this as a failure + // and do not run any batches. + return Err(anyhow::anyhow!( + "Errors encountered while constructing smoketests; aborting parallel run" + )); + } + + let batches: HashSet = tests + .into_iter() + .map(|t| { + let name = t.as_str().unwrap(); + let parts = name.split('.').collect::>(); + parts[2].to_string() + }) + .collect(); + + let mut any_failed_batch = false; + for batch in batches { + println!("Running smoketests batch {batch}.."); + // TODO: this doesn't work properly if the user passed multiple batches as input. + let mut batch_args: Vec = Vec::new(); + batch_args.push(batch.clone()); + batch_args.extend(args.iter().cloned()); + + // TODO: capture output and print it only in contiguous blocks + let result = run_smoketests_batch(start_server.clone(), &batch_args, python); + + if result.is_err() { + any_failed_batch = true; + } + } + + if any_failed_batch { + anyhow::bail!("One or more smoketest batches failed"); + } + } else { + run_smoketests_batch(start_server, &args, python)?; + } } Some(CiCmd::UpdateFlow { From 45ee326998bc289af1c668dd524f2d34f0a1bb5b Mon Sep 17 00:00:00 2001 From: Zeke Foppa Date: Mon, 24 Nov 2025 13:04:39 -0800 Subject: [PATCH 32/41] [bfops/parallel-smoketests]: custom python path option --- tools/ci/src/main.rs | 31 +++++++++++++++++++++++-------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/tools/ci/src/main.rs b/tools/ci/src/main.rs index c4016b299a5..b5b3eff2fa9 100644 --- a/tools/ci/src/main.rs +++ b/tools/ci/src/main.rs @@ -74,6 +74,12 @@ enum CiCmd { long_help = "Run smoketests in parallel batches grouped by test suite" )] parallel: bool, + #[arg( + long = "python", + value_name = "PYTHON_PATH", + long_help = "Python interpreter to use for smoketests" + )] + python: Option, #[arg( trailing_var_arg = true, long_help = "Additional arguments to pass to the smoketests runner. These are usually set by the CI environment, such as `-- --docker`" @@ -316,6 +322,7 @@ fn main() -> Result<()> { start_server, docker, parallel, + python, args, }) => { let start_server = match (start_server, docker.as_ref()) { @@ -339,12 +346,20 @@ fn main() -> Result<()> { args.push(compose_file.to_string()); } - // TODO: does this work on windows? - let py3_available = cmd!("bash", "-lc", "command -v python3 >/dev/null 2>&1") - .run() - .map(|s| s.status.success()) - .unwrap_or(false); - let python = if py3_available { "python3" } else { "python" }; + let python = if let Some(p) = python { + p + } else { + // TODO: does this work on windows? + let py3_available = cmd!("bash", "-lc", "command -v python3 >/dev/null 2>&1") + .run() + .map(|s| s.status.success()) + .unwrap_or(false); + if py3_available { + "python3".to_string() + } else { + "python".to_string() + } + }; if parallel { println!("Listing smoketests for parallel execution.."); @@ -400,7 +415,7 @@ fn main() -> Result<()> { batch_args.extend(args.iter().cloned()); // TODO: capture output and print it only in contiguous blocks - let result = run_smoketests_batch(start_server.clone(), &batch_args, python); + let result = run_smoketests_batch(start_server.clone(), &batch_args, &python); if result.is_err() { any_failed_batch = true; @@ -411,7 +426,7 @@ fn main() -> Result<()> { anyhow::bail!("One or more smoketest batches failed"); } } else { - run_smoketests_batch(start_server, &args, python)?; + run_smoketests_batch(start_server, &args, &python)?; } } From 46793a4077d6818086110408b80e7e53a2aa893a Mon Sep 17 00:00:00 2001 From: Zeke Foppa Date: Mon, 24 Nov 2025 13:07:03 -0800 Subject: [PATCH 33/41] [bfops/parallel-smoketests]: actual parallel --- tools/ci/src/main.rs | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/tools/ci/src/main.rs b/tools/ci/src/main.rs index b5b3eff2fa9..9acfaa0fa70 100644 --- a/tools/ci/src/main.rs +++ b/tools/ci/src/main.rs @@ -406,17 +406,29 @@ fn main() -> Result<()> { }) .collect(); - let mut any_failed_batch = false; + // Run each batch in parallel threads. + let mut handles = Vec::new(); for batch in batches { - println!("Running smoketests batch {batch}.."); - // TODO: this doesn't work properly if the user passed multiple batches as input. + let start_server_clone = start_server.clone(); + let python_clone = python.clone(); let mut batch_args: Vec = Vec::new(); + // TODO: this doesn't work properly if the user passed multiple batches as input. batch_args.push(batch.clone()); batch_args.extend(args.iter().cloned()); - // TODO: capture output and print it only in contiguous blocks - let result = run_smoketests_batch(start_server.clone(), &batch_args, &python); + handles.push(std::thread::spawn(move || { + println!("Running smoketests batch {batch}.."); + // TODO: capture output and print it only in contiguous blocks + run_smoketests_batch(start_server_clone, &batch_args, &python_clone) + })); + } + let mut any_failed_batch = false; + for handle in handles { + // If the thread panicked or the batch failed, treat it as a failure. + let result = handle + .join() + .unwrap_or_else(|_| Err(anyhow::anyhow!("smoketest batch thread panicked",))); if result.is_err() { any_failed_batch = true; } From f3944f3a8f11dda0e4492b08dbb5fde33617651d Mon Sep 17 00:00:00 2001 From: Zeke Foppa Date: Mon, 24 Nov 2025 13:09:36 -0800 Subject: [PATCH 34/41] [bfops/parallel-smoketests]: TODO --- tools/ci/src/main.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/ci/src/main.rs b/tools/ci/src/main.rs index 9acfaa0fa70..d02a780ac49 100644 --- a/tools/ci/src/main.rs +++ b/tools/ci/src/main.rs @@ -215,6 +215,7 @@ fn run_smoketests_batch(server_mode: StartServer, args: &[String], python: &str) StartServer::Yes { random_port } => { // Pre-build so that `cargo run -p spacetimedb-cli` will immediately start. Otherwise we risk starting the tests // before the server is up. + // TODO: The `cargo run` invocation still seems to rebuild a bunch? investigate.. maybe we infer the binary path from cargo metadata. bash!("cargo build -p spacetimedb-cli -p spacetimedb-standalone")?; // TODO: Make sure that this isn't brittle / multiple parallel batches don't grab the same port From e98d52563edebc97941b940fde93f495f317b5d0 Mon Sep 17 00:00:00 2001 From: Zeke Foppa Date: Mon, 24 Nov 2025 16:30:52 -0800 Subject: [PATCH 35/41] [bfops/parallel-smoketests]: review --- tools/ci/src/main.rs | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/tools/ci/src/main.rs b/tools/ci/src/main.rs index d02a780ac49..fc7dbe00cec 100644 --- a/tools/ci/src/main.rs +++ b/tools/ci/src/main.rs @@ -276,6 +276,7 @@ fn run_smoketests_batch(server_mode: StartServer, args: &[String], python: &str) pid )); } else { + // TODO: I keep getting errors about the pid not existing.. but the servers seem to shut down? let _ = bash!(&format!("kill {}", pid)); } } @@ -417,26 +418,29 @@ fn main() -> Result<()> { batch_args.push(batch.clone()); batch_args.extend(args.iter().cloned()); - handles.push(std::thread::spawn(move || { - println!("Running smoketests batch {batch}.."); - // TODO: capture output and print it only in contiguous blocks - run_smoketests_batch(start_server_clone, &batch_args, &python_clone) - })); + handles.push(( + batch.clone(), + std::thread::spawn(move || { + println!("Running smoketests batch {batch}.."); + // TODO: capture output and print it only in contiguous blocks + run_smoketests_batch(start_server_clone, &batch_args, &python_clone) + }), + )); } - let mut any_failed_batch = false; - for handle in handles { + let mut failed_batches = vec![]; + for (batch, handle) in handles { // If the thread panicked or the batch failed, treat it as a failure. let result = handle .join() .unwrap_or_else(|_| Err(anyhow::anyhow!("smoketest batch thread panicked",))); if result.is_err() { - any_failed_batch = true; + failed_batches.push(batch); } } - if any_failed_batch { - anyhow::bail!("One or more smoketest batches failed"); + if !failed_batches.is_empty() { + anyhow::bail!("Smoketest batch(es) failed: {}", failed_batches.join(", ")); } } else { run_smoketests_batch(start_server, &args, &python)?; From 52e3f6edcbc4f178f5d47e90fc3ac7abf490aa93 Mon Sep 17 00:00:00 2001 From: Zeke Foppa Date: Mon, 24 Nov 2025 16:32:41 -0800 Subject: [PATCH 36/41] [bfops/parallel-smoketests]: use parallel smoketests in CI --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b1928b0a600..0b547ee7e9c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -108,10 +108,10 @@ jobs: run: | # Our .dockerignore omits `target`, which our CI Dockerfile needs. rm .dockerignore - cargo ci smoketests --docker .github/docker-compose.yml -- ${SMOKETEST_ARGS} + cargo ci smoketests --docker .github/docker-compose.yml --parallel -- ${SMOKETEST_ARGS} - name: Run smoketests (Windows) if: runner.os == 'Windows' - run: cargo ci smoketests -- ${SMOKETEST_ARGS} + run: cargo ci smoketests --parallel -- ${SMOKETEST_ARGS} test: name: Test Suite From 40cb0deb16d492dfab0d95e1ee4039013e89dabc Mon Sep 17 00:00:00 2001 From: Zeke Foppa Date: Mon, 24 Nov 2025 16:37:39 -0800 Subject: [PATCH 37/41] [bfops/parallel-smoketests]: Don't build CLI in smoketests if we're already building it --- tools/ci/src/main.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/ci/src/main.rs b/tools/ci/src/main.rs index fc7dbe00cec..246270eee71 100644 --- a/tools/ci/src/main.rs +++ b/tools/ci/src/main.rs @@ -187,6 +187,7 @@ fn find_free_port() -> Result { } fn run_smoketests_batch(server_mode: StartServer, args: &[String], python: &str) -> Result<()> { + let mut args: Vec<_> = args.iter().cloned().collect(); let server_state = match server_mode { StartServer::No => ServerState::None, StartServer::Docker { @@ -217,6 +218,7 @@ fn run_smoketests_batch(server_mode: StartServer, args: &[String], python: &str) // before the server is up. // TODO: The `cargo run` invocation still seems to rebuild a bunch? investigate.. maybe we infer the binary path from cargo metadata. bash!("cargo build -p spacetimedb-cli -p spacetimedb-standalone")?; + args.push("--no-build-cli".into()); // TODO: Make sure that this isn't brittle / multiple parallel batches don't grab the same port let arg_string = if random_port { From 83f3fffb3ecbb52313edabe2cf9d44fab22a2c1f Mon Sep 17 00:00:00 2001 From: Zeke Foppa Date: Mon, 24 Nov 2025 16:43:25 -0800 Subject: [PATCH 38/41] [bfops/parallel-smoketests]: consolidate builds, and fix --- tools/ci/src/main.rs | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/tools/ci/src/main.rs b/tools/ci/src/main.rs index 246270eee71..b37ee642ed3 100644 --- a/tools/ci/src/main.rs +++ b/tools/ci/src/main.rs @@ -187,7 +187,6 @@ fn find_free_port() -> Result { } fn run_smoketests_batch(server_mode: StartServer, args: &[String], python: &str) -> Result<()> { - let mut args: Vec<_> = args.iter().cloned().collect(); let server_state = match server_mode { StartServer::No => ServerState::None, StartServer::Docker { @@ -209,17 +208,11 @@ fn run_smoketests_batch(server_mode: StartServer, args: &[String], python: &str) }; let compose_str = compose_file.to_string_lossy(); bash!(&format!( - "{env_string} docker compose -f {compose_str} --project {project} up -d" + "{env_string} docker compose -f {compose_str} --project-name {project} up -d" ))?; ServerState::Docker { compose_file, project } } StartServer::Yes { random_port } => { - // Pre-build so that `cargo run -p spacetimedb-cli` will immediately start. Otherwise we risk starting the tests - // before the server is up. - // TODO: The `cargo run` invocation still seems to rebuild a bunch? investigate.. maybe we infer the binary path from cargo metadata. - bash!("cargo build -p spacetimedb-cli -p spacetimedb-standalone")?; - args.push("--no-build-cli".into()); - // TODO: Make sure that this isn't brittle / multiple parallel batches don't grab the same port let arg_string = if random_port { let server_port = find_free_port()?; @@ -268,7 +261,9 @@ fn run_smoketests_batch(server_mode: StartServer, args: &[String], python: &str) ServerState::Docker { compose_file, project } => { println!("Shutting down server.."); let compose_str = compose_file.to_string_lossy(); - let _ = bash!(&format!("docker compose -f {compose_str} --project {project} down")); + let _ = bash!(&format!( + "docker compose -f {compose_str} --project-name {project} down" + )); } ServerState::Yes { pid } => { println!("Shutting down server.."); @@ -410,6 +405,14 @@ fn main() -> Result<()> { }) .collect(); + if matches!(start_server, StartServer::Yes { .. }) { + // Pre-build so that `cargo run -p spacetimedb-cli` will immediately start. Otherwise we risk starting the tests + // before the server is up. + // TODO: The `cargo run` invocation still seems to rebuild a bunch? investigate.. maybe we infer the binary path from cargo metadata. + bash!("cargo build -p spacetimedb-cli -p spacetimedb-standalone")?; + args.push("--no-build-cli".into()); + } + // Run each batch in parallel threads. let mut handles = Vec::new(); for batch in batches { From 75b47acc4f6232ebb83dedda58efe23454ca77a6 Mon Sep 17 00:00:00 2001 From: Zeke Foppa Date: Mon, 24 Nov 2025 16:54:21 -0800 Subject: [PATCH 39/41] [bfops/parallel-smoketests]: todos --- tools/ci/src/main.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/ci/src/main.rs b/tools/ci/src/main.rs index b37ee642ed3..5a58cb2edb6 100644 --- a/tools/ci/src/main.rs +++ b/tools/ci/src/main.rs @@ -235,6 +235,7 @@ fn run_smoketests_batch(server_mode: StartServer, args: &[String], python: &str) .read() .unwrap_or_default(); } else { + // TODO: Maybe we do this in a thread instead? Then it's easier to kill pid_str = cmd!( "bash", "-lc", @@ -253,6 +254,7 @@ fn run_smoketests_batch(server_mode: StartServer, args: &[String], python: &str) }; println!("Running smoketests.."); + // TODO: Don't we need to _use_ the port here?! let test_result = bash!(&format!("{python} -m smoketests {}", args.join(" "))); // TODO: Make an effort to run the wind-down behavior if we ctrl-C this process From 73548c1f239fa9ff2f9d0b14c248a83d1738eb91 Mon Sep 17 00:00:00 2001 From: Zeke Foppa Date: Mon, 24 Nov 2025 23:01:51 -0800 Subject: [PATCH 40/41] [bfops/parallel-smoketests]: Pre-build in non-parallel case too --- tools/ci/src/main.rs | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/tools/ci/src/main.rs b/tools/ci/src/main.rs index 5a58cb2edb6..aafc445399c 100644 --- a/tools/ci/src/main.rs +++ b/tools/ci/src/main.rs @@ -362,6 +362,16 @@ fn main() -> Result<()> { } }; + if matches!(start_server, StartServer::Yes { .. }) { + println!("Building SpacetimeDB.."); + + // Pre-build so that `cargo run -p spacetimedb-cli` will immediately start. Otherwise we risk starting the tests + // before the server is up. + // TODO: The `cargo run` invocation still seems to rebuild a bunch? investigate.. maybe we infer the binary path from cargo metadata. + bash!("cargo build -p spacetimedb-cli -p spacetimedb-standalone")?; + args.push("--no-build-cli".into()); + } + if parallel { println!("Listing smoketests for parallel execution.."); @@ -407,14 +417,6 @@ fn main() -> Result<()> { }) .collect(); - if matches!(start_server, StartServer::Yes { .. }) { - // Pre-build so that `cargo run -p spacetimedb-cli` will immediately start. Otherwise we risk starting the tests - // before the server is up. - // TODO: The `cargo run` invocation still seems to rebuild a bunch? investigate.. maybe we infer the binary path from cargo metadata. - bash!("cargo build -p spacetimedb-cli -p spacetimedb-standalone")?; - args.push("--no-build-cli".into()); - } - // Run each batch in parallel threads. let mut handles = Vec::new(); for batch in batches { From 04eeb8c0e8c6d7ddac4e347b807e9d3f4c83e5df Mon Sep 17 00:00:00 2001 From: Zeke Foppa Date: Mon, 24 Nov 2025 23:01:59 -0800 Subject: [PATCH 41/41] [bfops/parallel-smoketests]: sanitize cargo env --- tools/ci/src/main.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tools/ci/src/main.rs b/tools/ci/src/main.rs index aafc445399c..93cf0028312 100644 --- a/tools/ci/src/main.rs +++ b/tools/ci/src/main.rs @@ -287,6 +287,13 @@ fn run_smoketests_batch(server_mode: StartServer, args: &[String], python: &str) fn main() -> Result<()> { let cli = Cli::parse(); + // Remove all Cargo-provided env vars from the subcommand + for (key, _) in std::env::vars() { + if key.starts_with("CARGO_") && key != "CARGO_TARGET_DIR" { + std::env::remove_var(key); + } + } + match cli.cmd { Some(CiCmd::Test) => { bash!("cargo test --all -- --skip unreal")?;