pytest: use the worksteal scheduler and run on all logical cores, not just physical.

Author: mr-c

conformance-test.sh

@@ -78,7 +78,7 @@ else
 	pip uninstall -y cwltool
 	pip install -r"${SCRIPT_DIRECTORY}/mypy-requirements.txt"
 	CWLTOOL_USE_MYPYC=1 MYPYPATH="${SCRIPT_DIRECTORY}/mypy-stubs" pip install "${SCRIPT_DIRECTORY}" -r"${SCRIPT_DIRECTORY}/requirements.txt"
-	pip install 'cwltest>=2.5' pytest-cov pytest-xdist
+	pip install 'cwltest>=2.5' pytest-cov pytest-xdist>=3.2.0 psutil
 fi
 
 # Set conformance test filename
@@ -121,7 +121,7 @@ if (( "${#exclusions[*]}" > 0 )); then
 fi
 
 # Build command
-TEST_COMMAND="python -m pytest ${CONFORMANCE_TEST} -n auto -rs --junit-xml=${TMP_DIR}/cwltool_conf_${VERSION}_${GIT_TARGET}_${CONTAINER}.xml -o junit_suite_name=cwltool_$(echo "${CWLTOOL_OPTIONS}" | tr "[:blank:]-" _)"
+TEST_COMMAND="python -m pytest ${CONFORMANCE_TEST} -n logical --dist worksteal -rs --junit-xml=${TMP_DIR}/cwltool_conf_${VERSION}_${GIT_TARGET}_${CONTAINER}.xml -o junit_suite_name=cwltool_$(echo "${CWLTOOL_OPTIONS}" | tr "[:blank:]-" _)"
 if [[ -n "${EXCLUDE}" ]] ; then
   TEST_COMMAND="${TEST_COMMAND} --cwl-exclude ${EXCLUDE}"
 fi

release-test.sh

@@ -23,7 +23,7 @@ run_tests() {
 	mod_loc=$(pip show ${package} |
 		grep ^Location | awk '{print $2}')/${module}
 	"${test_prefix}"bin/py.test "--ignore=${mod_loc}/schemas/" \
-		--pyargs -x ${module} -n auto --dist=loadfile
+		--pyargs -x ${module} -n logical --dist=worksteal
 }
 pipver=23.1  # minimum required version of pip for Python 3.12
 setuptoolsver=67.6.1  # required for Python 3.12

test-requirements.txt

@@ -1,6 +1,7 @@
 bagit>=1.6.4,<1.9
 pytest>= 6.2,< 8.2
-pytest-xdist
+pytest-xdist>=3.2.0 # for the worksteal scheduler
+psutil # enhances pytest-xdist to allow "-n logical"
 pytest-httpserver
 pytest-retry;python_version>'3.9'
 mock>=2.0.0

tox.ini

@@ -11,7 +11,7 @@ envlist =
 skip_missing_interpreters = True
 
 [pytest]
-addopts=--ignore cwltool/schemas -n auto
+addopts=--ignore cwltool/schemas -n logical --dist worksteal
 testpaths = tests
 
 [gh-actions]