Merge branch 'main' into input_annotations

Author: mr-c

.github/workflows/ci-tests.yml

@@ -77,7 +77,7 @@ jobs:
           key: mypy-${{ env.py-semver }}
 
       - name: Test with tox
-        run: tox
+        run: APPTAINER_TMPDIR=${RUNNER_TEMP} tox
 
       - name: Upload coverage to Codecov
         if: ${{ matrix.step == 'unit' }}
@@ -156,7 +156,7 @@ jobs:
           chmod a-w .
 
       - name: run tests
-        run: make test
+        run: APPTAINER_TMPDIR=${RUNNER_TEMP} make test
 
 
   conformance_tests:
@@ -203,9 +203,15 @@ jobs:
         env:
           VERSION: ${{ matrix.cwl-version }}
           CONTAINER: ${{ matrix.container }}
-          GIT_TARGET: main
-          CWLTOOL_OPTIONS: ${{ matrix.extras }}
+          GIT_TARGET: ${{ matrix.cwl-version == 'v1.2' && '1.2.1_proposed' || 'main' }}
+          CWLTOOL_OPTIONS: ${{ matrix.cwl-version == 'v1.2' && '--relax-path-checks' || '' }} ${{ matrix.extras }}
         run: ./conformance-test.sh
+      - name: Archive test results
+        uses: actions/upload-artifact@v3
+        with:
+          name: cwl-${{ matrix.cwl-version }}-${{ matrix.container }}${{ matrix.extras }}-conformance-results
+          path: |
+            **/cwltool_conf*.xml
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v3
         with:

.github/workflows/quay-publish.yml

@@ -14,7 +14,7 @@ jobs:
       - name: Get image tags
         id: image_tags
         run: |
-          echo -n "IMAGE_TAGS=${GITHUB_REF#refs/*/}" >> $GITHUB_OUTPUT
+          echo -n "IMAGE_TAGS=${GITHUB_REF#refs/*/}" >> "$GITHUB_OUTPUT"
       - name: record cwltool version
         run: |
           pip install "setuptools>=61"

conformance-test.sh

@@ -41,6 +41,8 @@ set -x
 # The directory where this script resides
 SCRIPT_DIRECTORY="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 
+TMP_DIR=${TMP_DIR:-${SCRIPT_DIRECTORY}}
+
 # Download archive from GitHub
 if [[ "${VERSION}" = "v1.0" ]] ; then
     REPO=common-workflow-language
@@ -49,38 +51,40 @@ else
     REPO=cwl-$(echo "$VERSION" | sed 's/\(v[0-9]*\.\)\([0-9]*\).*/\1\2/')
 fi
 
-if [ ! -d "${REPO}-${GIT_TARGET}" ] ; then
-  if [ ! -f "${GIT_TARGET}.tar.gz" ] ; then
-	  wget "https://github.com/common-workflow-language/${REPO}/archive/${GIT_TARGET}.tar.gz"
+if [ ! -d "${TMP_DIR}/${REPO}-${GIT_TARGET}" ] ; then
+	if [ ! -f "${TMP_DIR}/${GIT_TARGET}.tar.gz" ] ; then
+	  wget --directory-prefix "${TMP_DIR}" "https://github.com/common-workflow-language/${REPO}/archive/${GIT_TARGET}.tar.gz"
   fi
-  tar xzf "${GIT_TARGET}.tar.gz"
+  tar --directory "${TMP_DIR}" -x -f "${TMP_DIR}/${GIT_TARGET}.tar.gz"
 fi
 
-if [ "${CONTAINER}" == "docker" ]; then
+if [ -v SKIP_PULL ] ; then
+    echo Skipping node container pull.
+elif [ "${CONTAINER}" == "docker" ]; then
     docker pull docker.io/node:slim
-fi
-
-if [ "${CONTAINER}" == "podman" ]; then
+elif [ "${CONTAINER}" == "podman" ]; then
     podman pull docker.io/node:slim
-fi
-
-if [ "${CONTAINER}" == "singularity" ]; then
+elif [ "${CONTAINER}" == "singularity" ]; then
     export CWL_SINGULARITY_CACHE="$SCRIPT_DIRECTORY/sifcache"
     mkdir --parents "${CWL_SINGULARITY_CACHE}"
 fi
 
 # Setup environment
-venv cwl-conformance-venv
-pip install -U setuptools wheel pip
-pip uninstall -y cwltool
-pip install "${SCRIPT_DIRECTORY}" -r"${SCRIPT_DIRECTORY}/requirements.txt"
-pip install 'cwltest>=2.3' pytest-cov pytest-xdist
+if [ -v SKIP_INSTALL ] ; then
+   echo 'Skip installing dependencies; cwltool & cwltest must already be installed'
+else
+	venv "${TMP_DIR}/cwl-conformance-venv"
+	pip install -U setuptools wheel pip
+	pip uninstall -y cwltool
+	pip install "${SCRIPT_DIRECTORY}" -r"${SCRIPT_DIRECTORY}/requirements.txt"
+	pip install 'cwltest>=2.3' pytest-cov pytest-xdist
+fi
 
 # Set conformance test filename
 if [[ "${VERSION}" = "v1.0" ]] ; then
-  CONFORMANCE_TEST="${SCRIPT_DIRECTORY}/${REPO}-${GIT_TARGET}/${VERSION}/conformance_test_v1.0.yaml"
+  CONFORMANCE_TEST="${TMP_DIR}/${REPO}-${GIT_TARGET}/${VERSION}/conformance_test_v1.0.yaml"
 else
-  CONFORMANCE_TEST="${SCRIPT_DIRECTORY}/${REPO}-${GIT_TARGET}/conformance_tests.yaml"
+  CONFORMANCE_TEST="${TMP_DIR}/${REPO}-${GIT_TARGET}/conformance_tests.yaml"
 fi
 cp "${CONFORMANCE_TEST}" "${CONFORMANCE_TEST%".yaml"}.cwltest.yaml"
 CONFORMANCE_TEST="${CONFORMANCE_TEST%".yaml"}.cwltest.yaml"
@@ -116,14 +120,18 @@ if (( "${#exclusions[*]}" > 0 )); then
 fi
 
 # Build command
-TEST_COMMAND="python -m pytest ${CONFORMANCE_TEST} -n auto -rs --junit-xml=${SCRIPT_DIRECTORY}/cwltool_conf_${VERSION}_${GIT_TARGET}_${CONTAINER}.xml -o junit_suite_name=cwltool_$(echo "${CWLTOOL_OPTIONS}" | tr "[:blank:]-" _)"
+TEST_COMMAND="python -m pytest ${CONFORMANCE_TEST} -n auto -rs --junit-xml=${TMP_DIR}/cwltool_conf_${VERSION}_${GIT_TARGET}_${CONTAINER}.xml -o junit_suite_name=cwltool_$(echo "${CWLTOOL_OPTIONS}" | tr "[:blank:]-" _)"
 if [[ -n "${EXCLUDE}" ]] ; then
   TEST_COMMAND="${TEST_COMMAND} --cwl-exclude ${EXCLUDE}"
 fi
-TEST_COMMAND="${TEST_COMMAND} --cov --cov-config ${SCRIPT_DIRECTORY}/.coveragerc --cov-report= ${PYTEST_EXTRA}"
+if [ -v SKIP_COV ] ; then
+	echo Skipping gathering of coverage information
+else
+	TEST_COMMAND="${TEST_COMMAND} --cov --cov-config ${SCRIPT_DIRECTORY}/.coveragerc --cov-report= ${PYTEST_EXTRA}"
+	# Clean up all old coverage data
+	find "${SCRIPT_DIRECTORY}" \( -type f -name .coverage -or -name '.coverage.*' -or -name coverage.xml \) -delete
+fi
 
-# Clean up all old coverage data
-find "${SCRIPT_DIRECTORY}" \( -type f -name .coverage -or -name '.coverage.*' -or -name coverage.xml \) -delete
 
 if [ "$GIT_BRANCH" = "origin/main" ] && [[ "$VERSION" = "v1.0" ]] && [[ "$CONTAINER" = "docker" ]]
 then
@@ -153,11 +161,11 @@ echo CWLTOOL_OPTIONS="${CWLTOOL_OPTIONS}"
 
 # Run test
 cp "${SCRIPT_DIRECTORY}/tests/cwl-conformance/cwltool-conftest.py" "$(dirname "${CONFORMANCE_TEST}")/conftest.py"
-bash -c "${TEST_COMMAND}"
+bash -c "cd ${TMP_DIR} && ${TEST_COMMAND}"
 RETURN_CODE=$?
 
 # Coverage report
-if [ "${RETURN_CODE}" -eq "0" ] ; then
+if [ ! -v SKIP_COV ] && [ "${RETURN_CODE}" -eq "0" ] ; then
   coverage report
   coverage xml
 fi
@@ -172,7 +180,11 @@ then
 fi
 
 # Cleanup
-deactivate
+if [ -z "$SKIP_INSTALL" ] ; then
+	echo Skipping venv cleanup
+else
+	deactivate
+fi
 #rm -rf "${GIT_TARGET}.tar.gz" "${SCRIPT_DIRECTORY}/${REPO}-${GIT_TARGET}" "${SCRIPT_DIRECTORY}/cwl-conformance-venv"
 
 # Exit

cwltool/argparser.py

@@ -378,6 +378,7 @@ def arg_parser() -> argparse.ArgumentParser:
 
     volumegroup = parser.add_mutually_exclusive_group()
     volumegroup.add_argument("--verbose", action="store_true", help="Default logging")
+    volumegroup.add_argument("--no-warnings", action="store_true", help="Only print errors.")
     volumegroup.add_argument("--quiet", action="store_true", help="Only print warnings and errors.")
     volumegroup.add_argument("--debug", action="store_true", help="Print even more logging")
 

cwltool/checker.py

@@ -21,7 +21,7 @@
 from .errors import WorkflowException
 from .loghandler import _logger
 from .process import shortname
-from .utils import CWLObjectType, CWLOutputAtomType, CWLOutputType, SinkType, aslist
+from .utils import CWLObjectType, CWLOutputType, SinkType, aslist
 
 
 def _get_type(tp):
@@ -90,8 +90,8 @@ def can_assign_src_to_sink(src: SinkType, sink: Optional[SinkType], strict: bool
             return False
         if src["type"] == "array" and sink["type"] == "array":
             return can_assign_src_to_sink(
-                cast(MutableSequence[CWLOutputAtomType], src["items"]),
-                cast(MutableSequence[CWLOutputAtomType], sink["items"]),
+                cast(MutableSequence[CWLOutputType], src["items"]),
+                cast(MutableSequence[CWLOutputType], sink["items"]),
                 strict,
             )
         if src["type"] == "record" and sink["type"] == "record":

cwltool/docker.py

@@ -207,7 +207,13 @@ def get_from_requirements(
         raise WorkflowException("Docker image %s not found" % r["dockerImageId"])
 
     @staticmethod
-    def append_volume(runtime: List[str], source: str, target: str, writable: bool = False) -> None:
+    def append_volume(
+        runtime: List[str],
+        source: str,
+        target: str,
+        writable: bool = False,
+        skip_mkdirs: bool = False,
+    ) -> None:
         """Add binding arguments to the runtime list."""
         options = [
             "type=bind",
@@ -221,7 +227,7 @@ def append_volume(runtime: List[str], source: str, target: str, writable: bool =
         mount_arg = output.getvalue().strip()
         runtime.append(f"--mount={mount_arg}")
         # Unlike "--volume", "--mount" will fail if the volume doesn't already exist.
-        if not os.path.exists(source):
+        if (not skip_mkdirs) and (not os.path.exists(source)):
             os.makedirs(source)
 
     def add_file_or_directory_volume(

cwltool/job.py

@@ -66,6 +66,11 @@
     from .cwlprov.provenance_profile import (
         ProvenanceProfile,  # pylint: disable=unused-import
     )
+
+    CollectOutputsType = Union[
+        Callable[[str, int], CWLObjectType], functools.partial[CWLObjectType]
+    ]
+
 needs_shell_quoting_re = re.compile(r"""(^$|[\s|&;()<>\'"$@])""")
 
 FORCE_SHELLED_POPEN = os.getenv("CWLTOOL_FORCE_SHELL_POPEN", "0") == "1"
@@ -112,9 +117,6 @@ def neverquote(string: str, pos: int = 0, endpos: int = 0) -> Optional[Match[str
     return None
 
 
-CollectOutputsType = Union[Callable[[str, int], CWLObjectType], functools.partial]
-
-
 class JobBase(HasReqsHints, metaclass=ABCMeta):
     def __init__(
         self,
@@ -144,7 +146,7 @@ def __init__(
         self.generatemapper: Optional[PathMapper] = None
 
         # set in CommandLineTool.job(i)
-        self.collect_outputs = cast(CollectOutputsType, None)
+        self.collect_outputs = cast("CollectOutputsType", None)
         self.output_callback: Optional[OutputCallbackType] = None
         self.outdir = ""
         self.tmpdir = ""

cwltool/loghandler.py

@@ -11,6 +11,7 @@
 
 def configure_logging(
     stderr_handler: logging.Handler,
+    no_warnings: bool,
     quiet: bool,
     debug: bool,
     enable_color: bool,
@@ -25,6 +26,8 @@ def configure_logging(
     deps_logger.addHandler(stderr_handler)
     ss_logger = logging.getLogger("salad")
     ss_logger.addHandler(stderr_handler)
+    if no_warnings:
+        stderr_handler.setLevel(logging.ERROR)
     if quiet:
         # Silence STDERR, not an eventual provenance log file
         stderr_handler.setLevel(logging.WARN)

cwltool/main.py

@@ -34,7 +34,6 @@
 import argcomplete
 import coloredlogs
 import ruamel.yaml
-from importlib_resources import files
 from ruamel.yaml.comments import CommentedMap, CommentedSeq
 from ruamel.yaml.main import YAML
 from schema_salad.exceptions import ValidationException
@@ -105,10 +104,10 @@
 from .utils import (
     DEFAULT_TMP_PREFIX,
     CWLObjectType,
-    CWLOutputAtomType,
     CWLOutputType,
     HasReqsHints,
     adjustDirObjs,
+    files,
     normalizeFilesDirs,
     processes_to_kill,
     trim_listing,
@@ -289,7 +288,7 @@ def realize_input_schema(
                 _, input_type_name = entry["type"].split("#")
                 if input_type_name in schema_defs:
                     entry["type"] = cast(
-                        CWLOutputAtomType,
+                        CWLOutputType,
                         realize_input_schema(
                             cast(
                                 MutableSequence[Union[str, CWLObjectType]],
@@ -300,29 +299,29 @@ def realize_input_schema(
                     )
             if isinstance(entry["type"], MutableSequence):
                 entry["type"] = cast(
-                    CWLOutputAtomType,
+                    CWLOutputType,
                     realize_input_schema(
                         cast(MutableSequence[Union[str, CWLObjectType]], entry["type"]),
                         schema_defs,
                     ),
                 )
             if isinstance(entry["type"], Mapping):
                 entry["type"] = cast(
-                    CWLOutputAtomType,
+                    CWLOutputType,
                     realize_input_schema([cast(CWLObjectType, entry["type"])], schema_defs),
                 )
             if entry["type"] == "array":
                 items = entry["items"] if not isinstance(entry["items"], str) else [entry["items"]]
                 entry["items"] = cast(
-                    CWLOutputAtomType,
+                    CWLOutputType,
                     realize_input_schema(
                         cast(MutableSequence[Union[str, CWLObjectType]], items),
                         schema_defs,
                     ),
                 )
             if entry["type"] == "record":
                 entry["fields"] = cast(
-                    CWLOutputAtomType,
+                    CWLOutputType,
                     realize_input_schema(
                         cast(MutableSequence[Union[str, CWLObjectType]], entry["fields"]),
                         schema_defs,
@@ -612,7 +611,7 @@ def loadref(base: str, uri: str) -> Union[CommentedMap, CommentedSeq, str, None]
         nestdirs=nestdirs,
     )
     if sfs is not None:
-        deps["secondaryFiles"] = cast(MutableSequence[CWLOutputAtomType], mergedirs(sfs))
+        deps["secondaryFiles"] = cast(MutableSequence[CWLOutputType], mergedirs(sfs))
 
     return deps
 
@@ -1014,6 +1013,7 @@ def main(
 
         configure_logging(
             stderr_handler,
+            args.no_warnings,
             args.quiet,
             runtimeContext.debug,
             args.enable_color,