Merge remote-tracking branch 'refs/remotes/upstream/main' into on-error-abort

# Conflicts:
#	cwltool/errors.py

Author: AlexTate

.github/workflows/ci-tests.yml

@@ -173,6 +173,8 @@ jobs:
           - cwl-version: v1.2
             container: docker
             extras: "--fast-parser"
+          - cwl-version: v1.3.0-dev1
+            extras: "--relax-path-checks"
 
     steps:
       - uses: actions/checkout@v4
@@ -266,7 +268,7 @@ jobs:
 
   macos:
     name: Test on macos-latest
-    runs-on: macos-latest
+    runs-on: macos-13  # not latest, that is now an Apple Silicon M1, for which seqtk is not yet built on bioconda
     env:
       TOXENV: py312-unit
     steps:

.github/workflows/quay-publish.yml

@@ -38,7 +38,7 @@ jobs:
           username: ${{ secrets.REGISTRY_USERNAME }}
           password: ${{ secrets.REGISTRY_PASSWORD }}
       - name: Build and publish cwltool_module image to Quay
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v6
         with:
           context: .
           file: cwltool.Dockerfile
@@ -49,7 +49,7 @@ jobs:
           cache-from: type=gha
           cache-to: type=gha,mode=max
       - name: Build and publish cwltool image to Quay
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v6
         with:
           context: .
           file: cwltool.Dockerfile

MANIFEST.in

@@ -7,6 +7,7 @@ recursive-include mypy-stubs *.pyi *.py
 include tests/*
 include tests/cwl-conformance/cwltool-conftest.py
 include tests/loop/*
+include tests/loop-ext/*
 include tests/tmp1/tmp2/tmp3/.gitkeep
 include tests/tmp4/alpha/*
 include tests/wf/*
@@ -54,6 +55,10 @@ include cwltool/schemas/v1.2/*.yml
 include cwltool/schemas/v1.2/*.md
 include cwltool/schemas/v1.2/salad/schema_salad/metaschema/*.yml
 include cwltool/schemas/v1.2/salad/schema_salad/metaschema/*.md
+include cwltool/schemas/v1.3.0-dev1/*.yml
+include cwltool/schemas/v1.3.0-dev1/*.md
+include cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/*.yml
+include cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/*.md
 include cwltool/extensions.yml
 include cwltool/extensions-v1.1.yml
 include cwltool/extensions-v1.2.yml

Makefile

@@ -160,11 +160,11 @@ diff-cover.html: coverage.xml
 
 ## test                   : run the cwltool test suite
 test: $(PYSOURCES)
-	python3 -m pytest -rs ${PYTEST_EXTRA}
+	python3 -m pytest -rsfE ${PYTEST_EXTRA}
 
 ## testcov                : run the cwltool test suite and collect coverage
 testcov: $(PYSOURCES)
-	python3 -m pytest -rs --cov --cov-config=.coveragerc --cov-report= ${PYTEST_EXTRA}
+	python3 -m pytest -rsfE --cov --cov-config=.coveragerc --cov-report= ${PYTEST_EXTRA}
 
 sloccount.sc: $(PYSOURCES) Makefile
 	sloccount --duplicates --wide --details $^ > $@
@@ -183,7 +183,7 @@ mypy: $(PYSOURCES)
 
 mypyc: $(PYSOURCES)
 	MYPYPATH=mypy-stubs CWLTOOL_USE_MYPYC=1 pip install --verbose -e . \
-		 && pytest -rs -vv ${PYTEST_EXTRA}
+		 && pytest -rsfE -vv ${PYTEST_EXTRA}
 
 shellcheck: FORCE
 	shellcheck build-cwltool-docker.sh cwl-docker.sh release-test.sh conformance-test.sh \

conformance-test.sh

@@ -16,7 +16,7 @@ venv() {
 # VERSION=v1.2 GIT_TARGET=main CONTAINER=podman ./conformance_test.sh
 
 # Version of the standard to test against
-# Current options: v1.0, v1.1, v1.2
+# Current options: v1.0, v1.1, v1.2, v1.3.0-dev1
 VERSION=${VERSION:-"v1.2"}
 
 # Which commit of the standard's repo to use
@@ -78,7 +78,7 @@ else
 	pip uninstall -y cwltool
 	pip install -r"${SCRIPT_DIRECTORY}/mypy-requirements.txt"
 	CWLTOOL_USE_MYPYC=1 MYPYPATH="${SCRIPT_DIRECTORY}/mypy-stubs" pip install "${SCRIPT_DIRECTORY}" -r"${SCRIPT_DIRECTORY}/requirements.txt"
-	pip install 'cwltest>=2.5' pytest-cov pytest-xdist>=3.2.0 psutil
+	pip install 'cwltest>=2.5,!=2.5.20240709060407' pytest-cov pytest-xdist>=3.2.0 psutil
 fi
 
 # Set conformance test filename
@@ -121,7 +121,7 @@ if (( "${#exclusions[*]}" > 0 )); then
 fi
 
 # Build command
-TEST_COMMAND="python -m pytest ${CONFORMANCE_TEST} -n logical --dist worksteal -rs --junit-xml=${TMP_DIR}/cwltool_conf_${VERSION}_${GIT_TARGET}_${CONTAINER}.xml -o junit_suite_name=cwltool_$(echo "${CWLTOOL_OPTIONS}" | tr "[:blank:]-" _)"
+TEST_COMMAND="python -m pytest ${CONFORMANCE_TEST} -n logical --dist worksteal -rsfE --junit-xml=${TMP_DIR}/cwltool_conf_${VERSION}_${GIT_TARGET}_${CONTAINER}.xml -o junit_suite_name=cwltool_$(echo "${CWLTOOL_OPTIONS}" | tr "[:blank:]-" _)"
 if [[ -n "${EXCLUDE}" ]] ; then
   TEST_COMMAND="${TEST_COMMAND} --cwl-exclude ${EXCLUDE}"
 fi

cwltool/builder.py

@@ -86,7 +86,7 @@ def content_limit_respected_read(f: IO[bytes]) -> str:
     :returns: the file contents
     :raises WorkflowException: if the file is too large
     """
-    return content_limit_respected_read_bytes(f).decode("utf-8")
+    return str(content_limit_respected_read_bytes(f), "utf-8")
 
 
 def substitute(value: str, replace: str) -> str:

cwltool/checker.py

@@ -517,42 +517,33 @@ def is_conditional_step(param_to_step: Dict[str, CWLObjectType], parm_id: str) -
 
 
 def is_all_output_method_loop_step(param_to_step: Dict[str, CWLObjectType], parm_id: str) -> bool:
-    """Check if a step contains a http://commonwl.org/cwltool#Loop requirement with `all` outputMethod."""
+    """Check if a step contains a `loop` directive with `all` outputMethod."""
     source_step: Optional[MutableMapping[str, Any]] = param_to_step.get(parm_id)
     if source_step is not None:
-        for requirement in source_step.get("requirements", []):
-            if (
-                requirement["class"] == "http://commonwl.org/cwltool#Loop"
-                and requirement.get("outputMethod") == "all"
-            ):
-                return True
+        if source_step.get("loop") is not None and source_step.get("outputMethod") == "all":
+            return True
     return False
 
 
 def loop_checker(steps: Iterator[MutableMapping[str, Any]]) -> None:
     """
-    Check http://commonwl.org/cwltool#Loop requirement compatibility with other directives.
+    Check `loop` compatibility with other directives.
 
-    :raises ValidationException: If there is an incompatible combination between
-            cwltool:loop and 'scatter' or 'when'.
+    :raises ValidationException: If there is an incompatible combination between `loop` and `scatter`.
     """
     exceptions = []
     for step in steps:
-        requirements = {
-            **{h["class"]: h for h in step.get("hints", [])},
-            **{r["class"]: r for r in step.get("requirements", [])},
-        }
-        if "http://commonwl.org/cwltool#Loop" in requirements:
-            if "when" in step:
+        if "loop" in step:
+            if "when" not in step:
                 exceptions.append(
                     SourceLine(step, "id").makeError(
-                        "The `cwltool:Loop` clause is not compatible with the `when` directive."
+                        "The `when` clause is mandatory when the `loop` directive is defined."
                     )
                 )
             if "scatter" in step:
                 exceptions.append(
                     SourceLine(step, "id").makeError(
-                        "The `cwltool:Loop` clause is not compatible with the `scatter` directive."
+                        "The `loop` clause is not compatible with the `scatter` directive."
                     )
                 )
     if exceptions:

cwltool/command_line_tool.py

@@ -1369,8 +1369,8 @@ def collect_output(
                     else:
                         if binding.get("loadContents"):
                             with fs_access.open(cast(str, rfile["location"]), "rb") as f:
-                                files["contents"] = content_limit_respected_read_bytes(f).decode(
-                                    "utf-8"
+                                files["contents"] = str(
+                                    content_limit_respected_read_bytes(f), "utf-8"
                                 )
                         if compute_checksum:
                             with fs_access.open(cast(str, rfile["location"]), "rb") as f:

cwltool/cwlprov/ro.py

@@ -106,6 +106,7 @@ def __str__(self) -> str:
         return f"ResearchObject <{self.ro_uuid}> in <{self.folder}>"
 
     def _initialize(self) -> None:
+        """Initialize the bagit folder structure."""
         for research_obj_folder in (
             METADATA,
             DATA,
@@ -359,6 +360,7 @@ def add_annotation(
         return uri
 
     def _ro_annotations(self) -> List[Annotation]:
+        """Append base RO and provenance annotations to the list of annotations."""
         annotations: List[Annotation] = []
         annotations.append(
             {
@@ -414,6 +416,7 @@ def _ro_annotations(self) -> List[Annotation]:
 
     def _authored_by(self) -> Optional[AuthoredBy]:
         authored_by: AuthoredBy = {}
+        """Returns the authoredBy metadata if it was supplied on CLI"""
         if self.orcid:
             authored_by["orcid"] = self.orcid
         if self.full_name:
@@ -542,6 +545,7 @@ def add_to_manifest(self, rel_path: str, checksums: Dict[str, str]) -> None:
                 checksum_file.write(line)
 
     def _add_to_bagit(self, rel_path: str, **checksums: str) -> None:
+        """Compute file size and checksums and adds to bagit manifest."""
         if PurePosixPath(rel_path).is_absolute():
             raise ValueError(f"rel_path must be relative: {rel_path}")
         lpath = os.path.join(self.folder, local_path(rel_path))

cwltool/cwlrdf.py

@@ -27,7 +27,7 @@ def printrdf(wflow: Process, ctx: ContextType, style: str) -> str:
     rdf = gather(wflow, ctx).serialize(format=style, encoding="utf-8")
     if not rdf:
         return ""
-    return rdf.decode("utf-8")
+    return str(rdf, "utf-8")
 
 
 def lastpart(uri: Any) -> str: