Packing handles mixed document versions in a workflow (#1320)

* Support packing mixed versions.  Now automatically updates to greatest version used by the workflow.
* scandeps finds file literals under IWD 'listing'

Author: tetron

cwltool/load_tool.py

@@ -377,10 +377,15 @@ def resolve_and_validate_document(
 
     document_loader.resolve_all(workflowobj, fileuri)
     processobj, metadata = document_loader.resolve_ref(uri)
-    if loadingContext.metadata:
-        metadata = loadingContext.metadata
     if not isinstance(processobj, (CommentedMap, CommentedSeq)):
         raise ValidationException("Workflow must be a CommentedMap or CommentedSeq.")
+
+    if not hasattr(processobj.lc, "filename"):
+        processobj.lc.filename = fileuri
+
+    if loadingContext.metadata:
+        metadata = loadingContext.metadata
+
     if not isinstance(metadata, CommentedMap):
         raise ValidationException(
             "metadata must be a CommentedMap, was %s" % type(metadata)

cwltool/pack.py

@@ -22,6 +22,8 @@
 from .load_tool import fetch_document, resolve_and_validate_document
 from .process import shortname, uniquename
 from .utils import CWLObjectType, CWLOutputType
+from .loghandler import _logger
+from .update import update, ORDERED_VERSIONS
 
 LoadRefType = Callable[[Optional[str], str], ResolveType]
 
@@ -151,15 +153,37 @@ def pack(
             document_loader.idx[po["id"]] = CommentedMap(po.items())
         document_loader.idx[metadata["id"]] = CommentedMap(metadata.items())
 
-    def loadref(base: Optional[str], uri: str) -> ResolveType:
-        return document_loader.resolve_ref(uri, base_url=base)[0]
+    found_versions = {
+        cast(str, loadingContext.metadata["cwlVersion"])
+    }  # type: Set[str]
+
+    def loadref(base: Optional[str], lr_uri: str) -> ResolveType:
+        lr_loadingContext = loadingContext.copy()
+        lr_loadingContext.metadata = {}
+        lr_loadingContext, lr_workflowobj, lr_uri = fetch_document(
+            lr_uri, lr_loadingContext
+        )
+        lr_loadingContext, lr_uri = resolve_and_validate_document(
+            lr_loadingContext, lr_workflowobj, lr_uri
+        )
+        found_versions.add(cast(str, lr_loadingContext.metadata["cwlVersion"]))
+        if lr_loadingContext.loader is None:
+            raise Exception("loader should not be None")
+        return lr_loadingContext.loader.resolve_ref(lr_uri, base_url=base)[0]
 
     ids = set()  # type: Set[str]
     find_ids(processobj, ids)
 
     runs = {uri}
     find_run(processobj, loadref, runs)
 
+    # Figure out the highest version, everything needs to be updated
+    # to it.
+    m = 0
+    for fv in found_versions:
+        m = max(m, ORDERED_VERSIONS.index(fv))
+    update_to_version = ORDERED_VERSIONS[m]
+
     for f in runs:
         find_ids(document_loader.resolve_ref(f)[0], ids)
 
@@ -193,7 +217,7 @@ def rewrite_id(r: str, mainuri: str) -> None:
         rewrite_id(r, uri)
 
     packed = CommentedMap(
-        (("$graph", CommentedSeq()), ("cwlVersion", metadata["cwlVersion"]))
+        (("$graph", CommentedSeq()), ("cwlVersion", update_to_version))
     )
     namespaces = metadata.get("$namespaces", None)
 
@@ -208,6 +232,21 @@ def rewrite_id(r: str, mainuri: str) -> None:
             dcr = cast(CommentedMap, dcr)
         if not isinstance(dcr, MutableMapping):
             continue
+
+        dcr = update(
+            dcr,
+            document_loader,
+            r,
+            loadingContext.enable_dev,
+            metadata,
+            update_to_version,
+        )
+
+        if "http://commonwl.org/cwltool#original_cwlVersion" in metadata:
+            del metadata["http://commonwl.org/cwltool#original_cwlVersion"]
+        if "http://commonwl.org/cwltool#original_cwlVersion" in dcr:
+            del dcr["http://commonwl.org/cwltool#original_cwlVersion"]
+
         if "$schemas" in metadata:
             for s in metadata["$schemas"]:
                 schemas.add(s)

cwltool/process.py

@@ -1235,7 +1235,10 @@ def scandeps(
                     if nestdirs:
                         deps = nestdir(base, deps)
                     r.append(deps)
-            elif k not in ("listing", "secondaryFiles"):
+            elif doc.get("class") in ("File", "Directory") and k in ("listing", "secondaryFiles"):
+                # should be handled earlier.
+                pass
+            else:
                 r.extend(
                     scandeps(
                         base,

cwltool/update.py

@@ -181,6 +181,16 @@ def v1_2_0dev3todev4(
     return (doc, "v1.2.0-dev4")
 
 
+ORDERED_VERSIONS = [
+    "v1.0",
+    "v1.1.0-dev1",
+    "v1.1",
+    "v1.2.0-dev1",
+    "v1.2.0-dev2",
+    "v1.2.0-dev3",
+    "v1.2.0-dev4",
+]
+
 UPDATES = {
     u"v1.0": v1_0to1_1,
     u"v1.1": v1_1to1_2,
@@ -268,16 +278,20 @@ def update(
     baseuri: str,
     enable_dev: bool,
     metadata: CommentedMap,
+    update_to: Optional[str] = None,
 ) -> CommentedMap:
 
+    if update_to is None:
+        update_to = INTERNAL_VERSION
+
     (cdoc, version) = checkversion(doc, metadata, enable_dev)
     originalversion = copy.copy(version)
 
     nextupdate = (
         identity
     )  # type: Optional[Callable[[CommentedMap, Loader, str], Tuple[CommentedMap, str]]]
 
-    while nextupdate:
+    while version != update_to and nextupdate:
         (cdoc, version) = nextupdate(cdoc, loader, baseuri)
         nextupdate = ALLUPDATES[version]