Merge pull request #1193 from common-workflow-language/cwlprov_conf_fixes

--provenance: conformance tests fixes

Author: mr-c

cwltool/main.py

@@ -836,6 +836,21 @@ def my_represent_none(self, data):  # pylint: disable=unused-argument
                 if runtimeContext.research_obj is not None:
                     runtimeContext.research_obj.create_job(
                         out, None, True)
+                    def remove_at_id(doc):  # type: (MutableMapping[Text, Any]) -> None
+                        for key in list(doc.keys()):
+                            if key == '@id':
+                                del doc[key]
+                            else:
+                                value = doc[key]
+                                if isinstance(value, MutableMapping):
+                                    remove_at_id(value)
+                                elif isinstance(value, MutableSequence):
+                                    for entry in value:
+                                        if isinstance(entry, MutableMapping):
+                                            remove_at_id(entry)
+                    remove_at_id(out)
+                    visit_class(out, ("File",), functools.partial(
+                        add_sizes, runtimeContext.make_fs_access('')))
 
                 def loc_to_path(obj):  # type: (Dict[Text, Any]) -> None
                     for field in ("path", "nameext", "nameroot", "dirname"):

cwltool/provenance.py

@@ -545,7 +545,12 @@ def declare_file(self, value):
         # Check for secondaries
         for sec in value.get("secondaryFiles", ()):
             # TODO: Record these in a specializationOf entity with UUID?
-            (sec_entity, _, _) = self.declare_file(sec)
+            if sec['class'] == "File":
+                (sec_entity, _, _) = self.declare_file(sec)
+            elif sec['class'] == "Directory":
+                sec_entity = self.declare_directory(sec)
+            else:
+                raise ValueError("Got unexpected secondaryFiles value: {}".format(sec))
             # We don't know how/when/where the secondary file was generated,
             # but CWL convention is a kind of summary/index derived
             # from the original file. As its generally in a different format
@@ -796,10 +801,13 @@ def used_artefacts(self,
                 base += "/" + name
             for key, value in job_order.items():
                 prov_role = self.wf_ns["%s/%s" % (base, key)]
-                entity = self.declare_artefact(value)
-                self.document.used(
-                    process_run_id, entity, datetime.datetime.now(), None,
-                    {"prov:role": prov_role})
+                try:
+                    entity = self.declare_artefact(value)
+                    self.document.used(
+                        process_run_id, entity, datetime.datetime.now(), None,
+                        {"prov:role": prov_role})
+                except OSError:
+                    pass
 
     def generate_output_prov(self,
                              final_output,    # type: Union[Dict[Text, Any], List[Dict[Text, Any]]]
@@ -1572,7 +1580,10 @@ def _relativise_files(self, structure):
                 del structure["location"]
 
             for val in structure.values():
-                self._relativise_files(val)
+                try:
+                    self._relativise_files(val)
+                except OSError:
+                    pass
             return
 
         if isinstance(structure, (str, Text)):