Add include_traceback on SourceLine so nested exceptions are not suppressed. (#547)

Author: tetron

cwltool/builder.py

@@ -207,7 +207,7 @@ def tostr(self, value):  # type: (Any) -> Text
     def generate_arg(self, binding):  # type: (Dict[Text,Any]) -> List[Text]
         value = binding.get("datum")
         if "valueFrom" in binding:
-            with SourceLine(binding, "valueFrom", WorkflowException):
+            with SourceLine(binding, "valueFrom", WorkflowException, _logger.isEnabledFor(logging.DEBUG)):
                 value = self.do_eval(binding["valueFrom"], context=value)
 
         prefix = binding.get("prefix")

cwltool/draft2tool.py

@@ -346,7 +346,9 @@ def rm_pending_output_callback(output_callbacks, jobcachepending,
         j.hints = self.hints
         j.name = jobname
 
-        if _logger.isEnabledFor(logging.DEBUG):
+        debug = _logger.isEnabledFor(logging.DEBUG)
+
+        if debug:
             _logger.debug(u"[job %s] initializing from %s%s",
                           j.name,
                           self.tool.get("id", ""),
@@ -405,28 +407,28 @@ def rm_pending_output_callback(output_callbacks, jobcachepending,
                 self.updatePathmap(builder.outdir, builder.pathmapper, l)
             visit_class([builder.files, builder.bindings], ("File", "Directory"), _check_adjust)
 
-        if _logger.isEnabledFor(logging.DEBUG):
+        if debug:
             _logger.debug(u"[job %s] path mappings is %s", j.name,
                           json.dumps({p: builder.pathmapper.mapper(p) for p in builder.pathmapper.files()}, indent=4))
 
         if self.tool.get("stdin"):
-            with SourceLine(self.tool, "stdin", validate.ValidationException):
+            with SourceLine(self.tool, "stdin", validate.ValidationException, debug):
                 j.stdin = builder.do_eval(self.tool["stdin"])
                 reffiles.append({"class": "File", "path": j.stdin})
 
         if self.tool.get("stderr"):
-            with SourceLine(self.tool, "stderr", validate.ValidationException):
+            with SourceLine(self.tool, "stderr", validate.ValidationException, debug):
                 j.stderr = builder.do_eval(self.tool["stderr"])
                 if os.path.isabs(j.stderr) or ".." in j.stderr:
                     raise validate.ValidationException("stderr must be a relative path, got '%s'" % j.stderr)
 
         if self.tool.get("stdout"):
-            with SourceLine(self.tool, "stdout", validate.ValidationException):
+            with SourceLine(self.tool, "stdout", validate.ValidationException, debug):
                 j.stdout = builder.do_eval(self.tool["stdout"])
                 if os.path.isabs(j.stdout) or ".." in j.stdout or not j.stdout:
                     raise validate.ValidationException("stdout must be a relative path, got '%s'" % j.stdout)
 
-        if _logger.isEnabledFor(logging.DEBUG):
+        if debug:
             _logger.debug(u"[job %s] command line bindings is %s", j.name, json.dumps(builder.bindings, indent=4))
 
         dockerReq = self.get_requirement("DockerRequirement")[0]
@@ -505,17 +507,18 @@ def register_reader(f):
     def collect_output_ports(self, ports, builder, outdir, compute_checksum=True, jobname="", readers=None):
         # type: (Set[Dict[Text, Any]], Builder, Text, bool, Text, Dict[Text, Any]) -> Dict[Text, Union[Text, List[Any], Dict[Text, Any]]]
         ret = {}  # type: Dict[Text, Union[Text, List[Any], Dict[Text, Any]]]
+        debug = _logger.isEnabledFor(logging.DEBUG)
         try:
             fs_access = builder.make_fs_access(outdir)
             custom_output = fs_access.join(outdir, "cwl.output.json")
             if fs_access.exists(custom_output):
                 with fs_access.open(custom_output, "r") as f:
                     ret = json.load(f)
-                if _logger.isEnabledFor(logging.DEBUG):
+                if debug:
                     _logger.debug(u"Raw output from %s: %s", custom_output, json.dumps(ret, indent=4))
             else:
                 for i, port in enumerate(ports):
-                    with SourceLine(ports, i, WorkflowException):
+                    with SourceLine(ports, i, WorkflowException, debug):
                         fragment = shortname(port["id"])
                         try:
                             ret[fragment] = self.collect_output(port, builder, outdir, fs_access,
@@ -554,14 +557,15 @@ def collect_output_ports(self, ports, builder, outdir, compute_checksum=True, jo
     def collect_output(self, schema, builder, outdir, fs_access, compute_checksum=True):
         # type: (Dict[Text, Any], Builder, Text, StdFsAccess, bool) -> Union[Dict[Text, Any], List[Union[Dict[Text, Any], Text]]]
         r = []  # type: List[Any]
+        debug = _logger.isEnabledFor(logging.DEBUG)
         if "outputBinding" in schema:
             binding = schema["outputBinding"]
             globpatterns = []  # type: List[Text]
 
             revmap = partial(revmap_file, builder, outdir)
 
             if "glob" in binding:
-                with SourceLine(binding, "glob", WorkflowException):
+                with SourceLine(binding, "glob", WorkflowException, debug):
                     for gb in aslist(binding["glob"]):
                         gb = builder.do_eval(gb)
                         if gb:
@@ -626,12 +630,12 @@ def collect_output(self, schema, builder, outdir, fs_access, compute_checksum=Tr
                 single = True
 
             if "outputEval" in binding:
-                with SourceLine(binding, "outputEval", WorkflowException):
+                with SourceLine(binding, "outputEval", WorkflowException, debug):
                     r = builder.do_eval(binding["outputEval"], context=r)
 
             if single:
                 if not r and not optional:
-                    with SourceLine(binding, "glob", WorkflowException):
+                    with SourceLine(binding, "glob", WorkflowException, debug):
                         raise WorkflowException("Did not find output file with glob pattern: '{}'".format(globpatterns))
                 elif not r and optional:
                     pass
@@ -642,7 +646,7 @@ def collect_output(self, schema, builder, outdir, fs_access, compute_checksum=Tr
                         r = r[0]
 
             if "secondaryFiles" in schema:
-                with SourceLine(schema, "secondaryFiles", WorkflowException):
+                with SourceLine(schema, "secondaryFiles", WorkflowException, debug):
                     for primary in aslist(r):
                         if isinstance(primary, dict):
                             primary.setdefault("secondaryFiles", [])

cwltool/pathmapper.py

@@ -254,7 +254,7 @@ def visit(self, obj, stagedir, basedir, copy=False, staged=False):
             if "contents" in obj and obj["location"].startswith("_:"):
                 self._pathmap[obj["location"]] = MapperEnt(obj["contents"], tgt, "CreateFile", staged)
             else:
-                with SourceLine(obj, "location", validate.ValidationException):
+                with SourceLine(obj, "location", validate.ValidationException, _logger.isEnabledFor(logging.DEBUG)):
                     deref = ab
                     if urllib.parse.urlsplit(deref).scheme in ['http','https']:
                         deref = downloadHttpFile(path)

cwltool/process.py

@@ -392,7 +392,7 @@ def checkFormat(actualFile, inputFormats, ontology):
 def fillInDefaults(inputs, job):
     # type: (List[Dict[Text, Text]], Dict[Text, Union[Dict[Text, Any], List, Text]]) -> None
     for e, inp in enumerate(inputs):
-        with SourceLine(inputs, e, WorkflowException):
+        with SourceLine(inputs, e, WorkflowException, _logger.isEnabledFor(logging.DEBUG)):
             fieldname = shortname(inp[u"id"])
             if job.get(fieldname) is not None:
                 pass

cwltool/workflow.py

@@ -449,7 +449,7 @@ def job(self, joborder, output_callback, **kwargs):
             del kwargs["outdir"]
 
         for e, i in enumerate(self.tool["inputs"]):
-            with SourceLine(self.tool["inputs"], e, WorkflowException):
+            with SourceLine(self.tool["inputs"], e, WorkflowException, _logger.isEnabledFor(logging.DEBUG)):
                 iid = shortname(i["id"])
                 if iid in joborder:
                     self.state[i["id"]] = WorkflowStateItem(i, copy.deepcopy(joborder[iid]), "success")

setup.py

@@ -53,7 +53,7 @@
           'ruamel.yaml >= 0.12.4, < 0.15',
           'rdflib >= 4.2.2, < 4.3.0',
           'shellescape >= 3.4.1, < 3.5',
-          'schema-salad >= 2.6, < 3',
+          'schema-salad >= 2.6.20170927145003, < 3',
           'typing >= 3.5.3',
           'six >= 1.8.0',
       ],