Merge pull request #2919 from oesteban/enh/robuster-node-tear-up

ENH: Do not override caught exceptions with FileNotFoundError from unfinished hashfile

Author: satra

nipype/pipeline/engine/nodes.py

@@ -27,7 +27,7 @@
 from ...utils.filemanip import (md5, FileNotFoundError, ensure_list,
                                 simplify_list, copyfiles, fnames_presuffix,
                                 loadpkl, split_filename, load_json, makedirs,
-                                emptydirs, savepkl, to_str, indirectory)
+                                emptydirs, savepkl, to_str, indirectory, silentrm)
 
 from ...interfaces.base import (traits, InputMultiPath, CommandLine, Undefined,
                                 DynamicTraitedSpec, Bunch, InterfaceResult,
@@ -440,7 +440,6 @@ def run(self, updatehash=False):
             for outdatedhash in glob(op.join(self.output_dir(), '_0x*.json')):
                 os.remove(outdatedhash)
 
-
         # Hashfile while running
         hashfile_unfinished = op.join(
             outdir, '_0x%s_unfinished.json' % self._hashvalue)
@@ -474,7 +473,11 @@ def run(self, updatehash=False):
         except Exception:
             logger.warning('[Node] Error on "%s" (%s)', self.fullname, outdir)
             # Tear-up after error
-            os.remove(hashfile_unfinished)
+            if not silentrm(hashfile_unfinished):
+                logger.warning("""\
+Interface finished unexpectedly and the corresponding unfinished hashfile %s \
+does not exist. Another nipype instance may be running against the same work \
+directory. Please ensure no other concurrent workflows are racing""", hashfile_unfinished)
             raise
 
         # Tear-up after success

nipype/utils/filemanip.py

@@ -832,6 +832,27 @@ def emptydirs(path, noexist_ok=False):
     makedirs(path)
 
 
+def silentrm(filename):
+    """
+    Equivalent to ``rm -f``, returns ``False`` if the file did not
+    exist.
+
+    Parameters
+    ----------
+
+    filename : str
+        file to be deleted
+
+    """
+    try:
+        os.remove(filename)
+    except OSError as e:
+        if e.errno != errno.ENOENT:
+            raise
+        return False
+    return True
+
+
 def which(cmd, env=None, pathext=None):
     """
     Return the path to an executable which would be run if the given