ENH: Add sync option to savepkl, sync when saving result file

Author: effigies

nipype/pipeline/engine/utils.py

@@ -236,17 +236,17 @@ def save_resultfile(result, cwd, name, rebase=None):
 
     if result.outputs is None:
         logger.warning('Storing result file without outputs')
-        savepkl(resultsfile, result)
+        savepkl(resultsfile, result, sync=True)
         return
     try:
         output_names = result.outputs.copyable_trait_names()
     except AttributeError:
         logger.debug('Storing non-traited results, skipping rebase of paths')
-        savepkl(resultsfile, result)
+        savepkl(resultsfile, result, sync=True)
         return
 
     if not rebase:
-        savepkl(resultsfile, result)
+        savepkl(resultsfile, result, sync=True)
         return
 
     backup_traits = {}
@@ -262,7 +262,7 @@ def save_resultfile(result, cwd, name, rebase=None):
                     backup_traits[key] = old
                     val = rebase_path_traits(result.outputs.trait(key), old, cwd)
                     setattr(result.outputs, key, val)
-        savepkl(resultsfile, result)
+        savepkl(resultsfile, result, sync=True)
     finally:
         # Restore resolved paths from the outputs dict no matter what
         for key, val in list(backup_traits.items()):

nipype/utils/filemanip.py

@@ -789,19 +789,18 @@ def read_stream(stream, logger=None, encoding=None):
     return out.splitlines()
 
 
-def savepkl(filename, record, versioning=False):
-    with SoftFileLock('%s.lock' % filename):
-        with open(filename, 'wb') as fobj:
-            pkl_file = gzip.GzipFile(fileobj=fobj) if filename.endswith('.pklz') else fobj
-            if versioning:
-                from nipype import __version__ as version
-                metadata = json.dumps({'version': version})
-
-                pkl_file.write(metadata.encode('utf-8'))
-                pkl_file.write('\n'.encode('utf-8'))
-
-            pickle.dump(record, pkl_file)
-            # Pickle files need to be available immediately, so force a sync
+def savepkl(filename, record, versioning=False, sync=False):
+    with open(filename, 'wb') as fobj:
+        pkl_file = gzip.GzipFile(fileobj=fobj) if filename.endswith('.pklz') else fobj
+        if versioning:
+            from nipype import __version__ as version
+            metadata = json.dumps({'version': version})
+
+            pkl_file.write(metadata.encode('utf-8'))
+            pkl_file.write('\n'.encode('utf-8'))
+
+        pickle.dump(record, pkl_file)
+        if sync:
             fobj.flush()
             os.fsync(fobj.fileno())