RF: ArrayProxy.KEEP_FILE_OPEN default value changed to 'auto'. Opener

keep_open flag passed through to indexed gzip drop_handles flag. indexed_gzip
versions > 0.6 all supported.

Author: pauldmccarthy

nibabel/arrayproxy.py

@@ -52,7 +52,7 @@
 specifying the ``keep_file_open`` flag will result in a ``ValueError`` being
 raised.
 """
-KEEP_FILE_OPEN_DEFAULT = False
+KEEP_FILE_OPEN_DEFAULT = 'auto'
 
 
 class ArrayProxy(object):
@@ -186,10 +186,10 @@ def _should_keep_file_open(self, file_like, keep_file_open):
         The return value is derived from these rules:
 
           - If ``file_like`` is a file(-like) object, ``False`` is returned.
-            Otherwise, ``file_like`` is assumed to be a file name.
-          - if ``file_like`` ends with ``'gz'``, and the ``indexed_gzip``
-            library is available, ``True`` is returned.
-          - Otherwise, ``False`` is returned.
+            Otherwise, ``file_like`` is assumed to be a file name
+          - If ``keep_file_open`` is ``auto``, and ``indexed_gzip`` is
+            not available, ``False`` is returned.
+          - Otherwise, the value of ``keep_file_open`` is returned unchanged.
 
         Parameters
         ----------
@@ -203,23 +203,20 @@ def _should_keep_file_open(self, file_like, keep_file_open):
         -------
 
         The value of ``keep_file_open`` that will be used by this
-        ``ArrayProxy``.
+        ``ArrayProxy``, and passed through to ``ImageOpener`` instances.
         """
         if keep_file_open is None:
             keep_file_open = KEEP_FILE_OPEN_DEFAULT
-        # if keep_file_open is True/False, we do what the user wants us to do
-        if isinstance(keep_file_open, bool):
-            return keep_file_open
-        if keep_file_open != 'auto':
+        if keep_file_open not in ('auto', True, False):
             raise ValueError('keep_file_open should be one of {None, '
                              '\'auto\', True, False}')
-
         # file_like is a handle - keep_file_open is irrelevant
         if hasattr(file_like, 'read') and hasattr(file_like, 'seek'):
             return False
-        # Otherwise, if file_like is gzipped, and we have_indexed_gzip, we set
-        # keep_file_open to True, else we set it to False
-        return HAVE_INDEXED_GZIP and file_like.endswith('gz')
+        # don't have indexed_gzip - auto -> False
+        if keep_file_open == 'auto' and not HAVE_INDEXED_GZIP:
+            return False
+        return keep_file_open
 
     @property
     @deprecate_with_version('ArrayProxy.header deprecated', '2.2', '3.0')
@@ -263,12 +260,13 @@ def _get_fileobj(self):
             A newly created ``ImageOpener`` instance, or an existing one,
             which provides access to the file.
         """
-        if self._keep_file_open:
+        if bool(self._keep_file_open):
             if not hasattr(self, '_opener'):
-                self._opener = ImageOpener(self.file_like, keep_open=True)
+                self._opener = ImageOpener(
+                    self.file_like, keep_open=self._keep_file_open)
             yield self._opener
         else:
-            with ImageOpener(self.file_like, keep_open=False) as opener:
+            with ImageOpener(self.file_like) as opener:
                 yield opener
 
     def get_unscaled(self):

nibabel/openers.py

@@ -18,16 +18,28 @@
 
 # is indexed_gzip present and modern?
 try:
-    from indexed_gzip import SafeIndexedGzipFile, __version__ as version
+    import indexed_gzip as igzip
+    version = igzip.__version__
 
     HAVE_INDEXED_GZIP = True
 
+    # < 0.6 - no good
     if StrictVersion(version) < StrictVersion('0.6.0'):
         warnings.warn('indexed_gzip is present, but too old '
                       '(>= 0.6.0 required): {})'.format(version))
         HAVE_INDEXED_GZIP = False
-
-    del version
+    # < 0.7 - does not support drop_handles
+    elif StrictVersion(version) < StrictVersion('0.7.0'):
+        class IndexedGzipFile(igzip.SafeIndexedGzipFile):
+            def __init__(self, *args, **kwargs):
+                kwargs.pop('drop_handles', None)
+                super(IndexedGzipFile, self).__init__(*args, **kwargs)
+    # >= 0.8 SafeIndexedGzipFile renamed to IndexedGzipFile
+    elif StrictVersion(version) < StrictVersion('0.8.0'):
+        IndexedGzipFile = igzip.SafeIndexedGzipFile
+    else:
+        IndexedGzipFile = igzip.IndexedGzipFile
+    del igzip, version
 
 except ImportError:
     HAVE_INDEXED_GZIP = False
@@ -80,9 +92,12 @@ def readinto(self, buf):
 
 def _gzip_open(filename, mode='rb', compresslevel=9, keep_open=False):
 
-    # use indexed_gzip if possible for faster read access
-    if keep_open and mode == 'rb' and HAVE_INDEXED_GZIP:
-        gzip_file = SafeIndexedGzipFile(filename)
+    # use indexed_gzip if possible for faster read access.  If keep_open ==
+    # True, we tell IndexedGzipFile to keep the file handle open. Otherwise
+    # the IndexedGzipFile will close/open the file on each read.
+    if HAVE_INDEXED_GZIP and keep_open in (True, 'auto') and mode == 'rb':
+        gzip_file = IndexedGzipFile(
+            filename, drop_handles=keep_open == 'auto')
 
     # Fall-back to built-in GzipFile (wrapped with the BufferedGzipFile class
     # defined above)