ENH: Scaffolding to support mem-mapping of external data files for GIFTI images

Author: pauldmccarthy

nibabel/gifti/gifti.py

@@ -881,27 +881,41 @@ def to_file_map(self, file_map=None):
         f.write(self.to_xml())
 
     @classmethod
-    def from_file_map(klass, file_map, buffer_size=35000000):
-        """ Load a Gifti image from a file_map
+    def from_file_map(klass, file_map, buffer_size=35000000, mmap=True):
+        """Load a Gifti image from a file_map
 
         Parameters
         ----------
         file_map : dict
             Dictionary with single key ``image`` with associated value which is
             a :class:`FileHolder` instance pointing to the image file.
 
+        buffer_size: None or int, optional
+            size of read buffer. None uses default buffer_size
+            from xml.parsers.expat.
+
+        mmap : {True, False, 'c', 'r', 'r+'}
+            Controls the use of numpy memory mapping for reading data.  Only
+            has an effect when loading GIFTI images with data stored in
+            external files (``DataArray`` elements with an ``Encoding`` equal
+            to ``ExternalFileBinary``).  If ``False``, do not try numpy
+            ``memmap`` for data array.  If one of ``{'c', 'r', 'r+'}``, try
+            numpy ``memmap`` with ``mode=mmap``.  A `mmap` value of ``True``
+            gives the same behavior as ``mmap='c'``.  If the file cannot be
+            memory-mapped, ignore `mmap` value and read array from file.
+
         Returns
         -------
         img : GiftiImage
         """
-        parser = klass.parser(buffer_size=buffer_size)
+        parser = klass.parser(buffer_size=buffer_size, mmap=mmap)
         parser.parse(fptr=file_map['image'].get_prepare_fileobj('rb'))
         return parser.img
 
     @classmethod
-    def from_filename(klass, filename, buffer_size=35000000):
+    def from_filename(klass, filename, buffer_size=35000000, mmap=True):
         file_map = klass.filespec_to_file_map(filename)
-        img = klass.from_file_map(file_map, buffer_size=buffer_size)
+        img = klass.from_file_map(file_map, buffer_size=buffer_size, mmap=mmap)
         return img
 
 

nibabel/gifti/parse_gifti_fast.py

@@ -31,7 +31,7 @@ class GiftiParseError(ExpatError):
     """ Gifti-specific parsing error """
 
 
-def read_data_block(darray, fname, data):
+def read_data_block(darray, fname, data, mmap):
     """Parses data from a <Data> element, or loads from an external file.
 
     Parameters
@@ -46,10 +46,25 @@ def read_data_block(darray, fname, data):
     data : str or None
          Data to parse, or None if data is in an external file
 
+    mmap : {True, False, 'c', 'r', 'r+'}
+        Controls the use of numpy memory mapping for reading data.  Only has
+        an effect when loading GIFTI images with data stored in external files
+        (``DataArray`` elements with an ``Encoding`` equal to
+        ``ExternalFileBinary``).  If ``False``, do not try numpy ``memmap``
+        for data array.  If one of ``{'c', 'r', 'r+'}``, try numpy ``memmap``
+        with ``mode=mmap``.  A `mmap` value of ``True`` gives the same
+        behavior as ``mmap='c'``.  If the file cannot be memory-mapped, ignore
+        `mmap` value and read array from file.
+
     Returns
     -------
     numpy.ndarray containing the parsed data
     """
+    if mmap not in (True, False, 'c', 'r', 'r+'):
+        raise ValueError("mmap value should be one of True, False, 'c', "
+                         "'r', 'r+'")
+    if mmap is True:
+        mmap = 'c'
     enclabel = gifti_encoding_codes.label[darray.encoding]
     dtype = data_type_codes.type[darray.datatype]
 
@@ -114,13 +129,17 @@ def _str2int(in_str):
 
 class GiftiImageParser(XmlParser):
 
-    def __init__(self, encoding=None, buffer_size=35000000, verbose=0):
+    def __init__(self, encoding=None, buffer_size=35000000, verbose=0,
+                 mmap=True):
         super(GiftiImageParser, self).__init__(encoding=encoding,
                                                buffer_size=buffer_size,
                                                verbose=verbose)
         # output
         self.img = None
 
+        # Queried when loading data from <Data> elements - see read_data_block
+        self.mmap = mmap
+
         # finite state machine stack
         self.fsm_state = []
 
@@ -358,7 +377,8 @@ def flush_chardata(self):
             c.close()
 
         elif self.write_to == 'Data':
-            self.da.data = read_data_block(self.da, self.fname, data)
+            self.da.data = read_data_block(self.da, self.fname, data,
+                                           self.mmap)
             # update the endianness according to the
             # current machine setting
             self.endian = gifti_endian_codes.code[sys.byteorder]