Merged in mikael/io (pull request #4)

Mikael/io

Author: testmikaem

docs/source/howtocite.rst

@@ -10,7 +10,7 @@ Please cite mpi4py-fft using
         year = {{2019}},
         title = {{Fast parallel multidimensional FFT using advanced MPI}},
         journal = {{Journal of Parallel and Distributed Computing}},
-        volume = {{in press}}
+        doi = {10.1016/j.jpdc.2019.02.006}
     }
     @electronic{mpi4py-fft,
         author = {{Lisandro Dalcin and Mikael Mortensen}},

docs/source/io.rst

@@ -4,7 +4,7 @@ Storing datafiles
 mpi4py-fft works with regular Numpy arrays. However, since arrays in parallel
 can become very large, and the arrays live on multiple processors, we require
 parallel IO capabilities that goes beyond Numpys regular methods.
-In the :mod:`mpi4py_fft.utilities` module there are two helper classes for dumping
+In the :mod:`mpi4py_fft.io` module there are two helper classes for dumping
 dataarrays to either `HDF5 <https://www.hdf5.org>`_ or
 `NetCDF <https://www.unidata.ucar.edu/software/netcdf/>`_ format:
 
@@ -17,56 +17,74 @@ reads data in parallel. A simple example of usage is::
     from mpi4py import MPI
     import numpy as np
     from mpi4py_fft import PFFT, HDF5File, NCFile, newDistArray
-
     N = (128, 256, 512)
     T = PFFT(MPI.COMM_WORLD, N)
     u = newDistArray(T, forward_output=False)
     v = newDistArray(T, forward_output=False, val=2)
-    u[:] = np.random.random(N)
-
+    u[:] = np.random.random(u.shape)
+    # Store by first creating output files
     fields = {'u': [u], 'v': [v]}
-    f0 = HDF5File('h5test.h5', T)
-    f1 = NCFile('nctest.nc', T)
+    f0 = HDF5File('h5test.h5', mode='w')
+    f1 = NCFile('nctest.nc', mode='w')
     f0.write(0, fields)
     f1.write(0, fields)
     v[:] = 3
     f0.write(1, fields)
     f1.write(1, fields)
 
-Note that we are creating two datafiles ``h5test.h5`` and ``nctest.nc``,
+Note that we are here creating two datafiles ``h5test.h5`` and ``nctest.nc``,
 for storing in HDF5 or NetCDF4 formats respectively. Normally, one would be
 satisfied using only one format, so this is only for illustration. We store
-the fields ``u`` and ``v`` using method ``write`` on two different occasions,
-so the datafiles will contain two snapshots of each field ``u`` and ``v``.
+the fields ``u`` and ``v`` on three different occasions,
+so the datafiles will contain three snapshots of each field ``u`` and ``v``.
+
+Also note that an alternative and perhaps simpler approach is to just use
+the ``write`` method of each distributed array::
+
+    u.write('h5test.h5', 'u', step=2)
+    v.write('h5test.h5', 'v', step=2)
+    u.write('nctest.nc', 'u', step=2)
+    v.write('nctest.nc', 'v', step=2)
 
-The stored dataarrays can be retrieved later on::
+The two different approaches can be used on the same output files.
+
+The stored dataarrays can also be retrieved later on::
 
-    f0 = HDF5File('h5test.h5', T, mode='r')
-    f1 = NCFile('nctest.nc', T, mode='r')
     u0 = newDistArray(T, forward_output=False)
     u1 = newDistArray(T, forward_output=False)
-    f0.read(u0, 'u', 0)
-    f0.read(u1, 'u', 1)
-    f1.read(u0, 'u', 0)
-    f1.read(u1, 'u', 1)
+    u0.read('h5test.h5', 'u', 0)
+    u1.read('h5test.h5', 'u', 1)
+    # or alternatively for netcdf
+    #u0.read('nctest.nc', 'u', 0)
+    #u1.read('nctest.nc', 'u', 1)
 
 Note that one does not have to use the same number of processors when
 retrieving the data as when they were stored.
 
 It is also possible to store only parts of the, potentially large, arrays.
-Any chosen slice may be stored, using a *global* view of the arrays::
+Any chosen slice may be stored, using a *global* view of the arrays. It is
+possible to store both complete fields and slices in one single call by
+using the following appraoch::
 
-    f2 = HDF5File('variousfields.h5', T, mode='w')
+    f2 = HDF5File('variousfields.h5', mode='w')
     fields = {'u': [u,
                     (u, [slice(None), slice(None), 4]),
                     (u, [5, 5, slice(None)])],
               'v': [v,
                     (v, [slice(None), 6, slice(None)])]}
     f2.write(0, fields)
     f2.write(1, fields)
-    f2.write(2, fields)
 
-This will lead to an hdf5-file with groups::
+Alternatively, one can use the write method of each field with the ``global_slice``
+keyword argument::
+
+    u.write('variousfields.h5', 'u', 2)
+    u.write('variousfields.h5', 'u', 2, global_slice=[slice(None), slice(None), 4])
+    u.write('variousfields.h5', 'u', 2, global_slice=[5, 5, slice(None)])
+    v.write('variousfields.h5', 'v', 2)
+    v.write('variousfields.h5', 'v', 2, global_slice=[slice(None), 6, slice(None)])
+
+In the end this will lead to an hdf5-file with groups::
 
     variousfields.h5/
     ├─ u/
@@ -80,41 +98,49 @@ This will lead to an hdf5-file with groups::
     |  |     ├─ 0
     |  |     ├─ 1
     |  |     └─ 2
-    |  └─ 3D/
-    |     ├─ 0
-    |     ├─ 1
-    |     └─ 2
-    ├─ v/
-    |  ├─ 2D/
-    |  |  └─ slice_6_slice/
-    |  |     ├─ 0
-    |  |     ├─ 1
-    |  |     └─ 2
-    |  └─ 3D/
-    |     ├─ 0
-    |     ├─ 1
-    |     └─ 2
-    └─ mesh/
-       ├─ x0
-       ├─ x1
-       └─ x2
-
-Note that a mesh is stored along with all the data. This mesh can be given in
-two different ways when creating the datafiles:
+    |  ├─ 3D/
+    |  |   ├─ 0
+    |  |   ├─ 1
+    |  |   └─ 2
+    |  └─ mesh/
+    |      ├─ x0
+    |      ├─ x1
+    |      └─ x2
+    └─ v/
+       ├─ 2D/
+       |  └─ slice_6_slice/
+       |     ├─ 0
+       |     ├─ 1
+       |     └─ 2
+       ├─ 3D/
+       |  ├─ 0
+       |  ├─ 1
+       |  └─ 2
+       └─ mesh/
+          ├─ x0
+          ├─ x1
+          └─ x2
+
+Note that a mesh is stored along with each group of data. This mesh can be
+given in two different ways when creating the datafiles:
 
     1) A sequence of 2-tuples, where each 2-tuple contains the (origin, length)
        of the domain along its dimension. For example, a uniform mesh that
        originates from the origin, with lengths :math:`\pi, 2\pi, 3\pi`, can be
-       given as::
+       given when creating the output file as::
+
+        f0 = HDF5File('filename.h5', domain=((0, pi), (0, 2*np.pi), (0, 3*np.pi)))
+
+        or, using the write method of the distributed array:
 
-        f0 = HDF5File('filename.h5', T, domain=((0, pi), (0, 2*np.pi), (0, 3*np.pi)))
+        u.write('filename.h5', 'u', 0, domain=((0, pi), (0, 2*np.pi), (0, 3*np.pi)))
 
     2) A sequence of arrays giving the coordinates for each dimension. For example::
 
         d = (np.arange(N[0], dtype=np.float)*1*np.pi/N[0],
              np.arange(N[1], dtype=np.float)*2*np.pi/N[1],
              np.arange(N[2], dtype=np.float)*2*np.pi/N[2])
-        f0 = HDF5File('filename.h5', T, domain=d)
+        f0 = HDF5File('filename.h5', domain=d)
 
 With NetCDF4 the layout is somewhat different. For ``variousfields`` above,
 if we were using :class:`.NCFile` instead of :class:`.HDF5File`,
@@ -147,9 +173,9 @@ opened with `Visit <https://www.visitusers.org>`_.
 
 To view the HDF5-files we first need to generate some light-weight *xdmf*-files that can
 be understood by both Paraview and Visit. To generate such files, simply throw the
-module :mod:`.utilities.generate_xdmf` on the HDF5-files::
+module :mod:`.io.generate_xdmf` on the HDF5-files::
 
-    from mpi4py_fft.utilities import generate_xdmf
+    from mpi4py_fft.io import generate_xdmf
     generate_xdmf('variousfields.h5')
 
 This will create a number of xdmf-files, one for each group that contains 2D

docs/source/mpi4py_fft.utilities.rst renamed to docs/source/mpi4py_fft.io.rst

@@ -1,45 +1,45 @@
-mpi4py_fft.utilities package
+mpi4py_fft.io package
 =============================
 
 Submodules
 ----------
 
-mpi4py_fft.utilities.generate_xdmf module
+mpi4py_fft.io.generate_xdmf module
 -----------------------------------------
 
-.. automodule:: mpi4py_fft.utilities.generate_xdmf
+.. automodule:: mpi4py_fft.io.generate_xdmf
     :members:
     :undoc-members:
     :show-inheritance:
 
-mpi4py_fft.utilities.h5py_file module
+mpi4py_fft.io.h5py_file module
 -------------------------------------
 
-.. automodule:: mpi4py_fft.utilities.h5py_file
+.. automodule:: mpi4py_fft.io.h5py_file
     :members:
     :undoc-members:
     :show-inheritance:
 
-mpi4py_fft.utilities.nc_file module
+mpi4py_fft.io.nc_file module
 -----------------------------------
 
-.. automodule:: mpi4py_fft.utilities.nc_file
+.. automodule:: mpi4py_fft.io.nc_file
     :members:
     :undoc-members:
     :show-inheritance:
 
-mpi4py_fft.utilities.file_base module
+mpi4py_fft.io.file_base module
 -------------------------------------
 
-.. automodule:: mpi4py_fft.utilities.file_base
+.. automodule:: mpi4py_fft.io.file_base
     :members:
     :undoc-members:
     :show-inheritance:
 
 Module contents
 ---------------
 
-.. automodule:: mpi4py_fft.utilities
+.. automodule:: mpi4py_fft.io
     :members:
     :undoc-members:
     :show-inheritance:

docs/source/mpi4py_fft.rst

@@ -7,7 +7,7 @@ Subpackages
 .. toctree::
 
     mpi4py_fft.fftw
-    mpi4py_fft.utilities
+    mpi4py_fft.io
 
 
 Submodules

examples/darray.py

@@ -50,7 +50,7 @@
 z[:] = MPI.COMM_WORLD.Get_rank()
 g0 = z.get((0, slice(None), 0))
 z2 = z.redistribute(2)
-z = z2.redistribute(darray=z)
+z = z2.redistribute(out=z)
 g1 = z.get((0, slice(None), 0))
 assert np.all(g0 == g1)
 s0 = MPI.COMM_WORLD.reduce(np.linalg.norm(z)**2)
@@ -69,14 +69,14 @@
 if MPI.COMM_WORLD.Get_rank() == 0:
     assert abs(s0-s1) < 1e-12
 
-z1 = z0.redistribute(darray=z1)
-z0 = z1.redistribute(darray=z0)
+z1 = z0.redistribute(out=z1)
+z0 = z1.redistribute(out=z0)
 
 N = (6, 6, 6, 6, 6)
 m0 = DistArray(N, dtype=float, alignment=2)
 m0[:] = MPI.COMM_WORLD.Get_rank()
 m1 = m0.redistribute(4)
-m0 = m1.redistribute(darray=m0)
+m0 = m1.redistribute(out=m0)
 s0 = MPI.COMM_WORLD.reduce(np.linalg.norm(m0)**2)
 s1 = MPI.COMM_WORLD.reduce(np.linalg.norm(m1)**2)
 if MPI.COMM_WORLD.Get_rank() == 0:

examples/transforms.py

@@ -1,7 +1,7 @@
 import functools
 import numpy as np
 from mpi4py import MPI
-from mpi4py_fft import PFFT, DistArray
+from mpi4py_fft import PFFT, newDistArray
 from mpi4py_fft.fftw import dctn, idctn
 
 # Set global size of the computational box
@@ -17,16 +17,16 @@
 
 assert fft.axes == pfft.axes
 
-u = DistArray(pfft=fft, forward_output=False)
+u = newDistArray(fft, forward_output=False)
 u[:] = np.random.random(u.shape).astype(u.dtype)
 
-u_hat = DistArray(pfft=fft, forward_output=True)
+u_hat = newDistArray(fft, forward_output=True)
 u_hat = fft.forward(u, u_hat)
 uj = np.zeros_like(u)
 uj = fft.backward(u_hat, uj)
 assert np.allclose(uj, u)
 
-u_padded = DistArray(pfft=pfft, forward_output=False)
+u_padded = newDistArray(pfft, forward_output=False)
 uc = u_hat.copy()
 u_padded = pfft.backward(u_hat, u_padded)
 u_hat = pfft.forward(u_padded, u_hat)

mpi4py_fft/__init__.py

@@ -16,7 +16,7 @@
 For more information, see `documentation <https://mpi4py-fft.readthedocs.io>`_.
 
 """
-__version__ = '2.0.0'
+__version__ = '2.0.1'
 __author__ = 'Lisandro Dalcin and Mikael Mortensen'
 
 from .distarray import DistArray, newDistArray, Function