Merge branch 'master' into enh/NewDipyInterfaces

Author: oesteban

.travis.yml

@@ -24,6 +24,7 @@ before_install:
   fi
 - if $INSTALL_DEB_DEPENDECIES; then source /etc/fsl/fsl.sh; fi
 - if $INSTALL_DEB_DEPENDECIES; then source /etc/afni/afni.sh; fi
+- export FSLOUTPUTTYPE=NIFTI_GZ
 install:
 - conda update --yes conda
 - conda create -n testenv --yes pip python=$TRAVIS_PYTHON_VERSION

CHANGES

@@ -3,6 +3,7 @@ Next release
 
 * ENH: New interfaces in dipy: RESTORE, EstimateResponseSH, CSD and StreamlineTractography
        (https://github.com/nipy/nipype/pull/1090)
+* ENH: Merge S3DataSink into DataSink, added AWS documentation (https://github.com/nipy/nipype/pull/1316)
 * TST: Cache APT in CircleCI (https://github.com/nipy/nipype/pull/1333)
 * ENH: Add new flags to the BRAINSABC for new features (https://github.com/nipy/nipype/pull/1322)
 * ENH: Provides a Nipype wrapper for ANTs DenoiseImage (https://github.com/nipy/nipype/pull/1291)

doc/users/aws.rst

@@ -0,0 +1,102 @@
+.. _aws:
+
+============================================
+Using Nipype with Amazon Web Services (AWS)
+============================================
+Several groups have been successfully using Nipype on AWS. This procedure
+involves setting a temporary cluster using StarCluster and potentially
+transferring files to/from S3. The latter is supported by Nipype through
+DataSink and S3DataGrabber.
+
+
+Using DataSink with S3
+======================
+The DataSink class now supports sending output data directly to an AWS S3
+bucket. It does this through the introduction of several input attributes to the
+DataSink interface and by parsing the `base_directory` attribute. This class
+uses the `boto3 <https://boto3.readthedocs.org/en/latest/>`_ and
+`botocore <https://botocore.readthedocs.org/en/latest/>`_ Python packages to
+interact with AWS. To configure the DataSink to write data to S3, the user must
+set the ``base_directory`` property to an S3-style filepath. For example:
+
+::
+
+	import nipype.interfaces.io as nio
+	ds = nio.DataSink()
+	ds.inputs.base_directory = 's3://mybucket/path/to/output/dir'
+
+With the "s3://" prefix in the path, the DataSink knows that the output
+directory to send files is on S3 in the bucket "mybucket". "path/to/output/dir"
+is the relative directory path within the bucket "mybucket" where output data
+will be uploaded to (NOTE: if the relative path specified contains folders that
+don’t exist in the bucket, the DataSink will create them). The DataSink treats
+the S3 base directory exactly as it would a local directory, maintaining support
+for containers, substitutions, subfolders, "." notation, etc to route output
+data appropriately.
+
+There are four new attributes introduced with S3-compatibility: ``creds_path``,
+``encrypt_bucket_keys``, ``local_copy``, and ``bucket``.
+
+::
+
+	ds.inputs.creds_path = '/home/user/aws_creds/credentials.csv'
+	ds.inputs.encrypt_bucket_keys = True
+	ds.local_copy = '/home/user/workflow_outputs/local_backup'
+
+``creds_path`` is a file path where the user's AWS credentials file (typically
+a csv) is stored. This credentials file should contain the AWS access key id and
+secret access key and should be formatted as one of the following (these formats
+are how Amazon provides the credentials file by default when first downloaded).
+
+Root-account user:
+
+::
+
+	AWSAccessKeyID=ABCDEFGHIJKLMNOP
+	AWSSecretKey=zyx123wvu456/ABC890+gHiJk
+
+IAM-user:
+
+::
+
+	User Name,Access Key Id,Secret Access Key
+	"username",ABCDEFGHIJKLMNOP,zyx123wvu456/ABC890+gHiJk
+
+The ``creds_path`` is necessary when writing files to a bucket that has
+restricted access (almost no buckets are publicly writable). If ``creds_path``
+is not specified, the DataSink will check the ``AWS_ACCESS_KEY_ID`` and
+``AWS_SECRET_ACCESS_KEY`` environment variables and use those values for bucket
+access.
+
+``encrypt_bucket_keys`` is a boolean flag that indicates whether to encrypt the
+output data on S3, using server-side AES-256 encryption. This is useful if the
+data being output is sensitive and one desires an extra layer of security on the
+data. By default, this is turned off.
+
+``local_copy`` is a string of the filepath where local copies of the output data
+are stored in addition to those sent to S3. This is useful if one wants to keep
+a backup version of the data stored on their local computer. By default, this is
+turned off.
+
+``bucket`` is a boto3 Bucket object that the user can use to overwrite the
+bucket specified in their ``base_directory``. This can be useful if one has to
+manually create a bucket instance on their own using special credentials (or
+using a mock server like `fakes3 <https://github.com/jubos/fake-s3>`_). This is
+typically used for developers unit-testing the DataSink class. Most users do not
+need to use this attribute for actual workflows. This is an optional argument.
+
+Finally, the user needs only to specify the input attributes for any incoming
+data to the node, and the outputs will be written to their S3 bucket.
+
+::
+
+	workflow.connect(inputnode, 'subject_id', ds, 'container')
+	workflow.connect(realigner, 'realigned_files', ds, 'motion')
+
+So, for example, outputs for sub001’s realigned_file1.nii.gz will be in:
+s3://mybucket/path/to/output/dir/sub001/motion/realigned_file1.nii.gz
+
+
+Using S3DataGrabber
+======================
+Coming soon...

doc/users/index.rst

@@ -38,6 +38,7 @@
    spmmcr
    mipav
    nipypecmd
+   aws
 
 
 

nipype/algorithms/misc.py

@@ -263,6 +263,14 @@ class TSNRInputSpec(BaseInterfaceInputSpec):
     in_file = InputMultiPath(File(exists=True), mandatory=True,
                              desc='realigned 4D file or a list of 3D files')
     regress_poly = traits.Range(low=1, desc='Remove polynomials')
+    tsnr_file = File('tsnr.nii.gz', usedefault=True, hash_files=False,
+                     desc='output tSNR file')
+    mean_file = File('mean.nii.gz', usedefault=True, hash_files=False,
+                     desc='output mean file')
+    stddev_file = File('stdev.nii.gz', usedefault=True, hash_files=False,
+                       desc='output tSNR file')
+    detrended_file = File('detrend.nii.gz', usedefault=True, hash_files=False,
+                          desc='input file after detrending')
 
 
 class TSNROutputSpec(TraitedSpec):
@@ -288,24 +296,18 @@ class TSNR(BaseInterface):
     input_spec = TSNRInputSpec
     output_spec = TSNROutputSpec
 
-    def _gen_output_file_name(self, suffix=None):
-        _, base, ext = split_filename(self.inputs.in_file[0])
-        if suffix in ['mean', 'stddev']:
-            return os.path.abspath(base + "_tsnr_" + suffix + ext)
-        elif suffix in ['detrended']:
-            return os.path.abspath(base + "_" + suffix + ext)
-        else:
-            return os.path.abspath(base + "_tsnr" + ext)
-
     def _run_interface(self, runtime):
         img = nb.load(self.inputs.in_file[0])
         header = img.header.copy()
         vollist = [nb.load(filename) for filename in self.inputs.in_file]
         data = np.concatenate([vol.get_data().reshape(
-            vol.shape[:3] + (-1,)) for vol in vollist], axis=3)
+            vol.get_shape()[:3] + (-1,)) for vol in vollist], axis=3)
+        data = data.nan_to_num()
+
         if data.dtype.kind == 'i':
             header.set_data_dtype(np.float32)
             data = data.astype(np.float32)
+
         if isdefined(self.inputs.regress_poly):
             timepoints = img.shape[-1]
             X = np.ones((timepoints, 1))
@@ -318,26 +320,28 @@ def _run_interface(self, runtime):
                                              betas[1:, :, :, :], 0, 3)),
                                   0, 4)
             data = data - datahat
-            img = nb.Nifti1Image(data, img.affine, header)
-            nb.save(img, self._gen_output_file_name('detrended'))
+            img = nb.Nifti1Image(data, img.get_affine(), header)
+            nb.save(img, op.abspath(self.inputs.detrended_file))
+
         meanimg = np.mean(data, axis=3)
         stddevimg = np.std(data, axis=3)
-        tsnr = meanimg / stddevimg
-        img = nb.Nifti1Image(tsnr, img.affine, header)
-        nb.save(img, self._gen_output_file_name())
-        img = nb.Nifti1Image(meanimg, img.affine, header)
-        nb.save(img, self._gen_output_file_name('mean'))
-        img = nb.Nifti1Image(stddevimg, img.affine, header)
-        nb.save(img, self._gen_output_file_name('stddev'))
+        tsnr = np.zeros_like(meanimg)
+        tsnr[stddevimg > 1.e-3] = meanimg[stddevimg > 1.e-3] / stddevimg[stddevimg > 1.e-3]
+        img = nb.Nifti1Image(tsnr, img.get_affine(), header)
+        nb.save(img, op.abspath(self.inputs.tsnr_file))
+        img = nb.Nifti1Image(meanimg, img.get_affine(), header)
+        nb.save(img, op.abspath(self.inputs.mean_file))
+        img = nb.Nifti1Image(stddevimg, img.get_affine(), header)
+        nb.save(img, op.abspath(self.inputs.stddev_file))
         return runtime
 
     def _list_outputs(self):
         outputs = self._outputs().get()
-        outputs['tsnr_file'] = self._gen_output_file_name()
-        outputs['mean_file'] = self._gen_output_file_name('mean')
-        outputs['stddev_file'] = self._gen_output_file_name('stddev')
+        for k in ['tsnr_file', 'mean_file', 'stddev_file']:
+            outputs[k] = op.abspath(getattr(self.inputs, k))
+
         if isdefined(self.inputs.regress_poly):
-            outputs['detrended_file'] = self._gen_output_file_name('detrended')
+            outputs['detrended_file'] = op.abspath(self.inputs.detrended_file)
         return outputs
 
 

nipype/algorithms/tests/test_auto_TSNR.py

@@ -4,12 +4,24 @@
 
 
 def test_TSNR_inputs():
-    input_map = dict(ignore_exception=dict(nohash=True,
+    input_map = dict(detrended_file=dict(hash_files=False,
+    usedefault=True,
+    ),
+    ignore_exception=dict(nohash=True,
     usedefault=True,
     ),
     in_file=dict(mandatory=True,
     ),
+    mean_file=dict(hash_files=False,
+    usedefault=True,
+    ),
     regress_poly=dict(),
+    stddev_file=dict(hash_files=False,
+    usedefault=True,
+    ),
+    tsnr_file=dict(hash_files=False,
+    usedefault=True,
+    ),
     )
     inputs = TSNR.input_spec()
 

nipype/interfaces/fsl/tests/test_auto_Smooth.py

@@ -9,9 +9,10 @@ def test_Smooth_inputs():
     environ=dict(nohash=True,
     usedefault=True,
     ),
-    fwhm=dict(argstr='-kernel gauss %f -fmean',
+    fwhm=dict(argstr='-kernel gauss %.03f -fmean',
     mandatory=True,
     position=1,
+    xor=['sigma'],
     ),
     ignore_exception=dict(nohash=True,
     usedefault=True,
@@ -21,9 +22,15 @@ def test_Smooth_inputs():
     position=0,
     ),
     output_type=dict(),
+    sigma=dict(argstr='-kernel gauss %.03f -fmean',
+    mandatory=True,
+    position=1,
+    xor=['fwhm'],
+    ),
     smoothed_file=dict(argstr='%s',
-    genfile=True,
     hash_files=False,
+    name_source=['in_file'],
+    name_template='%s_smooth',
     position=2,
     ),
     terminal_output=dict(nohash=True,

nipype/interfaces/fsl/utils.py

@@ -139,38 +139,59 @@ def _gen_filename(self, name):
 
 class SmoothInputSpec(FSLCommandInputSpec):
     in_file = File(exists=True, argstr="%s", position=0, mandatory=True)
-    fwhm = traits.Float(argstr="-kernel gauss %f -fmean", position=1,
-                        mandatory=True)
+    sigma = traits.Float(
+        argstr="-kernel gauss %.03f -fmean", position=1, xor=['fwhm'], mandatory=True,
+        desc='gaussian kernel sigma in mm (not voxels)')
+    fwhm = traits.Float(
+        argstr="-kernel gauss %.03f -fmean", position=1, xor=['sigma'], mandatory=True,
+        desc='gaussian kernel fwhm, will be converted to sigma in mm (not voxels)')
     smoothed_file = File(
-        argstr="%s", position=2, genfile=True, hash_files=False)
+        argstr="%s", position=2, name_source=['in_file'], name_template='%s_smooth', hash_files=False)
 
 
 class SmoothOutputSpec(TraitedSpec):
     smoothed_file = File(exists=True)
 
 
 class Smooth(FSLCommand):
-    '''Use fslmaths to smooth the image
-    '''
+    """
+    Use fslmaths to smooth the image
+
+    Examples
+    --------
+
+    Setting the kernel width using sigma:
+
+    >>> sm = Smooth()
+    >>> sm.inputs.in_file = 'functional2.nii'
+    >>> sm.inputs.sigma = 8.0
+    >>> sm.cmdline #doctest: +ELLIPSIS
+    'fslmaths functional2.nii -kernel gauss 8.000 -fmean functional2_smooth.nii.gz'
+
+    Setting the kernel width using fwhm:
+
+    >>> sm = Smooth()
+    >>> sm.inputs.in_file = 'functional2.nii'
+    >>> sm.inputs.fwhm = 8.0
+    >>> sm.cmdline #doctest: +ELLIPSIS
+    'fslmaths functional2.nii -kernel gauss 3.397 -fmean functional2_smooth.nii.gz'
+
+    One of sigma or fwhm must be set:
+
+    >>> from nipype.interfaces.fsl import Smooth
+    >>> sm = Smooth()
+    >>> sm.inputs.in_file = 'functional2.nii'
+    >>> sm.cmdline #doctest: +ELLIPSIS
+    Traceback (most recent call last):
+     ...
+    ValueError: Smooth requires a value for one of the inputs ...
+
+    """
 
     input_spec = SmoothInputSpec
     output_spec = SmoothOutputSpec
     _cmd = 'fslmaths'
 
-    def _gen_filename(self, name):
-        if name == 'smoothed_file':
-            return self._list_outputs()['smoothed_file']
-        return None
-
-    def _list_outputs(self):
-        outputs = self._outputs().get()
-        outputs['smoothed_file'] = self.inputs.smoothed_file
-        if not isdefined(outputs['smoothed_file']):
-            outputs['smoothed_file'] = self._gen_fname(self.inputs.in_file,
-                                                       suffix='_smooth')
-        outputs['smoothed_file'] = os.path.abspath(outputs['smoothed_file'])
-        return outputs
-
     def _format_arg(self, name, trait_spec, value):
         if name == 'fwhm':
             sigma = float(value) / np.sqrt(8 * np.log(2))