Merge pull request #1289 from pearsonlab/s3io-fix

FIX: S3io

Author: chrisgorgo

nipype/interfaces/io.py

@@ -475,7 +475,8 @@ def localtos3(self, paths):
 
 class S3DataGrabberInputSpec(DynamicTraitedSpec, BaseInterfaceInputSpec):
     anon = traits.Bool(False, usedefault=True,
-                       desc='Use anonymous connection to s3')
+                       desc='Use anonymous connection to s3.  If this is set to True, boto may print' +
+                            ' a urlopen error, but this does not prevent data from being downloaded.')
     region = traits.Str('us-east-1', usedefault=True,
                         desc='Region of s3 bucket')
     bucket = traits.Str(mandatory=True,
@@ -656,14 +657,17 @@ def _list_outputs(self):
         # Outputs are currently stored as locations on S3.
         # We must convert to the local location specified
         # and download the files.
-        for key in outputs:
-            if type(outputs[key]) == list:
-                paths = outputs[key]
-                for i in range(len(paths)):
-                    path = paths[i]
+        for key,val in outputs.iteritems():
+            #This will basically be either list-like or string-like:
+            #if it has the __iter__ attribute, it's list-like (list,
+            #tuple, numpy array) and we iterate through each of its
+            #values. If it doesn't, it's string-like (string,
+            #unicode), and we convert that value directly.
+            if hasattr(val,'__iter__'):
+                for i,path in enumerate(val):
                     outputs[key][i] = self.s3tolocal(path, bkt)
-            elif type(outputs[key]) == str:
-                outputs[key] = self.s3tolocal(outputs[key], bkt)
+            else:
+                outputs[key] = self.s3tolocal(val, bkt)
 
         return outputs
 

nipype/interfaces/tests/test_io.py

@@ -96,7 +96,8 @@ def test_selectfiles_valueerror():
 
 @skipif(noboto)
 def test_s3datagrabber_communication():
-    dg = nio.S3DataGrabber(infields=['subj_id', 'run_num'], outfields=['func', 'struct'])
+    dg = nio.S3DataGrabber(
+        infields=['subj_id', 'run_num'], outfields=['func', 'struct'])
     dg.inputs.anon = True
     dg.inputs.bucket = 'openfmri'
     dg.inputs.bucket_path = 'ds001/'
@@ -108,20 +109,20 @@ def test_s3datagrabber_communication():
                                     struct='%s/anatomy/highres001_brain.nii.gz')
     dg.inputs.subj_id = ['sub001', 'sub002']
     dg.inputs.run_num = ['run001', 'run003']
-    dg.inputs.template_args = dg.inputs.template_args = dict(
+    dg.inputs.template_args = dict(
         func=[['subj_id', 'run_num']], struct=[['subj_id']])
     res = dg.run()
     func_outfiles = res.outputs.func
     struct_outfiles = res.outputs.struct
 
     # check for all files
-    yield assert_true, '/sub001/BOLD/task001_run001/bold.nii.gz' in func_outfiles[0]
+    yield assert_true, os.path.join(dg.inputs.local_directory, '/sub001/BOLD/task001_run001/bold.nii.gz') in func_outfiles[0]
     yield assert_true, os.path.exists(func_outfiles[0])
-    yield assert_true, '/sub001/anatomy/highres001_brain.nii.gz' in struct_outfiles[0]
+    yield assert_true, os.path.join(dg.inputs.local_directory, '/sub001/anatomy/highres001_brain.nii.gz') in struct_outfiles[0]
     yield assert_true, os.path.exists(struct_outfiles[0])
-    yield assert_true, '/sub002/BOLD/task001_run003/bold.nii.gz' in func_outfiles[1]
+    yield assert_true, os.path.join(dg.inputs.local_directory, '/sub002/BOLD/task001_run003/bold.nii.gz') in func_outfiles[1]
     yield assert_true, os.path.exists(func_outfiles[1])
-    yield assert_true, '/sub002/anatomy/highres001_brain.nii.gz' in struct_outfiles[1]
+    yield assert_true, os.path.join(dg.inputs.local_directory, '/sub002/anatomy/highres001_brain.nii.gz') in struct_outfiles[1]
     yield assert_true, os.path.exists(struct_outfiles[1])
 
     shutil.rmtree(tempdir)
@@ -220,7 +221,8 @@ def test_s3datasink_substitutions():
     # run fakes3 server and set up bucket
     fakes3dir = op.expanduser('~/fakes3')
     try:
-        proc = Popen(['fakes3', '-r', fakes3dir, '-p', '4567'], stdout=open(os.devnull, 'wb'))
+        proc = Popen(
+            ['fakes3', '-r', fakes3dir, '-p', '4567'], stdout=open(os.devnull, 'wb'))
     except OSError as ose:
         if 'No such file or directory' in str(ose):
             return  # fakes3 not installed. OK!