Fix ssh2 client sftp push/pull with absolute paths

Author: Pan

pssh/ssh2_client.py

@@ -396,11 +396,13 @@ def copy_file(self, local_file, remote_file, recurse=False,
         :raises: :py:class:`OSError` on local OS errors like permission denied
         """
         sftp = self._make_sftp() if sftp is None else sftp
+        # import ipdb; ipdb.set_trace()
         if os.path.isdir(local_file) and recurse:
             return self._copy_dir(local_file, remote_file, sftp)
         elif os.path.isdir(local_file) and not recurse:
             raise ValueError("Recurse must be true if local_file is a "
                              "directory.")
+        # import ipdb; ipdb.set_trace()
         destination = self._remote_paths_split(remote_file)
         if destination is not None:
             try:
@@ -433,7 +435,7 @@ def sftp_put(self, sftp, local_file, remote_file):
                 logger.error(msg, remote_file, ex)
                 raise SFTPIOError(msg, remote_file, ex)
 
-    def mkdir(self, sftp, directory, parent_path=None):
+    def mkdir(self, sftp, directory, _parent_path=None):
         """Make directory via SFTP channel.
 
         Parent paths in the directory are created if they do not exist.
@@ -445,21 +447,32 @@ def mkdir(self, sftp, directory, parent_path=None):
 
         Catches and logs at error level remote IOErrors on creating directory.
         """
+        # import ipdb; ipdb.set_trace()
+        # if _parent_path is None and directory.startswith('/'):
+        #     directory = self._eagain(sftp.realpath, directory)
         try:
-            _parent_path, sub_dirs = directory.split('/', 1)
+            _dir, sub_dirs = directory.split('/', 1)
         except ValueError:
-            _parent_path = directory.split('/', 1)[0]
+            _dir = directory.split('/', 1)[0]
             sub_dirs = None
-        _directory = _parent_path if parent_path is None else \
-            '/'.join([parent_path, _parent_path])
+        if not _dir and directory.startswith('/'):
+            try:
+                _dir, sub_dirs = sub_dirs.split(os.path.sep, 1)
+            except ValueError:
+                return True
+        if _parent_path is not None:
+            _dir = '/'.join((_parent_path, _dir))
         try:
-            self._eagain(sftp.stat, _directory)
+            self._eagain(sftp.stat, _dir)
         except SFTPHandleError:
-            self._mkdir(sftp, _directory)
+            self._mkdir(sftp, _dir)
         if sub_dirs is not None:
-            _parent_path = _parent_path if parent_path is None else \
-                           '/'.join([parent_path, _parent_path])
-            return self.mkdir(sftp, sub_dirs, parent_path=_parent_path)
+            # import ipdb; ipdb.set_trace()c
+            if directory.startswith('/'):
+                _dir = ''.join(('/', _dir))
+            # _parent_path = _dir if _parent_path is None else \
+            #                '/'.join([_parent_path, _dir])
+            return self.mkdir(sftp, sub_dirs, _parent_path=_dir)
 
     def _copy_dir(self, local_dir, remote_dir, sftp):
         """Call copy_file on every file in the specified directory, copying
@@ -496,6 +509,7 @@ def copy_remote_file(self, remote_file, local_file, recurse=False,
         :raises: :py:class:`IOError` on local file IO errors
         :raises: :py:class:`OSError` on local OS errors like permission denied
         """
+        # import ipdb; ipdb.set_trace()
         sftp = self._make_sftp() if sftp is None else sftp
         try:
             self._eagain(sftp.stat, remote_file)
@@ -585,6 +599,6 @@ def _remote_paths_split(self, file_path):
                  if _dir][:-1])
         except IndexError:
             return
-        if destination == '':
-            return
+        if file_path.startswith('/') or destination == '':
+            return '/' + destination
         return destination

tests/test_pssh_ssh2_client.py

@@ -319,7 +319,9 @@ def test_pssh_client_directory(self):
         test_file_data = 'test'
         local_test_path = 'directory_test'
         remote_test_path = 'directory_test_copied'
-        remote_test_path_abs = os.path.expanduser('~/' + remote_test_path)
+        dir_name = os.path.dirname(__file__)
+        remote_test_path_abs = os.sep.join(
+            (dir_name, remote_test_path))
         for path in [local_test_path, remote_test_path_abs]:
             try:
                 shutil.rmtree(path)
@@ -334,23 +336,25 @@ def test_pssh_client_directory(self):
             local_file_path = os.path.join(local_file_path_dir, 'foo' + str(i))
             remote_file_path = os.path.join(
                 remote_test_path, 'sub_dir1', 'sub_dir2', 'dir_foo' + str(i), 'foo' + str(i))
-            remote_file_paths.append(os.path.expanduser('~/' + remote_file_path))
+            remote_file_paths.append(
+                os.sep.join((os.path.dirname(__file__), remote_file_path)))
             test_file = open(local_file_path, 'w')
             test_file.write(test_file_data)
             test_file.close()
-        cmds = self.client.copy_file(local_test_path, remote_test_path, recurse=True)
+        cmds = self.client.copy_file(local_test_path, remote_test_path_abs, recurse=True)
         gevent.joinall(cmds, raise_error=True)
         for path in remote_file_paths:
             self.assertTrue(os.path.isfile(path))
         shutil.rmtree(local_test_path)
-        shutil.rmtree(remote_test_path_abs)
+        shutil.rmtree(os.sep.join((os.path.dirname(__file__), remote_test_path)))
 
     def test_pssh_client_copy_file_failure(self):
         """Test failure scenarios of file copy"""
         test_file_data = 'test'
         local_test_path = 'directory_test'
         remote_test_path = 'directory_test_copied'
-        remote_test_path_abs = os.path.expanduser('~/' + remote_test_path)
+        dir_name = os.path.dirname(__file__)
+        remote_test_path_abs = os.sep.join((dir_name, remote_test_path))
         for path in [local_test_path, remote_test_path_abs]:
             mask = int('0700') if sys.version_info <= (2,) else 0o700
             if os.path.isdir(path):
@@ -367,32 +371,32 @@ def test_pssh_client_copy_file_failure(self):
         os.mkdir(remote_test_path_abs)
         local_file_path = os.path.join(local_test_path, 'test_file')
         remote_file_path = os.path.join(remote_test_path, 'test_file')
-        remote_file_path_abs = os.path.expanduser('~/' + remote_file_path)
+        remote_test_path_abs = os.sep.join((dir_name, remote_test_path))
         test_file = open(local_file_path, 'w')
         test_file.write('testing\n')
         test_file.close()
         # Permission errors on writing into dir
         mask = int('0111') if sys.version_info <= (2,) else 0o111
         os.chmod(remote_test_path_abs, mask)
-        cmds = self.client.copy_file(local_test_path, remote_test_path, recurse=True)
+        cmds = self.client.copy_file(local_test_path, remote_test_path_abs, recurse=True)
         try:
             gevent.joinall(cmds, raise_error=True)
             raise Exception("Expected SFTPError exception")
         except SFTPError:
             pass
-        self.assertFalse(os.path.isfile(remote_file_path_abs))
+        self.assertFalse(os.path.isfile(remote_test_path_abs))
         # Create directory tree failure test
         local_file_path = os.path.join(local_test_path, 'test_file')
         remote_file_path = os.path.join(remote_test_path, 'test_dir', 'test_file')
-        remote_file_path_abs = os.path.expanduser('~/' + remote_file_path)
-        cmds = self.client.copy_file(local_file_path, remote_file_path, recurse=True)
+        remote_test_path_abs = os.sep.join((dir_name, remote_test_path))
+        cmds = self.client.copy_file(local_file_path, remote_test_path_abs, recurse=True)
         try:
             gevent.joinall(cmds, raise_error=True)
             raise Exception("Expected SFTPError exception on creating remote "
                             "directory")
         except SFTPError:
             pass
-        self.assertFalse(os.path.isfile(remote_file_path_abs))
+        self.assertFalse(os.path.isfile(remote_test_path_abs))
         mask = int('0600') if sys.version_info <= (2,) else 0o600
         os.chmod(remote_test_path_abs, mask)
         for path in [local_test_path, remote_test_path_abs]:
@@ -406,9 +410,10 @@ def test_pssh_copy_remote_file_failure(self):
     def test_pssh_copy_remote_file(self):
         """Test parallel copy file to local host"""
         test_file_data = 'test'
-        local_test_path = os.path.expanduser('~/' + 'directory_test_local_remote_copied')
+        dir_name = os.path.dirname(__file__)
+        local_test_path = os.sep.join((dir_name, 'directory_test_local_remote_copied'))
         remote_test_path = 'directory_test_remote_copy'
-        remote_test_path_abs = os.path.expanduser('~/' + remote_test_path)
+        remote_test_path_abs = os.sep.join((dir_name, remote_test_path))
         local_copied_dir = '_'.join([local_test_path, self.host])
         new_local_copied_dir = '.'.join([local_test_path, self.host])
         for path in [local_test_path, remote_test_path_abs, local_copied_dir,
@@ -434,9 +439,9 @@ def test_pssh_copy_remote_file(self):
             test_file = open(remote_file_path, 'w')
             test_file.write(test_file_data)
             test_file.close()
-        cmds = self.client.copy_remote_file(remote_test_path, local_test_path)
+        cmds = self.client.copy_remote_file(remote_test_path_abs, local_test_path)
         self.assertRaises(ValueError, gevent.joinall, cmds, raise_error=True)
-        cmds = self.client.copy_remote_file(remote_test_path, local_test_path,
+        cmds = self.client.copy_remote_file(remote_test_path_abs, local_test_path,
                                             recurse=True)
         gevent.joinall(cmds, raise_error=True)
         try:
@@ -447,7 +452,7 @@ def test_pssh_copy_remote_file(self):
             shutil.rmtree(remote_test_path_abs)
         finally:
             shutil.rmtree(local_copied_dir)
-        cmds = self.client.copy_remote_file(remote_test_path, local_test_path,
+        cmds = self.client.copy_remote_file(remote_test_path_abs, local_test_path,
                                             suffix_separator='.', recurse=True)
         gevent.joinall(cmds, raise_error=True)
         new_local_copied_dir = '.'.join([local_test_path, self.host])