Skip to content

Commit aa3e90b

Browse files
pkittenispkittenis
authored
Sftp mkdir fix (#211)
* Added SFTP mkdir test, copy and scp file with absolute remote path tests. * Re-wrote sftp mkdir implementation - resolves #197 * Updated changelog
1 parent c1ca671 commit aa3e90b

File tree

6 files changed

+356
-43
lines changed

6 files changed

+356
-43
lines changed

‎.circleci/config.yml‎

Lines changed: 141 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,141 @@
1+
version: 2.1
2+
3+
orbs:
4+
python: circleci/python@0.3.2
5+
6+
jobs:
7+
python_test:
8+
parameters:
9+
python_ver:
10+
type: string
11+
default: "3.6"
12+
docker:
13+
- image: circleci/python:<< parameters.python_ver >>
14+
steps:
15+
- checkout
16+
- python/load-cache:
17+
dependency-file: requirements_dev.txt
18+
key: depsv3-{{ .Branch }}.{{ arch }}-PY<< parameters.python_ver >>
19+
- run:
20+
name: Deps
21+
command: |
22+
sudo apt-get install openssh-server
23+
- python/save-cache:
24+
dependency-file: requirements_dev.txt
25+
key: depsv3-{{ .Branch }}.{{ arch }}-PY<< parameters.python_ver >>
26+
- run:
27+
command: |
28+
pip install -U -r requirements_dev.txt
29+
name: Build
30+
- run:
31+
command: |
32+
eval "$(ssh-agent -s)"
33+
pytest --cov-append --cov=pssh tests/test_imports.py tests/test_output.py tests/test_utils.py
34+
pytest --reruns 5 --cov-append --cov=pssh tests/miko
35+
pytest --reruns 10 --cov-append --cov=pssh tests/native/test_tunnel.py tests/native/test_agent.py
36+
pytest --reruns 5 --cov-append --cov=pssh tests/native/test_*_client.py
37+
pytest --reruns 5 --cov-append --cov=pssh tests/ssh
38+
flake8 pssh
39+
cd doc; make html; cd ..
40+
# Test building from source distribution
41+
python setup.py sdist
42+
cd dist; pip install *; cd ..
43+
python setup.py check --restructuredtext
44+
name: Test
45+
- run:
46+
command: codecov
47+
name: Coverage
48+
49+
osx:
50+
parameters:
51+
xcode_ver:
52+
type: string
53+
default: "11.6.0"
54+
macos:
55+
xcode: << parameters.xcode_ver >>
56+
environment:
57+
HOMEBREW_NO_AUTO_UPDATE: 1
58+
steps:
59+
- checkout
60+
- run:
61+
name: deps
62+
command: |
63+
pip3 install twine
64+
which twine
65+
- run:
66+
name: Build Wheel
67+
command: |
68+
./ci/osx-wheel.sh
69+
- store_artifacts:
70+
path: wheels
71+
- run:
72+
name: Upload Wheel
73+
command: |
74+
twine upload --skip-existing -u $PYPI_U -p $PYPI_P wheels/*
75+
76+
manylinux:
77+
machine:
78+
image: ubuntu-1604:201903-01
79+
steps:
80+
- checkout
81+
- run:
82+
name: sdist
83+
command: python setup.py sdist
84+
- python/load-cache:
85+
key: manylinuxdepsv6-{{ .Branch }}.{{ arch }}
86+
dependency-file: requirements.txt
87+
- run:
88+
name: Deps
89+
command: |
90+
sudo apt-get install python-pip
91+
pip install -U pip
92+
pip install twine
93+
which twine
94+
- python/save-cache:
95+
key: manylinuxdepsv6-{{ .Branch }}.{{ arch }}
96+
dependency-file: requirements.txt
97+
- run:
98+
name: Build Wheels
99+
command: |
100+
if [[ -z "${CIRCLE_PULL_REQUEST}" ]]; then
101+
echo "$DOCKER_PASSWORD" | docker login -u="$DOCKER_USERNAME" --password-stdin;
102+
fi
103+
./ci/travis/build-manylinux.sh
104+
- run:
105+
name: PyPi Upload
106+
command: |
107+
twine upload --skip-existing -u $PYPI_USER -p $PYPI_PASSWORD dist/* wheelhouse/*
108+
109+
workflows:
110+
version: 2.1
111+
main:
112+
jobs:
113+
- python_test:
114+
matrix:
115+
parameters:
116+
python_ver:
117+
- "3.6"
118+
- "3.7"
119+
- "3.8"
120+
filters:
121+
tags:
122+
ignore: /.*/
123+
- manylinux:
124+
context: Docker
125+
filters:
126+
tags:
127+
only: /.*/
128+
branches:
129+
ignore: /.*/
130+
- osx:
131+
matrix:
132+
parameters:
133+
xcode_ver:
134+
- "11.6.0"
135+
- "11.1.0"
136+
context: Docker
137+
filters:
138+
tags:
139+
only: /.*/
140+
branches:
141+
ignore: /.*/

‎Changelog.rst‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,15 @@
11
Change Log
22
============
33

4+
1.12.2
5+
++++++
6+
7+
Fixes
8+
------
9+
10+
* `ParallelSSHClient.copy_file` with recurse enabled and absolute destination path would create empty directory in home directory of user - #197.
11+
* `ParallelSSHClient.copy_file` and `scp_recv` with recurse enabled would not create remote directories when copying empty local directories.
12+
413
1.12.1
514
++++++
615

‎pssh/clients/base/single.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -294,6 +294,7 @@ def mkdir(self, sftp, directory, _parent_path=None):
294294
def _copy_dir(self, local_dir, remote_dir, sftp):
295295
"""Call copy_file on every file in the specified directory, copying
296296
them to the specified remote directory."""
297+
self.mkdir(sftp, remote_dir)
297298
file_list = os.listdir(local_dir)
298299
for file_name in file_list:
299300
local_path = os.path.join(local_dir, file_name)

‎pssh/clients/native/single.py‎

Lines changed: 21 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717

1818
import logging
1919
import os
20+
from collections import deque
2021
from warnings import warn
2122

2223
from gevent import sleep, spawn
@@ -325,8 +326,7 @@ def _mkdir(self, sftp, directory):
325326
raise SFTPIOError(msg, directory, self.host, error)
326327
logger.debug("Created remote directory %s", directory)
327328

328-
def copy_file(self, local_file, remote_file, recurse=False,
329-
sftp=None, _dir=None):
329+
def copy_file(self, local_file, remote_file, recurse=False, sftp=None):
330330
"""Copy local file to host via SFTP.
331331
332332
:param local_file: Local filepath to copy to remote host
@@ -383,7 +383,7 @@ def sftp_put(self, sftp, local_file, remote_file):
383383
logger.error(msg, remote_file, ex)
384384
raise SFTPIOError(msg, remote_file, ex)
385385

386-
def mkdir(self, sftp, directory, _parent_path=None):
386+
def mkdir(self, sftp, directory):
387387
"""Make directory via SFTP channel.
388388
389389
Parent paths in the directory are created if they do not exist.
@@ -395,27 +395,20 @@ def mkdir(self, sftp, directory, _parent_path=None):
395395
396396
Catches and logs at error level remote IOErrors on creating directory.
397397
"""
398-
try:
399-
_dir, sub_dirs = directory.split('/', 1)
400-
except ValueError:
401-
_dir = directory.split('/', 1)[0]
402-
sub_dirs = None
403-
if not _dir and directory.startswith('/'):
398+
_paths_to_create = deque()
399+
for d in directory.split('/'):
400+
if not d:
401+
continue
402+
_paths_to_create.append(d)
403+
cwd = '' if directory.startswith('/') else '.'
404+
while _paths_to_create:
405+
cur_dir = _paths_to_create.popleft()
406+
cwd = '/'.join([cwd, cur_dir])
404407
try:
405-
_dir, sub_dirs = sub_dirs.split(os.path.sep, 1)
406-
except ValueError:
407-
return True
408-
if _parent_path is not None:
409-
_dir = '/'.join((_parent_path, _dir))
410-
try:
411-
self._eagain(sftp.stat, _dir)
412-
except (SFTPHandleError, SFTPProtocolError) as ex:
413-
logger.debug("Stat for %s failed with %s", _dir, ex)
414-
self._mkdir(sftp, _dir)
415-
if sub_dirs is not None:
416-
if directory.startswith('/'):
417-
_dir = ''.join(('/', _dir))
418-
return self.mkdir(sftp, sub_dirs, _parent_path=_dir)
408+
self._eagain(sftp.stat, cwd)
409+
except (SFTPHandleError, SFTPProtocolError) as ex:
410+
logger.debug("Stat for %s failed with %s", cwd, ex)
411+
self._mkdir(sftp, cwd)
419412

420413
def copy_remote_file(self, remote_file, local_file, recurse=False,
421414
sftp=None, encoding='utf-8'):
@@ -470,7 +463,7 @@ def scp_recv(self, remote_file, local_file, recurse=False, sftp=None,
470463
encoding='utf-8'):
471464
"""Copy remote file to local host via SCP.
472465
473-
Note - Remote directory listings are gather via SFTP when
466+
Note - Remote directory listings are gathered via SFTP when
474467
``recurse`` is enabled - SCP lacks directory list support.
475468
Enabling recursion therefore involves creating an extra SFTP channel
476469
and requires SFTP support on the server.
@@ -504,6 +497,10 @@ def scp_recv(self, remote_file, local_file, recurse=False, sftp=None,
504497
except SFTPError:
505498
pass
506499
else:
500+
try:
501+
os.makedirs(local_file)
502+
except OSError:
503+
pass
507504
file_list = self._sftp_readdir(dir_h)
508505
return self._scp_recv_dir(file_list, remote_file,
509506
local_file, sftp,

0 commit comments

Comments
 (0)