Python 3.4-only update. Bump to 2.9.0

[x] partial migration to pytest for parametrized tests
[x] Fix no stdin in exec result
(cherry picked from commit 038ba13)

Subprocess is not singleton anymore: only lock property is shared. (#96)

`SingleLock` metaclass added. `Singleton` saved for historical reasons.
All Subprocess tests and SSHAuth tests ported to pytest.
(cherry picked from commit f120cae)
Signed-off-by: Alexey Stepanov <penguinolog@gmail.com>

Author: penguinolog

.gitignore

@@ -6,7 +6,7 @@
 .pytest_cache/*
 
 ### Generated code
-/exec_helpers/*.c
+/exec_helpers/**/*.c
 
 ### TortoiseGit template
 # Project-level settings

.travis.yml

@@ -3,28 +3,19 @@ language: python
 os: linux
 install:
 - &upgrade_python_toolset pip install --upgrade pip setuptools wheel
-- &install_test_deps pip install --upgrade pytest pytest-sugar
+- &install_test_deps pip install --upgrade mock pytest pytest-mock pytest-sugar
 - &install_deps pip install -r CI_REQUIREMENTS.txt
 - pip install --upgrade pytest-cov coveralls
 
 _python:
 - &python34
   name: "Python 3.4"
   python: 3.4
-- &python35
-  name: "Python 3.5"
-  python: 3.5
-- &python36
-  name: "Python 3.6"
-  python: 3.6
 - &python37
   name: "Python 3.7"
   python: 3.7
   dist: xenial
   sudo: true
-- &pypy3
-  name: "PyPy3"
-  python: pypy3.5
 
 _helpers:
 - &install_cython pip install --upgrade Cython
@@ -46,12 +37,12 @@ _helpers:
 
 - &static_analysis
   stage: Static analysis
-  <<: *python37
+  <<: *python34
   after_success: skip
 
 - &code_style_check
   stage: Code style check
-  <<: *python37
+  <<: *python34
   after_success: skip
 
 script:
@@ -62,7 +53,6 @@ after_success:
 - coveralls
 
 jobs:
-  fast_finish: true
   include:
   - <<: *static_analysis
     name: "PyLint"
@@ -80,6 +70,7 @@ jobs:
     script:
     - bandit -r exec_helpers
   - <<: *static_analysis
+    <<: *python37
     name: "MyPy"
     install:
     - *upgrade_python_toolset
@@ -103,6 +94,7 @@ jobs:
     script:
     - pydocstyle exec_helpers
   - <<: *code_style_check
+    <<: *python37
     name: "Black formatting"
     install:
     - *upgrade_python_toolset
@@ -112,29 +104,15 @@ jobs:
 
   - stage: test
     <<: *python34
-  - stage: test
-    <<: *python35
-  - stage: test
-    <<: *python36
-  - stage: test
-    <<: *python37
-  - stage: test
-    <<: *pypy3
 
   - <<: *test_cythonized
     <<: *python34
-  - <<: *test_cythonized
-    <<: *python35
-  - <<: *test_cythonized
-    <<: *python36
-  - <<: *test_cythonized
-    <<: *python37
 
   - stage: deploy
     # This prevents job from appearing in test plan unless commit is tagged:
     if: tag IS present
     # Run on pypy to build not cythonized wheel
-    <<: *pypy3
+    <<: *python34
     name: Build universal and cythonized bdist_wheel. Deploy bdist and sdist.
     services:
     - docker
@@ -143,7 +121,7 @@ jobs:
     script:
     - ./tools/run_docker.sh "exec_helpers"
     before_deploy:
-    - pip install -r build_requirements.txt
+    - pip install -r requirements.txt
     - *build_package
     deploy:
     - provider: pypi

README.rst

@@ -44,13 +44,12 @@ Pros:
 
 ::
 
-    Python 3.4
     Python 3.5
     Python 3.6
     Python 3.7
     PyPy3 3.5+
 
-.. note:: For Python 2.7 and PyPy please use versions 1.x.x
+.. note:: For Python 2.7 and PyPy please use versions 1.x.x. For python 3.5+ use versions 3.0+
 
 This package includes:
 

build_requirements.txt

@@ -1,4 +1,4 @@
 Cython; platform_python_implementation == "CPython"
-wheel
+wheel==0.31.1
 -r CI_REQUIREMENTS.txt
 -r requirements.txt

doc/source/ExecResult.rst

@@ -6,7 +6,7 @@ API: ExecResult
 .. py:module:: exec_helpers
 .. py:currentmodule:: exec_helpers
 
-.. py:class:: ExecResult(object)
+.. py:class:: ExecResult()
 
     Command execution result.
 
@@ -27,6 +27,7 @@ API: ExecResult
 
         ``threading.RLock``
         Lock object for thread-safe operation.
+
         .. versionadded:: 2.2.0
 
     .. py:attribute:: stderr_lock

doc/source/SSHClient.rst

@@ -325,7 +325,7 @@ API: SSHClient and SSHAuth.
         :rtype: ``bool``
 
 
-.. py:class:: SSHAuth(object)
+.. py:class:: SSHAuth()
 
     SSH credentials object.
 

doc/source/Subprocess.rst

@@ -16,6 +16,7 @@ API: Subprocess
         :type log_mask_re: typing.Optional[str]
 
         .. versionchanged:: 1.2.0 log_mask_re regex rule for masking cmd
+        .. versionchanged:: 2.9.0 Not singleton anymore. Only lock is shared between all instances.
 
     .. py:attribute:: log_mask_re
 

exec_helpers/__init__.py

@@ -25,11 +25,11 @@
     ExecHelperTimeoutError,
 )
 
-from .exec_result import ExecResult
 from .api import ExecHelper
+from .exec_result import ExecResult
 from .ssh_auth import SSHAuth
-from ._ssh_client_base import SshExecuteAsyncResult
 from .ssh_client import SSHClient
+from ._ssh_client_base import SshExecuteAsyncResult
 from .subprocess_runner import Subprocess, SubprocessExecuteAsyncResult  # nosec  # Expected
 
 __all__ = (
@@ -49,7 +49,7 @@
     "ExecResult",
 )
 
-__version__ = "2.2.0"
+__version__ = "2.9.0"
 __author__ = "Alexey Stepanov"
 __author_email__ = "penguinolog@gmail.com"
 __maintainers__ = {

exec_helpers/_ssh_client_base.py

@@ -614,12 +614,10 @@ def execute_async(
 
         return SshExecuteAsyncResult(chan, _stdin, stderr, stdout)
 
-    def _exec_command(
+    def _exec_command(  # type: ignore
         self,
         command: str,
-        interface: paramiko.channel.Channel,
-        stdout: typing.Optional[paramiko.ChannelFile],
-        stderr: typing.Optional[paramiko.ChannelFile],
+        async_result: SshExecuteAsyncResult,
         timeout: typing.Union[int, float, None],
         verbose: bool = False,
         log_mask_re: typing.Optional[str] = None,
@@ -629,12 +627,8 @@ def _exec_command(
 
         :param command: executed command (for logs)
         :type command: str
-        :param interface: interface to control execution
-        :type interface: paramiko.channel.Channel
-        :param stdout: source for STDOUT read
-        :type stdout: typing.Optional[paramiko.ChannelFile]
-        :param stderr: source for STDERR read
-        :type stderr: typing.Optional[paramiko.ChannelFile]
+        :param async_result: execute_async result
+        :type async_result: SshExecuteAsyncResult
         :param timeout: timeout before stop execution with TimeoutError
         :type timeout: typing.Union[int, float, None]
         :param verbose: produce log.info records for STDOUT/STDERR
@@ -653,10 +647,10 @@ def _exec_command(
 
         def poll_streams() -> None:
             """Poll FIFO buffers if data available."""
-            if stdout and interface.recv_ready():
-                result.read_stdout(src=stdout, log=self.logger, verbose=verbose)
-            if stderr and interface.recv_stderr_ready():
-                result.read_stderr(src=stderr, log=self.logger, verbose=verbose)
+            if async_result.stdout and async_result.interface.recv_ready():
+                result.read_stdout(src=async_result.stdout, log=self.logger, verbose=verbose)
+            if async_result.stderr and async_result.interface.recv_stderr_ready():
+                result.read_stderr(src=async_result.stderr, log=self.logger, verbose=verbose)
 
         @threaded.threadpooled
         def poll_pipes(stop: threading.Event) -> None:
@@ -666,21 +660,21 @@ def poll_pipes(stop: threading.Event) -> None:
             """
             while not stop.is_set():
                 time.sleep(0.1)
-                if stdout or stderr:
+                if async_result.stdout or async_result.stderr:
                     poll_streams()
 
-                if interface.status_event.is_set():
-                    result.read_stdout(src=stdout, log=self.logger, verbose=verbose)
-                    result.read_stderr(src=stderr, log=self.logger, verbose=verbose)
-                    result.exit_code = interface.exit_status
+                if async_result.interface.status_event.is_set():
+                    result.read_stdout(src=async_result.stdout, log=self.logger, verbose=verbose)
+                    result.read_stderr(src=async_result.stderr, log=self.logger, verbose=verbose)
+                    result.exit_code = async_result.interface.exit_status
 
                     stop.set()
 
         # channel.status_event.wait(timeout)
         cmd_for_log = self._mask_command(cmd=command, log_mask_re=log_mask_re)
 
         # Store command with hidden data
-        result = exec_result.ExecResult(cmd=cmd_for_log)
+        result = exec_result.ExecResult(cmd=cmd_for_log, stdin=kwargs.get("stdin"))
 
         stop_event = threading.Event()
 
@@ -693,11 +687,11 @@ def poll_pipes(stop: threading.Event) -> None:
 
         # Process closed?
         if stop_event.is_set():
-            interface.close()
+            async_result.interface.close()
             return result
 
         stop_event.set()
-        interface.close()
+        async_result.interface.close()
         future.cancel()
 
         wait_err_msg = _log_templates.CMD_WAIT_ERROR.format(result=result, timeout=timeout)
@@ -774,9 +768,15 @@ def execute_through_host(
 
         channel.exec_command(command)  # nosec  # Sanitize on caller side
 
+        async_result = SshExecuteAsyncResult(interface=channel, stdin=None, stdout=stdout, stderr=stderr)
+
         # noinspection PyDictCreation
         result = self._exec_command(
-            command, channel, stdout, stderr, timeout, verbose=verbose, log_mask_re=kwargs.get("log_mask_re", None)
+            command,
+            async_result=async_result,
+            timeout=timeout,
+            verbose=verbose,
+            log_mask_re=kwargs.get("log_mask_re", None),
         )
 
         intermediate_channel.close()

exec_helpers/_subprocess_helpers.py

@@ -0,0 +1,58 @@
+#    Copyright 2018 Alexey Stepanov aka penguinolog.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""Python subprocess shared code."""
+
+__all__ = ("kill_proc_tree", "subprocess_kw")
+
+import platform
+
+# pylint: disable=unused-import
+import typing  # noqa: F401
+
+# pylint: enable=unused-import
+
+import psutil  # type: ignore
+
+
+# Adopt from:
+# https://stackoverflow.com/questions/1230669/subprocess-deleting-child-processes-in-windows
+def kill_proc_tree(pid: int, including_parent: bool = True) -> None:  # pragma: no cover
+    """Kill process tree.
+
+    :param pid: PID of parent process to kill
+    :type pid: int
+    :param including_parent: kill also parent process
+    :type including_parent: bool
+    """
+    parent = psutil.Process(pid)
+    children = parent.children(recursive=True)
+    for child in children:  # type: psutil.Process
+        child.kill()
+    _, alive = psutil.wait_procs(children, timeout=5)
+    for proc in alive:  # type: psutil.Process
+        proc.kill()  # 2nd shot
+    if including_parent:
+        parent.kill()
+        parent.wait(5)
+
+
+# Subprocess extra arguments.
+# Flags from:
+# https://stackoverflow.com/questions/13243807/popen-waiting-for-child-process-even-when-the-immediate-child-has-terminated
+subprocess_kw = {}  # type: typing.Dict[str, typing.Any]
+if "Windows" == platform.system():  # pragma: no cover
+    subprocess_kw["creationflags"] = 0x00000200
+else:  # pragma: no cover
+    subprocess_kw["start_new_session"] = True