Subprocess is not singleton anymore: only lock property is shared. (#96)

`SingleLock` metaclass added. `Singleton` saved for historical reasons.
All Subprocess tests and SSHAuth tests ported to pytest.

Author: penguinolog

doc/source/Subprocess.rst

@@ -16,6 +16,7 @@ API: Subprocess
         :type log_mask_re: typing.Optional[str]
 
         .. versionchanged:: 1.2.0 log_mask_re regex rule for masking cmd
+        .. versionchanged:: 3.1.0 Not singleton anymore. Only lock is shared between all instances.
 
     .. py:attribute:: log_mask_re
 

exec_helpers/__init__.py

@@ -51,7 +51,7 @@
     "async_api",
 )
 
-__version__ = "3.0.0"
+__version__ = "3.1.0"
 __author__ = "Alexey Stepanov"
 __author_email__ = "penguinolog@gmail.com"
 __maintainers__ = {

exec_helpers/async_api/subprocess_runner.py

@@ -63,7 +63,7 @@ def stdout(self) -> typing.Optional[asyncio.StreamReader]:  # type: ignore
         return super(SubprocessExecuteAsyncResult, self).stdout
 
 
-class Subprocess(api.ExecHelper, metaclass=metaclasses.SingletonMeta):
+class Subprocess(api.ExecHelper, metaclass=metaclasses.SingleLock):
     """Subprocess helper with timeouts and lock-free FIFO."""
 
     __slots__ = ()
@@ -76,6 +76,8 @@ def __init__(self, logger: logging.Logger = logger, log_mask_re: typing.Optional
         :param log_mask_re: regex lookup rule to mask command for logger.
                             all MATCHED groups will be replaced by '<*masked*>'
         :type log_mask_re: typing.Optional[str]
+
+        .. versionchanged:: 3.1.0 Not singleton anymore. Only lock is shared between all instances.
         """
         super(Subprocess, self).__init__(logger=logger, log_mask_re=log_mask_re)
 

exec_helpers/metaclasses.py

@@ -47,3 +47,34 @@ def __prepare__(  # pylint: disable=unused-argument
         .. versionadded:: 1.2.0
         """
         return collections.OrderedDict()
+
+
+class SingleLock(abc.ABCMeta):
+    """Metaclass for creating classes with single lock instance per class."""
+
+    def __init__(cls, name: str, bases: typing.Tuple[type, ...], namespace: typing.Dict[str, typing.Any]) -> None:
+        """Create lock object for class."""
+        super(SingleLock, cls).__init__(name, bases, namespace)
+        cls.__lock = threading.RLock()
+
+    def __new__(
+        mcs, name: str, bases: typing.Tuple[type, ...], namespace: typing.Dict[str, typing.Any], **kwargs: typing.Any
+    ) -> typing.Type:
+        """Create lock property for class instances."""
+        namespace["lock"] = property(fget=lambda self: self.__class__.lock)
+        return super().__new__(mcs, name, bases, namespace, **kwargs)  # type: ignore
+
+    @property
+    def lock(cls) -> threading.RLock:
+        """Lock property for class."""
+        return cls.__lock
+
+    @classmethod
+    def __prepare__(  # pylint: disable=unused-argument
+        mcs: typing.Type["SingleLock"], name: str, bases: typing.Iterable[typing.Type], **kwargs: typing.Any
+    ) -> collections.OrderedDict:
+        """Metaclass magic for object storage.
+
+        .. versionadded:: 1.2.0
+        """
+        return collections.OrderedDict()

exec_helpers/subprocess_runner.py

@@ -61,7 +61,7 @@ def stdout(self) -> typing.Optional[typing.IO]:  # type: ignore
         return super(SubprocessExecuteAsyncResult, self).stdout
 
 
-class Subprocess(api.ExecHelper, metaclass=metaclasses.SingletonMeta):
+class Subprocess(api.ExecHelper, metaclass=metaclasses.SingleLock):
     """Subprocess helper with timeouts and lock-free FIFO."""
 
     def __init__(self, log_mask_re: typing.Optional[str] = None) -> None:
@@ -74,6 +74,7 @@ def __init__(self, log_mask_re: typing.Optional[str] = None) -> None:
         :type log_mask_re: typing.Optional[str]
 
         .. versionchanged:: 1.2.0 log_mask_re regex rule for masking cmd
+        .. versionchanged:: 3.1.0 Not singleton anymore. Only lock is shared between all instances.
         """
         super(Subprocess, self).__init__(logger=logger, log_mask_re=log_mask_re)
 

test/async_api/test_subprocess.py

@@ -22,7 +22,6 @@
 import pytest
 
 import exec_helpers
-from exec_helpers import metaclasses
 from exec_helpers import _subprocess_helpers
 
 # All test coroutines will be treated as marked.
@@ -256,7 +255,6 @@ async def test_002_execute(create_subprocess_shell, logger, exec_result, run_par
 
 
 async def test_003_context_manager(monkeypatch, create_subprocess_shell, logger, exec_result, run_parameters) -> None:
-    metaclasses.SingletonMeta._instances.clear()  # prepare
     lock = asynctest.CoroutineMock()
     lock.attach_mock(asynctest.CoroutineMock("acquire"), "acquire")
     lock.attach_mock(mock.Mock("release"), "release")

test/async_api/test_subprocess_special.py

@@ -91,6 +91,16 @@ async def read_stream(stream: FakeFileStream):
     "stdin_error": dict(ec=(0xDEADBEEF,), stdout=(), stdin="Failed", write=OSError(), expect_exc=OSError),
     "stdin_close_closed": dict(stdout=(b" \n", b"2\n", b"3\n", b" \n"), stdin="Stdin", stdin_close=eshutdown_exc),
     "stdin_close_fail": dict(ec=(0xDEADBEEF,), stdout=(), stdin="Failed", stdin_close=OSError(), expect_exc=OSError),
+    "mask_global": dict(
+        command="USE='secret=secret_pass' do task",
+        init_log_mask_re=r"secret\s*=\s*([A-Z-a-z0-9_\-]+)",
+        masked_cmd="USE='secret=<*masked*>' do task",
+    ),
+    "mask_local": dict(
+        command="USE='secret=secret_pass' do task",
+        log_mask_re=r"secret\s*=\s*([A-Z-a-z0-9_\-]+)",
+        masked_cmd="USE='secret=<*masked*>' do task",
+    ),
 }
 
 
@@ -112,6 +122,8 @@ def pytest_generate_tests(metafunc):
                 "stdin_error",
                 "stdin_close_closed",
                 "stdin_close_fail",
+                "mask_global",
+                "mask_local",
             ],
             indirect=True,
         )
@@ -131,7 +143,7 @@ def exec_result(run_parameters):
         stdout_res = tuple([elem for elem in run_parameters["stdout"] if isinstance(elem, bytes)])
 
     return exec_helpers.async_api.ExecResult(
-        cmd=command,
+        cmd=run_parameters.get("masked_cmd", command),
         stdin=run_parameters.get("stdin", None),
         stdout=stdout_res,
         stderr=(),
@@ -190,16 +202,24 @@ def logger(mocker):
 
 
 async def test_special_cases(create_subprocess_shell, exec_result, logger, run_parameters) -> None:
-    runner = exec_helpers.async_api.Subprocess()
+    runner = exec_helpers.async_api.Subprocess(log_mask_re=run_parameters.get("init_log_mask_re", None))
     if "expect_exc" not in run_parameters:
         res = await runner.execute(
-            command, stdin=run_parameters.get("stdin", None), verbose=run_parameters.get("verbose", None)
+            command=run_parameters.get("command", command),
+            stdin=run_parameters.get("stdin", None),
+            verbose=run_parameters.get("verbose", None),
+            log_mask_re=run_parameters.get("log_mask_re", None),
         )
         level = logging.INFO if run_parameters.get("verbose", False) else logging.DEBUG
-        assert logger.mock_calls[0] == mock.call.log(level=level, msg=command_log)
-        assert logger.mock_calls[-1] == mock.call.log(
-            level=level, msg="Command {result.cmd!r} exit code: {result.exit_code!s}".format(result=res)
+
+        command_for_log = run_parameters.get("masked_cmd", command)
+        command_log = "Executing command:\n{!r}\n".format(command_for_log.rstrip())
+        result_log = "Command {command!r} exit code: {result.exit_code!s}".format(
+            command=command_for_log.rstrip(), result=res
         )
+
+        assert logger.mock_calls[0] == mock.call.log(level=level, msg=command_log)
+        assert logger.mock_calls[-1] == mock.call.log(level=level, msg=result_log)
         assert res == exec_result
     else:
         with pytest.raises(run_parameters["expect_exc"]):

test/test_ssh_client_init.py

@@ -53,7 +53,6 @@ def __iter__(self):
 port = 22
 username = "user"
 password = "pass"
-private_keys = []
 
 
 # noinspection PyTypeChecker