pre-commit and mypy fixes

Author: RonnyPfannschmidt

.pre-commit-config.yaml

@@ -25,11 +25,13 @@ repos:
     hooks:
     -   id: reorder-python-imports
         args: ['--application-directories=execnet', --py37-plus]
--   repo: local
+-   repo: https://github.com/PyCQA/doc8
+    rev: 'v1.1.1'
     hooks:
-    -   id: rst
-        name: rst
-        entry: rst-lint --encoding utf-8
-        files: ^(CHANGELOG.rst|HOWTORELEASE.rst|README.rst)$
-        language: python
-        additional_dependencies: [pygments, restructuredtext_lint]
+    -   id: doc8
+        args: ["--ignore", "D001"]
+
+-   repo: https://github.com/pre-commit/mirrors-mypy
+    rev: 'v0.991'
+    hooks:
+    -   id: mypy

CHANGELOG.rst

@@ -3,9 +3,11 @@
 
 * removed support for Python < 3.7
   * apply pyupgrade --py37-plus
+  * minimal mypy fixes and python2 support code drop
 * migrate packaging to hatch
 
 
+
 1.9.0 (2021-06-13)
 ------------------
 

README.rst

@@ -4,9 +4,6 @@ execnet: distributed Python deployment and communication
 Important
 ---------
 
-**execnet currently is in maintenance-only mode, mostly because it is still the backend
-of the pytest-xdist plugin. Do not use in new projects.**
-
 .. image:: https://img.shields.io/pypi/v/execnet.svg
     :target: https://pypi.org/project/execnet/
 

doc/example/remote1.py

@@ -1,4 +1,4 @@
 # content of a module remote1.py
 
 if __name__ == "__channelexec__":
-    channel.send("initialization complete")
+    channel.send("initialization complete")  # type: ignore[name-defined]

doc/example/remotecmd.py

@@ -10,5 +10,5 @@ def listdir(path):
 
 
 if __name__ == "__channelexec__":
-    for item in channel:
-        channel.send(eval(item))
+    for item in channel:  # type: ignore[name-defined]
+        channel.send(eval(item))  # type: ignore[name-defined]

doc/example/servefiles.py

@@ -9,4 +9,4 @@ def servefiles(channel):
 
 
 if __name__ == "__channelexec__":
-    servefiles(channel)
+    servefiles(channel)  # type: ignore[name-defined]

doc/example/taskserver.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import execnet
 
 group = execnet.Group()
@@ -24,7 +26,7 @@ def process_item(channel):
 
 # get a queue that gives us results
 q = mch.make_receive_queue(endmarker=-1)
-tasks = range(10)  # a list of tasks, here just integers
+tasks: list[int] | None = list(range(10))  # a list of tasks, here just integers
 terminated = 0
 while 1:
     channel, item = q.get()
@@ -37,11 +39,11 @@ def process_item(channel):
         continue
     if item != "ready":
         print(f"other side {channel.gateway.id} returned {item!r}")
-    if not tasks:
+    if not tasks and tasks is not None:
         print("no tasks remain, sending termination request to all")
         mch.send_each(None)
-        tasks = -1
-    if tasks and tasks != -1:
+        tasks = None
+    if tasks:
         task = tasks.pop()
         channel.send(task)
         print(f"sent task {task!r} to {channel.gateway.id}")

execnet/gateway.py

@@ -199,12 +199,8 @@ def _source_of_function(function):
     if not args or args[0] != "channel":
         raise ValueError("expected first function argument to be `channel`")
 
-    if gateway_base.ISPY3:
-        closure = function.__closure__
-        codeobj = function.__code__
-    else:
-        closure = function.func_closure
-        codeobj = function.func_code
+    closure = function.__closure__
+    codeobj = function.__code__
 
     if closure is not None:
         raise ValueError("functions with closures can't be passed")

execnet/gateway_base.py

@@ -11,52 +11,26 @@
     - Ronny Pfannschmidt
     - many others
 """
+from __future__ import annotations
+
 import os
 import struct
 import sys
 import traceback
 import weakref
+from io import BytesIO
+from typing import Callable
 
-# NOTE that we want to avoid try/except style importing
-# to avoid setting sys.exc_info() during import
-#
-
-ISPY3 = sys.version_info >= (3, 0)
-if ISPY3:
-    from io import BytesIO
-
-    exec("do_exec = exec")
-
-    def reraise(cls, val, tb):
-        raise val.with_traceback(tb)
-
-    unicode = str
-    _long_type = int
-    from _thread import interrupt_main
 
-    SUBPROCESS32 = False
-else:
-    from StringIO import StringIO as BytesIO
-
-    exec(
-        "def do_exec(co, loc): exec co in loc\n"
-        "def reraise(cls, val, tb): raise cls, val, tb\n"
-    )
-    bytes = str
-    _long_type = long
-    try:
-        from thread import interrupt_main
-    except ImportError:
-        interrupt_main = None
-    try:
-        import subprocess32  # NOQA
+def reraise(cls, val, tb):
+    raise val.with_traceback(tb)
 
-        SUBPROCESS32 = True
-    except ImportError:
-        SUBPROCESS32 = False
-        sys.exc_clear()
 
+unicode = str
+_long_type = int
+from _thread import interrupt_main
 
+SUBPROCESS32 = False
 # f = open("/tmp/execnet-%s" % os.getpid(), "w")
 # def log_extra(*msg):
 #     f.write(" ".join([str(x) for x in msg]) + "\n")
@@ -77,9 +51,9 @@ def get_execmodel(backend):
                 "_thread::start_new_thread",
             ],
             "threading": ["threading"],
-            "queue": ["queue" if ISPY3 else "Queue"],
+            "queue": ["queue"],
             "sleep": ["time::sleep"],
-            "subprocess": ["subprocess32" if SUBPROCESS32 else "subprocess"],
+            "subprocess": ["subprocess"],
             "socket": ["socket"],
             "_fdopen": ["os::fdopen"],
             "_lock": ["threading"],
@@ -414,7 +388,7 @@ def close_write(self):
 class Message:
     """encapsulates Messages and their wire protocol."""
 
-    _types = []
+    _types: list[Callable[[Message, BaseGateway], None]] = []
 
     def __init__(self, msgcode, channelid=0, data=""):
         self.msgcode = msgcode
@@ -554,6 +528,7 @@ class Channel:
 
     def __init__(self, gateway, id):
         assert isinstance(id, int)
+        assert not isinstance(gateway, type)
         self.gateway = gateway
         # XXX: defaults copied from Unserializer
         self._strconfig = getattr(gateway, "_strconfig", (True, False))
@@ -1062,21 +1037,12 @@ def trace(msg):
     def executetask(self, item):
         try:
             channel, (source, file_name, call_name, kwargs) = item
-            if not ISPY3 and kwargs:
-                # some python2 versions do not accept unicode keyword params
-                # note: Unserializer generally turns py2-str to py3-str objects
-                newkwargs = {}
-                for name, value in kwargs.items():
-                    if isinstance(name, unicode):
-                        name = name.encode("ascii")
-                    newkwargs[name] = value
-                kwargs = newkwargs
             loc = {"channel": channel, "__name__": "__channelexec__"}
             self._trace(f"execution starts[{channel.id}]: {repr(source)[:50]}")
             channel._executing = True
             try:
                 co = compile(source + "\n", file_name or "<remote exec>", "exec")
-                do_exec(co, loc)  # noqa
+                exec(co, loc)
                 if call_name:
                     self._trace("calling %s(**%60r)" % (call_name, kwargs))
                     function = loc[call_name]
@@ -1116,15 +1082,11 @@ class LoadError(DataFormatError):
     """Error while unserializing an object."""
 
 
-if ISPY3:
+def bchr(n):
+    return bytes([n])
 
-    def bchr(n):
-        return bytes([n])
 
-else:
-    bchr = chr
-
-DUMPFORMAT_VERSION = bchr(1)
+DUMPFORMAT_VERSION = bchr(2)
 
 FOUR_BYTE_INT_MAX = 2147483647
 
@@ -1139,7 +1101,9 @@ class _Stop(Exception):
 
 
 class Unserializer:
-    num2func = {}  # is filled after this class definition
+    num2func: dict[
+        int, Callable[[Unserializer], None]
+    ] = {}  # is filled after this class definition
     py2str_as_py3str = True  # True
     py3str_as_py2str = False  # false means py2 will get unicode
 
@@ -1193,18 +1157,8 @@ def load_longint(self):
         s = self._read_byte_string()
         self.stack.append(int(s))
 
-    if ISPY3:
-        load_long = load_int
-        load_longlong = load_longint
-    else:
-
-        def load_long(self):
-            i = self._read_int4()
-            self.stack.append(long(i))
-
-        def load_longlong(self):
-            l = self._read_byte_string()
-            self.stack.append(long(l))
+    load_long = load_int
+    load_longlong = load_longint
 
     def load_float(self):
         binary = self.stream.read(FLOAT_FORMAT_SIZE)
@@ -1224,15 +1178,15 @@ def _read_byte_string(self):
 
     def load_py3string(self):
         as_bytes = self._read_byte_string()
-        if not ISPY3 and self.py3str_as_py2str:
+        if self.py3str_as_py2str:
             # XXX Should we try to decode into latin-1?
             self.stack.append(as_bytes)
         else:
             self.stack.append(as_bytes.decode("utf-8"))
 
     def load_py2string(self):
         as_bytes = self._read_byte_string()
-        if ISPY3 and self.py2str_as_py3str:
+        if self.py2str_as_py3str:
             s = as_bytes.decode("latin-1")
         else:
             s = as_bytes
@@ -1366,7 +1320,7 @@ def dumps_internal(obj):
 
 
 class _Serializer:
-    _dispatch = {}
+    _dispatch = {object: Callable[["_Serializer", object], None]}
 
     def __init__(self, write=None):
         if write is None:
@@ -1413,21 +1367,9 @@ def save_bytes(self, bytes_):
         self._write(opcode.BYTES)
         self._write_byte_sequence(bytes_)
 
-    if ISPY3:
-
-        def save_str(self, s):
-            self._write(opcode.PY3STRING)
-            self._write_unicode_string(s)
-
-    else:
-
-        def save_str(self, s):
-            self._write(opcode.PY2STRING)
-            self._write_byte_sequence(s)
-
-        def save_unicode(self, s):
-            self._write(opcode.UNICODE)
-            self._write_unicode_string(s)
+    def save_str(self, s):
+        self._write(opcode.PY3STRING)
+        self._write_unicode_string(s)
 
     def _write_unicode_string(self, s):
         try:

execnet/gateway_io.py

@@ -10,7 +10,7 @@
 try:
     from execnet.gateway_base import Popen2IO, Message
 except ImportError:
-    from __main__ import Popen2IO, Message
+    from __main__ import Popen2IO, Message  # type: ignore[no-redef]
 
 from functools import partial
 
@@ -246,4 +246,4 @@ def controll(data):
 
 
 if __name__ == "__channelexec__":
-    serve_proxy_io(channel)  # noqa
+    serve_proxy_io(channel)  # type: ignore[name-defined]