Merge remote-tracking branch 'upstream/master' into MagicPro1994/master

Author: nicoddemus

CHANGELOG.rst

@@ -1,3 +1,9 @@
+1.6.3 (08-08-2019)
+------------------
+
+* `#102 <https://github.com/pytest-dev/execnet/pull/102>`__: Show paths in stack traces
+  generated by ``remote_exec()``.
+
 1.6.1 (2019-07-22)
 ------------------
 

execnet/gateway.py

@@ -7,7 +7,6 @@
 import os
 import inspect
 import types
-import linecache
 import textwrap
 import execnet
 from execnet.gateway_base import Message
@@ -111,22 +110,27 @@ def remote_exec(self, source, **kwargs):
             executing code.
         """
         call_name = None
+        file_name = None
         if isinstance(source, types.ModuleType):
-            linecache.updatecache(inspect.getsourcefile(source))
-            source = inspect.getsource(source)
+            file_name = inspect.getsourcefile(source)
+            if not file_name:
+                source = inspect.getsource(source)
+            else:
+                source = None
         elif isinstance(source, types.FunctionType):
             call_name = source.__name__
+            file_name = inspect.getsourcefile(source)
             source = _source_of_function(source)
         else:
             source = textwrap.dedent(str(source))
 
-        if call_name is None and kwargs:
+        if not call_name and kwargs:
             raise TypeError("can't pass kwargs to non-function remote_exec")
 
         channel = self.newchannel()
         self._send(Message.CHANNEL_EXEC,
                    channel.id,
-                   gateway_base.dumps_internal((source, call_name, kwargs)))
+                   gateway_base.dumps_internal((source, file_name, call_name, kwargs)))
         return channel
 
     def remote_init_threads(self, num=None):
@@ -186,7 +190,7 @@ def _source_of_function(function):
         args = inspect.getargspec(function)[0]
     else:
         args = sig.args
-    if args[0] != 'channel':
+    if not args or args[0] != 'channel':
         raise ValueError('expected first function argument to be `channel`')
 
     if gateway_base.ISPY3:
@@ -213,4 +217,5 @@ def _source_of_function(function):
             used_globals,
         )
 
-    return source
+    leading_ws = "\n" * (codeobj.co_firstlineno - 1)
+    return leading_ws + source

execnet/gateway_base.py

@@ -13,6 +13,7 @@
 """
 from __future__ import with_statement
 import sys
+import linecache
 import os
 import weakref
 import traceback
@@ -25,8 +26,9 @@
 ISPY3 = sys.version_info >= (3, 0)
 if ISPY3:
     from io import BytesIO
-    exec("def do_exec(co, loc): exec(co, loc)\n"
-         "def reraise(cls, val, tb): raise val\n")
+    exec("do_exec = exec")
+    def reraise(cls, val, tb):
+        raise val.with_traceback(tb)
     unicode = str
     _long_type = int
     from _thread import interrupt_main
@@ -1044,7 +1046,7 @@ def trace(msg):
 
     def executetask(self, item):
         try:
-            channel, (source, call_name, kwargs) = item
+            channel, (source, file_name, call_name, kwargs) = item
             if not ISPY3 and kwargs:
                 # some python2 versions do not accept unicode keyword params
                 # note: Unserializer generally turns py2-str to py3-str objects
@@ -1054,12 +1056,15 @@ def executetask(self, item):
                         name = name.encode('ascii')
                     newkwargs[name] = value
                 kwargs = newkwargs
+            if source is None:
+                assert file_name, file_name
+                source = "".join(linecache.updatecache(file_name))
             loc = {'channel': channel, '__name__': '__channelexec__'}
             self._trace("execution starts[%s]: %s" %
                         (channel.id, repr(source)[:50]))
             channel._executing = True
             try:
-                co = compile(source+'\n', '<remote exec>', 'exec')
+                co = compile(source+'\n', file_name or '<remote exec>', 'exec')
                 do_exec(co, loc)  # noqa
                 if call_name:
                     self._trace('calling %s(**%60r)' % (call_name, kwargs))

execnet/script/socketserver.py

@@ -63,7 +63,7 @@ def exec_from_one_connection(serversock):
     }
     source = eval(source)
     if source:
-        co = compile(source+'\n', source, 'exec')
+        co = compile(source+'\n', '<socket server>', 'exec')
         print_(progname, 'compiled source, executing')
         try:
             exec_(co, g)  # noqa

testing/conftest.py

@@ -186,8 +186,8 @@ def gw(request, execmodel, group):
 def execmodel(request):
     if request.param != "thread":
         pytest.importorskip(request.param)
-    if sys.platform == "win32":
-        pytest.xfail("eventlet/gevent do not work onwin32")
+    if request.param in ("eventlet", "gevent") and sys.platform == "win32":
+        pytest.xfail(request.param + " does not work on win32")
     return get_execmodel(request.param)
 
 

testing/test_basics.py

@@ -218,8 +218,6 @@ def test_stdouterrin_setnull(execmodel):
     cap = py.io.StdCaptureFD()
     gateway_base.init_popen_io(execmodel)
     os.write(1, "hello".encode('ascii'))
-    if os.name == "nt":
-        os.write(2, "world")
     os.read(0, 1)
     out, err = cap.reset()
     assert not out
@@ -326,7 +324,7 @@ def test_source_of_nested_function(self):
         def working(channel):
             pass
 
-        send_source = gateway._source_of_function(working)
+        send_source = gateway._source_of_function(working).lstrip("\r\n")
         expected = 'def working(channel):\n    pass\n'
         assert send_source == expected
 

testing/test_gateway.py

@@ -2,6 +2,8 @@
 mostly functional tests of gateways.
 """
 import os
+from textwrap import dedent
+
 import py
 import pytest
 import socket
@@ -15,6 +17,7 @@
 needs_osdup = py.test.mark.skipif("not hasattr(os, 'dup')")
 
 
+flakytest = pytest.mark.xfail(reason='on some systems this test fails due to timing problems')
 skip_win_pypy = pytest.mark.xfail(condition=hasattr(sys, 'pypy_version_info') and sys.platform.startswith('win'),
                                   reason='failing on Windows on PyPy (#63)')
 
@@ -76,6 +79,7 @@ def test_gateway_status_no_real_channel(self, gw):
         # closure of temporary channels
         assert numchan2 == numchan
 
+    @flakytest
     def test_gateway_status_busy(self, gw):
         numchannels = gw.remote_status().numchannels
         ch1 = gw.remote_exec("channel.send(1); channel.receive()")
@@ -110,6 +114,38 @@ def test_remote_exec_module(self, tmpdir, gw):
         name = channel.receive()
         assert name == 2
 
+    def test_remote_exec_module_with_traceback(self, gw, tmpdir, monkeypatch):
+        remotetest = tmpdir.join("remotetest.py")
+        remotetest.write(dedent("""
+            def run_me(channel=None):
+                raise ValueError('me')
+
+            if __name__ == '__channelexec__':
+                run_me()
+            """)
+        )
+
+        monkeypatch.syspath_prepend(tmpdir)
+        import remotetest
+
+        ch = gw.remote_exec(remotetest)
+        try:
+            ch.receive()
+        except execnet.gateway_base.RemoteError as e:
+            assert 'remotetest.py", line 3, in run_me' in str(e)
+            assert "ValueError: me" in str(e)
+        finally:
+            ch.close()
+
+        ch = gw.remote_exec(remotetest.run_me)
+        try:
+            ch.receive()
+        except execnet.gateway_base.RemoteError as e:
+            assert 'remotetest.py", line 3, in run_me' in str(e)
+            assert "ValueError: me" in str(e)
+        finally:
+            ch.close()
+
     def test_correct_setup_no_py(self, gw):
         channel = gw.remote_exec("""
             import sys
@@ -347,6 +383,7 @@ def test_threads_race_sending(self, makegateway):
         for ch in channels:
             ch.waitclose(TESTTIMEOUT)
 
+    @flakytest
     def test_status_with_threads(self, makegateway):
         gw = makegateway('popen')
         c1 = gw.remote_exec("channel.send(1) ; channel.receive()")
@@ -391,6 +428,7 @@ def test_popen_filetracing(self, testdir, monkeypatch, makegateway):
         gw.exit()
 
     @skip_win_pypy
+    @flakytest
     def test_popen_stderr_tracing(self, capfd, monkeypatch, makegateway):
         monkeypatch.setenv('EXECNET_DEBUG', "2")
         gw = makegateway("popen")

testing/test_rsync.py

@@ -232,7 +232,7 @@ def filter(self, x):
         assert len(dest.listdir()) == 1
         assert len(source.listdir()) == 1
 
-    @py.test.mark.skip_if('sys.version_info >= (3)')
+    @py.test.mark.skipif('sys.version_info >= (3,)')
     def test_2_to_3_bridge_can_send_binary_files(self, tmpdir, makegateway):
         python = _find_version('3')
         gw = makegateway('popen//python=%s' % python)

testing/test_xspec.py

@@ -125,6 +125,7 @@ def test_popen_default(self, makegateway):
         assert rinfo.version_info == sys.version_info
 
     @pytest.mark.skipif("not hasattr(os, 'nice')")
+    @pytest.mark.xfail(reason='fails due to timing problems on busy single-core VMs')
     def test_popen_nice(self, makegateway):
         gw = makegateway("popen")