Merge branch 'main' into cxx-compat

Author: Christopher Doris

docs/src/releasenotes.md

@@ -1,5 +1,14 @@
 # Release Notes
 
+## Unreleased
+* Experimental new function `juliacall.interactive()` allows the Julia async event loop to
+  run in the background of the Python REPL.
+* Experimental new IPython extension `juliacall.ipython` providing the `%jl` and `%%jl`
+  magics for executing Julia code.
+* Experimental new module `juliacall.importer` allowing you to write Python modules in
+  Julia.
+* Bug fixes.
+
 ## 0.9.4 (2022-07-26)
 * Bug fixes.
 

pysrc/juliacall/__init__.py

@@ -21,6 +21,13 @@ def convert(T, x):
         _convert = PythonCall.seval("pyjlcallback((T,x)->pyjl(pyconvert(pyjlvalue(T)::Type,x)))")
     return _convert(T, x)
 
+def interactive(enable=True):
+    "Allow the Julia event loop to run in the background of the Python REPL."
+    if enable:
+        PythonCall._set_python_input_hook()
+    else:
+        PythonCall._unset_python_input_hook()
+
 class JuliaError(Exception):
     "An error arising in Julia code."
     def __init__(self, exception, backtrace=None):
@@ -122,6 +129,7 @@ def args_from_config():
     CONFIG['opt_sysimage'] = sysimg = path_option('sysimage', check_exists=True)[0]
     CONFIG['opt_threads'] = int_option('threads', accept_auto=True)[0]
     CONFIG['opt_warn_overwrite'] = choice('warn_overwrite', ['yes', 'no'])[0]
+    CONFIG['opt_handle_signals'] = 'no'
 
     # Stop if we already initialised
     if CONFIG['inited']:
@@ -178,6 +186,7 @@ def jlstr(x):
             return 'raw"' + x.replace('"', '\\"').replace('\\', '\\\\') + '"'
         script = '''
         try
+            Base.require(Main, :CompilerSupportLibraries_jll)
             import Pkg
             ENV["JULIA_PYTHONCALL_LIBPTR"] = {}
             ENV["JULIA_PYTHONCALL_EXE"] = {}

pysrc/juliacall/importer.py

@@ -0,0 +1,70 @@
+import base64
+import io
+import os
+import sys
+
+from . import newmodule, Base
+from importlib.machinery import ModuleSpec, SourceFileLoader
+
+class Finder:
+    def __init__(self, jlext='.jl', pyext='.py'):
+        self.jlext = jlext
+        self.pyext = pyext
+
+    def find_spec(self, fullname, path, target=None):
+        if path is None:
+            path = sys.path
+            if '.' in fullname:
+                return
+            name = fullname
+        else:
+            name = fullname.split('.')[-1]
+        for root in path:
+            jlfile = os.path.join(root, name + self.jlext)
+            if os.path.isfile(jlfile):
+                jlfile = os.path.realpath(jlfile)
+                pyfile = os.path.join(root, name + self.pyext)
+                gen_file(jlfile, pyfile)
+                return ModuleSpec(fullname, SourceFileLoader(fullname, pyfile), origin=jlfile)
+
+def install(**kw):
+    finder = Finder(**kw)
+    sys.meta_path.insert(0, finder)
+    return finder
+
+def uninstall(finder):
+    sys.meta_path.remove(finder)
+
+def gen_code(jl):
+    buf = io.StringIO()
+    pr = lambda x: print(x, file=buf)
+    jl2 = jl.replace('\\', '\\\\').replace("'", "\\'")
+    pr('# This file was automatically generated by juliacall.importer')
+    pr('import juliacall.importer')
+    pr('juliacall.importer.exec_module(__name__,')
+    pr("'''"+jl2+"''')")
+    return buf.getvalue()
+
+def gen_file(jl, py):
+    with open(jl, encoding='utf-8') as fp:
+        jlcode = fp.read()
+    pycode = gen_code(jlcode)
+    with open(py, 'w', encoding='utf-8') as fp:
+        fp.write(pycode)
+
+def exec_module(name, code):
+    pymod = sys.modules[name]
+    jlmod = newmodule(name)
+    jlmod.seval('begin\n' + code + '\nend')
+    delattr(pymod, 'juliacall')
+    setattr(pymod, '__jl_code__', code)
+    setattr(pymod, '__jl_module__', jlmod)
+    ks = [str(k) for k in Base.names(jlmod)]
+    ks = [k for k in ks if k != name]
+    if not ks:
+        ks = [str(k) for k in Base.names(jlmod, all=True)]
+        ks = [k for k in ks if not (k == name or k == 'include' or k == 'eval' or k.startswith('_') or '#' in k)]
+    setattr(pymod, '__all__', ks)
+    setattr(pymod, '__doc__', str(Base.Docs.doc(jlmod)))
+    for k in ks:
+        setattr(pymod, k, getattr(jlmod, k))

pysrc/juliacall/ipython.py

@@ -0,0 +1,27 @@
+from IPython.core.magic import Magics, magics_class, line_cell_magic
+from . import Main, Base, PythonCall
+
+@magics_class
+class JuliaMagics(Magics):
+    
+    @line_cell_magic
+    def julia(self, line, cell=None):
+        code = line if cell is None else cell
+        ans = Main.seval('begin\n' + code + '\nend')
+        Base.flush(Base.stdout)
+        Base.flush(Base.stderr)
+        if not code.strip().endswith(';'):
+            return ans
+
+def load_ipython_extension(ip):
+    # register magics
+    ip.register_magics(JuliaMagics(ip))
+    # redirect stdout/stderr
+    PythonCall.seval("""begin
+        const _redirected_stdout = redirect_stdout()
+        const _redirected_stderr = redirect_stderr()
+        const _py_stdout = PyIO(pyimport("sys" => "stdout"), buflen=1)
+        const _py_stderr = PyIO(pyimport("sys" => "stderr"), buflen=1)
+        const _redirect_stdout_task = @async write($_py_stdout, $_redirected_stdout)
+        const _redirect_stderr_task = @async write($_py_stderr, $_redirected_stderr)
+    end""")

pysrc/juliacall/juliapkg-dev.json

@@ -3,7 +3,7 @@
     "packages": {
         "PythonCall": {
             "uuid": "6099a3de-0909-46bc-b1f4-468b9a2dfc0d",
-            "version": "0.9.4",
+            "version": "=0.9.4",
             "path": "../..",
             "dev": true
         }

src/compat/gui.jl

@@ -165,3 +165,36 @@ function event_loop_on(g::Symbol; interval::Real = 0.04, fix::Bool = false)
     callback = new_event_loop_callback(string(g), Float64(interval))
     EVENT_LOOPS[g] = Timer(t -> callback(), 0; interval = interval)
 end
+
+function _python_input_hook()
+    try
+        @static if Sys.iswindows()
+            # on windows, we can call yield in a loop because _kbhit() lets us know
+            # when to stop
+            while true
+                yield()
+                if ccall(:_kbhit, Cint, ()) != 0
+                    break
+                end
+                sleep(0.01)
+            end
+        else
+            # on other platforms, if readline is enabled, the input hook is called
+            # repeatedly so the loop is not required
+            yield()
+        end
+    catch
+        return Cint(1)
+    end
+    return Cint(0)
+end
+
+function _set_python_input_hook()
+    C.PyOS_SetInputHook(@cfunction(_python_input_hook, Cint, ()))
+    return
+end
+
+function _unset_python_input_hook()
+    C.PyOS_SetInputHook(C_NULL)
+    return
+end

src/cpython/extras.jl

@@ -41,8 +41,17 @@ PyBuffer_Release(_b) = begin
     return
 end
 
+function PyOS_SetInputHook(hook::Ptr{Cvoid})
+    Base.unsafe_store!(POINTERS.PyOS_InputHookPtr, hook)
+    return
+end
+
+function PyOS_GetInputHook()
+    return Base.unsafe_load(POINTERS.PyOS_InputHookPtr)
+end
+
 function PyOS_RunInputHook()
-    hook = Base.unsafe_load(Ptr{Ptr{Cvoid}}(dlsym(CTX.lib_ptr, :PyOS_InputHook)))
+    hook = PyOS_GetInputHook()
     if hook == C_NULL
         return false
     else

src/cpython/pointers.jl

@@ -273,6 +273,7 @@ const CAPI_OBJECTS = Set([
     $([:($name :: Ptr{Cvoid} = C_NULL) for name in CAPI_FUNCS]...)
     $([:($name :: PyPtr = C_NULL) for name in CAPI_EXCEPTIONS]...)
     $([:($name :: PyPtr = C_NULL) for name in CAPI_OBJECTS]...)
+    PyOS_InputHookPtr :: Ptr{Ptr{Cvoid}} = C_NULL
     PyJuliaBase_Type :: PyPtr = C_NULL
     PyExc_JuliaError :: PyPtr = C_NULL
 end
@@ -290,6 +291,7 @@ const POINTERS = CAPIPointers()
     ]...)
     $([:(p.$name = Base.unsafe_load(Ptr{PyPtr}(dlsym(lib, $(QuoteNode(name)))))) for name in CAPI_EXCEPTIONS]...)
     $([:(p.$name = dlsym(lib, $(QuoteNode(name)))) for name in CAPI_OBJECTS]...)
+    p.PyOS_InputHookPtr = dlsym(CTX.lib_ptr, :PyOS_InputHook)
 end
 
 for (name, (argtypes, rettype)) in CAPI_FUNC_SIGS

src/jlwrap/callback.jl

@@ -10,14 +10,19 @@ function pyjlcallback_call(self, args_::Py, kwargs_::Py)
         args = pyconvert(Vector{Py}, args_)
         kwargs = pyconvert(Dict{Symbol,Py}, kwargs_)
         Py(self(args...; kwargs...))
-    elseif pylen(args_) > 0
+    elseif (nargs = pylen(args_)) > 0
         args = pyconvert(Vector{Py}, args_)
-        if length(args) == 1
+        @assert length(args) == nargs
+        if nargs == 1
             Py(self(args[1]))
-        elseif length(args) == 2
+        elseif nargs == 2
             Py(self(args[1], args[2]))
-        elseif length(args) == 3
+        elseif nargs == 3
             Py(self(args[1], args[2], args[3]))
+        elseif nargs == 4
+            Py(self(args[1], args[2], args[3], args[4]))
+        elseif nargs == 5
+            Py(self(args[1], args[2], args[3], args[4], args[5]))
         else
             Py(self(args...))
         end
@@ -53,7 +58,7 @@ end
 pyjlcallback(f) = pyjl(pyjlcallbacktype, f)
 
 """
-    pyfunc(f; name=nothing, qualname=name, doc=nothing, signature=nothing)
+    pyfunc(f; [name], [qualname], [doc], [signature])
 
 Wrap the callable `f` as an ordinary Python function.
 
@@ -63,10 +68,18 @@ The name, qualname, docstring or signature can optionally be set with `name`, `q
 Unlike `Py(f)` (or `pyjl(f)`), the arguments passed to `f` are always of type `Py`, i.e.
 they are never converted.
 """
-function pyfunc(f; name=nothing, qualname=name, doc=nothing, signature=nothing)
+function pyfunc(f; name=nothing, qualname=name, doc=nothing, signature=nothing, wrap=pywrapcallback)
     f2 = ispy(f) ? f : pyjlcallback(f)
-    f3 = pywrapcallback(f2)
-    pydel!(f2)
+    if wrap isa Pair
+        wrapargs, wrapfunc = wrap
+    else
+        wrapargs, wrapfunc = (), wrap
+    end
+    if wrapfunc isa AbstractString
+        f3 = pybuiltins.eval(wrapfunc, pydict())(f2, wrapargs...)
+    else
+        f3 = wrapfunc(f2, wrapargs...)
+    end
     f3.__name__ = name === nothing ? "<lambda>" : name
     f3.__qualname__ = name === nothing ? "<lambda>" : qualname
     if doc !== nothing