implement juliacall dependencies

Christopher Doris · Christopher Doris · commit 6288f2ef2bc2 · 2021-09-09T19:03:39.000+01:00
diff --git a/juliacall/deps.py b/juliacall/deps.py
@@ -1,4 +1,13 @@
-from . import CONFIG
+import os
+import semantic_version as semver
+import sys
+
+from collections import namedtuple
+from time import time
+
+from . import CONFIG, __version__
+
+### META
 
 def load_meta():
     import json, os.path
@@ -35,33 +44,134 @@ def set_meta(*args):
     meta2[keys[-1]] = value
     save_meta(meta)
 
-def get_dep(package, name):
-    return get_meta('pydeps', package, name)
+### VERSION PARSING
 
-def set_dep(package, name, value):
-    set_meta('pydeps', package, name, value)
+@semver.base.BaseSpec.register_syntax
+class JuliaVersionSpec(semver.SimpleSpec):
+    SYNTAX = 'julia'
+    class Parser(semver.SimpleSpec.Parser):
+        PREFIX_ALIASES = {'=': '==', '': '^'}
+        @classmethod
+        def parse(cls, expression):
+            blocks = expression.split(',')
+            clause = semver.base.Never()
+            for block in blocks:
+                block = block.strip()
+                if not cls.NAIVE_SPEC.match(block):
+                    raise ValueError('Invalid simple block %r' % block)
+                clause |= cls.parse_block(block)
+            return clause
 
-def require(package, name, value, func=None):
-    if func is None:
-        return Require(package, name, value)
-    elif get_dep(package, name) != value:
-        func()
-        set_dep(package, name, value)
+### RESOLVE
 
-class Require:
-    def __init__(self, package, name, value):
-        self.package = package
+class PackageSpec:
+    def __init__(self, name, uuid, dev=False, compat=None, path=None, url=None, rev=None, version=None):
         self.name = name
-        self.value = value
-    def __enter__(self):
-        self.required = get_dep(self.package, self.name) != self.value
-        return self.required
-    def __exit__(self, t, v, b):
-        if t is None and self.required:
-            set_dep(self.package, self.name, self.value)
-
-def require_julia(package, name, version):
-    with require(package, name, version) as required:
-        if required:
-            from juliacall import Pkg
-            Pkg.add(name=name, version=version)
+        self.uuid = uuid
+        self.dev = dev
+        self.compat = compat
+        self.path = path
+        self.url = url
+        self.rev = rev
+        self.version = version
+
+    def jlstr(self):
+        args = ['name="{}"'.format(self.name), 'uuid="{}"'.format(self.uuid)]
+        if self.path is not None:
+            args.append('path=raw"{}"'.format(self.path))
+        if self.url is not None:
+            args.append('url=raw"{}"'.format(self.url))
+        if self.rev is not None:
+            args.append('rev=raw"{}"'.format(self.rev))
+        if self.version is not None:
+            args.append('version=raw"{}"'.format(self.version))
+        return "Pkg.PackageSpec({})".format(', '.join(args))
+
+    def dict(self):
+        ans = {
+            "name": self.name,
+            "uuid": self.uuid,
+            "dev": self.dev,
+            "compat": self.compat,
+            "path": self.path,
+            "url": self.url,
+            "rev": self.rev,
+            "version": self.version,
+        }
+        return {k:v for (k,v) in ans.items() if v is not None}
+
+def can_skip_resolve():
+    # resolve if we haven't resolved before
+    deps = get_meta("pydeps")
+    if deps is None:
+        return False
+    # resolve whenever the version changes
+    version = deps.get("version")
+    if version is None or version != __version__:
+        return False
+    # resolve whenever swapping between dev/not dev
+    isdev = deps.get("dev")
+    if isdev is None or isdev != CONFIG["dev"]:
+        return False
+    # resolve whenever anything in sys.path changes
+    timestamp = deps.get("timestamp")
+    if timestamp is None:
+        return False
+    sys_path = deps.get("sys_path")
+    if sys_path is None or sys_path != sys.path:
+        return False
+    for path in sys.path:
+        if not path:
+            path = os.getcwd()
+        if not os.path.exists(path):
+            continue
+        if os.path.getmtime(path) > timestamp:
+            return False
+        if os.path.isdir(path):
+            fn = os.path.join(path, "juliacalldeps.json")
+            if os.path.exists(fn) and os.path.getmtime(fn) > timestamp:
+                return False
+    return True
+
+def deps_files():
+    ans = []
+    for path in sys.path:
+        if not path:
+            path = os.getcwd()
+        if not os.path.isdir(path):
+            continue
+        fn = os.path.join(path, "juliacalldeps.json")
+        if os.path.isfile(fn):
+            ans.append(fn)
+        for subdir in os.listdir(path):
+            fn = os.path.join(path, subdir, "juliacalldeps.json")
+            if os.path.isfile(fn):
+                ans.append(fn)
+    return list(set(ans))
+
+def required_packages():
+    import json
+    ans = {}
+    for fn in deps_files():
+        with open(fn) as fp:
+            deps = json.load(fp)
+        for (name, kw) in deps.get("packages", {}).items():
+            if name in ans:
+                p = ans[name]
+                if p.uuid != kw["uuid"]:
+                    raise Exception("found multiple UUIDs for package '{}'".format(name))
+                # todo: dev, compat, path, url, rev, version
+                raise NotImplementedError("need to merge repeated dependency '{}'")
+            else:
+                p = PackageSpec(name=name, **kw)
+                ans[p.name] = p
+    return list(ans.values())
+
+def record_resolve(pkgs):
+    set_meta("pydeps", {
+        "version": __version__,
+        "dev": CONFIG["dev"],
+        "timestamp": time(),
+        "sys_path": sys.path,
+        "pkgs": [pkg.dict() for pkg in pkgs],
+    })
diff --git a/juliacall/init.py b/juliacall/init.py
@@ -1,6 +1,5 @@
-import os, os.path, sys, ctypes as c, types, shutil, subprocess, jill.install as jli
-from . import CONFIG, __version__
-from .deps import get_dep, set_dep
+import os, os.path, ctypes as c, shutil, subprocess, jill.install as jli
+from . import CONFIG, __version__, deps
 
 # Determine if this is a development version of juliacall
 # i.e. it is installed from the github repo, which contains Project.toml
@@ -36,6 +35,7 @@
     jlenv = os.path.join(jldepot, "environments", "PythonCall")
 else:
     jlenv = os.path.join(prefix, "julia_env")
+CONFIG['jlenv'] = os.path.join(jlenv)
 CONFIG['meta'] = os.path.join(jlenv, "PythonCallMeta.json")
 
 # Find the Julia library
@@ -103,12 +103,47 @@
     lib.jl_init__threading()
     lib.jl_eval_string.argtypes = [c.c_char_p]
     lib.jl_eval_string.restype = c.c_void_p
-    if isdev:
-        install = 'Pkg.develop(path="{}")'.format(reporoot.replace('\\', '\\\\'))
-    elif get_dep('juliacall', 'PythonCall') != __version__:
-        install = 'Pkg.add(name="PythonCall", version="{}")'.format(__version__)
-    else:
+    if deps.can_skip_resolve():
+        pkgs = None
         install = ''
+    else:
+        # get required packages
+        pkgs = deps.required_packages()
+        # add PythonCall
+        if isdev:
+            pkgs.append(deps.PackageSpec(name="PythonCall", uuid="6099a3de-0909-46bc-b1f4-468b9a2dfc0d", path=reporoot, dev=True))
+        else:
+            pkgs.append(deps.PackageSpec(name="PythonCall", uuid="6099a3de-0909-46bc-b1f4-468b9a2dfc0d", compat="= "+__version__))
+        # check if pkgs has changed at all
+        prev_pkgs = deps.get_meta("pydeps", "pkgs")
+        if prev_pkgs is not None and sorted(prev_pkgs, key=lambda p: p['name']) == sorted([p.dict() for p in pkgs], key=lambda p: p['name']):
+            install = ''
+        else:
+            # Write a Project.toml specifying UUIDs and compatibility of required packages
+            if os.path.exists(jlenv):
+                shutil.rmtree(jlenv)
+            os.makedirs(jlenv)
+            with open(os.path.join(jlenv, "Project.toml"), "wt") as fp:
+                print('[deps]', file=fp)
+                for pkg in pkgs:
+                    print('{} = "{}"'.format(pkg.name, pkg.uuid), file=fp)
+                print(file=fp)
+                print('[compat]', file=fp)
+                for pkg in pkgs:
+                    if pkg.compat:
+                        print('{} = "{}"'.format(pkg.name, pkg.compat), file=fp)
+                print(file=fp)
+            # Create install command
+            dev_pkgs = [pkg.jlstr() for pkg in pkgs if pkg.dev]
+            add_pkgs = [pkg.jlstr() for pkg in pkgs if not pkg.dev]
+            if dev_pkgs and add_pkgs:
+                install = 'Pkg.develop([{}]); Pkg.add([{}])'.format(', '.join(dev_pkgs), ', '.join(add_pkgs))
+            elif dev_pkgs:
+                install = 'Pkg.develop([{}])'.format(', '.join(dev_pkgs))
+            elif add_pkgs:
+                install = 'Pkg.add([{}])'.format(', '.join(add_pkgs))
+            else:
+                install = ''
     script = '''
         try
             import Pkg
@@ -124,9 +159,7 @@
     res = lib.jl_eval_string(script.encode('utf8'))
     if res is None:
         raise Exception('PythonCall.jl did not start properly')
-    if isdev:
-        set_dep('juliacall', 'PythonCall', 'dev')
-    elif install:
-        set_dep('juliacall', 'PythonCall', __version__)
+    if pkgs is not None:
+        deps.record_resolve(pkgs)
 finally:
     os.chdir(d)
