Patches pyld to prevent contexts from overwhelming its cache

Author: Ralf Grubenmann

renku/core/commands/checks/validate_shacl.py

@@ -17,11 +17,11 @@
 # limitations under the License.
 """Check KG structure using SHACL."""
 import yaml
-from pyld import jsonld as ld
 from rdflib.namespace import Namespace
 from rdflib.term import BNode
 
 from renku.core.commands.echo import WARNING
+from renku.core.compat import pyld
 from renku.core.models.jsonld import NoDatesSafeLoader
 from renku.core.utils.shacl import validate_graph
 
@@ -100,7 +100,7 @@ def check_shacl_structure(path):
     with path.open(mode='r') as fp:
         source = yaml.load(fp, Loader=NoDatesSafeLoader) or {}
 
-    rdf = ld.to_rdf(
+    rdf = pyld.jsonld.to_rdf(
         source,
         options={
             'format': 'application/n-quads',

renku/core/commands/format/graph.py

@@ -39,10 +39,10 @@ def _jsonld(graph, format, *args, **kwargs):
     """Return formatted graph in JSON-LD ``format`` function."""
     import json
 
-    from pyld import jsonld
+    from renku.core.compat import pyld
     from renku.core.models.jsonld import asjsonld
 
-    output = getattr(jsonld, format)([
+    output = getattr(pyld.jsonld, format)([
         asjsonld(action) for action in graph.activities.values()
     ])
     return json.dumps(output, indent=2)

renku/core/compat.py

@@ -18,10 +18,14 @@
 """Compatibility layer for different Python versions."""
 
 import contextlib
+import json
 import os
 import sys
+from collections import deque
 from pathlib import Path
 
+import pyld
+
 if sys.version_info < (3, 6):
     original_resolve = Path.resolve
 
@@ -63,4 +67,36 @@ def __exit__(self, *excinfo):
 except NameError:  # pragma: no cover
     FileNotFoundError = IOError
 
-__all__ = ('FileNotFoundError', 'Path', 'contextlib')
+
+class ActiveContextCache(object):
+    """Pyld context cache without issue of missing contexts."""
+
+    def __init__(self, size=100):
+        self.order = deque()
+        self.cache = {}
+        self.size = size
+
+    def get(self, active_ctx, local_ctx):
+        key1 = json.dumps(active_ctx)
+        key2 = json.dumps(local_ctx)
+        return self.cache.get(key1, {}).get(key2)
+
+    def set(self, active_ctx, local_ctx, result):
+        if len(self.order) == self.size:
+            entry = self.order.popleft()
+            if sum(
+                e['activeCtx'] == entry['activeCtx'] and
+                e['localCtx'] == entry['localCtx'] for e in self.order
+            ) == 0:
+                # only delete from cache if it doesn't exist in context deque
+                print("this totally works")
+                del self.cache[entry['activeCtx']][entry['localCtx']]
+        key1 = json.dumps(active_ctx)
+        key2 = json.dumps(local_ctx)
+        self.order.append({'activeCtx': key1, 'localCtx': key2})
+        self.cache.setdefault(key1, {})[key2] = json.loads(json.dumps(result))
+
+
+pyld._cache = {'activeCtx': ActiveContextCache()}
+
+__all__ = ('FileNotFoundError', 'Path', 'contextlib', 'pyld')

renku/core/models/jsonld.py

@@ -30,8 +30,8 @@
 from attr._compat import iteritems
 from attr._funcs import has
 from attr._make import Factory, fields
-from pyld import jsonld as ld
 
+from renku.core.compat import pyld
 from renku.core.models.locals import ReferenceMixin, with_reference
 from renku.core.models.migrations import JSONLD_MIGRATIONS
 
@@ -149,7 +149,7 @@ def wrap(cls):
 
         # Register class for given JSON-LD @type
         try:
-            type_ = ld.expand({
+            type_ = pyld.jsonld.expand({
                 '@type': jsonld_cls._jsonld_type,
                 '@context': context
             })[0]['@type']
@@ -473,10 +473,10 @@ def from_jsonld(
 
         if cls._jsonld_translate:
             # perform the translation
-            data = ld.compact(data, cls._jsonld_translate)
+            data = pyld.jsonld.compact(data, cls._jsonld_translate)
             # compact using the class json-ld context
             data.pop('@context', None)
-            data = ld.compact(data, cls._jsonld_context)
+            data = pyld.jsonld.compact(data, cls._jsonld_context)
 
         data.setdefault('@context', cls._jsonld_context)
 
@@ -504,7 +504,7 @@ def from_jsonld(
                 data['@context'] = {'@base': data['@context']}
             data['@context'].update(cls._jsonld_context)
             try:
-                compacted = ld.compact(data, cls._jsonld_context)
+                compacted = pyld.jsonld.compact(data, cls._jsonld_context)
             except Exception:
                 compacted = data
         else:

tests/cli/test_log.py

@@ -62,5 +62,6 @@ def test_dataset_log_strict(tmpdir, runner, project, client, format):
     result = runner.invoke(
         cli, ['log', '--strict', '--format={}'.format(format)]
     )
+
     assert 0 == result.exit_code, result.output
     assert all(p in result.output for p in paths)

tests/core/models/test_shacl_schema.py

@@ -17,10 +17,8 @@
 # limitations under the License.
 """test KG against SHACL shape."""
 
-from pyld import jsonld as ld
-
 from renku.cli import cli
-from renku.core.compat import Path
+from renku.core.compat import Path, pyld
 from renku.core.utils.shacl import validate_graph
 
 
@@ -61,7 +59,7 @@ def test_dataset_shacl(tmpdir, runner, project, client):
 
     with client.with_dataset('dataset') as dataset:
         g = dataset.asjsonld()
-        rdf = ld.to_rdf(
+        rdf = pyld.jsonld.to_rdf(
             g,
             options={
                 'format': 'application/n-quads',
@@ -94,7 +92,7 @@ def test_project_shacl(project, client):
     project.creator = Creator(email='johndoe@example.com', name='Johnny Doe')
 
     g = project.asjsonld()
-    rdf = ld.to_rdf(
+    rdf = pyld.jsonld.to_rdf(
         g,
         options={
             'format': 'application/n-quads',