SHACL Validation

MANIFEST.in

@@ -56,6 +56,7 @@ recursive-include renku *.html
 recursive-include renku *.sh
 recursive-include renku *.txt
 recursive-include renku *.yml
+recursive-include renku *.json
 recursive-include renku Dockerfile
-recursive-include tests *.py *.gz *.yml
+recursive-include tests *.py *.gz *.yml *.json
 prune .github

renku/cli/log.py

@@ -86,9 +86,15 @@
     default=False,
     help='Display commands without output files.'
 )
+@click.option(
+    '--strict',
+    is_flag=True,
+    default=False,
+    help='Validate triples before output.'
+)
 @click.argument('paths', type=click.Path(exists=True), nargs=-1)
 @pass_local_client
-def log(client, revision, format, no_output, paths):
+def log(client, revision, format, no_output, strict, paths):
     """Show logs for a file."""
     graph = Graph(client)
     if not paths:
@@ -108,4 +114,4 @@ def log(client, revision, format, no_output, paths):
     # NOTE shall we warn when "not no_output and not paths"?
     graph.build(paths=paths, revision=revision, can_be_cwl=no_output)
 
-    FORMATS[format](graph)
+    FORMATS[format](graph, strict=strict)

renku/core/commands/checks/__init__.py

@@ -19,11 +19,14 @@
 
 from .migration import check_dataset_metadata, check_missing_files
 from .references import check_missing_references
+from .validate_shacl import check_project_structure, check_datasets_structure
 
 # Checks will be executed in the order as they are listed in __all__.
 # They are mostly used in ``doctor`` command to inspect broken things.
 __all__ = (
     'check_dataset_metadata',
     'check_missing_files',
     'check_missing_references',
+    'check_project_structure',
+    'check_datasets_structure',
 )

renku/core/commands/checks/validate_shacl.py

@@ -0,0 +1,111 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright 2019 - Swiss Data Science Center (SDSC)
+# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and
+# Eidgenössische Technische Hochschule Zürich (ETHZ).
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Check KG structure using SHACL."""
+import yaml
+from pyld import jsonld as ld
+from rdflib.namespace import Namespace
+from rdflib.term import BNode
+
+from renku.core.commands.echo import WARNING
+from renku.core.models.jsonld import NoDatesSafeLoader
+from renku.core.utils.shacl import validate_graph
+
+
+def _shacl_graph_to_string(graph):
+    """Converts a shacl validation graph into human readable format."""
+    sh = Namespace('http://www.w3.org/ns/shacl#')
+
+    problems = []
+
+    for _, result in graph.subject_objects(sh.result):
+        path = graph.value(result, sh.resultPath)
+        res = graph.value(result, sh.resultMessage)
+
+        if res:
+            message = '{}: {}'.format(path, res)
+        else:
+            kind = graph.value(result, sh.sourceConstraintComponent)
+            focusNode = graph.value(result, sh.focusNode)
+
+            if isinstance(focusNode, BNode):
+                focusNode = '<Anonymous>'
+
+            message = '{}: Type: {}, Node ID: {}'.format(path, kind, focusNode)
+
+        problems.append(message)
+
+    return '\n\t'.join(problems)
+
+
+def check_project_structure(client):
+    """Validate project metadata against SHACL."""
+    project_path = client.renku_metadata_path
+
+    conform, graph, t = check_shacl_structure(project_path)
+
+    if conform:
+        return True, None
+
+    problems = (
+        WARNING + 'Invalid structure of project metadata\n\t' +
+        _shacl_graph_to_string(graph)
+    )
+
+    return False, problems
+
+
+def check_datasets_structure(client):
+    """Validate dataset metadata against SHACL."""
+    ok = True
+
+    problems = WARNING + 'Invalid structure of dataset metadata\n'
+
+    for path in client.renku_datasets_path.rglob(client.METADATA):
+        try:
+            conform, graph, t = check_shacl_structure(path)
+        except (Exception, BaseException) as e:
+            problems += 'Couldn\'t validate {}: {}\n\n'.format(path, e)
+            continue
+
+        if conform:
+            continue
+
+        ok = False
+
+        problems += str(path) + '\n\t' + _shacl_graph_to_string(graph) + '\n\n'
+
+    if ok:
+        return True, None
+
+    return False, problems
+
+
+def check_shacl_structure(path):
+    """Validates all metadata aginst the SHACL schema."""
+    with path.open(mode='r') as fp:
+        source = yaml.load(fp, Loader=NoDatesSafeLoader) or {}
+
+    rdf = ld.to_rdf(
+        source,
+        options={
+            'format': 'application/n-quads',
+            'produceGeneralizedRdf': True
+        }
+    )
+
+    return validate_graph(rdf)

renku/core/commands/format/graph.py

@@ -21,12 +21,17 @@
 
 import click
 
+from renku.core.utils.shacl import validate_graph
 
-def ascii(graph):
+
+def ascii(graph, strict=False):
     """Format graph as an ASCII art."""
     from ..ascii import DAG
     from ..echo import echo_via_pager
 
+    if strict:
+        raise click.BadParameter('--strict not supported for json-ld-graph')
+
     echo_via_pager(str(DAG(graph)))
 
 
@@ -43,21 +48,30 @@ def _jsonld(graph, format, *args, **kwargs):
     return json.dumps(output, indent=2)
 
 
-def dot(graph, simple=True, debug=False, landscape=False):
-    """Format graph as a dot file."""
-    import sys
-
+def _conjunctive_graph(graph):
+    """Convert a renku ``Graph`` to an rdflib ``ConjunctiveGraph``."""
     from rdflib import ConjunctiveGraph
     from rdflib.plugin import register, Parser
-    from rdflib.tools.rdf2dot import rdf2dot
 
     register('json-ld', Parser, 'rdflib_jsonld.parser', 'JsonLDParser')
 
-    g = ConjunctiveGraph().parse(
+    return ConjunctiveGraph().parse(
         data=_jsonld(graph, 'expand'),
         format='json-ld',
     )
 
+
+def dot(graph, simple=True, debug=False, landscape=False, strict=False):
+    """Format graph as a dot file."""
+    import sys
+
+    from rdflib.tools.rdf2dot import rdf2dot
+
+    if strict:
+        raise click.BadParameter('--strict not supported for json-ld-graph')
+
+    g = _conjunctive_graph(graph)
+
     g.bind('prov', 'http://www.w3.org/ns/prov#')
     g.bind('foaf', 'http://xmlns.com/foaf/0.1/')
     g.bind('wfdesc', 'http://purl.org/wf4ever/wfdesc#')
@@ -293,10 +307,13 @@ def color(p):
     stream.write('}\n')
 
 
-def makefile(graph):
+def makefile(graph, strict=False):
     """Format graph as Makefile."""
     from renku.core.models.provenance.activities import ProcessRun, WorkflowRun
 
+    if strict:
+        raise click.BadParameter('--strict not supported for json-ld-graph')
+
     for activity in graph.activities.values():
         if not isinstance(activity, ProcessRun):
             continue
@@ -316,44 +333,53 @@ def makefile(graph):
             )
 
 
-def jsonld(graph):
+def jsonld(graph, strict=False):
     """Format graph as JSON-LD file."""
-    click.echo(_jsonld(graph, 'expand'))
+    ld = _jsonld(graph, 'expand')
+
+    if strict:
+        r, _, t = validate_graph(ld, format='json-ld')
+
+        if not r:
+            click.BadParameter(
+                "{}\nCouldn't get log: Invalid Knowledge Graph data".format(t)
+            )
+    click.echo(ld)
 
 
-def jsonld_graph(graph):
+def jsonld_graph(graph, strict=False):
     """Format graph as JSON-LD graph file."""
+    if strict:
+        raise click.BadParameter('--strict not supported for json-ld-graph')
     click.echo(_jsonld(graph, 'flatten'))
 
 
-def nt(graph):
+def nt(graph, strict=False):
     """Format graph as n-tuples."""
-    from rdflib import ConjunctiveGraph
-    from rdflib.plugin import register, Parser
+    nt = _conjunctive_graph(graph).serialize(format='nt')
+    if strict:
+        r, _, t = validate_graph(nt, format='nt')
 
-    register('json-ld', Parser, 'rdflib_jsonld.parser', 'JsonLDParser')
+        if not r:
+            click.BadParameter(
+                "{}\nCouldn't get log: Invalid Knowledge Graph data".format(t)
+            )
 
-    click.echo(
-        ConjunctiveGraph().parse(
-            data=_jsonld(graph, 'expand'),
-            format='json-ld',
-        ).serialize(format='nt')
-    )
+    click.echo(nt)
 
 
-def rdf(graph):
+def rdf(graph, strict=False):
     """Output the graph as RDF."""
-    from rdflib import ConjunctiveGraph
-    from rdflib.plugin import register, Parser
+    xml = _conjunctive_graph(graph).serialize(format='application/rdf+xml')
+    if strict:
+        r, _, t = validate_graph(xml, format='xml')
 
-    register('json-ld', Parser, 'rdflib_jsonld.parser', 'JsonLDParser')
+        if not r:
+            click.BadParameter(
+                "{}\nCouldn't get log: Invalid Knowledge Graph data".format(t)
+            )
 
-    click.echo(
-        ConjunctiveGraph().parse(
-            data=_jsonld(graph, 'expand'),
-            format='json-ld',
-        ).serialize(format='application/rdf+xml')
-    )
+    click.echo()
 
 
 FORMATS = {

renku/core/models/provenance/entities.py

@@ -62,7 +62,9 @@ def default_id(self):
             hexsha = self.commit.hexsha
         else:
             hexsha = 'UNCOMMITTED'
-        return 'blob/{hexsha}/{self.path}'.format(hexsha=hexsha, self=self)
+        return 'file://blob/{hexsha}/{self.path}'.format(
+            hexsha=hexsha, self=self
+        )
 
     @_label.default
     def default_label(self):

renku/core/utils/shacl.py

@@ -0,0 +1,46 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright 2018-2019- Swiss Data Science Center (SDSC)
+# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and
+# Eidgenössische Technische Hochschule Zürich (ETHZ).
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""JSON-LD SHACL validation."""
+
+import json
+
+from pkg_resources import resource_string
+from pyshacl import validate
+
+
+def validate_graph(graph, shacl_path=None, format='nquads'):
+    """Validate the current graph with a SHACL schema.
+
+    uses default schema if not supplied.
+    """
+    if shacl_path:
+        with open(shacl_path, 'r', encoding='utf-8') as f:
+            shacl = f.read()
+    else:
+        shacl = resource_string('renku', 'data/shacl_shape.json')
+
+    return validate(
+        graph,
+        shacl_graph=shacl,
+        inference='rdfs',
+        meta_shacl=True,
+        debug=False,
+        data_graph_format=format,
+        shacl_graph_format='json-ld',
+        advanced=True
+    )