Merge branch 'master' into 755-kg-validation

Author: Panaetius

CHANGES.rst

@@ -18,6 +18,60 @@
 Changes
 =======
 
+
+`0.8.0 <https://github.com/SwissDataScienceCenter/renku-python/compare/v0.7.0...v0.8.0>`__ (2019-11-21)
+-------------------------------------------------------------------------------------------------------
+
+Bug Fixes
+~~~~~~~~~
+
+-  addressed CI problems with git submodules
+   (`#783 <https://github.com/SwissDataScienceCenter/renku-python/issues/783>`__)
+   (`0d3eeb7 <https://github.com/SwissDataScienceCenter/renku-python/commit/0d3eeb7>`__)
+-  adds simple check on empty filename
+   (`#786 <https://github.com/SwissDataScienceCenter/renku-python/issues/786>`__)
+   (`8cd061b <https://github.com/SwissDataScienceCenter/renku-python/commit/8cd061b>`__)
+-  ensure all Person instances have valid ids
+   (`4f80efc <https://github.com/SwissDataScienceCenter/renku-python/commit/4f80efc>`__),
+   closes
+   `#812 <https://github.com/SwissDataScienceCenter/renku-python/issues/812>`__
+-  Fixes jsonld issue when importing from dataverse
+   (`#759 <https://github.com/SwissDataScienceCenter/renku-python/issues/759>`__)
+   (`ffe36c6 <https://github.com/SwissDataScienceCenter/renku-python/commit/ffe36c6>`__)
+-  fixes nested type scoped handling if a class only has a single class
+   (`#804 <https://github.com/SwissDataScienceCenter/renku-python/issues/804>`__)
+   (`16d03b6 <https://github.com/SwissDataScienceCenter/renku-python/commit/16d03b6>`__)
+-  ignore deleted paths in generated entities
+   (`86fedaf <https://github.com/SwissDataScienceCenter/renku-python/commit/86fedaf>`__),
+   closes
+   `#806 <https://github.com/SwissDataScienceCenter/renku-python/issues/806>`__
+-  integration tests
+   (`#831 <https://github.com/SwissDataScienceCenter/renku-python/issues/831>`__)
+   (`a4ad7f9 <https://github.com/SwissDataScienceCenter/renku-python/commit/a4ad7f9>`__)
+-  make Creator a subclass of Person
+   (`ac9bac3 <https://github.com/SwissDataScienceCenter/renku-python/commit/ac9bac3>`__),
+   closes
+   `#793 <https://github.com/SwissDataScienceCenter/renku-python/issues/793>`__
+-  Redesign scoped context in jsonld
+   (`#750 <https://github.com/SwissDataScienceCenter/renku-python/issues/750>`__)
+   (`2b1948d <https://github.com/SwissDataScienceCenter/renku-python/commit/2b1948d>`__)
+
+Features
+~~~~~~~~
+
+-  avoid creation of nested datasets
+   (`#796 <https://github.com/SwissDataScienceCenter/renku-python/issues/796>`__)
+   (`6084c87 <https://github.com/SwissDataScienceCenter/renku-python/commit/6084c87>`__)
+-  do not create dataset implicitly
+   (`#779 <https://github.com/SwissDataScienceCenter/renku-python/issues/779>`__)
+   (`84e59d0 <https://github.com/SwissDataScienceCenter/renku-python/commit/84e59d0>`__)
+-  local git repo not treated as remote
+   (`8cc2834 <https://github.com/SwissDataScienceCenter/renku-python/commit/8cc2834>`__)
+-  renku clone command
+   (`#828 <https://github.com/SwissDataScienceCenter/renku-python/issues/828>`__)
+   (`4b3b615 <https://github.com/SwissDataScienceCenter/renku-python/commit/4b3b615>`__)
+
+
 `0.7.0 <https://github.com/SwissDataScienceCenter/renku-python/compare/v0.6.0...v0.6.1>`__ (2019-10-15)
 -------------------------------------------------------------------------------------------------------
 
@@ -26,13 +80,10 @@ Bug Fixes
 
 -  use UI-resolved project path as project ID (`#701`_) (`dfcc9e6`_)
 
-.. _section-1:
 
 `0.6.1 <https://github.com/SwissDataScienceCenter/renku-python/compare/v0.6.0...v0.6.1>`__ (2019-10-10)
 -------------------------------------------------------------------------------------------------------
 
-.. _bug-fixes-1:
-
 Bug Fixes
 ~~~~~~~~~
 

renku/cli/__init__.py

@@ -69,6 +69,7 @@
 import click_completion
 import yaml
 
+from renku.cli.clone import clone
 from renku.cli.config import config
 from renku.cli.dataset import dataset
 from renku.cli.doctor import doctor
@@ -187,6 +188,7 @@ def help(ctx):
 
 
 # Register subcommands:
+cli.add_command(clone)
 cli.add_command(config)
 cli.add_command(dataset)
 cli.add_command(doctor)

renku/cli/clone.py

@@ -0,0 +1,53 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright 2018-2019- Swiss Data Science Center (SDSC)
+# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and
+# Eidgenössische Technische Hochschule Zürich (ETHZ).
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Clone a Renku project.
+
+Cloning a Renku project
+~~~~~~~~~~~~~~~~~~~~~~~
+
+To clone a Renku project and set up required Git hooks and Git LFS use
+``renku clone`` command.
+
+.. code-block:: console
+
+    $ renku clone git+ssh://host.io/namespace/project.git
+        <destination-directory>
+
+"""
+
+import click
+
+from renku.core.commands.clone import renku_clone
+from renku.core.commands.echo import GitProgress
+
+
+@click.command()
+@click.option(
+    '--pull-data', is_flag=True, help='Pull data from Git-LFS.', default=False
+)
+@click.argument('url')
+@click.argument('path', required=False, default=None)
+def clone(pull_data, url, path):
+    """Clone a Renku repository."""
+    click.echo('Cloning {} ...'.format(url))
+
+    skip_smudge = not pull_data
+    renku_clone(
+        url=url, path=path, skip_smudge=skip_smudge, progress=GitProgress()
+    )
+    click.secho('OK', fg='green')

renku/cli/dataset.py

@@ -410,7 +410,10 @@ def edit(dataset_id):
     '--force', is_flag=True, help='Allow adding otherwise ignored files.'
 )
 @click.option(
-    '--create', is_flag=True, help='Create dataset if it does not exist.'
+    '-c',
+    '--create',
+    is_flag=True,
+    help='Create dataset if it does not exist.'
 )
 @click.option(
     '-s',

renku/cli/githooks.py

@@ -38,14 +38,9 @@
 
 """
 
-import stat
-from pathlib import Path
-
 import click
 
-from renku.core.commands.client import pass_local_client
-
-HOOKS = ('pre-commit', )
+from renku.core.commands.githooks import install_githooks, uninstall_githooks
 
 
 @click.group()
@@ -55,42 +50,14 @@ def githooks():
 
 @githooks.command()
 @click.option('--force', is_flag=True, help='Override existing hooks.')
-@pass_local_client
-def install(client, force):
+def install(force):
     """Install Git hooks."""
-    import pkg_resources
-    from git.index.fun import hook_path as get_hook_path
-
-    for hook in HOOKS:
-        hook_path = Path(get_hook_path(hook, client.repo.git_dir))
-        if hook_path.exists():
-            if not force:
-                click.echo(
-                    'Hook already exists. Skipping {0}'.format(str(hook_path)),
-                    err=True
-                )
-                continue
-            else:
-                hook_path.unlink()
-
-        # Make sure the hooks directory exists.
-        hook_path.parent.mkdir(parents=True, exist_ok=True)
-
-        Path(hook_path).write_bytes(
-            pkg_resources.resource_string(
-                'renku.data', '{hook}.sh'.format(hook=hook)
-            )
-        )
-        hook_path.chmod(hook_path.stat().st_mode | stat.S_IEXEC)
+    install_githooks(force)
+    click.secho('OK', fg='green')
 
 
 @githooks.command()
-@pass_local_client
-def uninstall(client):
+def uninstall():
     """Uninstall Git hooks."""
-    from git.index.fun import hook_path as get_hook_path
-
-    for hook in HOOKS:
-        hook_path = Path(get_hook_path(hook, client.repo.git_dir))
-        if hook_path.exists():
-            hook_path.unlink()
+    uninstall_githooks()
+    click.secho('OK', fg='green')

renku/core/commands/clone.py

@@ -0,0 +1,43 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright 2018-2019- Swiss Data Science Center (SDSC)
+# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and
+# Eidgenössische Technische Hochschule Zürich (ETHZ).
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Clone a Renku repo along with all Renku-specific initializations."""
+
+from renku.core.management.clone import clone
+
+from .client import pass_local_client
+
+
+@pass_local_client
+def renku_clone(
+    client,
+    url,
+    path=None,
+    install_githooks=True,
+    skip_smudge=True,
+    progress=None
+):
+    """Clone Renku project repo, install Git hooks and LFS."""
+    install_lfs = client.use_external_storage
+    clone(
+        url=url,
+        path=path,
+        install_githooks=install_githooks,
+        install_lfs=install_lfs,
+        skip_smudge=skip_smudge,
+        progress=progress
+    )

renku/core/commands/dataset.py

@@ -47,6 +47,7 @@
 from renku.core.models.refs import LinkReference
 from renku.core.models.tabulate import tabulate
 from renku.core.utils.doi import extract_doi
+from renku.core.utils.urls import remove_credentials
 
 from .client import pass_local_client
 from .echo import WARNING
@@ -131,6 +132,8 @@ def edit_dataset(client, dataset_id, transform_fn):
     clean=False,
     commit=True,
     commit_only=COMMIT_DIFF_STRATEGY,
+    commit_empty=False,
+    raise_if_empty=True
 )
 def add_file(
     client,
@@ -166,13 +169,12 @@ def add_to_dataset(
     urlscontext=contextlib.nullcontext
 ):
     """Add data to a dataset."""
-    if sources or destination:
-        if len(urls) == 0:
-            raise UsageError('No URL is specified')
-        elif len(urls) > 1:
-            raise UsageError(
-                'Cannot add multiple URLs with --source or --destination'
-            )
+    if len(urls) == 0:
+        raise UsageError('No URL is specified')
+    if (sources or destination) and len(urls) > 1:
+        raise UsageError(
+            'Cannot add multiple URLs with --source or --destination'
+        )
 
     # check for identifier before creating the dataset
     identifier = extract_doi(
@@ -183,7 +185,7 @@ def add_to_dataset(
             name=name, identifier=identifier, create=create
         ) as dataset:
             with urlscontext(urls) as bar:
-                client.add_data_to_dataset(
+                warning_message = client.add_data_to_dataset(
                     dataset,
                     bar,
                     link=link,
@@ -193,6 +195,9 @@ def add_to_dataset(
                     ref=ref
                 )
 
+            if warning_message:
+                click.echo(WARNING + warning_message)
+
             if with_metadata:
                 for file_ in with_metadata.files:
                     for added_ in dataset.files:
@@ -202,6 +207,7 @@ def add_to_dataset(
                                 file_.url = file_.url.geturl()
 
                             file_.path = added_.path
+                            file_.url = remove_credentials(file_.url)
                             file_.creator = with_metadata.creator
                             file_._label = added_._label
                             file_.commit = added_.commit
@@ -501,6 +507,7 @@ def import_dataset(
         pool.close()
 
         dataset_name = name or dataset.display_name
+        dataset.url = remove_credentials(dataset.url)
         add_to_dataset(
             client,
             urls=[str(p) for p in Path(data_folder).glob('*')],

renku/core/commands/echo.py

@@ -21,6 +21,7 @@
 import os
 
 import click
+from git.remote import RemoteProgress
 
 WARNING = click.style('Warning: ', bold=True, fg='yellow')
 
@@ -45,3 +46,23 @@ def echo_via_pager(*args, **kwargs):
     show_pos=True,
     item_show_func=lambda x: x,
 )
+
+
+class GitProgress(RemoteProgress):
+    """Progress printing for GitPython."""
+
+    def __init__(self):
+        """Initialize a Git progress printer."""
+        super().__init__()
+        self._previous_line_length = 0
+
+    def update(self, op_code, cur_count, max_count=None, message=''):
+        """Callback for printing Git operation status."""
+        self._clear_line()
+        print(self._cur_line, end='\r')
+        self._previous_line_length = len(self._cur_line)
+        if (op_code & RemoteProgress.END) != 0:
+            print()
+
+    def _clear_line(self):
+        print(self._previous_line_length * ' ', end='\r')