feat: renku clone command (#828)

mohammad-alisafaee · web-flow · commit 4b3b615a8817 · 2019-11-21T08:29:42.000+01:00
* refactor: move githooks functionality to commands
* feat: renku clone command
* refactor: use Renku clone instead of Git clone
diff --git a/renku/cli/__init__.py b/renku/cli/__init__.py
@@ -69,6 +69,7 @@
 import click_completion
 import yaml
 
+from renku.cli.clone import clone
 from renku.cli.config import config
 from renku.cli.dataset import dataset
 from renku.cli.doctor import doctor
@@ -187,6 +188,7 @@ def help(ctx):
 
 
 # Register subcommands:
+cli.add_command(clone)
 cli.add_command(config)
 cli.add_command(dataset)
 cli.add_command(doctor)
diff --git a/renku/cli/clone.py b/renku/cli/clone.py
@@ -0,0 +1,53 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright 2018-2019- Swiss Data Science Center (SDSC)
+# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and
+# Eidgenössische Technische Hochschule Zürich (ETHZ).
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Clone a Renku project.
+
+Cloning a Renku project
+~~~~~~~~~~~~~~~~~~~~~~~
+
+To clone a Renku project and set up required Git hooks and Git LFS use
+``renku clone`` command.
+
+.. code-block:: console
+
+    $ renku clone git+ssh://host.io/namespace/project.git
+        <destination-directory>
+
+"""
+
+import click
+
+from renku.core.commands.clone import renku_clone
+from renku.core.commands.echo import GitProgress
+
+
+@click.command()
+@click.option(
+    '--pull-data', is_flag=True, help='Pull data from Git-LFS.', default=False
+)
+@click.argument('url')
+@click.argument('path', required=False, default=None)
+def clone(pull_data, url, path):
+    """Clone a Renku repository."""
+    click.echo('Cloning {} ...'.format(url))
+
+    skip_smudge = not pull_data
+    renku_clone(
+        url=url, path=path, skip_smudge=skip_smudge, progress=GitProgress()
+    )
+    click.secho('OK', fg='green')
diff --git a/renku/cli/githooks.py b/renku/cli/githooks.py
@@ -38,14 +38,9 @@
 
 """
 
-import stat
-from pathlib import Path
-
 import click
 
-from renku.core.commands.client import pass_local_client
-
-HOOKS = ('pre-commit', )
+from renku.core.commands.githooks import install_githooks, uninstall_githooks
 
 
 @click.group()
@@ -55,42 +50,14 @@ def githooks():
 
 @githooks.command()
 @click.option('--force', is_flag=True, help='Override existing hooks.')
-@pass_local_client
-def install(client, force):
+def install(force):
     """Install Git hooks."""
-    import pkg_resources
-    from git.index.fun import hook_path as get_hook_path
-
-    for hook in HOOKS:
-        hook_path = Path(get_hook_path(hook, client.repo.git_dir))
-        if hook_path.exists():
-            if not force:
-                click.echo(
-                    'Hook already exists. Skipping {0}'.format(str(hook_path)),
-                    err=True
-                )
-                continue
-            else:
-                hook_path.unlink()
-
-        # Make sure the hooks directory exists.
-        hook_path.parent.mkdir(parents=True, exist_ok=True)
-
-        Path(hook_path).write_bytes(
-            pkg_resources.resource_string(
-                'renku.data', '{hook}.sh'.format(hook=hook)
-            )
-        )
-        hook_path.chmod(hook_path.stat().st_mode | stat.S_IEXEC)
+    install_githooks(force)
+    click.secho('OK', fg='green')
 
 
 @githooks.command()
-@pass_local_client
-def uninstall(client):
+def uninstall():
     """Uninstall Git hooks."""
-    from git.index.fun import hook_path as get_hook_path
-
-    for hook in HOOKS:
-        hook_path = Path(get_hook_path(hook, client.repo.git_dir))
-        if hook_path.exists():
-            hook_path.unlink()
+    uninstall_githooks()
+    click.secho('OK', fg='green')
diff --git a/renku/core/commands/clone.py b/renku/core/commands/clone.py
@@ -0,0 +1,43 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright 2018-2019- Swiss Data Science Center (SDSC)
+# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and
+# Eidgenössische Technische Hochschule Zürich (ETHZ).
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Clone a Renku repo along with all Renku-specific initializations."""
+
+from renku.core.management.clone import clone
+
+from .client import pass_local_client
+
+
+@pass_local_client
+def renku_clone(
+    client,
+    url,
+    path=None,
+    install_githooks=True,
+    skip_smudge=True,
+    progress=None
+):
+    """Clone Renku project repo, install Git hooks and LFS."""
+    install_lfs = client.use_external_storage
+    clone(
+        url=url,
+        path=path,
+        install_githooks=install_githooks,
+        install_lfs=install_lfs,
+        skip_smudge=skip_smudge,
+        progress=progress
+    )
diff --git a/renku/core/commands/echo.py b/renku/core/commands/echo.py
@@ -21,6 +21,7 @@
 import os
 
 import click
+from git.remote import RemoteProgress
 
 WARNING = click.style('Warning: ', bold=True, fg='yellow')
 
@@ -45,3 +46,23 @@ def echo_via_pager(*args, **kwargs):
     show_pos=True,
     item_show_func=lambda x: x,
 )
+
+
+class GitProgress(RemoteProgress):
+    """Progress printing for GitPython."""
+
+    def __init__(self):
+        """Initialize a Git progress printer."""
+        super().__init__()
+        self._previous_line_length = 0
+
+    def update(self, op_code, cur_count, max_count=None, message=''):
+        """Callback for printing Git operation status."""
+        self._clear_line()
+        print(self._cur_line, end='\r')
+        self._previous_line_length = len(self._cur_line)
+        if (op_code & RemoteProgress.END) != 0:
+            print()
+
+    def _clear_line(self):
+        print(self._previous_line_length * ' ', end='\r')
diff --git a/renku/core/commands/githooks.py b/renku/core/commands/githooks.py
@@ -0,0 +1,40 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright 2018-2019- Swiss Data Science Center (SDSC)
+# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and
+# Eidgenössische Technische Hochschule Zürich (ETHZ).
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Install and uninstall Git hooks."""
+
+import click
+
+from renku.core.management.githooks import install, uninstall
+
+from .client import pass_local_client
+from .echo import WARNING
+
+
+@pass_local_client
+def install_githooks(client, force):
+    """Install Git hooks."""
+    warning_messages = install(client=client, force=force)
+    if warning_messages:
+        for message in warning_messages:
+            click.echo(WARNING + message)
+
+
+@pass_local_client
+def uninstall_githooks(client):
+    """Uninstall Git hooks."""
+    uninstall(client=client)
diff --git a/renku/core/management/clone.py b/renku/core/management/clone.py
@@ -0,0 +1,68 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright 2018-2019- Swiss Data Science Center (SDSC)
+# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and
+# Eidgenössische Technische Hochschule Zürich (ETHZ).
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Clone a Renku repo along with all Renku-specific initializations."""
+
+import os
+
+from git import GitCommandError, Repo
+
+from renku.core import errors
+from renku.core.management.githooks import install
+
+
+def clone(
+    url,
+    path=None,
+    install_githooks=True,
+    install_lfs=True,
+    skip_smudge=True,
+    recursive=True,
+    depth=None,
+    progress=None
+):
+    """Clone Renku project repo, install Git hooks and LFS."""
+    from renku.core.management.client import LocalClient
+
+    path = path or '.'
+    # Clone the project
+    if skip_smudge:
+        os.environ['GIT_LFS_SKIP_SMUDGE'] = '1'
+    try:
+        repo = Repo.clone_from(
+            url, path, recursive=recursive, depth=depth, progress=progress
+        )
+    except GitCommandError as e:
+        raise errors.GitError(
+            'Cannot clone remote Renku project: {}'.format(url)
+        ) from e
+
+    client = LocalClient(path)
+
+    if install_githooks:
+        install(client=client, force=True)
+
+    if install_lfs:
+        command = ['git', 'lfs', 'install', '--local', '--force']
+        if skip_smudge:
+            command += ['--skip-smudge']
+        try:
+            repo.git.execute(command=command, with_exceptions=True)
+        except GitCommandError as e:
+            raise errors.GitError('Cannot install Git LFS') from e
+
+    return repo
diff --git a/renku/core/management/datasets.py b/renku/core/management/datasets.py
@@ -34,6 +34,7 @@
 from git import GitCommandError, GitError, Repo
 
 from renku.core import errors
+from renku.core.management.clone import clone
 from renku.core.management.config import RENKU_HOME
 from renku.core.models.datasets import Dataset, DatasetFile, DatasetTag
 from renku.core.models.git import GitURL
@@ -755,9 +756,13 @@ def checkout(repo, ref):
             repo = Repo(str(repo_path))
             if repo.remotes.origin.url == url:
                 try:
-                    repo.git.fetch()
+                    repo.git.fetch(all=True)
                     repo.git.checkout(ref)
-                    repo.git.pull()
+                    try:
+                        repo.git.pull()
+                    except GitError:
+                        # When ref is not a branch, an error is thrown
+                        pass
                 except GitError:
                     # ignore the error and try re-cloning
                     pass
@@ -772,25 +777,17 @@ def checkout(repo, ref):
                     format(repo_path)
                 )
 
+        repo = clone(url, path=str(repo_path), install_githooks=False)
+
+        # Because the name of the default branch is not always 'master', we
+        # create an alias of the default branch when cloning the repo. It
+        # is used to refer to the default branch later.
+        renku_ref = 'refs/heads/' + RENKU_BRANCH
         try:
-            os.environ['GIT_LFS_SKIP_SMUDGE'] = '1'
-            repo = Repo.clone_from(url, str(repo_path), recursive=True)
-            # Because the name of the default branch is not always 'master', we
-            # create an alias of the default branch when cloning the repo. It
-            # is used to refer to the default branch later.
-            renku_ref = 'refs/heads/' + RENKU_BRANCH
             repo.git.execute([
                 'git', 'symbolic-ref', renku_ref, repo.head.reference.path
             ])
             checkout(repo, ref)
-            # Disable Git LFS smudge filter
-            repo.git.execute(
-                command=[
-                    'git', 'lfs', 'install', '--local', '--skip-smudge',
-                    '--force'
-                ],
-                with_exceptions=False
-            )
         except GitCommandError as e:
             raise errors.GitError(
                 'Cannot clone remote Git repo: {}'.format(url)
diff --git a/renku/core/management/githooks.py b/renku/core/management/githooks.py
diff --git a/tests/cli/test_integration_datasets.py b/tests/cli/test_integration_datasets.py
