code mostly ready

Author: valfrancois

gitlab_submodule/__init__.py

@@ -0,0 +1,18 @@
+"""List project submodules and get the commits they point to with python-gitlab
+"""
+
+__version__ = '0.1.0'
+__all__ = [
+    'GitmodulesSubmodule', 'ProjectSubmodule',
+    'list_all_project_submodules',
+    'read_gitlab_submodule',
+    'iterate_gitlab_project_submodules', 'list_gitlab_project_submodules'
+]
+
+from gitlab_submodule.objects import GitmodulesSubmodule, ProjectSubmodule
+from gitlab_submodule.gitlab_submodule import (
+    list_all_project_submodules,
+    read_gitlab_submodule,
+    iterate_gitlab_project_submodules,
+    list_gitlab_project_submodules
+)

gitlab_submodule/gitlab_submodule.py

@@ -0,0 +1,55 @@
+from typing import List, Optional, Iterable
+
+from gitlab.v4.objects import Project
+
+from gitlab_submodule.objects import GitmodulesSubmodule, ProjectSubmodule
+from gitlab_submodule.read_gitmodules import list_project_submodules
+from gitlab_submodule.gitmodules_to_project import gitmodules_to_project
+from gitlab_submodule.submodule_commit import get_submodule_commit
+
+
+def list_all_project_submodules(
+        project: Project,
+        ref: Optional[str] = None) -> List[GitmodulesSubmodule]:
+    return list_project_submodules(project, ref)
+
+
+def read_gitlab_submodule(
+        gitmodules_submodule: GitmodulesSubmodule,
+        get_latest_commit_possible_if_not_found: bool = True,
+        get_latest_commit_possible_ref: Optional[str] = None
+) -> ProjectSubmodule:
+    submodule_project = gitmodules_to_project(gitmodules_submodule)
+    submodule_commit, commit_is_exact = get_submodule_commit(
+        gitmodules_submodule,
+        submodule_project,
+        get_latest_commit_possible_if_not_found,
+        get_latest_commit_possible_ref)
+    return ProjectSubmodule(
+        gitmodules_submodule,
+        submodule_project,
+        submodule_commit,
+        commit_is_exact
+    )
+
+
+def iterate_gitlab_project_submodules(
+        project: Project,
+        ref: Optional[str] = None,
+        get_latest_commit_possible_if_not_found: bool = True,
+        get_latest_commit_possible_ref: Optional[str] = None
+) -> Iterable[ProjectSubmodule]:
+    for gitmodules_submodule in list_all_project_submodules(project, ref):
+        try:
+            yield read_gitlab_submodule(
+                gitmodules_submodule,
+                get_latest_commit_possible_if_not_found,
+                get_latest_commit_possible_ref)
+        except ValueError:
+            continue
+        except Exception:
+            raise
+
+
+def list_gitlab_project_submodules(**kwargs) -> List[ProjectSubmodule]:
+    return list(iterate_gitlab_project_submodules(**kwargs))

gitlab_submodule/gitmodules_to_project.py

@@ -0,0 +1,50 @@
+from typing import Optional
+
+from posixpath import join, normpath
+from giturlparse import parse
+
+from gitlab import Gitlab
+from gitlab.v4.objects import Project
+
+from gitlab_submodule.objects import GitmodulesSubmodule
+
+
+def gitmodules_to_project(submodule: GitmodulesSubmodule) -> Project:
+    submodule_project_path_with_namespace = \
+        _submodule_url_to_path_with_namespace(submodule.url,
+                                              submodule.parent_project)
+    if not submodule_project_path_with_namespace:
+        raise ValueError(
+            f'submodule at {submodule.url} is not hosted on Gitlab')
+    gl = Gitlab()
+    submodule_project = gl.projects.get(submodule_project_path_with_namespace)
+    return submodule_project
+
+
+def _submodule_url_to_path_with_namespace(
+        url: str,
+        parent_project: Project
+) -> Optional[str]:
+    """Returns a path pointing to a Gitlab project, or None if the submodule
+    is hosted elsewhere
+    """
+    try:
+        parsed = parse(url)
+        if parsed.platform != 'gitlab':
+            return None
+        if parsed.groups:
+            to_join = [parsed.owner, join(*parsed.groups), parsed.repo]
+        else:
+            to_join = [parsed.owner, parsed.repo]
+        path_with_namespace = join(*to_join)
+        return path_with_namespace
+    except Exception as e:
+        print(e)
+        # check if the submodule url is a relative path to the project path
+        if url.startswith('./') or url.startswith('../'):
+            # we build the path of the submodule project using the path of
+            # the current project
+            path_with_namespace = normpath(
+                join(parent_project.path_with_namespace, url))
+            return path_with_namespace
+    return None

gitlab_submodule/objects.py

@@ -0,0 +1,108 @@
+from gitlab.v4.objects import Project, ProjectCommit
+
+
+class GitmodulesSubmodule:
+
+    def __init__(self,
+                 parent_project: Project,
+                 parent_ref: str,
+                 name: str,
+                 path: str,
+                 url: str):
+        self.parent_project = parent_project
+        self.parent_ref = parent_ref
+        self.name = name
+        self.path = path
+        self.url = url
+
+    def keys(self):
+        return {'parent_project', 'parent_ref', 'name', 'path', 'url'}
+
+    def __getitem__(self, key):
+        if key in self.keys():
+            return getattr(self, key)
+        else:
+            raise KeyError(key)
+
+    def __str__(self):
+        keys = sorted(self.keys())
+        class_part = f"<class '{self.__class__.__name__}'>"
+
+        def to_str(key):
+            if isinstance(self[key], str):
+                return f"'{self[key]}'"
+            else:
+                return str(self[key])
+
+        attributes = [f"'{key}': {to_str(key)}" for key in keys]
+        return class_part + ' => {' + ', '.join(attributes) + '}'
+
+    def __repr__(self):
+        return '{} ({}, {}, {}, {}, {})'.format(
+            self.__class__.__name__,
+            repr(self.parent_project),
+            f"'{self.parent_ref}'",
+            f"'{self.name}'",
+            f"'{self.path}'",
+            f"'{self.url}'",
+        )
+
+
+def lstrip(string: str, pattern: str) -> str:
+    if string[:len(pattern)] == pattern:
+        return string[len(pattern):]
+    else:
+        return string
+
+
+class ProjectSubmodule:
+    def __init__(self,
+                 submodule: GitmodulesSubmodule,
+                 project: Project,
+                 commit: ProjectCommit,
+                 commit_is_exact: bool):
+        self.submodule = submodule
+        self.project = project
+        self.commit = commit
+        self.commit_is_exact = commit_is_exact
+
+    def __getattribute__(self, item: str):
+        try:
+            return super().__getattribute__(item)
+        except AttributeError:
+            for attribute in {'submodule', 'project', 'commit'}:
+                if item.startswith(f'{attribute}_'):
+                    return getattr(super().__getattribute__(attribute),
+                                   lstrip(item, f'{attribute}_'))
+
+        raise AttributeError("'{} object has no attribute '{}'".format(
+            self.__class__.__name__, item))
+
+    def __setattr__(self, key, value):
+        for attribute in {'submodule', 'project', 'commit'}:
+            if key == attribute:
+                return super().__setattr__(key, value)
+            if key.startswith(f'{attribute}_'):
+                return setattr(getattr(self, attribute),
+                               lstrip(key, f'{attribute}_'),
+                               value)
+
+        try:
+            super().__setattr__(key, value)
+        except AttributeError:
+            raise AttributeError("'{} object has no attribute '{}'".format(
+                self.__class__.__name__, key))
+
+    def __str__(self):
+        class_part = f"<class '{self.__class__.__name__}'>"
+        attributes = [f"\n    '{key}': {getattr(self, key)}"
+                      for key in ['submodule', 'project', 'commit']]
+        return class_part + ' => {' + ','.join(attributes) + '\n}'
+
+    def __repr__(self):
+        return '{} (\n    {},\n    {},\n    {}\n)'.format(
+            self.__class__.__name__,
+            repr(self.submodule),
+            repr(self.project),
+            repr(self.commit),
+        )

gitlab_submodule/read_gitmodules.py

@@ -0,0 +1,55 @@
+from typing import List, Optional, Iterable, Tuple
+
+import re
+
+from gitlab.v4.objects import Project
+
+from gitlab_submodule.objects import GitmodulesSubmodule
+
+
+def list_project_submodules(
+        project: Project,
+        ref: Optional[str] = None) -> List[GitmodulesSubmodule]:
+    return list(_get_project_submodules(project, ref))
+
+
+def _get_project_submodules(
+        project: Project,
+        ref: Optional[str] = None) -> Iterable[GitmodulesSubmodule]:
+    gitmodules_file_content = _get_gitmodules_file_content(project, ref)
+    if not gitmodules_file_content:
+        return []
+    for (name, url, path) in _read_gitmodules_file_content(
+            gitmodules_file_content):
+        yield GitmodulesSubmodule(
+            parent_project=project,
+            parent_ref=ref if ref else project.default_branch,
+            name=name,
+            url=url,
+            path=path)
+
+
+def _get_gitmodules_file_content(project: Project,
+                                 ref: Optional[str] = None) -> Optional[str]:
+    try:
+        gitmodules = project.files.get(
+            '.gitmodules',
+            ref=ref if ref else project.default_branch)
+        return gitmodules.decode().decode('utf-8')
+    except Exception:
+        return None
+
+
+def _read_gitmodules_file_content(
+        gitmodules_file_content: str) -> List[Tuple[str, str, str]]:
+    """Some basic regex extractions to parse content of .gitmodules file
+    """
+    name_regex = r'\[submodule "([a-zA-Z0-9\.\-/_]+)"\]'
+    path_regex = r'path ?= ?([a-zA-Z0-9\.\-/_]+)'
+    url_regex = r'url ?= ?([a-zA-Z0-9\.\-/_:@]+)'
+    names = re.findall(name_regex, gitmodules_file_content)
+    paths = re.findall(path_regex, gitmodules_file_content)
+    urls = re.findall(url_regex, gitmodules_file_content)
+    if not (len(names) == len(paths) == len(urls)):
+        raise RuntimeError('Failed parsing the .gitmodules content')
+    return list(zip(names, urls, paths))

gitlab_submodule/submodule_commit.py

@@ -0,0 +1,90 @@
+from typing import Optional, Tuple
+
+import re
+
+from gitlab.v4.objects import Project, ProjectCommit
+
+from gitlab_submodule.objects import GitmodulesSubmodule
+
+
+def get_submodule_commit(
+        submodule: GitmodulesSubmodule,
+        submodule_project: Project,
+        get_latest_commit_possible_if_not_found: bool = True,
+        get_latest_commit_possible_ref: Optional[str] = None
+ ) -> Tuple[ProjectCommit, bool]:
+    commit_id, is_exact = _get_submodule_commit_id(
+        submodule.parent_project,
+        submodule.path,
+        submodule.parent_ref,
+        submodule_project,
+        get_latest_commit_possible_if_not_found,
+        get_latest_commit_possible_ref
+    )
+    commit = submodule_project.commits.get(commit_id)
+    return commit, is_exact
+
+
+def _get_submodule_commit_id(
+    project: Project,
+    submodule_path: str,
+    ref: Optional[str] = None,
+    submodule_project: Optional[Project] = None,
+    get_latest_commit_possible_if_not_found: bool = True,
+    get_latest_commit_possible_ref: Optional[str] = None
+) -> Tuple[str, bool]:
+    """This uses a trick:
+    - The .gitmodules files doesn't contain the actual commit sha that the
+      submodules points to.
+    - Accessing the `<submodule_path>` dir via the ProjectFileManager
+      doesn't bring any useful info, EXCEPT: the id of the last commit that
+      modified the file (i.e. that updated the submodule commit sha)
+
+    => We use that info to get the diff of the last commit that updated the
+       submodule commit
+    => We parse the diff to get the new submodule commit sha
+
+    NOTE: in some weird cases I observed without really understanding,
+    a commit which created a .gitmodules file can contain zero submodule
+    commit sha in its entire diff.
+    In that case, we can only try to guess which was the latest commit in
+    the submodule project at the datetime of the commit.
+    """
+    submodule_dir = project.files.get(
+        submodule_path,
+        ref=ref if ref else project.default_branch)
+    last_commit_id = submodule_dir.last_commit_id
+    update_submodule_commit = project.commits.get(last_commit_id)
+
+    submodule_commit_regex = r'Submodule commit ([a-zA-Z0-9]+)\n'
+    for diff_file in update_submodule_commit.diff():
+        if diff_file['new_path'] == submodule_path:
+            # either the commit id was added for the first time,
+            # or it was updated -> we can find one or two matches
+            # (or 0 in these weird cases)
+            matches = re.findall(submodule_commit_regex, diff_file['diff'])
+            # submodule commit id was updated
+            if len(matches) == 2:
+                return matches[1], True
+            # submodule was added
+            if len(matches) == 1:
+                return matches[0], True
+
+    # If the commit diff doesn't contain the submodule commit info, we still
+    # know the date of the last commit in the project that updated the
+    # submodule, so we can fallback to the last commit in the submodule that
+    # was created before this date.
+    # This requires a Project object for the submodule so if it wasn't
+    # passed we cannot guess anything.
+    if not get_latest_commit_possible_if_not_found:
+        raise ValueError(
+            f'Could not find commit id for submodule {submodule_path} of '
+            f'project {project.path_with_namespace}.')
+    else:
+        last_subproject_commits = submodule_project.commits.list(
+            ref_name=(get_latest_commit_possible_ref
+                      if get_latest_commit_possible_ref
+                      else submodule_project.default_branch),
+            until=update_submodule_commit.created_at
+        )
+    return last_subproject_commits[0].id, False

requirements.txt

@@ -0,0 +1,2 @@
+python-gitlab==3.1.0
+giturlparse==0.10.0