start of work to add private remotes

Signed-off-by: vsoch <vsoch@users.noreply.github.com>

Author: vsoch

CHANGELOG.md

@@ -14,6 +14,7 @@ and **Merged pull requests**. Critical items to know are:
 The versions coincide with releases on pip. Only major versions will be released as tags on Github.
 
 ## [0.0.x](https://github.com/singularityhub/singularity-hpc/tree/main) (0.0.x)
+ - Support for remotes that do not expose library.json (0.0.12)
  - Update add to return container yaml (0.1.11)
  - Fixing bug with writing package file in update (0.1.1)
  - Add support for remote registry and sync commands --all (0.1.0)

shpc/client/sync.py

@@ -2,9 +2,10 @@
 __copyright__ = "Copyright 2021-2022, Vanessa Sochat"
 __license__ = "MPL 2.0"
 
+import os
+
 import shpc.logger as logger
 import shpc.utils
-import os
 
 
 def sync_registry(args, parser, extra, subparser):

shpc/main/modules/base.py

@@ -15,7 +15,7 @@
 import shpc.main.modules.template as templatectl
 import shpc.main.modules.versions as versionfile
 import shpc.main.modules.views as views
-import shpc.main.registry as registry
+import shpc.main.registry
 import shpc.utils as utils
 from shpc.logger import logger
 from shpc.main.client import Client as BaseClient
@@ -198,7 +198,7 @@ def add(self, image, module_name=None, **kwargs):
 
         # Load config (but don't validate yet!)
         config = container.ContainerConfig(
-            registry.FilesystemResult(module_name, template), validate=False
+            shpc.main.registry.FilesystemResult(module_name, template), validate=False
         )
         return self.container.add(
             module_name, image, config, container_yaml=dest, **kwargs
@@ -236,16 +236,8 @@ def docgen(self, module_name, registry=None, out=None, branch="main"):
         template = self.template.load("docs.md")
         registry = registry or defaults.github_url
         github_url = "%s/blob/%s/%s/container.yaml" % (registry, branch, module_name)
-        registry_bare = registry.split(".com")[-1]
-        raw = (
-            "https://gitlab.com/%s/-/raw/%s/%s/container.yaml"
-            if "gitlab" in registry
-            else "https://raw.githubusercontent.com/%s/%s/%s/container.yaml"
-        )
-        raw_github_url = raw % (
-            registry_bare,
-            branch,
-            module_name,
+        raw_github_url = shpc.main.registry.get_module_config_url(
+            registry, module_name, branch
         )
 
         # Currently one doc is rendered for all containers

shpc/main/registry/__init__.py

@@ -14,7 +14,7 @@
 from shpc.main.settings import SettingsBase
 
 from .filesystem import Filesystem, FilesystemResult
-from .remote import GitHub, GitLab
+from .remote import GitHub, GitLab, get_module_config_url
 
 
 def update_container_module(module, from_path, existing_path):

shpc/main/registry/remote.py

@@ -5,8 +5,8 @@
 
 import os
 import re
+import shutil
 import subprocess as sp
-import sys
 
 import requests
 
@@ -16,6 +16,23 @@
 from .provider import Provider, Result
 
 
+def get_module_config_url(registry, module_name, branch="main"):
+    """
+    Get the raw address of the config (container.yaml)
+    """
+    registry_bare = registry.split(".com")[-1]
+    raw = (
+        "https://gitlab.com/%s/-/raw/%s/%s/container.yaml"
+        if "gitlab" in registry
+        else "https://raw.githubusercontent.com/%s/%s/%s/container.yaml"
+    )
+    return raw % (
+        registry_bare,
+        branch,
+        module_name,
+    )
+
+
 class RemoteResult(Result):
     """
     A remote result provides courtesy functions for interacting with
@@ -117,6 +134,9 @@ def exists(self, name):
         """
         Determine if a module exists in the registry.
         """
+        name = name.split(":")[0]
+        if self._cache and name in self._cache:
+            return True
         dirname = self.source
         if self.subdir:
             dirname = os.path.join(dirname, self.subdir)
@@ -158,7 +178,8 @@ def find(self, name):
         """
         Find a particular entry in a registry
         """
-        self._update_cache()
+        if not self._cache:
+            self._update_cache()
         if name in self._cache:
             return RemoteResult(name, self._cache[name])
 
@@ -172,12 +193,29 @@ def _update_cache(self, force=False):
         # Check for exposed library API on GitHub or GitLab pages
         response = requests.get(self.web_url)
         if response.status_code != 200:
-            sys.exit(
-                "Remote %s is not deploying a Registry API (%s). Open a GitHub issue to ask for help."
-                % (self.source, self.web_url)
-            )
+            return self._update_clone_cache()
         self._cache = response.json()
 
+    def _update_clone_cache(self):
+        """
+        Given a remote that does not expose a library.json, handle via clone.
+        """
+        logger.warning(
+            "Remote %s is not deploying a Registry API, falling back to clone."
+            % self.source
+        )
+        tmpdir = self.clone()
+        for dirname, module in self.iter_modules():
+            # Minimum amount of metadata to function here
+            config_url = get_module_config_url(self.source, module)
+            self._cache[module] = {
+                "config": shpc.utils.read_yaml(
+                    os.path.join(dirname, module, "container.yaml")
+                ),
+                "config_url": config_url,
+            }
+        shutil.rmtree(tmpdir)
+
     def iter_registry(self, filter_string=None):
         """
         Yield metadata about containers in a remote registry.

shpc/tests/test_sync.py

@@ -106,6 +106,8 @@ def test_sync_from_file(tmp_path):
     [
         "https://github.com/singularityhub/shpc-registry",
         "https://gitlab.com/singularityhub/shpc-registry",
+        # This registry does not expose a web UI
+        "https://github.com/researchapps/shpc-test-registry",
     ],
 )
 def test_remote_upgrade(tmp_path, remote):
@@ -133,3 +135,30 @@ def test_remote_upgrade(tmp_path, remote):
 
     client.registry.sync(sync_registry=remote)
     assert list(client.registry.iter_modules())
+
+
+@pytest.mark.parametrize(
+    "remote",
+    [
+        "https://github.com/singularityhub/shpc-registry",
+        "https://gitlab.com/singularityhub/shpc-registry",
+        # This registry does not expose a web UI
+        "https://github.com/researchapps/shpc-test-registry",
+    ],
+)
+def test_registry_interaction(tmp_path, remote):
+    """
+    Test interactions with registries of different types
+    """
+    client = init_client(str(tmp_path), "lmod", "singularity")
+    reg = client.registry.get_registry(remote)
+
+    assert not reg.is_filesystem_registry
+
+    # This will hit the underlying logic to list/show
+    mods = list(reg.iter_registry())
+    assert mods
+
+    # Should use the cache
+    assert reg.exists("vanessa/salad")
+    assert reg.find("vanessa/salad") is not None

shpc/version.py

@@ -2,7 +2,7 @@
 __copyright__ = "Copyright 2021-2022, Vanessa Sochat"
 __license__ = "MPL 2.0"
 
-__version__ = "0.1.11"
+__version__ = "0.1.12"
 AUTHOR = "Vanessa Sochat"
 EMAIL = "vsoch@users.noreply.github.com"
 NAME = "singularity-hpc"