Support remote images (#143)

* support remote images in challenge deployment
* add registry:// prefix to mark images as remote

Author: Miłosz Skaza

ctfcli/core/challenge.py

@@ -2,7 +2,7 @@
 import subprocess
 from os import PathLike
 from pathlib import Path
-from typing import Any, Dict, List, Tuple, Union
+from typing import Any, Dict, List, Optional, Tuple, Union
 
 import click
 import yaml
@@ -140,16 +140,58 @@ def __init__(self, challenge_yml: Union[str, PathLike], overrides=None):
         # API is not initialized before running an API-related operation, but should be reused later
         self._api = None
 
-        # Set Image to None if the challenge does not provide one
-        self.image = None
-
-        # Get name and a build path for the image if the challenge provides one
-        if self.get("image"):
-            self.image = Image(slugify(self["name"]), self.challenge_directory / self["image"])
+        # Assign an image if the challenge provides one, otherwise this will be set to None
+        self.image = self._process_challenge_image(self.get("image"))
 
     def __str__(self):
         return self["name"]
 
+    def _process_challenge_image(self, challenge_image: Optional[str]) -> Optional[Image]:
+        if not challenge_image:
+            return None
+
+        # Check if challenge_image is explicitly marked with registry:// prefix
+        if challenge_image.startswith("registry://"):
+            challenge_image = challenge_image.replace("registry://", "")
+            return Image(challenge_image)
+
+        # Check if it's a library image
+        if challenge_image.startswith("library/"):
+            return Image(f"docker.io/{challenge_image}")
+
+        # Check if it defines a known registry
+        known_registries = [
+            "docker.io",
+            "gcr.io",
+            "ecr.aws",
+            "ghcr.io",
+            "azurecr.io",
+            "registry.digitalocean.com",
+            "registry.gitlab.com",
+            "registry.ctfd.io",
+        ]
+        for registry in known_registries:
+            if registry in challenge_image:
+                return Image(challenge_image)
+
+        # Check if it's a path to dockerfile to be built
+        if (self.challenge_directory / challenge_image / "Dockerfile").exists():
+            return Image(slugify(self["name"]), self.challenge_directory / self["image"])
+
+        # Check if it's a local pre-built image
+        if (
+            subprocess.call(
+                ["docker", "inspect", challenge_image], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL
+            )
+            == 0
+        ):
+            return Image(challenge_image)
+
+        # If the image is set, but we fail to determine whether it's local / remote - raise an exception
+        raise InvalidChallengeFile(
+            f"Challenge file at {self.challenge_file_path} defines an image, but it couldn't be resolved"
+        )
+
     def _load_challenge_id(self):
         remote_challenges = self.load_installed_challenges()
         if not remote_challenges:

ctfcli/core/deployment/cloud.py

@@ -23,7 +23,7 @@ def __init__(self, *args, **kwargs):
         # Do not fail here if challenge does not provide an image
         # rather return a failed deployment result during deploy
         if self.challenge.get("image"):
-            self.image_name = self.challenge.image.name
+            self.image_name = self.challenge.image.basename
 
     def deploy(self, skip_login=False, *args, **kwargs) -> DeploymentResult:
         # Check whether challenge defines image
@@ -40,11 +40,15 @@ def deploy(self, skip_login=False, *args, **kwargs) -> DeploymentResult:
         # Get or create Image in CTFd
         image_data = self._get_or_create_image()
 
-        # Build new / initial version of the image
-        image_name = self.challenge.image.build()
-        if not image_name:
-            click.secho("Could not build the image. Please check docker output above.", fg="red")
-            return DeploymentResult(False)
+        # Build or Pull / Update the configured image
+        if self.challenge.image.built:
+            if not self.challenge.image.pull():
+                click.secho("Could not pull the image. Please check docker output above.", fg="red")
+                return DeploymentResult(False)
+        else:
+            if not self.challenge.image.build():
+                click.secho("Could not build the image. Please check docker output above.", fg="red")
+                return DeploymentResult(False)
 
         if skip_login:
             click.secho(

ctfcli/core/deployment/registry.py

@@ -29,7 +29,7 @@ def deploy(self, skip_login=False, *args, **kwargs) -> DeploymentResult:
         # e.g. registry.example.com/test-project/challenge-image-name
         # challenge image name is appended to the host provided for the deployment
         host_url = urlparse(self.host)
-        location = f"{host_url.netloc}{host_url.path.rstrip('/')}/{self.challenge.image.name}"
+        location = f"{host_url.netloc}{host_url.path.rstrip('/')}/{self.challenge.image.basename}"
 
         if skip_login:
             click.secho(
@@ -57,10 +57,14 @@ def deploy(self, skip_login=False, *args, **kwargs) -> DeploymentResult:
                 click.secho("Could not log in to the registry. Please check your configured credentials.", fg="red")
                 return DeploymentResult(False)
 
-        build_result = self.challenge.image.build()
-        if not build_result:
-            click.secho("Could not build the image. Please check docker output above.", fg="red")
-            return DeploymentResult(False)
+        if self.challenge.image.built:
+            if not self.challenge.image.pull():
+                click.secho("Could not pull the image. Please check docker output above.", fg="red")
+                return DeploymentResult(False)
+        else:
+            if not self.challenge.image.build():
+                click.secho("Could not build the image. Please check docker output above.", fg="red")
+                return DeploymentResult(False)
 
         push_result = self.challenge.image.push(location)
         if not push_result:
@@ -89,7 +93,9 @@ def _registry_login(self, username: str, password: str, registry: str):
         ]
 
         try:
-            log.debug(f"call({docker_login_command}, stderr=subprocess.PIPE, input=password)")
+            log.debug(
+                f"call({docker_login_command}, input=password, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)"
+            )
             subprocess.run(
                 docker_login_command, input=password.encode(), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL
             )

ctfcli/core/deployment/ssh.py

@@ -19,18 +19,23 @@ def deploy(self, *args, **kwargs) -> DeploymentResult:
             )
             return DeploymentResult(False)
 
-        build_result = self.challenge.image.build()
-        if not build_result:
-            click.secho("Could not build the image. Please check docker output above.", fg="red")
-            return DeploymentResult(False)
+        if self.challenge.image.built:
+            if not self.challenge.image.pull():
+                click.secho("Could not pull the image. Please check docker output above.", fg="red")
+                return DeploymentResult(False)
+        else:
+            if not self.challenge.image.build():
+                click.secho("Could not build the image. Please check docker output above.", fg="red")
+                return DeploymentResult(False)
 
         image_name = self.challenge.image.name
-        export_result = self.challenge.image.export()
-        if not export_result:
+        image_basename = self.challenge.image.basename
+        image_export = self.challenge.image.export()
+        if not image_export:
             click.secho("Could not export the image. Please check docker output above.", fg="red")
             return DeploymentResult(False)
 
-        image_export_path = Path(export_result)
+        image_export_path = Path(image_export)
         host_url = urlparse(self.host)
         target_path = host_url.path or "/tmp"
         target_file = f"{target_path}/{image_export_path.name}"
@@ -54,14 +59,15 @@ def deploy(self, *args, **kwargs) -> DeploymentResult:
                 [
                     "ssh",
                     host_url.netloc,
-                    f"docker stop {image_name} 2>/dev/null; docker rm {image_name} 2>/dev/null",
+                    f"docker stop {image_basename} 2>/dev/null; docker rm {image_basename} 2>/dev/null",
                 ]
             )
             subprocess.run(
                 [
                     "ssh",
                     host_url.netloc,
-                    f"docker run -d -p{exposed_port}:{exposed_port} --name {image_name} --restart always {image_name}",
+                    f"docker run -d -p{exposed_port}:{exposed_port} --name {image_basename} "
+                    f"--restart always {image_name}",
                 ]
             )
 

ctfcli/core/image.py

@@ -7,10 +7,21 @@
 
 
 class Image:
-    def __init__(self, name: str, build_path: Union[str, PathLike]):
+    def __init__(self, name: str, build_path: Optional[Union[str, PathLike]] = None):
+        # name can be either a new name to assign or an existing image name
         self.name = name
-        self.build_path = Path(build_path)
-        self.built = False
+
+        # if the image is a remote image (eg. ghcr.io/.../...), extract the basename
+        self.basename = name
+        if "/" in self.name or ":" in self.name:
+            self.basename = self.name.split(":")[0].split("/")[-1]
+
+        self.built = True
+
+        # if the image provides a build path, assume it is not built yet
+        if build_path:
+            self.build_path = Path(build_path)
+            self.built = False
 
     def build(self) -> Optional[str]:
         docker_build = subprocess.call(["docker", "build", "-t", self.name, "."], cwd=self.build_path.absolute())
@@ -20,6 +31,13 @@ def build(self) -> Optional[str]:
         self.built = True
         return self.name
 
+    def pull(self) -> Optional[str]:
+        docker_pull = subprocess.call(["docker", "pull", self.name])
+        if docker_pull != 0:
+            return
+
+        return self.name
+
     def push(self, location: str) -> Optional[str]:
         if not self.built:
             self.build()
@@ -36,7 +54,7 @@ def export(self) -> Optional[str]:
         if not self.built:
             self.build()
 
-        image_tar = tempfile.NamedTemporaryFile(delete=False, suffix=f"_{self.name}.docker.tar")
+        image_tar = tempfile.NamedTemporaryFile(delete=False, suffix=f"_{self.basename}.docker.tar")
         docker_save = subprocess.call(["docker", "save", "--output", image_tar.name, self.name])
 
         if docker_save != 0:

tests/core/deployment/test_cloud_deployment.py

@@ -83,6 +83,8 @@ def test_fails_deployment_if_image_build_failed(
 
         mock_image: MagicMock = mock_image_constructor.return_value
         mock_image.name = "test-challenge"
+        mock_image.basename = "test-challenge"
+        mock_image.built = False
         mock_image.build.return_value = None
 
         mock_api: MagicMock = mock_api_constructor.return_value
@@ -134,6 +136,7 @@ def test_fails_deployment_if_image_push_failed(
 
         mock_image: MagicMock = mock_image_constructor.return_value
         mock_image.name = "test-challenge"
+        mock_image.basename = "test-challenge"
         mock_image.build.return_value = "test-challenge"
         mock_image.push.return_value = None
 
@@ -201,6 +204,7 @@ def test_fails_deployment_if_registry_login_unsuccessful(
 
         mock_image: MagicMock = mock_image_constructor.return_value
         mock_image.name = "test-challenge"
+        mock_image.basename = "test-challenge"
         mock_image.build.return_value = "test-challenge"
 
         mock_api: MagicMock = mock_api_constructor.return_value
@@ -282,6 +286,7 @@ def test_fails_deployment_if_instance_url_is_not_ctfd_assigned(
 
         mock_image: MagicMock = mock_image_constructor.return_value
         mock_image.name = "test-challenge"
+        mock_image.basename = "test-challenge"
         mock_image.build.return_value = "test-challenge"
 
         mock_api: MagicMock = mock_api_constructor.return_value
@@ -365,6 +370,7 @@ def test_allows_skipping_registry_login(
 
         mock_image: MagicMock = mock_image_constructor.return_value
         mock_image.name = "test-challenge"
+        mock_image.basename = "test-challenge"
         mock_image.build.return_value = "test-challenge"
         mock_image.push.return_value = "registry.ctfd.io/example-project/test-challenge"
 
@@ -495,6 +501,7 @@ def test_deploys_challenge_with_existing_image_service(
 
         mock_image: MagicMock = mock_image_constructor.return_value
         mock_image.name = "test-challenge"
+        mock_image.basename = "test-challenge"
         mock_image.build.return_value = "test-challenge"
         mock_image.push.return_value = "registry.ctfd.io/example-project/test-challenge"
 
@@ -637,6 +644,7 @@ def test_deploys_challenge_with_new_image_service(
 
         mock_image: MagicMock = mock_image_constructor.return_value
         mock_image.name = "test-challenge"
+        mock_image.basename = "test-challenge"
         mock_image.build.return_value = "test-challenge"
         mock_image.push.return_value = "registry.ctfd.io/example-project/test-challenge"
 
@@ -830,6 +838,7 @@ def test_fails_deployment_after_timeout(
 
         mock_image = mock_image_constructor.return_value
         mock_image.name = "test-challenge"
+        mock_image.basename = "test-challenge"
         mock_image.build.return_value = "test-challenge"
         mock_image.push.return_value = "registry.ctfd.io/example-project/test-challenge"
 
@@ -1019,6 +1028,7 @@ def test_exposes_tcp_port(
 
         mock_image = mock_image_constructor.return_value
         mock_image.name = "test-challenge"
+        mock_image.basename = "test-challenge"
         mock_image.build.return_value = "test-challenge"
         mock_image.push.return_value = "registry.ctfd.io/example-project/test-challenge"
 

tests/core/deployment/test_registry_deployment.py

@@ -30,6 +30,7 @@ def test_builds_and_pushes_image(
 
         mock_image: MagicMock = mock_image_constructor.return_value
         mock_image.name = "test-challenge"
+        mock_image.basename = "test-challenge"
         mock_image.build.return_value = "test-challenge"
         mock_image.push.return_value = "registry.example.com/example-project/test-challenge"
 
@@ -79,6 +80,7 @@ def test_fails_deployment_if_no_registry_config(
 
         mock_image: MagicMock = mock_image_constructor.return_value
         mock_image.name = "test-challenge"
+        mock_image.basename = "test-challenge"
         mock_image.build.return_value = "test-challenge"
 
         handler = RegistryDeploymentHandler(challenge, host="registry://registry.example.com/example-project")
@@ -104,6 +106,7 @@ def test_fails_if_no_credentials(
 
         mock_image: MagicMock = mock_image_constructor.return_value
         mock_image.name = "test-challenge"
+        mock_image.basename = "test-challenge"
         mock_image.build.return_value = "test-challenge"
 
         mock_config_constructor.return_value = {"registry": {"username": "test"}}
@@ -133,6 +136,7 @@ def test_fails_if_registry_credentials_invalid(
 
         mock_image: MagicMock = mock_image_constructor.return_value
         mock_image.name = "test-challenge"
+        mock_image.basename = "test-challenge"
         mock_image.build.return_value = "test-challenge"
 
         mock_config_constructor.return_value = {"registry": {"username": "test", "password": "test"}}
@@ -164,6 +168,8 @@ def test_fails_deployment_if_image_build_failed(
 
         mock_image: MagicMock = mock_image_constructor.return_value
         mock_image.name = "test-challenge"
+        mock_image.basename = "test-challenge"
+        mock_image.built = False
         mock_image.build.return_value = None
 
         mock_config_constructor.return_value = {"registry": {"username": "test", "password": "test"}}
@@ -192,6 +198,7 @@ def test_fails_deployment_if_image_push_failed(
 
         mock_image: MagicMock = mock_image_constructor.return_value
         mock_image.name = "test-challenge"
+        mock_image.basename = "test-challenge"
         mock_image.build.return_value = "test-challenge"
         mock_image.push.return_value = None
 
@@ -224,6 +231,7 @@ def test_allows_skipping_login(
 
         mock_image: MagicMock = mock_image_constructor.return_value
         mock_image.name = "test-challenge"
+        mock_image.basename = "test-challenge"
         mock_image.build.return_value = "test-challenge"
         mock_image.push.return_value = "registry.example.com/example-project/test-challenge"
 
@@ -257,6 +265,7 @@ def test_warns_about_logging_in_with_skip_login(
 
         mock_image: MagicMock = mock_image_constructor.return_value
         mock_image.name = "test-challenge"
+        mock_image.basename = "test-challenge"
         mock_image.build.return_value = "test-challenge"
         mock_image.push.return_value = None