feat: handle local singularity sandbox image

feat: handle local singularity sandbox image

fix: skip searching if singularity inspect found image

improve singularity inspect

test: add test for singularity sandbox image

remove unused normalization

add comment

Author: natthan-pigoux

cwltool/singularity.py

@@ -1,13 +1,15 @@
 """Support for executing Docker format containers using Singularity {2,3}.x or Apptainer 1.x."""
 
+import json
 import logging
 import os
 import os.path
 import re
 import shutil
+import subprocess
 import sys
 from collections.abc import Callable, MutableMapping
-from subprocess import check_call, check_output  # nosec
+from subprocess import DEVNULL, check_call, check_output, run  # nosec
 from typing import cast
 
 from schema_salad.sourceline import SourceLine
@@ -144,6 +146,27 @@ def _normalize_sif_id(string: str) -> str:
         string += "_latest"
     return string.replace("/", "_") + ".sif"
 
+def _inspect_singularity_image(path: str) -> bool:
+    """Inspect singularity image to be sure it is not an empty directory."""
+    cmd = [
+        "singularity",
+        "inspect",
+        "--json",
+        path,
+    ]
+    try:
+        result = run(cmd, capture_output=True, text=True)
+    except Exception:
+        return False
+    
+    if result.returncode == 0:
+        try:
+            output = json.loads(result.stdout)
+        except json.JSONDecodeError:
+            return False
+        if output.get('data', {}).get('attributes', {}):
+            return True
+    return False
 
 class SingularityCommandLineJob(ContainerCommandLineJob):
     def __init__(
@@ -229,24 +252,40 @@ def get_image(
                     )
                 found = True
         elif "dockerImageId" not in dockerRequirement and "dockerPull" in dockerRequirement:
-            match = re.search(pattern=r"([a-z]*://)", string=dockerRequirement["dockerPull"])
-            img_name = _normalize_image_id(dockerRequirement["dockerPull"])
-            candidates.append(img_name)
-            if is_version_3_or_newer():
-                sif_name = _normalize_sif_id(dockerRequirement["dockerPull"])
-                candidates.append(sif_name)
-                dockerRequirement["dockerImageId"] = sif_name
+            # looking for local singularity sandbox image and handle it as a local image
+            if os.path.isdir(dockerRequirement["dockerPull"]) and _inspect_singularity_image(dockerRequirement["dockerPull"]):
+                dockerRequirement["dockerImageId"] = dockerRequirement["dockerPull"]
+                _logger.info(
+                    "Using local Singularity sandbox image found in %s",
+                    dockerRequirement["dockerImageId"],
+                )
+                return True
             else:
-                dockerRequirement["dockerImageId"] = img_name
-            if not match:
-                dockerRequirement["dockerPull"] = "docker://" + dockerRequirement["dockerPull"]
+                match = re.search(pattern=r"([a-z]*://)", string=dockerRequirement["dockerPull"])
+                img_name = _normalize_image_id(dockerRequirement["dockerPull"])
+                candidates.append(img_name)
+                if is_version_3_or_newer():
+                    sif_name = _normalize_sif_id(dockerRequirement["dockerPull"])
+                    candidates.append(sif_name)
+                    dockerRequirement["dockerImageId"] = sif_name
+                else:
+                    dockerRequirement["dockerImageId"] = img_name
+                if not match:
+                    dockerRequirement["dockerPull"] = "docker://" + dockerRequirement["dockerPull"]
         elif "dockerImageId" in dockerRequirement:
             if os.path.isfile(dockerRequirement["dockerImageId"]):
                 found = True
-            candidates.append(dockerRequirement["dockerImageId"])
-            candidates.append(_normalize_image_id(dockerRequirement["dockerImageId"]))
-            if is_version_3_or_newer():
-                candidates.append(_normalize_sif_id(dockerRequirement["dockerImageId"]))
+                candidates.append(dockerRequirement["dockerImageId"])
+                candidates.append(_normalize_image_id(dockerRequirement["dockerImageId"]))
+                if is_version_3_or_newer():
+                    candidates.append(_normalize_sif_id(dockerRequirement["dockerImageId"]))
+            # handling local singularity sandbox image
+            elif os.path.isdir(dockerRequirement["dockerImageId"]) and _inspect_singularity_image(dockerRequirement["dockerImageId"]):
+                _logger.info(
+                    "Using local Singularity sandbox image found in %s",
+                    dockerRequirement["dockerImageId"],
+                )
+                return True
 
         targets = [os.getcwd()]
         if "CWL_SINGULARITY_CACHE" in os.environ:

tests/sing_local_sandbox_test.cwl

@@ -0,0 +1,14 @@
+#!/usr/bin/env cwl-runner
+cwlVersion: v1.0
+class: CommandLineTool
+
+requirements:
+  DockerRequirement:
+    dockerPull: container_repo/alpine
+
+inputs:
+  message: string
+
+outputs: []
+
+baseCommand: echo

tests/test_singularity.py

@@ -159,3 +159,34 @@ def test_singularity3_docker_image_id_in_tool(tmp_path: Path) -> None:
             ]
         )
         assert result_code1 == 0
+
+@needs_singularity
+def test_singularity_local_sandbox_image(tmp_path: Path):
+    import subprocess
+    workdir = tmp_path / "working_dir"
+    workdir.mkdir()
+    with working_directory(workdir):
+        # build a sandbox image
+        container_path = Path(f"{workdir}/container_repo/")
+        container_path.mkdir()
+        cmd = [
+            "apptainer",
+            "build",
+            "--sandbox",
+            container_path / "alpine",
+            "docker://alpine:latest"
+        ]
+        build = subprocess.run(cmd, capture_output=True, text=True)
+        if build.returncode == 0:
+            result_code, stdout, stderr = get_main_output(
+                [
+                    "--singularity",
+                    "--disable-pull",
+                    get_data("tests/sing_local_sandbox_test.cwl"),
+                    "--message",
+                    "hello",
+                ]
+            )
+            assert result_code == 0
+        else:
+            pytest.skip(f"Failed to build the Singularity image: {build.stderr}")