diff --git a/cwltool/singularity.py b/cwltool/singularity.py index 76f1fc488..419d1dc61 100644 --- a/cwltool/singularity.py +++ b/cwltool/singularity.py @@ -1,5 +1,6 @@ """Support for executing Docker format containers using Singularity {2,3}.x or Apptainer 1.x.""" +import json import logging import os import os.path @@ -7,7 +8,7 @@ import shutil import sys from collections.abc import Callable, MutableMapping -from subprocess import check_call, check_output # nosec +from subprocess import check_call, check_output, run # nosec from typing import cast from schema_salad.sourceline import SourceLine @@ -145,6 +146,29 @@ def _normalize_sif_id(string: str) -> str: return string.replace("/", "_") + ".sif" +def _inspect_singularity_image(path: str) -> bool: + """Inspect singularity image to be sure it is not an empty directory.""" + cmd = [ + "singularity", + "inspect", + "--json", + path, + ] + try: + result = run(cmd, capture_output=True, text=True) + except Exception: + return False + + if result.returncode == 0: + try: + output = json.loads(result.stdout) + except json.JSONDecodeError: + return False + if output.get("data", {}).get("attributes", {}): + return True + return False + + class SingularityCommandLineJob(ContainerCommandLineJob): def __init__( self, @@ -229,6 +253,16 @@ def get_image( ) found = True elif "dockerImageId" not in dockerRequirement and "dockerPull" in dockerRequirement: + # looking for local singularity sandbox image and handle it as a local image + if os.path.isdir(dockerRequirement["dockerPull"]) and _inspect_singularity_image( + dockerRequirement["dockerPull"] + ): + dockerRequirement["dockerImageId"] = dockerRequirement["dockerPull"] + _logger.info( + "Using local Singularity sandbox image found in %s", + dockerRequirement["dockerImageId"], + ) + return True match = re.search(pattern=r"([a-z]*://)", string=dockerRequirement["dockerPull"]) img_name = _normalize_image_id(dockerRequirement["dockerPull"]) candidates.append(img_name) @@ -243,6 +277,15 @@ def get_image( elif "dockerImageId" in dockerRequirement: if os.path.isfile(dockerRequirement["dockerImageId"]): found = True + # handling local singularity sandbox image + elif os.path.isdir(dockerRequirement["dockerImageId"]) and _inspect_singularity_image( + dockerRequirement["dockerImageId"] + ): + _logger.info( + "Using local Singularity sandbox image found in %s", + dockerRequirement["dockerImageId"], + ) + return True candidates.append(dockerRequirement["dockerImageId"]) candidates.append(_normalize_image_id(dockerRequirement["dockerImageId"])) if is_version_3_or_newer(): diff --git a/tests/sing_local_sandbox_test.cwl b/tests/sing_local_sandbox_test.cwl new file mode 100755 index 000000000..64d6f6b1c --- /dev/null +++ b/tests/sing_local_sandbox_test.cwl @@ -0,0 +1,14 @@ +#!/usr/bin/env cwl-runner +cwlVersion: v1.0 +class: CommandLineTool + +requirements: + DockerRequirement: + dockerPull: container_repo/alpine + +inputs: + message: string + +outputs: [] + +baseCommand: echo diff --git a/tests/test_singularity.py b/tests/test_singularity.py index 1139dfbc7..af9d704e5 100644 --- a/tests/test_singularity.py +++ b/tests/test_singularity.py @@ -1,11 +1,13 @@ """Tests to find local Singularity image.""" import shutil +import subprocess from pathlib import Path import pytest from cwltool.main import main +from cwltool.singularity import _inspect_singularity_image from .util import ( get_data, @@ -159,3 +161,88 @@ def test_singularity3_docker_image_id_in_tool(tmp_path: Path) -> None: ] ) assert result_code1 == 0 + + +@needs_singularity +def test_singularity_local_sandbox_image(tmp_path: Path): + workdir = tmp_path / "working_dir" + workdir.mkdir() + with working_directory(workdir): + # build a sandbox image + container_path = workdir / "container_repo" + container_path.mkdir() + cmd = [ + "singularity", + "build", + "--sandbox", + str(container_path / "alpine"), + "docker://alpine:latest", + ] + + build = subprocess.run(cmd, capture_output=True, text=True) + if build.returncode == 0: + result_code, stdout, stderr = get_main_output( + [ + "--singularity", + "--disable-pull", + get_data("tests/sing_local_sandbox_test.cwl"), + "--message", + "hello", + ] + ) + assert result_code == 0 + else: + pytest.skip(f"Failed to build the singularity image: {build.stderr}") + + +@needs_singularity +def test_singularity_inspect_image(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): + workdir = tmp_path / "working_dir" + workdir.mkdir() + repo_path = workdir / "container_repo" + image_path = repo_path / "alpine" + + # test image exists + repo_path.mkdir() + cmd = [ + "singularity", + "build", + "--sandbox", + str(image_path), + "docker://alpine:latest", + ] + build = subprocess.run(cmd, capture_output=True, text=True) + if build.returncode == 0: + # Verify the path is a correct container image + res_inspect = _inspect_singularity_image(image_path) + assert res_inspect is True + + # test wrong json output + def mock_subprocess_run(*args, **kwargs): + class Result: + returncode = 0 + stdout = "not-json" + + return Result() + + monkeypatch.setattr("cwltool.singularity.run", mock_subprocess_run) + res_inspect = _inspect_singularity_image(image_path) + assert res_inspect is False + else: + pytest.skip(f"singularity sandbox image build didn't worked: {build.stderr}") + + +def test_singularity_sandbox_image_not_exists(): + image_path = "/tmp/not_existing/image" + res_inspect = _inspect_singularity_image(image_path) + assert res_inspect is False + + +def test_inspect_image_wrong_sb_call(monkeypatch: pytest.MonkeyPatch): + + def mock_failed_subprocess(*args, **kwargs): + raise subprocess.CalledProcessError(returncode=1, cmd=args[0]) + + monkeypatch.setattr("cwltool.singularity.run", mock_failed_subprocess) + res_inspect = _inspect_singularity_image("/tmp/container_repo/alpine") + assert res_inspect is False diff --git a/tests/test_tmpdir.py b/tests/test_tmpdir.py index 18a588cf8..810379706 100644 --- a/tests/test_tmpdir.py +++ b/tests/test_tmpdir.py @@ -285,6 +285,98 @@ def test_dockerfile_singularity_build(monkeypatch: pytest.MonkeyPatch, tmp_path: shutil.rmtree(subdir) +@needs_singularity +def test_singularity_get_image_from_sandbox(monkeypatch: pytest.MonkeyPatch, tmp_path: Path): + """Test that SingularityCommandLineJob.get_image correctly handle sandbox image.""" + + (tmp_path / "out").mkdir(exist_ok=True) + tmp_outdir_prefix = tmp_path / "out" + tmp_outdir_prefix.mkdir(exist_ok=True) + (tmp_path / "tmp").mkdir(exist_ok=True) + tmpdir_prefix = str(tmp_path / "tmp") + runtime_context = RuntimeContext( + {"tmpdir_prefix": tmpdir_prefix, "user_space_docker_cmd": None} + ) + builder = Builder( + {}, + [], + [], + {}, + schema.Names(), + [], + [], + {}, + None, + None, + StdFsAccess, + StdFsAccess(""), + None, + 0.1, + True, + False, + False, + "no_listing", + runtime_context.get_outdir(), + runtime_context.get_tmpdir(), + runtime_context.get_stagedir(), + INTERNAL_VERSION, + "singularity", + ) + + workdir = tmp_path / "working_dir" + workdir.mkdir() + repo_path = workdir / "container_repo" + repo_path.mkdir() + image_path = repo_path / "alpine" + image_path.mkdir() + + # directory exists but is not an image + monkeypatch.setattr( + "cwltool.singularity._inspect_singularity_image", lambda *args, **kwargs: False + ) + req = {"class": "DockerRequirement", "dockerPull": f"{image_path}"} + res = SingularityCommandLineJob( + builder, {}, CommandLineTool.make_path_mapper, [], [], "" + ).get_image( + req, + pull_image=False, + tmp_outdir_prefix=str(tmp_outdir_prefix), + force_pull=False, + ) + assert req["dockerPull"].startswith("docker://") + assert res is False + + # directory exists and is an image: + monkeypatch.setattr( + "cwltool.singularity._inspect_singularity_image", lambda *args, **kwargs: True + ) + req = {"class": "DockerRequirement", "dockerPull": f"{image_path}"} + res = SingularityCommandLineJob( + builder, {}, CommandLineTool.make_path_mapper, [], [], "" + ).get_image( + req, + pull_image=False, + tmp_outdir_prefix=str(tmp_outdir_prefix), + force_pull=False, + ) + assert req["dockerPull"] == str(image_path) + assert req["dockerImageId"] == str(image_path) + assert res + + # test that dockerImageId is set and image exists: + req = {"class": "DockerRequirement", "dockerImageId": f"{image_path}"} + res = SingularityCommandLineJob( + builder, {}, CommandLineTool.make_path_mapper, [], [], "" + ).get_image( + req, + pull_image=False, + tmp_outdir_prefix=str(tmp_outdir_prefix), + force_pull=False, + ) + assert req["dockerImageId"] == str(image_path) + assert res + + def test_docker_tmpdir_prefix(tmp_path: Path) -> None: """Test that DockerCommandLineJob respects temp directory directives.""" (tmp_path / "3").mkdir()