From dba9616719cf755e0607f35f8de9fe7c8083f574 Mon Sep 17 00:00:00 2001 From: natthan-pigoux Date: Fri, 24 Oct 2025 15:36:52 +0200 Subject: [PATCH 1/7] feat: handle local singularity sandbox image feat: handle local singularity sandbox image fix: skip searching if singularity inspect found image improve singularity inspect test: add test for singularity sandbox image remove unused normalization add comment --- cwltool/singularity.py | 69 ++++++++++++++++++++++++------- tests/sing_local_sandbox_test.cwl | 14 +++++++ tests/test_singularity.py | 31 ++++++++++++++ 3 files changed, 99 insertions(+), 15 deletions(-) create mode 100755 tests/sing_local_sandbox_test.cwl diff --git a/cwltool/singularity.py b/cwltool/singularity.py index 76f1fc488..4031e8794 100644 --- a/cwltool/singularity.py +++ b/cwltool/singularity.py @@ -1,13 +1,15 @@ """Support for executing Docker format containers using Singularity {2,3}.x or Apptainer 1.x.""" +import json import logging import os import os.path import re import shutil +import subprocess import sys from collections.abc import Callable, MutableMapping -from subprocess import check_call, check_output # nosec +from subprocess import DEVNULL, check_call, check_output, run # nosec from typing import cast from schema_salad.sourceline import SourceLine @@ -144,6 +146,27 @@ def _normalize_sif_id(string: str) -> str: string += "_latest" return string.replace("/", "_") + ".sif" +def _inspect_singularity_image(path: str) -> bool: + """Inspect singularity image to be sure it is not an empty directory.""" + cmd = [ + "singularity", + "inspect", + "--json", + path, + ] + try: + result = run(cmd, capture_output=True, text=True) + except Exception: + return False + + if result.returncode == 0: + try: + output = json.loads(result.stdout) + except json.JSONDecodeError: + return False + if output.get('data', {}).get('attributes', {}): + return True + return False class SingularityCommandLineJob(ContainerCommandLineJob): def __init__( @@ -229,24 +252,40 @@ def get_image( ) found = True elif "dockerImageId" not in dockerRequirement and "dockerPull" in dockerRequirement: - match = re.search(pattern=r"([a-z]*://)", string=dockerRequirement["dockerPull"]) - img_name = _normalize_image_id(dockerRequirement["dockerPull"]) - candidates.append(img_name) - if is_version_3_or_newer(): - sif_name = _normalize_sif_id(dockerRequirement["dockerPull"]) - candidates.append(sif_name) - dockerRequirement["dockerImageId"] = sif_name + # looking for local singularity sandbox image and handle it as a local image + if os.path.isdir(dockerRequirement["dockerPull"]) and _inspect_singularity_image(dockerRequirement["dockerPull"]): + dockerRequirement["dockerImageId"] = dockerRequirement["dockerPull"] + _logger.info( + "Using local Singularity sandbox image found in %s", + dockerRequirement["dockerImageId"], + ) + return True else: - dockerRequirement["dockerImageId"] = img_name - if not match: - dockerRequirement["dockerPull"] = "docker://" + dockerRequirement["dockerPull"] + match = re.search(pattern=r"([a-z]*://)", string=dockerRequirement["dockerPull"]) + img_name = _normalize_image_id(dockerRequirement["dockerPull"]) + candidates.append(img_name) + if is_version_3_or_newer(): + sif_name = _normalize_sif_id(dockerRequirement["dockerPull"]) + candidates.append(sif_name) + dockerRequirement["dockerImageId"] = sif_name + else: + dockerRequirement["dockerImageId"] = img_name + if not match: + dockerRequirement["dockerPull"] = "docker://" + dockerRequirement["dockerPull"] elif "dockerImageId" in dockerRequirement: if os.path.isfile(dockerRequirement["dockerImageId"]): found = True - candidates.append(dockerRequirement["dockerImageId"]) - candidates.append(_normalize_image_id(dockerRequirement["dockerImageId"])) - if is_version_3_or_newer(): - candidates.append(_normalize_sif_id(dockerRequirement["dockerImageId"])) + candidates.append(dockerRequirement["dockerImageId"]) + candidates.append(_normalize_image_id(dockerRequirement["dockerImageId"])) + if is_version_3_or_newer(): + candidates.append(_normalize_sif_id(dockerRequirement["dockerImageId"])) + # handling local singularity sandbox image + elif os.path.isdir(dockerRequirement["dockerImageId"]) and _inspect_singularity_image(dockerRequirement["dockerImageId"]): + _logger.info( + "Using local Singularity sandbox image found in %s", + dockerRequirement["dockerImageId"], + ) + return True targets = [os.getcwd()] if "CWL_SINGULARITY_CACHE" in os.environ: diff --git a/tests/sing_local_sandbox_test.cwl b/tests/sing_local_sandbox_test.cwl new file mode 100755 index 000000000..64d6f6b1c --- /dev/null +++ b/tests/sing_local_sandbox_test.cwl @@ -0,0 +1,14 @@ +#!/usr/bin/env cwl-runner +cwlVersion: v1.0 +class: CommandLineTool + +requirements: + DockerRequirement: + dockerPull: container_repo/alpine + +inputs: + message: string + +outputs: [] + +baseCommand: echo diff --git a/tests/test_singularity.py b/tests/test_singularity.py index 1139dfbc7..6aa65f339 100644 --- a/tests/test_singularity.py +++ b/tests/test_singularity.py @@ -159,3 +159,34 @@ def test_singularity3_docker_image_id_in_tool(tmp_path: Path) -> None: ] ) assert result_code1 == 0 + +@needs_singularity +def test_singularity_local_sandbox_image(tmp_path: Path): + import subprocess + workdir = tmp_path / "working_dir" + workdir.mkdir() + with working_directory(workdir): + # build a sandbox image + container_path = Path(f"{workdir}/container_repo/") + container_path.mkdir() + cmd = [ + "apptainer", + "build", + "--sandbox", + container_path / "alpine", + "docker://alpine:latest" + ] + build = subprocess.run(cmd, capture_output=True, text=True) + if build.returncode == 0: + result_code, stdout, stderr = get_main_output( + [ + "--singularity", + "--disable-pull", + get_data("tests/sing_local_sandbox_test.cwl"), + "--message", + "hello", + ] + ) + assert result_code == 0 + else: + pytest.skip(f"Failed to build the Singularity image: {build.stderr}") From c51d312088f92aa69b85af7331587f96402d29d7 Mon Sep 17 00:00:00 2001 From: natthan-pigoux Date: Sat, 25 Oct 2025 13:31:21 +0200 Subject: [PATCH 2/7] test: improve test and linting --- cwltool/singularity.py | 17 +++++--- tests/test_singularity.py | 50 +++++++++++++++++++--- tests/test_tmpdir.py | 87 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 142 insertions(+), 12 deletions(-) diff --git a/cwltool/singularity.py b/cwltool/singularity.py index 4031e8794..f1f0f9c1f 100644 --- a/cwltool/singularity.py +++ b/cwltool/singularity.py @@ -6,10 +6,9 @@ import os.path import re import shutil -import subprocess import sys from collections.abc import Callable, MutableMapping -from subprocess import DEVNULL, check_call, check_output, run # nosec +from subprocess import check_call, check_output, run # nosec from typing import cast from schema_salad.sourceline import SourceLine @@ -146,6 +145,7 @@ def _normalize_sif_id(string: str) -> str: string += "_latest" return string.replace("/", "_") + ".sif" + def _inspect_singularity_image(path: str) -> bool: """Inspect singularity image to be sure it is not an empty directory.""" cmd = [ @@ -158,16 +158,17 @@ def _inspect_singularity_image(path: str) -> bool: result = run(cmd, capture_output=True, text=True) except Exception: return False - + if result.returncode == 0: try: output = json.loads(result.stdout) except json.JSONDecodeError: return False - if output.get('data', {}).get('attributes', {}): + if output.get("data", {}).get("attributes", {}): return True return False + class SingularityCommandLineJob(ContainerCommandLineJob): def __init__( self, @@ -253,7 +254,9 @@ def get_image( found = True elif "dockerImageId" not in dockerRequirement and "dockerPull" in dockerRequirement: # looking for local singularity sandbox image and handle it as a local image - if os.path.isdir(dockerRequirement["dockerPull"]) and _inspect_singularity_image(dockerRequirement["dockerPull"]): + if os.path.isdir(dockerRequirement["dockerPull"]) and _inspect_singularity_image( + dockerRequirement["dockerPull"] + ): dockerRequirement["dockerImageId"] = dockerRequirement["dockerPull"] _logger.info( "Using local Singularity sandbox image found in %s", @@ -280,7 +283,9 @@ def get_image( if is_version_3_or_newer(): candidates.append(_normalize_sif_id(dockerRequirement["dockerImageId"])) # handling local singularity sandbox image - elif os.path.isdir(dockerRequirement["dockerImageId"]) and _inspect_singularity_image(dockerRequirement["dockerImageId"]): + elif os.path.isdir(dockerRequirement["dockerImageId"]) and _inspect_singularity_image( + dockerRequirement["dockerImageId"] + ): _logger.info( "Using local Singularity sandbox image found in %s", dockerRequirement["dockerImageId"], diff --git a/tests/test_singularity.py b/tests/test_singularity.py index 6aa65f339..70b51a2af 100644 --- a/tests/test_singularity.py +++ b/tests/test_singularity.py @@ -1,11 +1,13 @@ """Tests to find local Singularity image.""" import shutil +import subprocess from pathlib import Path import pytest from cwltool.main import main +from cwltool.singularity import _inspect_singularity_image from .util import ( get_data, @@ -160,22 +162,23 @@ def test_singularity3_docker_image_id_in_tool(tmp_path: Path) -> None: ) assert result_code1 == 0 + @needs_singularity def test_singularity_local_sandbox_image(tmp_path: Path): - import subprocess workdir = tmp_path / "working_dir" workdir.mkdir() with working_directory(workdir): # build a sandbox image - container_path = Path(f"{workdir}/container_repo/") + container_path = workdir / "container_repo" container_path.mkdir() cmd = [ - "apptainer", + "singularity", "build", "--sandbox", - container_path / "alpine", - "docker://alpine:latest" + str(container_path / "alpine"), + "docker://alpine:latest", ] + build = subprocess.run(cmd, capture_output=True, text=True) if build.returncode == 0: result_code, stdout, stderr = get_main_output( @@ -189,4 +192,39 @@ def test_singularity_local_sandbox_image(tmp_path: Path): ) assert result_code == 0 else: - pytest.skip(f"Failed to build the Singularity image: {build.stderr}") + pytest.skip(f"Failed to build the singularity image: {build.stderr}") + + +@needs_singularity +def test_singularity_inspect_image(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): + workdir = tmp_path / "working_dir" + workdir.mkdir() + repo_path = workdir / "container_repo" + image_path = repo_path / "alpine" + + # test path does not exists + res_inspect = _inspect_singularity_image(str(image_path)) + assert res_inspect is False + + # test image exists + repo_path.mkdir() + cmd = [ + "singularity", + "build", + "--sandbox", + str(image_path), + "docker://alpine:latest", + ] + build = subprocess.run(cmd, capture_output=True, text=True) + if build.returncode == 0: + # Verify the path is a container image + res_inspect = _inspect_singularity_image(image_path) + assert res_inspect is True + + # test wrong subprocess call + def mock_failed_subprocess(*args, **kwargs): + raise subprocess.CalledProcessError(returncode=1, cmd=args[0]) + + monkeypatch.setattr("cwltool.singularity.run", mock_failed_subprocess) + res_inspect = _inspect_singularity_image(str(image_path)) + assert res_inspect is False \ No newline at end of file diff --git a/tests/test_tmpdir.py b/tests/test_tmpdir.py index 18a588cf8..f0476e7c9 100644 --- a/tests/test_tmpdir.py +++ b/tests/test_tmpdir.py @@ -285,6 +285,93 @@ def test_dockerfile_singularity_build(monkeypatch: pytest.MonkeyPatch, tmp_path: shutil.rmtree(subdir) +@needs_singularity +def test_singularity_get_image_from_sandbox(monkeypatch: pytest.MonkeyPatch, tmp_path: Path): + """Test that SingularityCommandLineJob.get_image correctly handle sandbox image.""" + + (tmp_path / "out").mkdir(exist_ok=True) + tmp_outdir_prefix = tmp_path / "out" + (tmp_path / "3").mkdir(exist_ok=True) + tmpdir_prefix = str(tmp_path / "tmp") + runtime_context = RuntimeContext( + {"tmpdir_prefix": tmpdir_prefix, "user_space_docker_cmd": None} + ) + builder = Builder( + {}, + [], + [], + {}, + schema.Names(), + [], + [], + {}, + None, + None, + StdFsAccess, + StdFsAccess(""), + None, + 0.1, + True, + False, + False, + "no_listing", + runtime_context.get_outdir(), + runtime_context.get_tmpdir(), + runtime_context.get_stagedir(), + INTERNAL_VERSION, + "singularity", + ) + + workdir = tmp_path / "working_dir" + workdir.mkdir() + repo_path = workdir / "container_repo" + repo_path.mkdir() + image_path = repo_path / "alpine" + image_path.mkdir() + + # directory exists but is not an image + monkeypatch.setattr("cwltool.singularity._inspect_singularity_image", lambda *args, **kwargs: False) + req = {"class": "DockerRequirement", "dockerPull": f"{image_path}"} + res = SingularityCommandLineJob( + builder, {}, CommandLineTool.make_path_mapper, [], [], "" + ).get_image( + req, + pull_image=False, + tmp_outdir_prefix=str(tmp_outdir_prefix), + force_pull=False, + ) + assert req["dockerPull"].startswith("docker://") + assert res is False + + # directory exists and is an image: + monkeypatch.setattr("cwltool.singularity._inspect_singularity_image", lambda *args, **kwargs: True) + req = {"class": "DockerRequirement", "dockerPull": f"{image_path}"} + res = SingularityCommandLineJob( + builder, {}, CommandLineTool.make_path_mapper, [], [], "" + ).get_image( + req, + pull_image=False, + tmp_outdir_prefix=str(tmp_outdir_prefix), + force_pull=False, + ) + assert req["dockerPull"] == str(image_path) + assert req["dockerImageId"] == str(image_path) + assert res + + # test that dockerImageId is set and image exists: + req = {"class": "DockerRequirement", "dockerImageId": f"{image_path}"} + res = SingularityCommandLineJob( + builder, {}, CommandLineTool.make_path_mapper, [], [], "" + ).get_image( + req, + pull_image=False, + tmp_outdir_prefix=str(tmp_outdir_prefix), + force_pull=False, + ) + assert req["dockerImageId"] == str(image_path) + assert res + + def test_docker_tmpdir_prefix(tmp_path: Path) -> None: """Test that DockerCommandLineJob respects temp directory directives.""" (tmp_path / "3").mkdir() From 6f96b54142ba97cf97111323b7699531e5fb8b1c Mon Sep 17 00:00:00 2001 From: natthan-pigoux Date: Wed, 5 Nov 2025 14:33:54 +0100 Subject: [PATCH 3/7] refactor: inspect_image test --- tests/test_singularity.py | 37 ++++++++++++++++++++++++++++--------- tests/test_tmpdir.py | 12 ++++++++---- 2 files changed, 36 insertions(+), 13 deletions(-) diff --git a/tests/test_singularity.py b/tests/test_singularity.py index 70b51a2af..1aa69c26d 100644 --- a/tests/test_singularity.py +++ b/tests/test_singularity.py @@ -1,5 +1,6 @@ """Tests to find local Singularity image.""" +import json import shutil import subprocess from pathlib import Path @@ -202,10 +203,6 @@ def test_singularity_inspect_image(tmp_path: Path, monkeypatch: pytest.MonkeyPat repo_path = workdir / "container_repo" image_path = repo_path / "alpine" - # test path does not exists - res_inspect = _inspect_singularity_image(str(image_path)) - assert res_inspect is False - # test image exists repo_path.mkdir() cmd = [ @@ -217,14 +214,36 @@ def test_singularity_inspect_image(tmp_path: Path, monkeypatch: pytest.MonkeyPat ] build = subprocess.run(cmd, capture_output=True, text=True) if build.returncode == 0: - # Verify the path is a container image + # Verify the path is a correct container image res_inspect = _inspect_singularity_image(image_path) assert res_inspect is True - # test wrong subprocess call + # test wrong json output + def mock_subprocess_run(*args, **kwargs): + class Result: + returncode = 0 + stdout = "not-json" + + return Result() + + monkeypatch.setattr("cwltool.singularity.run", mock_subprocess_run) + res_inspect = _inspect_singularity_image(image_path) + assert res_inspect is False + else: + pytest.skip(f"singularity sandbox image build didn't worked: {build.stderr}") + + +def test_singularity_sandbox_image_not_exists(): + image_path = "/tmp/not_existing/image" + res_inspect = _inspect_singularity_image(image_path) + assert res_inspect is False + + +def test_inspect_image_wrong_sb_call(monkeypatch: pytest.MonkeyPatch): + def mock_failed_subprocess(*args, **kwargs): raise subprocess.CalledProcessError(returncode=1, cmd=args[0]) - + monkeypatch.setattr("cwltool.singularity.run", mock_failed_subprocess) - res_inspect = _inspect_singularity_image(str(image_path)) - assert res_inspect is False \ No newline at end of file + res_inspect = _inspect_singularity_image("/tmp/container_repo/alpine") + assert res_inspect is False diff --git a/tests/test_tmpdir.py b/tests/test_tmpdir.py index f0476e7c9..e87f0c3a1 100644 --- a/tests/test_tmpdir.py +++ b/tests/test_tmpdir.py @@ -288,7 +288,7 @@ def test_dockerfile_singularity_build(monkeypatch: pytest.MonkeyPatch, tmp_path: @needs_singularity def test_singularity_get_image_from_sandbox(monkeypatch: pytest.MonkeyPatch, tmp_path: Path): """Test that SingularityCommandLineJob.get_image correctly handle sandbox image.""" - + (tmp_path / "out").mkdir(exist_ok=True) tmp_outdir_prefix = tmp_path / "out" (tmp_path / "3").mkdir(exist_ok=True) @@ -330,7 +330,9 @@ def test_singularity_get_image_from_sandbox(monkeypatch: pytest.MonkeyPatch, tmp image_path.mkdir() # directory exists but is not an image - monkeypatch.setattr("cwltool.singularity._inspect_singularity_image", lambda *args, **kwargs: False) + monkeypatch.setattr( + "cwltool.singularity._inspect_singularity_image", lambda *args, **kwargs: False + ) req = {"class": "DockerRequirement", "dockerPull": f"{image_path}"} res = SingularityCommandLineJob( builder, {}, CommandLineTool.make_path_mapper, [], [], "" @@ -344,9 +346,11 @@ def test_singularity_get_image_from_sandbox(monkeypatch: pytest.MonkeyPatch, tmp assert res is False # directory exists and is an image: - monkeypatch.setattr("cwltool.singularity._inspect_singularity_image", lambda *args, **kwargs: True) + monkeypatch.setattr( + "cwltool.singularity._inspect_singularity_image", lambda *args, **kwargs: True + ) req = {"class": "DockerRequirement", "dockerPull": f"{image_path}"} - res = SingularityCommandLineJob( + res = SingularityCommandLineJob( builder, {}, CommandLineTool.make_path_mapper, [], [], "" ).get_image( req, From 438b8ce0a61418fb24e03d2988d57ba36800a001 Mon Sep 17 00:00:00 2001 From: natthan-pigoux Date: Wed, 5 Nov 2025 14:54:21 +0100 Subject: [PATCH 4/7] test: fix mkdir --- tests/test_tmpdir.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_tmpdir.py b/tests/test_tmpdir.py index e87f0c3a1..810379706 100644 --- a/tests/test_tmpdir.py +++ b/tests/test_tmpdir.py @@ -291,7 +291,8 @@ def test_singularity_get_image_from_sandbox(monkeypatch: pytest.MonkeyPatch, tmp (tmp_path / "out").mkdir(exist_ok=True) tmp_outdir_prefix = tmp_path / "out" - (tmp_path / "3").mkdir(exist_ok=True) + tmp_outdir_prefix.mkdir(exist_ok=True) + (tmp_path / "tmp").mkdir(exist_ok=True) tmpdir_prefix = str(tmp_path / "tmp") runtime_context = RuntimeContext( {"tmpdir_prefix": tmpdir_prefix, "user_space_docker_cmd": None} From d21b9dd5ca808440938abd57a61d72b71f43e32e Mon Sep 17 00:00:00 2001 From: natthan-pigoux Date: Wed, 5 Nov 2025 14:55:07 +0100 Subject: [PATCH 5/7] refactor: replace fill candidates list --- cwltool/singularity.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cwltool/singularity.py b/cwltool/singularity.py index f1f0f9c1f..e2ded2ab0 100644 --- a/cwltool/singularity.py +++ b/cwltool/singularity.py @@ -278,10 +278,6 @@ def get_image( elif "dockerImageId" in dockerRequirement: if os.path.isfile(dockerRequirement["dockerImageId"]): found = True - candidates.append(dockerRequirement["dockerImageId"]) - candidates.append(_normalize_image_id(dockerRequirement["dockerImageId"])) - if is_version_3_or_newer(): - candidates.append(_normalize_sif_id(dockerRequirement["dockerImageId"])) # handling local singularity sandbox image elif os.path.isdir(dockerRequirement["dockerImageId"]) and _inspect_singularity_image( dockerRequirement["dockerImageId"] @@ -291,6 +287,10 @@ def get_image( dockerRequirement["dockerImageId"], ) return True + candidates.append(dockerRequirement["dockerImageId"]) + candidates.append(_normalize_image_id(dockerRequirement["dockerImageId"])) + if is_version_3_or_newer(): + candidates.append(_normalize_sif_id(dockerRequirement["dockerImageId"])) targets = [os.getcwd()] if "CWL_SINGULARITY_CACHE" in os.environ: From af2e24a3e20ac47ab2694c3fb66b0653e0538809 Mon Sep 17 00:00:00 2001 From: natthan-pigoux Date: Wed, 5 Nov 2025 14:55:36 +0100 Subject: [PATCH 6/7] test: fix import --- tests/test_singularity.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_singularity.py b/tests/test_singularity.py index 1aa69c26d..af9d704e5 100644 --- a/tests/test_singularity.py +++ b/tests/test_singularity.py @@ -1,6 +1,5 @@ """Tests to find local Singularity image.""" -import json import shutil import subprocess from pathlib import Path From ca94aa01fee084390985bce841c1ce87527c3070 Mon Sep 17 00:00:00 2001 From: natthan-pigoux Date: Wed, 5 Nov 2025 15:01:35 +0100 Subject: [PATCH 7/7] refactor: insert new code smoothly --- cwltool/singularity.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/cwltool/singularity.py b/cwltool/singularity.py index e2ded2ab0..419d1dc61 100644 --- a/cwltool/singularity.py +++ b/cwltool/singularity.py @@ -263,18 +263,17 @@ def get_image( dockerRequirement["dockerImageId"], ) return True + match = re.search(pattern=r"([a-z]*://)", string=dockerRequirement["dockerPull"]) + img_name = _normalize_image_id(dockerRequirement["dockerPull"]) + candidates.append(img_name) + if is_version_3_or_newer(): + sif_name = _normalize_sif_id(dockerRequirement["dockerPull"]) + candidates.append(sif_name) + dockerRequirement["dockerImageId"] = sif_name else: - match = re.search(pattern=r"([a-z]*://)", string=dockerRequirement["dockerPull"]) - img_name = _normalize_image_id(dockerRequirement["dockerPull"]) - candidates.append(img_name) - if is_version_3_or_newer(): - sif_name = _normalize_sif_id(dockerRequirement["dockerPull"]) - candidates.append(sif_name) - dockerRequirement["dockerImageId"] = sif_name - else: - dockerRequirement["dockerImageId"] = img_name - if not match: - dockerRequirement["dockerPull"] = "docker://" + dockerRequirement["dockerPull"] + dockerRequirement["dockerImageId"] = img_name + if not match: + dockerRequirement["dockerPull"] = "docker://" + dockerRequirement["dockerPull"] elif "dockerImageId" in dockerRequirement: if os.path.isfile(dockerRequirement["dockerImageId"]): found = True