Merge branch 'upstream-main' into on-error-abort

Author: AlexTate

.github/workflows/ci-tests.yml

@@ -81,10 +81,11 @@ jobs:
 
       - name: Upload coverage to Codecov
         if: ${{ matrix.step == 'unit' }}
-        uses: codecov/codecov-action@v3
+        uses: codecov/codecov-action@v4
         with:
-          fail_ci_if_error: false
-          token: ${{ secrets.CODECOV_TOKEN }}
+          fail_ci_if_error: true
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
 
   tox-style:
     name: Linters
@@ -213,10 +214,11 @@ jobs:
           path: |
             **/cwltool_conf*.xml
       - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v3
+        uses: codecov/codecov-action@v4
         with:
-          fail_ci_if_error: false
-          token: ${{ secrets.CODECOV_TOKEN }}
+          fail_ci_if_error: true
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
   release_test:
     name: cwltool release test
     runs-on: ubuntu-22.04
@@ -297,7 +299,8 @@ jobs:
       - name: Test with tox
         run: tox
       - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v3
+        uses: codecov/codecov-action@v4
         with:
-          fail_ci_if_error: false
-          token: ${{ secrets.CODECOV_TOKEN }}
+          fail_ci_if_error: true
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

cwltool/cwlprov/provenance_profile.py

@@ -498,7 +498,7 @@ def declare_artefact(self, value: Any) -> ProvEntity:
             self.research_object.add_uri(entity.identifier.uri)
             return entity
 
-        if isinstance(value, (str, str)):
+        if isinstance(value, str):
             (entity, _) = self.declare_string(value)
             return entity
 

cwltool/docker.py

@@ -429,7 +429,10 @@ def create_runtime(
                     "assurance.",
                     self.name,
                 )
-
+        shm_size_od, shm_bool = self.builder.get_requirement("http://commonwl.org/cwltool#ShmSize")
+        if shm_bool:
+            shm_size = cast(CWLObjectType, shm_size_od)["shmSize"]
+            runtime.append(f"--shm-size={shm_size}")
         return runtime, cidfile_path
 
 

cwltool/extensions-v1.1.yml

@@ -120,3 +120,21 @@ $graph:
       doc: |
         Maximum number of GPU devices to request.  If not specified,
         same as `cudaDeviceCountMin`.
+- name: ShmSize
+  type: record
+  extends: cwl:ProcessRequirement
+  inVocab: false
+  fields:
+    class:
+      type: string
+      doc: 'cwltool:ShmSize'
+      jsonldPredicate:
+        "_id": "@type"
+        "_type": "@vocab"
+    shmSize:
+      type: string
+      doc: |
+        Size of /dev/shm. The format is `<number><unit>`. <number> must be greater
+        than 0. Unit is optional and can be `b` (bytes), `k` (kilobytes), `m`
+        (megabytes), or `g` (gigabytes). If you omit the unit, the default is
+        bytes. If you omit the size entirely, the value is `64m`."

cwltool/extensions-v1.2.yml

@@ -240,4 +240,21 @@ $graph:
         - Specify the desired method of dealing with loop outputs
         - Default. Propagates only the last computed element to the subsequent steps when the loop terminates.
         - Propagates a single array with all output values to the subsequent steps when the loop terminates.
-
+- name: ShmSize
+  type: record
+  extends: cwl:ProcessRequirement
+  inVocab: false
+  fields:
+    class:
+      type: string
+      doc: 'cwltool:ShmSize'
+      jsonldPredicate:
+        "_id": "@type"
+        "_type": "@vocab"
+    shmSize:
+      type: string
+      doc: |
+        Size of /dev/shm. The format is `<number><unit>`. <number> must be greater
+        than 0. Unit is optional and can be `b` (bytes), `k` (kilobytes), `m`
+        (megabytes), or `g` (gigabytes). If you omit the unit, the default is
+        bytes. If you omit the size entirely, the value is `64m`."

cwltool/extensions.yml

@@ -1,6 +1,7 @@
 $base: http://commonwl.org/cwltool#
 $namespaces:
   cwl: "https://w3id.org/cwl/cwl#"
+  cwltool: "http://commonwl.org/cwltool#"
 $graph:
 - $import: https://w3id.org/cwl/CommonWorkflowLanguage.yml
 
@@ -229,3 +230,21 @@ $graph:
       doc: |
         Maximum number of GPU devices to request.  If not specified,
         same as `cudaDeviceCountMin`.
+- name: ShmSize
+  type: record
+  extends: cwl:ProcessRequirement
+  inVocab: false
+  fields:
+    class:
+      type: string
+      doc: 'cwltool:ShmSize'
+      jsonldPredicate:
+        "_id": "@type"
+        "_type": "@vocab"
+    shmSize:
+      type: string
+      doc: |
+        Size of /dev/shm. The format is `<number><unit>`. <number> must be greater
+        than 0. Unit is optional and can be `b` (bytes), `k` (kilobytes), `m`
+        (megabytes), or `g` (gigabytes). If you omit the unit, the default is
+        bytes. If you omit the size entirely, the value is `64m`."

cwltool/process.py

@@ -121,6 +121,7 @@ def filter(self, record: logging.LogRecord) -> bool:
     "http://commonwl.org/cwltool#LoadListingRequirement",
     "http://commonwl.org/cwltool#InplaceUpdateRequirement",
     "http://commonwl.org/cwltool#CUDARequirement",
+    "http://commonwl.org/cwltool#ShmSize",
 ]
 
 cwl_files = (

cwltool/singularity.py

@@ -390,7 +390,7 @@ def add_writable_file_volume(
             if self.inplace_update:
                 try:
                     os.link(os.path.realpath(volume.resolved), host_outdir_tgt)
-                except os.error:
+                except OSError:
                     shutil.copy(volume.resolved, host_outdir_tgt)
             else:
                 shutil.copy(volume.resolved, host_outdir_tgt)

tests/test_docker.py

@@ -264,3 +264,20 @@ def test_singularity_required_missing_secfile(
         "tests/secondary-files-required-missing-container.cwl:16:5: Missing required secondary file"
     )
     assert "file.ext3" in stderr
+
+
+@needs_docker
+def test_docker_shm_size(tmp_path: Path) -> None:
+    result_code, stdout, stderr = get_main_output(
+        [
+            "--enable-ext",
+            "--default-container",
+            "docker.io/debian:stable-slim",
+            "--outdir",
+            str(tmp_path),
+            get_data("tests/wf/shm_size.cwl"),
+        ]
+    )
+    stderr = re.sub(r"\s\s+", " ", stderr)
+    assert result_code == 0
+    assert "--shm-size=128m" in stderr

tests/wf/shm_size.cwl

@@ -0,0 +1,17 @@
+#!/usr/bin/env cwl-runner
+class: CommandLineTool
+cwlVersion: v1.2
+requirements:
+  cwltool:ShmSize:
+    shmSize: 128m
+inputs: []
+
+outputs:
+  output:
+    type: stdout
+
+baseCommand: echo
+
+stdout: shm-size.txt
+
+arguments: [ $(runtime) ]