Added loop support

Author: GlassOfWhiskey

Workflow.yml

@@ -443,6 +443,56 @@ $graph:
     to connect the output value to downstream parameters.
 
 
+- name: LoopInput
+  type: record
+  extends: [Identified, Sink]
+  fields:
+    - name: source
+      doc: |
+        Specifies one or more of the step output parameters that will
+        provide input to the loop iterations after the first one (inputs
+        of the first iteration are the step input parameters).
+      jsonldPredicate:
+        "_id": "cwl:source"
+        "_type": "@id"
+        refScope: 1
+      type:
+        - string?
+        - string[]?
+    - name: default
+      type: ["null", File, Directory, Any]
+      doc: |
+        The default value for this parameter to use if either there is no
+        `source` field, or the value produced by the `source` is `null`.  The
+        default must be applied prior to scattering or evaluating `valueFrom`.
+      jsonldPredicate:
+        _id: "sld:default"
+        noLinkCheck: true
+    - name: valueFrom
+      type:
+        - "null"
+        - string
+        - Expression
+      jsonldPredicate: "cwl:valueFrom"
+      doc: |
+        To use valueFrom, [StepInputExpressionRequirement](#StepInputExpressionRequirement) must
+        be specified in the workflow or workflow step requirements.
+
+        If `valueFrom` is a constant string value, use this as the value for
+        this input parameter.
+
+        If `valueFrom` is a parameter reference or expression, it must be
+        evaluated to yield the actual value to be assigned to the input field.
+
+        The `self` value in the parameter reference or expression must be
+        `null` if there is no `source` field, or the value of the
+        parameter(s) specified in the `source` field.
+
+        The value of `inputs` in the parameter reference or expression must be
+        the input object to the previous iteration of the workflow step (or the initial
+        inputs for the first iteration).
+    
+
 - name: ScatterMethod
   type: enum
   docParent: "#WorkflowStep"
@@ -453,6 +503,15 @@ $graph:
     - flat_crossproduct
 
 
+- name: LoopOutputMethod
+  type: enum
+  docParent: "#WorkflowStep"
+  doc: The loop output method, as described in [workflow step loop](#WorkflowStep).
+  symbols:
+    - last
+    - all
+
+
 - name: WorkflowStep
   type: record
   extends: [Identified, Labeled, sld:Documented]
@@ -505,7 +564,7 @@ $graph:
           output arrays must be flattened to a single level, but otherwise listed in the
           order that the input arrays are listed in the `scatter` field.
 
-    # Conditional execution (Optional)
+    # Conditional and iterative execution (Optional)
 
     Conditional execution makes execution of a step conditional on an
     expression.  A step that is not executed is "skipped".  A skipped
@@ -522,12 +581,37 @@ $graph:
     input object (or individual scatter job), and returns a boolean
     value.  It is an error if this expression returns a value other
     than `true` or `false`.
-
-    Conditionals in CWL are an optional feature and are not required
-    to be implemented by all consumers of CWL documents.  An
-    implementation that does not support conditionals must return a
+    
+    The `loop` field controls iterative execution. It defines the input
+    parameters of the loop iterations after the first one (inputs of the
+    first iteration are the step input parameters, as usual). If no
+    `loop` rule is specified for a given step `in` field, the initial
+    value is kept constant among all iterations.
+    
+    When a `loop` field is present, the `when` field is mandatory. It is
+    evaluated before each loop iteration and acts as a termination condition:
+    as soon as the `when` expression evaluates to `false`, the loop terminates
+    and the step outputs are propagated to the subsequent workflow steps.
+    
+    The `outputMethod` field describes how to deal with loop outputs after
+    termination:
+
+      * **last** specifies that only the last computed element for each output
+        parameter should be propagated to the subsequenct steps. This is the 
+        default value.
+    
+      * **all** specifies that a single ordered array with all output values
+        computed at the end of each loop iteration should be propagated to the
+        subsequent steps.
+
+    Conditionals and iterative execution in CWL are an optional features
+    and are not required to be implemented by all consumers of CWL documents.
+    An implementation that does not support conditionals must return a
     fatal error when attempting to execute a workflow that uses
     conditional constructs the implementation does not support.
+    
+    At this time, the `loop` field is not compatible with the `scatter` field.
+    Combining the two in the same step will produce an error.
 
     # Subworkflows
 
@@ -619,6 +703,24 @@ $graph:
       jsonldPredicate:
         "_id": "cwl:scatterMethod"
         "_type": "@vocab"
+    - name: loop
+      doc: |
+        Defines the input parameters of the loop iterations after the first one
+        (inputs of the first iteration are the step input parameters). If no
+        `loop` rule is specified for a given step `in` field, the initial value
+        is kept constant among all iterations.
+      type: LoopInput[]?
+      jsonldPredicate:
+        _id: "cwl:loop"
+        mapSubject: id
+        mapPredicate: source
+    - name: outputMethod
+      doc: |
+        Required if `loop` is defined.
+      type: LoopOutputMethod?
+      jsonldPredicate:
+        "_id": "cwl:outputMethod"
+        "_type": "@vocab"
 
 
 - name: Workflow

conformance_tests.yaml

@@ -3525,3 +3525,5 @@
       an_array_of_mixed_booleans: [ false, true, false ]
       an_array_of_trues: [ true, true, true ]
       an_int: 42
+
+- $import: tests/loop/test-index.yaml

tests/loop/all-output-loop-no-iteration.cwl

@@ -0,0 +1,28 @@
+#!/usr/bin/env cwl-runner
+cwlVersion: v1.3.0-dev1
+class: Workflow
+requirements:
+  InlineJavascriptRequirement: {}
+inputs:
+  i1: int
+outputs:
+  o1:
+    type: int[]
+    outputSource: subworkflow/o1
+steps:
+  subworkflow:
+    when: $(inputs.i1 < 1)
+    loop:
+      i1: o1
+    outputMethod: all
+    run:
+      class: ExpressionTool
+      inputs:
+        i1: int
+      outputs:
+        o1: int
+      expression: >
+        ${return {'o1': inputs.i1 + 1};}
+    in:
+      i1: i1
+    out: [o1]

tests/loop/all-output-loop.cwl

@@ -0,0 +1,28 @@
+#!/usr/bin/env cwl-runner
+cwlVersion: v1.3.0-dev1
+class: Workflow
+requirements:
+  InlineJavascriptRequirement: {}
+inputs:
+  i1: int
+outputs:
+  o1:
+    type: int[]
+    outputSource: subworkflow/o1
+steps:
+  subworkflow:
+    when: $(inputs.i1 < 10)
+    loop:
+      i1: o1
+    outputMethod: all
+    run:
+      class: ExpressionTool
+      inputs:
+        i1: int
+      outputs:
+        o1: int
+      expression: >
+        ${return {'o1': inputs.i1 + 1};}
+    in:
+      i1: i1
+    out: [o1]

tests/loop/default-value-loop.cwl

@@ -0,0 +1,47 @@
+#!/usr/bin/env cwl-runner
+cwlVersion: v1.3.0-dev1
+class: Workflow
+requirements:
+  InlineJavascriptRequirement: {}
+  ScatterFeatureRequirement: {}
+  SubworkflowFeatureRequirement: {}
+inputs:
+  i1: int
+outputs:
+  o1:
+    type: int[]
+    outputSource: loop/o1
+    pickValue: all_non_null
+steps:
+  loop:
+    when: $(inputs.i1 < 20)
+    loop:
+      i1:
+        source: o1
+        default: 5
+    outputMethod: all
+    run:
+      class: Workflow
+      inputs:
+        i1: int
+      outputs:
+        o1:
+          type: int?
+          outputSource: big_values/o1
+      steps:
+        big_values:
+          when: $(inputs.i1 >= 5)
+          run:
+            class: ExpressionTool
+            inputs:
+              i1: int
+            outputs:
+              o1: int
+            expression: >
+              ${return {'o1': inputs.i1 + 3};}
+          in:
+            i1: i1
+          out: [ o1 ]
+    in:
+      i1: i1
+    out: [ o1 ]

tests/loop/invalid-loop-scatter.cwl

@@ -0,0 +1,34 @@
+#!/usr/bin/env cwl-runner
+cwlVersion: v1.3.0-dev1
+class: Workflow
+requirements:
+  InlineJavascriptRequirement: {}
+  ScatterFeatureRequirement: {}
+  SubworkflowFeatureRequirement: {}
+inputs:
+  i1: int[]
+  i2: int
+outputs:
+  o1:
+    type: int[]
+    outputSource: subworkflow/o1
+steps:
+  subworkflow:
+    when: $(inputs.i1 < 10)
+    loop:
+      i1: o1
+    outputMethod: last
+    run:
+      class: ExpressionTool
+      inputs:
+        i1: int
+        i2: int
+      outputs:
+        o1: int
+      expression: >
+        ${return {'o1': inputs.i1 + inputs.i2};}
+    in:
+      i1: i1
+      i2: i2
+    scatter: i1
+    out: [o1]

tests/loop/invalid-multi-source-loop-no-requirement.cwl

@@ -0,0 +1,64 @@
+#!/usr/bin/env cwl-runner
+cwlVersion: v1.3.0-dev1
+class: Workflow
+requirements:
+  InlineJavascriptRequirement: {}
+  ScatterFeatureRequirement: {}
+  SubworkflowFeatureRequirement: {}
+inputs:
+  i1: int
+outputs:
+  o1:
+    type: int[]
+    outputSource: [loop/osmall, loop/obig]
+    linkMerge: merge_flattened
+    pickValue: all_non_null
+steps:
+  loop:
+    when: $(inputs.i1 < 20)
+    loop:
+      i1:
+        source: [ osmall, obig ]
+        pickValue: the_only_non_null
+    outputMethod: all
+    run:
+      class: Workflow
+      inputs:
+        i1: int
+      outputs:
+        osmall:
+          type: int?
+          outputSource: small_values/o1
+        obig:
+          type: int?
+          outputSource: big_values/o1
+      steps:
+        small_values:
+          when: $(inputs.i1 < 5)
+          run:
+            class: ExpressionTool
+            inputs:
+              i1: int
+            outputs:
+              o1: int
+            expression: >
+              ${return {'o1': inputs.i1 + 1};}
+          in:
+            i1: i1
+          out: [o1]
+        big_values:
+          when: $(inputs.i1 >= 5)
+          run:
+            class: ExpressionTool
+            inputs:
+              i1: int
+            outputs:
+              o1: int
+            expression: >
+              ${return {'o1': inputs.i1 + 3};}
+          in:
+            i1: i1
+          out: [ o1 ]
+    in:
+      i1: i1
+    out: [osmall, obig]

tests/loop/invalid-no-when.cwl

@@ -0,0 +1,32 @@
+#!/usr/bin/env cwl-runner
+cwlVersion: v1.3.0-dev1
+class: Workflow
+requirements:
+  InlineJavascriptRequirement: {}
+  ScatterFeatureRequirement: {}
+  SubworkflowFeatureRequirement: {}
+inputs:
+  i1: int
+  i2: int
+outputs:
+  o1:
+    type: int
+    outputSource: subworkflow/o1
+steps:
+  subworkflow:
+    loop:
+      i1: o1
+    outputMethod: last
+    run:
+      class: ExpressionTool
+      inputs:
+        i1: int
+        i2: int
+      outputs:
+        o1: int
+      expression: >
+        ${return {'o1': inputs.i1 + inputs.i2};}
+    in:
+      i1: i1
+      i2: i2
+    out: [o1]