Generalize "overrides" by putting them under "requirements" field. (#590)

* Generalize "overrides" by putting them under "requirements" field.

Author: tetron

README.rst

@@ -65,7 +65,7 @@ Running tests locally
 To run the basis tests after installing `cwltool` execute the following:
 
 .. code:: bash
-  
+
   pip install pytest mock
   py.test --ignore cwltool/schemas/ --pyarg cwltool
 
@@ -399,17 +399,16 @@ environment or with a particular dataset.  To avoid the need to modify the
 underlying workflow, cwltool supports requirement "overrides".
 
 The format of the "overrides" object is a mapping of item identifier (workflow,
-workflow step, or command line tool) followed by a list of ProcessRequirements
-that should be applied.
+workflow step, or command line tool) to the process requirements that should be applied.
 
 .. code:: yaml
 
   cwltool:overrides:
     echo.cwl:
-      - class: EnvVarRequirement
-        envDef:
-          MESSAGE: override_value
-
+      requirements:
+        EnvVarRequirement:
+          envDef:
+            MESSAGE: override_value
 
 Overrides can be specified either on the command line, or as part of the job
 input document.  Workflow steps are identified using the name of the workflow
@@ -426,9 +425,10 @@ Override identifiers are relative to the toplevel workflow document.
   input_parameter2: value2
   cwltool:overrides:
     workflow.cwl#step1:
-      - class: EnvVarRequirement
-        envDef:
-          MESSAGE: override_value
+      requirements:
+        EnvVarRequirement:
+          envDef:
+            MESSAGE: override_value
 
 .. code:: bash
 

cwltool/load_tool.py

@@ -45,10 +45,9 @@
     u"overrides": {
         "@id": "cwltool:overrides",
         "mapSubject": "overrideTarget",
-        "mapPredicate": "override"
     },
-    u"override": {
-        "@id": "cwltool:override",
+    "requirements": {
+        "@id": "https://w3id.org/cwl/cwl#requirements",
         "mapSubject": "class"
     }
 }  # type: ContextType

cwltool/process.py

@@ -422,13 +422,13 @@ def avroize_type(field_type, name_prefix=""):
             avroize_type(field_type["items"], name_prefix)
     return field_type
 
-def get_overrides(overrides, toolid):  # type: (List[Dict[Text, Any]], Text) -> List[Dict[Text, Any]]
-    req = []  # type: List[Dict[Text, Any]]
+def get_overrides(overrides, toolid):  # type: (List[Dict[Text, Any]], Text) -> Dict[Text, Any]
+    req = {}  # type: Dict[Text, Any]
     if not isinstance(overrides, list):
         raise validate.ValidationException("Expected overrides to be a list, but was %s" % type(overrides))
     for ov in overrides:
         if ov["overrideTarget"] == toolid:
-            req.extend(ov["override"])
+            req.update(ov)
     return req
 
 class Process(six.with_metaclass(abc.ABCMeta, object)):
@@ -467,7 +467,7 @@ def __init__(self, toolpath_object, **kwargs):
         self.tool = toolpath_object
         self.requirements = (kwargs.get("requirements", []) +
                              self.tool.get("requirements", []) +
-                             get_overrides(kwargs.get("overrides", []), self.tool["id"]))
+                             get_overrides(kwargs.get("overrides", []), self.tool["id"]).get("requirements", []))
         self.hints = kwargs.get("hints", []) + self.tool.get("hints", [])
         self.formatgraph = None  # type: Graph
         if "loader" in kwargs:

cwltool/workflow.py

@@ -673,7 +673,7 @@ def __init__(self, toolpath_object, pos, **kwargs):
 
         kwargs["requirements"] = (kwargs.get("requirements", []) +
                                   toolpath_object.get("requirements", []) +
-                                  get_overrides(kwargs.get("overrides", []), self.id))
+                                  get_overrides(kwargs.get("overrides", []), self.id).get("requirements", []))
         kwargs["hints"] = kwargs.get("hints", []) + toolpath_object.get("hints", [])
 
         try:

tests/override/echo-job-ov.yml

@@ -1,6 +1,7 @@
 m1: zing
 cwltool:overrides:
   echo.cwl:
-    - class: EnvVarRequirement
-      envDef:
-        MESSAGE: hello3
+    requirements:
+      EnvVarRequirement:
+        envDef:
+          MESSAGE: hello3

tests/override/echo-job-ov2.yml

@@ -1,7 +1,8 @@
 m1: zing
 cwltool:overrides:
   echo.cwl:
-    - class: EnvVarRequirement
-      envDef:
-        MESSAGE: hello4
+    requirements:
+      EnvVarRequirement:
+        envDef:
+          MESSAGE: hello4
 cwl:tool: echo.cwl

tests/override/ov.yml

@@ -1,5 +1,6 @@
 cwltool:overrides:
   echo.cwl:
-    - class: EnvVarRequirement
-      envDef:
-        MESSAGE: hello2
+    requirements:
+      EnvVarRequirement:
+        envDef:
+          MESSAGE: hello2

tests/override/ov2.yml

@@ -1,5 +1,6 @@
 cwltool:overrides:
   "echo-wf.cwl#step1":
-    - class: EnvVarRequirement
-      envDef:
-        MESSAGE: hello5
+    requirements:
+      EnvVarRequirement:
+        envDef:
+          MESSAGE: hello5

tests/override/ov3.yml

@@ -1,5 +1,6 @@
 cwltool:overrides:
-  "echo-wf.cwl":
-    - class: EnvVarRequirement
-      envDef:
-        MESSAGE: hello6
+  echo-wf.cwl:
+    requirements:
+      EnvVarRequirement:
+        envDef:
+          MESSAGE: hello6