use mypy 0.4.4 and new Text type

Author: mr-c

Makefile

@@ -155,6 +155,7 @@ mypy: ${PYSOURCES}
 	ln -s $(shell python -c 'from __future__ import print_function; import schema_salad; import os.path; print(os.path.dirname(schema_salad.__file__))') \
 		typeshed/2.7/schema_salad
 	MYPYPATH=typeshed/2.7 mypy --py2 --disallow-untyped-calls \
-		 --warn-redundant-casts --warn-unused-ignores cwltool
+		 --warn-redundant-casts --warn-unused-ignores --fast-parser \
+		 cwltool
 
 FORCE:

cwltool/builder.py

@@ -3,15 +3,15 @@
 from . import expression
 import avro
 import schema_salad.validate as validate
-from typing import Any, Union, AnyStr, Callable, Type
+from typing import Any, Callable, Text, Type, Union
 from .errors import WorkflowException
 from .stdfsaccess import StdFsAccess
 from .pathmapper import PathMapper, adjustFileObjs, adjustDirObjs, normalizeFilesDirs
 
 CONTENT_LIMIT = 64 * 1024
 
 
-def substitute(value, replace):  # type: (str, str) -> str
+def substitute(value, replace):  # type: (Text, Text) -> Text
     if replace[0] == "^":
         return substitute(value[0:value.rindex('.')], replace[1:])
     else:
@@ -21,24 +21,24 @@ class Builder(object):
 
     def __init__(self):  # type: () -> None
         self.names = None  # type: avro.schema.Names
-        self.schemaDefs = None  # type: Dict[str,Dict[unicode, Any]]
-        self.files = None  # type: List[Dict[unicode, unicode]]
+        self.schemaDefs = None  # type: Dict[Text, Dict[Text, Any]]
+        self.files = None  # type: List[Dict[Text, Text]]
         self.fs_access = None  # type: StdFsAccess
-        self.job = None  # type: Dict[unicode, Union[Dict[unicode, Any], List, unicode]]
-        self.requirements = None  # type: List[Dict[str,Any]]
-        self.outdir = None  # type: str
-        self.tmpdir = None  # type: str
-        self.resources = None  # type: Dict[str, Union[int, str]]
-        self.bindings = []  # type: List[Dict[str, Any]]
+        self.job = None  # type: Dict[Text, Union[Dict[Text, Any], List, Text]]
+        self.requirements = None  # type: List[Dict[Text, Any]]
+        self.outdir = None  # type: Text
+        self.tmpdir = None  # type: Text
+        self.resources = None  # type: Dict[Text, Union[int, Text]]
+        self.bindings = []  # type: List[Dict[Text, Any]]
         self.timeout = None  # type: int
         self.pathmapper = None  # type: PathMapper
-        self.stagedir = None  # type: unicode
+        self.stagedir = None  # type: Text
         self.make_fs_access = None  # type: Type[StdFsAccess]
 
     def bind_input(self, schema, datum, lead_pos=[], tail_pos=[]):
-        # type: (Dict[unicode, Any], Any, Union[int, List[int]], List[int]) -> List[Dict[str, Any]]
-        bindings = []  # type: List[Dict[str,str]]
-        binding = None  # type: Dict[str,Any]
+        # type: (Dict[Text, Any], Any, Union[int, List[int]], List[int]) -> List[Dict[Text, Any]]
+        bindings = []  # type: List[Dict[Text,Text]]
+        binding = None  # type: Dict[Text,Any]
         if "inputBinding" in schema and isinstance(schema["inputBinding"], dict):
             binding = copy.copy(schema["inputBinding"])
 
@@ -52,7 +52,7 @@ def bind_input(self, schema, datum, lead_pos=[], tail_pos=[]):
         # Handle union types
         if isinstance(schema["type"], list):
             for t in schema["type"]:
-                if isinstance(t, (str, unicode)) and self.names.has_name(t, ""):
+                if isinstance(t, (str, Text)) and self.names.has_name(t, ""):
                     avsc = self.names.get_name(t, "")
                 elif isinstance(t, dict) and "name" in t and self.names.has_name(t["name"], ""):
                     avsc = self.names.get_name(t["name"], "")
@@ -134,23 +134,23 @@ def _capture_files(f):
 
         return bindings
 
-    def tostr(self, value):  # type: (Any) -> str
+    def tostr(self, value):  # type: (Any) -> Text
         if isinstance(value, dict) and value.get("class") in ("File", "Directory"):
             if "path" not in value:
                 raise WorkflowException(u"%s object missing \"path\": %s" % (value["class"], value))
             return value["path"]
         else:
-            return str(value)
+            return Text(value)
 
-    def generate_arg(self, binding):  # type: (Dict[str,Any]) -> List[str]
+    def generate_arg(self, binding):  # type: (Dict[Text,Any]) -> List[Text]
         value = binding.get("datum")
         if "valueFrom" in binding:
             value = self.do_eval(binding["valueFrom"], context=value)
 
         prefix = binding.get("prefix")
         sep = binding.get("separate", True)
 
-        l = []  # type: List[Dict[str,str]]
+        l = []  # type: List[Dict[Text,Text]]
         if isinstance(value, list):
             if binding.get("itemSeparator"):
                 l = [binding["itemSeparator"].join([self.tostr(v) for v in value])]
@@ -182,7 +182,7 @@ def generate_arg(self, binding):  # type: (Dict[str,Any]) -> List[str]
         return [a for a in args if a is not None]
 
     def do_eval(self, ex, context=None, pull_image=True, recursive=False):
-        # type: (Union[Dict[str, str], unicode], Any, bool, bool) -> Any
+        # type: (Union[Dict[Text, Text], Text], Any, bool, bool) -> Any
         if recursive:
             if isinstance(ex, dict):
                 return {k: self.do_eval(v, context, pull_image, recursive) for k,v in ex.iteritems()}

cwltool/cwlrdf.py

@@ -4,14 +4,14 @@
 from schema_salad.jsonld_context import makerdf
 from rdflib import Graph, plugin, URIRef
 from rdflib.serializer import Serializer
-from typing import Any, Union, Dict, IO
+from typing import Any, Dict, IO, Text, Union
 
 def printrdf(workflow, wf, ctx, sr, stdout):
-    # type: (Union[str, unicode], Union[List[Dict[unicode, Any]], Dict[unicode, Any]], Loader.ContextType, str, IO[Any]) -> None
+    # type: (Union[Text, Text], Union[List[Dict[Text, Any]], Dict[Text, Any]], Loader.ContextType, Text, IO[Any]) -> None
     stdout.write(makerdf(workflow, wf, ctx).serialize(format=sr))
 
-def lastpart(uri):  # type: (Any) -> str
-    uri = str(uri)
+def lastpart(uri):  # type: (Any) -> Text
+    uri = Text(uri)
     if "/" in uri:
         return uri[uri.rindex("/")+1:]
     else:
@@ -75,7 +75,7 @@ def dot_with_parameters(g, stdout):  # type: (Graph, IO[Any]) -> None
         stdout.write(u'"%s" [shape=octagon]\n' % (lastpart(inp)))
 
 def dot_without_parameters(g, stdout):  # type: (Graph, IO[Any]) -> None
-    dotname = {}  # type: Dict[str,str]
+    dotname = {}  # type: Dict[Text,Text]
     clusternode = {}
 
     stdout.write("compound=true\n")
@@ -118,8 +118,8 @@ def dot_without_parameters(g, stdout):  # type: (Graph, IO[Any]) -> None
             else:
                 currentwf = None
 
-        if str(runtype) != "https://w3id.org/cwl/cwl#Workflow":
-            stdout.write(u'"%s" [label="%s"]\n' % (dotname[step], urlparse.urldefrag(str(step))[1]))
+        if Text(runtype) != "https://w3id.org/cwl/cwl#Workflow":
+            stdout.write(u'"%s" [label="%s"]\n' % (dotname[step], urlparse.urldefrag(Text(step))[1]))
 
     if currentwf is not None:
         stdout.write("}\n")
@@ -148,7 +148,7 @@ def dot_without_parameters(g, stdout):  # type: (Graph, IO[Any]) -> None
 
 
 def printdot(workflow, wf, ctx, stdout, include_parameters=False):
-    # type: (Union[str, unicode], Union[List[Dict[unicode, Any]], Dict[unicode, Any]], Loader.ContextType, Any, bool) -> None
+    # type: (Union[Text, Text], Union[List[Dict[Text, Any]], Dict[Text, Any]], Loader.ContextType, Any, bool) -> None
     g = makerdf(workflow, wf, ctx)
 
     stdout.write("digraph {")

cwltool/docker.py

@@ -6,18 +6,19 @@
 from .errors import WorkflowException
 import re
 import tempfile
-from typing import Any, Union
+from typing import Any, Text, Union
 
 _logger = logging.getLogger("cwltool")
 
 def get_image(dockerRequirement, pull_image, dry_run=False):
-    # type: (Dict[str,str], bool, bool) -> bool
+    # type: (Dict[Text, Text], bool, bool) -> bool
     found = False
 
     if "dockerImageId" not in dockerRequirement and "dockerPull" in dockerRequirement:
         dockerRequirement["dockerImageId"] = dockerRequirement["dockerPull"]
 
-    for ln in subprocess.check_output(["docker", "images", "--no-trunc", "--all"]).splitlines():
+    for ln in subprocess.check_output(
+            ["docker", "images", "--no-trunc", "--all"]).splitlines():
         try:
             m = re.match(r"^([^ ]+)\s+([^ ]+)\s+([^ ]+)", ln)
             sp = dockerRequirement["dockerImageId"].split(":")
@@ -31,24 +32,26 @@ def get_image(dockerRequirement, pull_image, dry_run=False):
             pass
 
     if not found and pull_image:
+        cmd = []  # type: List[str]
         if "dockerPull" in dockerRequirement:
-            cmd = ["docker", "pull", dockerRequirement["dockerPull"]]
-            _logger.info(str(cmd))
+            cmd = ["docker", "pull", str(dockerRequirement["dockerPull"])]
+            _logger.info(Text(cmd))
             if not dry_run:
                 subprocess.check_call(cmd, stdout=sys.stderr)
                 found = True
         elif "dockerFile" in dockerRequirement:
             dockerfile_dir = str(tempfile.mkdtemp())
             with open(os.path.join(dockerfile_dir, "Dockerfile"), "w") as df:
                 df.write(dockerRequirement["dockerFile"])
-            cmd = ["docker", "build", "--tag=%s" % dockerRequirement["dockerImageId"], dockerfile_dir]
-            _logger.info(str(cmd))
+            cmd = ["docker", "build", "--tag=%s" %
+                str(dockerRequirement["dockerImageId"]), dockerfile_dir]
+            _logger.info(Text(cmd))
             if not dry_run:
                 subprocess.check_call(cmd, stdout=sys.stderr)
                 found = True
         elif "dockerLoad" in dockerRequirement:
             cmd = ["docker", "load"]
-            _logger.info(str(cmd))
+            _logger.info(Text(cmd))
             if not dry_run:
                 if os.path.exists(dockerRequirement["dockerLoad"]):
                     _logger.info(u"Loading docker image from %s", dockerRequirement["dockerLoad"])
@@ -69,8 +72,9 @@ def get_image(dockerRequirement, pull_image, dry_run=False):
                     raise WorkflowException("Docker load returned non-zero exit status %i" % (rcode))
                 found = True
         elif "dockerImport" in dockerRequirement:
-            cmd = ["docker", "import", dockerRequirement["dockerImport"], dockerRequirement["dockerImageId"]]
-            _logger.info(str(cmd))
+            cmd = ["docker", "import", str(dockerRequirement["dockerImport"]),
+                str(dockerRequirement["dockerImageId"])]
+            _logger.info(Text(cmd))
             if not dry_run:
                 subprocess.check_call(cmd, stdout=sys.stderr)
                 found = True
@@ -79,15 +83,15 @@ def get_image(dockerRequirement, pull_image, dry_run=False):
 
 
 def get_from_requirements(r, req, pull_image, dry_run=False):
-    # type: (Dict[str,str], bool, bool, bool) -> Union[None,str]
+    # type: (Dict[Text, Text], bool, bool, bool) -> Text
     if r:
         errmsg = None
         try:
             subprocess.check_output(["docker", "version"])
         except subprocess.CalledProcessError as e:
-            errmsg = "Cannot communicate with docker daemon: " + str(e)
+            errmsg = "Cannot communicate with docker daemon: " + Text(e)
         except OSError as e:
-            errmsg = "'docker' executable not found: " + str(e)
+            errmsg = "'docker' executable not found: " + Text(e)
 
         if errmsg:
             if req:

cwltool/docker_uid.py

@@ -1,8 +1,8 @@
 import subprocess
-from typing import Union
+from typing import Text, Union
 
 
-def docker_vm_uid():  # type: () -> Union[int,None]
+def docker_vm_uid():  # type: () -> int
     """
     Returns the UID of the default docker user inside the VM
 
@@ -20,7 +20,7 @@ def docker_vm_uid():  # type: () -> Union[int,None]
         return None
 
 
-def check_output_and_strip(cmd):  # type: (List[str]) -> Union[str,None]
+def check_output_and_strip(cmd):  # type: (List[Text]) -> Text
     """
     Passes a command list to subprocess.check_output, returning None
     if an expected exception is raised
@@ -37,7 +37,7 @@ def check_output_and_strip(cmd):  # type: (List[str]) -> Union[str,None]
         return None
 
 
-def docker_machine_name():  # type: () -> Union[str,None]
+def docker_machine_name():  # type: () -> Text
     """
     Get the machine name of the active docker-machine machine
     :return: Name of the active machine or None if error
@@ -46,7 +46,7 @@ def docker_machine_name():  # type: () -> Union[str,None]
 
 
 def cmd_output_matches(check_cmd, expected_status):
-    # type: (List[str], str) -> bool
+    # type: (List[Text], Text) -> bool
     """
     Runs a command and compares output to expected
     :param check_cmd: Command list to execute
@@ -76,7 +76,7 @@ def docker_machine_running():  # type: () -> bool
     return cmd_output_matches(['docker-machine', 'status', machine_name], 'Running')
 
 
-def cmd_output_to_int(cmd):  # type: (List[str]) -> Union[int,None]
+def cmd_output_to_int(cmd):  # type: (List[Text]) -> int
     """
     Runs the provided command and returns the integer value of the result
     :param cmd: The command to run
@@ -91,15 +91,15 @@ def cmd_output_to_int(cmd):  # type: (List[str]) -> Union[int,None]
             return None
 
 
-def boot2docker_uid():  # type: () -> Union[int,None]
+def boot2docker_uid():  # type: () -> int
     """
     Gets the UID of the docker user inside a running boot2docker vm
     :return: the UID, or None if error (e.g. boot2docker not present or stopped)
     """
     return cmd_output_to_int(['boot2docker', 'ssh', 'id', '-u'])
 
 
-def docker_machine_uid():  # type: () -> Union[int,None]
+def docker_machine_uid():  # type: () -> int
     """
     Asks docker-machine for active machine and gets the UID of the docker user
     inside the vm