Support for user space docker (#542)

* Support for user space docker

* Initial draft of documentation for user space docker

Author: geetduggal

README.rst

@@ -500,3 +500,18 @@ logger_handler
       logging.Handler
 
   Handler object for logging.
+
+Running user-space implementations of Docker
+--------------------------------------------
+
+Some compute environments disallow user-space installation of Docker due to incompatiblities in libraries or to meet security requirements. The CWL reference supports using a user space implementation with the `--user-space-docker-cmd` option.
+
+Example using `dx-docker` (https://wiki.dnanexus.com/Developer-Tutorials/Using-Docker-Images):
+
+For use on Linux, install the DNAnexus toolkit (see https://wiki.dnanexus.com/Downloads for instructions).
+
+Run `cwltool` just as you normally would, but with the new option, e.g. from the conformance tests:
+
+```
+cwltool --user-space-docker-cmd=dx-docker --outdir=/tmp/tmpidytmp v1.0/test-cwl-out2.cwl v1.0/empty.json
+```

cwltool/job.py

@@ -321,15 +321,20 @@ def add_volumes(self, pathmapper, runtime):
             if not vol.staged:
                 continue
             if vol.target.startswith(container_outdir+"/"):
-                host_outdir_tgt = os.path.join(host_outdir, vol.target[len(container_outdir)+1:])
+                host_outdir_tgt = os.path.join(
+                    host_outdir, vol.target[len(container_outdir)+1:])
             else:
                 host_outdir_tgt = None
             if vol.type in ("File", "Directory"):
                 if not vol.resolved.startswith("_:"):
-                    runtime.append(u"--volume=%s:%s:ro" % (docker_windows_path_adjust(vol.resolved), docker_windows_path_adjust(vol.target)))
+                    runtime.append(u"--volume=%s:%s:ro" % (
+                        docker_windows_path_adjust(vol.resolved),
+                        docker_windows_path_adjust(vol.target)))
             elif vol.type == "WritableFile":
                 if self.inplace_update:
-                    runtime.append(u"--volume=%s:%s:rw" % (docker_windows_path_adjust(vol.resolved), docker_windows_path_adjust(vol.target)))
+                    runtime.append(u"--volume=%s:%s:rw" % (
+                        docker_windows_path_adjust(vol.resolved),
+                        docker_windows_path_adjust(vol.target)))
                 else:
                     shutil.copy(vol.resolved, host_outdir_tgt)
                     ensure_writable(host_outdir_tgt)
@@ -338,7 +343,9 @@ def add_volumes(self, pathmapper, runtime):
                     os.makedirs(vol.target, 0o0755)
                 else:
                     if self.inplace_update:
-                        runtime.append(u"--volume=%s:%s:rw" % (docker_windows_path_adjust(vol.resolved), docker_windows_path_adjust(vol.target)))
+                        runtime.append(u"--volume=%s:%s:rw" % (
+                            docker_windows_path_adjust(vol.resolved),
+                            docker_windows_path_adjust(vol.target)))
                     else:
                         shutil.copytree(vol.resolved, host_outdir_tgt)
                         ensure_writable(host_outdir_tgt)
@@ -350,7 +357,9 @@ def add_volumes(self, pathmapper, runtime):
                     fd, createtmp = tempfile.mkstemp(dir=self.tmpdir)
                     with os.fdopen(fd, "wb") as f:
                         f.write(vol.resolved.encode("utf-8"))
-                    runtime.append(u"--volume=%s:%s:rw" % (docker_windows_path_adjust(createtmp), docker_windows_path_adjust(vol.target)))
+                    runtime.append(u"--volume=%s:%s:rw" % (
+                        docker_windows_path_adjust(createtmp),
+                        docker_windows_path_adjust(vol.target)))
 
     def run(self, pull_image=True, rm_container=True,
             rm_tmpdir=True, move_outputs="move", **kwargs):
@@ -360,59 +369,89 @@ def run(self, pull_image=True, rm_container=True,
 
         img_id = None
         env = None  # type: MutableMapping[Text, Text]
-        try:
-            env = cast(MutableMapping[Text, Text], os.environ)
-            if docker_req and kwargs.get("use_container"):
-                img_id = docker.get_from_requirements(docker_req, True, pull_image)
-            if img_id is None:
-                if self.builder.find_default_container:
-                    default_container = self.builder.find_default_container()
-                    if default_container:
-                        img_id = default_container
-                        env = cast(MutableMapping[Text, Text], os.environ)
-
-            if docker_req and img_id is None and kwargs.get("use_container"):
-                raise Exception("Docker image not available")
-        except Exception as e:
-            _logger.debug("Docker error", exc_info=True)
-            if docker_is_req:
-                raise UnsupportedRequirement(
-                    "Docker is required to run this tool: %s" % e)
+        user_space_docker_cmd = kwargs.get("user_space_docker_cmd")
+        if docker_req and user_space_docker_cmd:
+            # For user-space docker implementations, a local image name or ID
+            # takes precedence over a network pull
+            if 'dockerImageId' in docker_req:
+                img_id = str(docker_req["dockerImageId"])
+            elif 'dockerPull' in docker_req:
+                img_id = str(docker_req["dockerPull"])
             else:
-                raise WorkflowException(
-                    "Docker is not available for this tool, try --no-container"
-                    " to disable Docker: %s" % e)
+                raise Exception("Docker image must be specified as "
+                        "'dockerImageId' or 'dockerPull' when using user "
+                        "space implementations of Docker")
+        else:
+            try:
+                env = cast(MutableMapping[Text, Text], os.environ)
+                if docker_req and kwargs.get("use_container"):
+                    img_id = str(docker.get_from_requirements(
+                        docker_req, True, pull_image))
+                if img_id is None:
+                    if self.builder.find_default_container:
+                        default_container = self.builder.find_default_container()
+                        if default_container:
+                            img_id = str(default_container)
+                            env = cast(MutableMapping[Text, Text], os.environ)
+
+                if docker_req and img_id is None and kwargs.get("use_container"):
+                    raise Exception("Docker image not available")
+            except Exception as e:
+                _logger.debug("Docker error", exc_info=True)
+                if docker_is_req:
+                    raise UnsupportedRequirement(
+                        "Docker is required to run this tool: %s" % e)
+                else:
+                    raise WorkflowException(
+                        "Docker is not available for this tool, try "
+                        "--no-container to disable Docker, or install "
+                        "a user space Docker replacement like uDocker with "
+                        "--user-space-docker-cmd.: %s" % e)
 
         self._setup(kwargs)
 
-        runtime = [u"docker", u"run", u"-i"]
+        if user_space_docker_cmd:
+            runtime = [user_space_docker_cmd, u"run"]
+        else:
+            runtime = [u"docker", u"run", u"-i"]
 
-        runtime.append(u"--volume=%s:%s:rw" % (docker_windows_path_adjust(os.path.realpath(self.outdir)), self.builder.outdir))
-        runtime.append(u"--volume=%s:%s:rw" % (docker_windows_path_adjust(os.path.realpath(self.tmpdir)), "/tmp"))
+        runtime.append(u"--volume=%s:%s:rw" % (
+            docker_windows_path_adjust(os.path.realpath(self.outdir)),
+            self.builder.outdir))
+        runtime.append(u"--volume=%s:%s:rw" % (
+            docker_windows_path_adjust(os.path.realpath(self.tmpdir)), "/tmp"))
 
         self.add_volumes(self.pathmapper, runtime)
         if self.generatemapper:
             self.add_volumes(self.generatemapper, runtime)
 
-        runtime.append(u"--workdir=%s" % (docker_windows_path_adjust(self.builder.outdir)))
+        if user_space_docker_cmd:
+            runtime = [x.replace(":ro", "") for x in runtime]
+            runtime = [x.replace(":rw", "") for x in runtime]
 
-        if not kwargs.get("no_read_only"):
-            runtime.append(u"--read-only=true")
+        runtime.append(u"--workdir=%s" % (
+            docker_windows_path_adjust(self.builder.outdir)))
+        if not user_space_docker_cmd:
 
-        if kwargs.get("custom_net", None) is not None:
-            runtime.append(u"--net={0}".format(kwargs.get("custom_net")))
-        elif kwargs.get("disable_net", None):
-            runtime.append(u"--net=none")
+            if not kwargs.get("no_read_only"):
+                runtime.append(u"--read-only=true")
 
-        if self.stdout:
-            runtime.append("--log-driver=none")
+            if kwargs.get("custom_net", None) is not None:
+                runtime.append(u"--net={0}".format(kwargs.get("custom_net")))
+            elif kwargs.get("disable_net", None):
+                runtime.append(u"--net=none")
+
+            if self.stdout:
+                runtime.append("--log-driver=none")
 
-        euid, egid = docker_vm_id()
-        if not onWindows():  # MS Windows does not have getuid() or geteuid() functions
-            euid, egid = euid or os.geteuid(), egid or os.getgid()
+            euid, egid = docker_vm_id()
+            if not onWindows():
+                # MS Windows does not have getuid() or geteuid() functions
+                euid, egid = euid or os.geteuid(), egid or os.getgid()
 
-        if kwargs.get("no_match_user", None) is False and (euid, egid) != (None, None):
-            runtime.append(u"--user=%d:%d" % (euid, egid))
+            if kwargs.get("no_match_user", None) is False \
+                    and (euid, egid) != (None, None):
+                runtime.append(u"--user=%d:%d" % (euid, egid))
 
         if rm_container:
             runtime.append(u"--rm")
@@ -429,7 +468,8 @@ def run(self, pull_image=True, rm_container=True,
 
         runtime.append(img_id)
 
-        self._execute(runtime, env, rm_tmpdir=rm_tmpdir, move_outputs=move_outputs)
+        self._execute(
+            runtime, env, rm_tmpdir=rm_tmpdir, move_outputs=move_outputs)
 
 
 def _job_popen(

cwltool/main.py

@@ -159,6 +159,10 @@ def arg_parser():  # type: () -> argparse.ArgumentParser
     exgroup.add_argument("--debug", action="store_true", help="Print even more logging")
 
     parser.add_argument("--js-console", action="store_true", help="Enable javascript console output")
+    parser.add_argument("--user-space-docker-cmd",
+                        help="(Linux/OS X only) Specify a user space docker "
+                        "command (like udocker or dx-docker) that will be "
+                        "used to call 'pull' and 'run'")
 
     dependency_resolvers_configuration_help = argparse.SUPPRESS
     dependencies_directory_help = argparse.SUPPRESS