pathmapper: don't use temporary lists

Should speed up items_exclude_children() a bit

Author: mr-c

cwltool/pathmapper.py

@@ -4,8 +4,17 @@
 import stat
 import urllib
 import uuid
-from pathlib import Path
-from typing import Dict, Iterator, List, Optional, Tuple, cast
+from typing import (
+    Dict,
+    ItemsView,
+    Iterable,
+    Iterator,
+    KeysView,
+    List,
+    Optional,
+    Tuple,
+    cast,
+)
 
 from mypy_extensions import mypyc_attr
 from schema_salad.exceptions import ValidationException
@@ -210,21 +219,29 @@ def mapper(self, src: str) -> MapperEnt:
             return MapperEnt(p.resolved, p.target + src[i:], p.type, p.staged)
         return self._pathmap[src]
 
-    def files(self) -> List[str]:
-        return list(self._pathmap.keys())
+    def files(self) -> KeysView[str]:
+        """Return a dictionary keys view of locations."""
+        return self._pathmap.keys()
 
-    def items(self) -> List[Tuple[str, MapperEnt]]:
-        return list(self._pathmap.items())
+    def items(self) -> ItemsView[str, MapperEnt]:
+        """Return a dictionary items view."""
+        return self._pathmap.items()
 
-    def items_exclude_children(self) -> List[Tuple[str, MapperEnt]]:
+    def items_exclude_children(self) -> ItemsView[str, MapperEnt]:
+        """Return a dictionary items view minus any entries which are children of other entries."""
         newitems = {}
-        keys = [key for key, entry in self.items()]
+
+        def parents(path: str) -> Iterable[str]:
+            result = path
+            while len(result) > 1:
+                result = os.path.dirname(result)
+                yield result
+
         for key, entry in self.items():
-            parents = Path(key).parents
-            if any([Path(key_) in parents for key_ in keys]):
+            if not self.files().isdisjoint(parents(key)):
                 continue
             newitems[key] = entry
-        return list(newitems.items())
+        return newitems.items()
 
     def reversemap(
         self,

cwltool/process.py

@@ -242,7 +242,7 @@ def stage_files(
     """
     items = pathmapper.items() if not symlink else pathmapper.items_exclude_children()
     targets: Dict[str, MapperEnt] = {}
-    for key, entry in items:
+    for key, entry in list(items):
         if "File" not in entry.type:
             continue
         if entry.target not in targets:
@@ -265,7 +265,7 @@ def stage_files(
                 )
     # refresh the items, since we may have updated the pathmapper due to file name clashes
     items = pathmapper.items() if not symlink else pathmapper.items_exclude_children()
-    for key, entry in items:
+    for key, entry in list(items):
         if not entry.staged:
             continue
         if not os.path.exists(os.path.dirname(entry.target)):