cache relative paths (#1685)

* cachedir: always calculate checksums

Author: mr-c

cwltool/argparser.py

@@ -831,7 +831,7 @@ def __init__(
         nargs: Any = None,
         **kwargs: Any,
     ) -> None:
-        """Intialize."""
+        """Initialize."""
         super().__init__(option_strings, dest, **kwargs)
         self._called = False
 

cwltool/command_line_tool.py

@@ -850,11 +850,18 @@ def job(
             _check_adjust = partial(
                 check_adjust, self.path_check_mode.value, cachebuilder
             )
+            _checksum = partial(
+                compute_checksums,
+                runtimeContext.make_fs_access(runtimeContext.basedir),
+            )
             visit_class(
                 [cachebuilder.files, cachebuilder.bindings],
                 ("File", "Directory"),
                 _check_adjust,
             )
+            visit_class(
+                [cachebuilder.files, cachebuilder.bindings], ("File"), _checksum
+            )
 
             cmdline = flatten(
                 list(map(cachebuilder.generate_arg, cachebuilder.bindings))
@@ -895,14 +902,19 @@ def calc_checksum(location: str) -> Optional[str]:
                         return cast(Optional[str], e["checksum"])
                 return None
 
+            def remove_prefix(s: str, prefix: str) -> str:
+                # replace with str.removeprefix when Python 3.9+
+                return s[len(prefix) :] if s.startswith(prefix) else s
+
             for location, fobj in cachebuilder.pathmapper.items():
                 if fobj.type == "File":
                     checksum = calc_checksum(location)
                     fobj_stat = os.stat(fobj.resolved)
+                    path = remove_prefix(fobj.resolved, runtimeContext.basedir + "/")
                     if checksum is not None:
-                        keydict[fobj.resolved] = [fobj_stat.st_size, checksum]
+                        keydict[path] = [fobj_stat.st_size, checksum]
                     else:
-                        keydict[fobj.resolved] = [
+                        keydict[path] = [
                             fobj_stat.st_size,
                             int(fobj_stat.st_mtime * 1000),
                         ]

docs/pythonversions.rst

@@ -22,7 +22,7 @@ and downstream users before making the decision to drop support for a
 Python version before the date outlined in this policy.  The reasoning
 for dropping support for a Python version should be outlined here.
 
-As of Feburary 2022, here are approximate cwltool support periods for
+As of February 2022, here are approximate cwltool support periods for
 across Python versions:
 
 ====== ======================
@@ -38,7 +38,7 @@ Python cwltool end of support
 ====== ======================
 
 Default Python version of supported Linux distributions, for reference
-(as of Feburary 2022)
+(as of February 2022)
 
 ====== =============================================
 Python Linux distros where it is the default version

tests/test_examples.py

@@ -1305,6 +1305,46 @@ def test_issue_740_fixed(tmp_path: Path, factor: str) -> None:
     assert error_code == 0, stderr
 
 
+@needs_docker
+@pytest.mark.parametrize("factor", test_factors)
+def test_cache_relative_paths(tmp_path: Path, factor: str) -> None:
+    """Confirm that re-running a particular workflow with caching succeeds."""
+    test_file = "secondary-files.cwl"
+    test_job_file = "secondary-files-job.yml"
+    cache_dir = str(tmp_path / "cwltool_cache")
+    commands = factor.split()
+    commands.extend(
+        [
+            "--cachedir",
+            cache_dir,
+            get_data(f"tests/{test_file}"),
+            get_data(f"tests/{test_job_file}"),
+        ]
+    )
+    error_code, _, stderr = get_main_output(commands)
+
+    stderr = re.sub(r"\s\s+", " ", stderr)
+    assert "completed success" in stderr
+    assert error_code == 0
+
+    commands = factor.split()
+    commands.extend(
+        [
+            "--cachedir",
+            cache_dir,
+            get_data(f"tests/{test_file}"),
+            get_data(f"tests/{test_job_file}"),
+        ]
+    )
+    error_code, _, stderr = get_main_output(commands)
+
+    stderr = re.sub(r"\s\s+", " ", stderr)
+    assert "Output of job will be cached in" not in stderr
+    assert error_code == 0, stderr
+
+    assert (tmp_path / "cwltool_cache" / "27903451fc1ee10c148a0bdeb845b2cf").exists()
+
+
 @needs_docker
 def test_compute_checksum() -> None:
     runtime_context = RuntimeContext()