Set last modification date of remote file as local file timestamp (#1676)

Co-authored-by: Michael R. Crusoe <1330696+mr-c@users.noreply.github.com>

Author: 2 people

Makefile

@@ -97,13 +97,14 @@ remove_unused_imports: $(PYSOURCES)
 	autoflake --in-place --remove-all-unused-imports $^
 
 pep257: pydocstyle
-## pydocstyle      : check Python code style
+## pydocstyle      : check Python docstring style
 pydocstyle: $(PYSOURCES)
 	pydocstyle --add-ignore=D100,D101,D102,D103 $^ || true
 
 pydocstyle_report.txt: $(PYSOURCES)
 	pydocstyle setup.py $^ > $@ 2>&1 || true
 
+## diff_pydocstyle_report	: check Python docstring style for changed files only
 diff_pydocstyle_report: pydocstyle_report.txt
 	diff-quality --compare-branch=main --violations=pydocstyle --fail-under=100 $^
 

cwltool/pathmapper.py

@@ -139,7 +139,7 @@ def visit(
                 ):
                     deref = ab
                     if urllib.parse.urlsplit(deref).scheme in ["http", "https"]:
-                        deref = downloadHttpFile(path)
+                        deref, _last_modified = downloadHttpFile(path)
                     else:
                         # Dereference symbolic links
                         st = os.lstat(deref)

cwltool/utils.py

@@ -1,5 +1,4 @@
 """Shared functions and other definitions."""
-
 import collections
 import os
 import random
@@ -11,6 +10,8 @@
 import tempfile
 import urllib
 import uuid
+from datetime import datetime
+from email.utils import parsedate_to_datetime
 from functools import partial
 from itertools import zip_longest
 from pathlib import Path, PurePosixPath
@@ -344,8 +345,14 @@ def trim_listing(obj):  # type: (Dict[str, Any]) -> None
         del obj["listing"]
 
 
-def downloadHttpFile(httpurl):
-    # type: (str) -> str
+def downloadHttpFile(httpurl: str) -> Tuple[str, Optional[datetime]]:
+    """
+    Download a remote file, possibly using a locally cached copy.
+
+    Returns a tuple:
+    - the local path for the downloaded file
+    - the Last-Modified timestamp if received from the remote server.
+    """
     cache_session = None
     if "XDG_CACHE_HOME" in os.environ:
         directory = os.environ["XDG_CACHE_HOME"]
@@ -365,7 +372,14 @@ def downloadHttpFile(httpurl):
             if chunk:  # filter out keep-alive new chunks
                 f.write(chunk)
     r.close()
-    return str(f.name)
+
+    date_raw: Optional[str] = r.headers.get("Last-Modified", None)
+    date: Optional[datetime] = parsedate_to_datetime(date_raw) if date_raw else None
+    if date:
+        date_epoch = date.timestamp()
+        os.utime(f.name, (date_epoch, date_epoch))
+
+    return str(f.name), date
 
 
 def ensure_writable(path: str, include_root: bool = False) -> None:

setup.py

@@ -134,6 +134,7 @@
         "pytest >= 6.2, < 7.2",
         "mock >= 2.0.0",
         "pytest-mock >= 1.10.0",
+        "pytest-httpserver",
         "arcp >= 0.2.0",
         "rdflib-jsonld>=0.4.0, <= 0.6.1;python_version<='3.6'",
     ],

test-requirements.txt

@@ -1,5 +1,6 @@
 pytest >= 6.2, < 7.2
 pytest-xdist
+pytest-httpserver
 mock >= 2.0.0
 pytest-mock >= 1.10.0
 pytest-cov

tests/test_http_input.py

@@ -1,7 +1,12 @@
 import os
+import sys
+from datetime import datetime
 from pathlib import Path
 from typing import List
 
+import pytest
+from pytest_httpserver import HTTPServer
+
 from cwltool.pathmapper import PathMapper
 from cwltool.utils import CWLObjectType
 
@@ -25,3 +30,47 @@ def test_http_path_mapping(tmp_path: Path) -> None:
         contents = file.read()
 
     assert ">Sequence 561 BP; 135 A; 106 C; 98 G; 222 T; 0 other;" in contents
+
+
+@pytest.mark.skipif(sys.version_info < (3, 7), reason="timesout on CI")
+def test_modification_date(tmp_path: Path) -> None:
+    """Local copies of remote files should preserve last modification date."""
+    # Initialize the server
+    headers = {
+        "Server": "nginx",
+        "Date": "Mon, 27 Jun 2022 14:26:17 GMT",
+        "Content-Type": "application/zip",
+        "Content-Length": "123906",
+        "Connection": "keep-alive",
+        "Last-Modified": "Tue, 14 Dec 2021 14:23:30 GMT",
+        "ETag": '"1e402-5d31beef49671"',
+        "Accept-Ranges": "bytes",
+        "Strict-Transport-Security": "max-age=31536000; includeSubDomains",
+    }
+
+    remote_file_name = "testfile.txt"
+
+    with HTTPServer() as httpserver:
+        httpserver.expect_request(f"/{remote_file_name}").respond_with_data(
+            response_data="Hello World", headers=headers
+        )
+        location = httpserver.url_for(f"/{remote_file_name}")
+
+        base_file: List[CWLObjectType] = [
+            {
+                "class": "File",
+                "location": location,
+                "basename": remote_file_name,
+            }
+        ]
+
+        date_now = datetime.now()
+
+        pathmap = PathMapper(base_file, os.getcwd(), str(tmp_path))._pathmap
+
+        assert location in pathmap
+        assert os.path.exists(pathmap[location].resolved)
+
+        last_modified = os.path.getmtime(pathmap[location].resolved)
+
+        assert date_now.timestamp() > last_modified