Added fs.copy.copy_metadata.

All logic for the added `preserve_time` parameter is contained in that function, so other pieces of code just have to make a single call. It's still not implemented at this point.

Author: atollk

fs/_bulk.py

@@ -56,20 +56,30 @@ def __call__(self):
 class _CopyTask(_Task):
     """A callable that copies from one file another."""
 
-    def __init__(self, src_file, dst_file):
-        # type: (IO, IO) -> None
-        self.src_file = src_file
-        self.dst_file = dst_file
+    def __init__(
+        self,
+        src_fs,  # type: FS
+        src_path,  # type: Text
+        dst_fs,  # type: FS
+        dst_path,  # type: Text
+        preserve_time,  # type: bool
+    ):
+        # type: (...) -> None
+        self.src_fs = src_fs
+        self.src_path = src_path
+        self.dst_fs = dst_fs
+        self.dst_path = dst_path
+        self.preserve_time = preserve_time
 
     def __call__(self):
         # type: () -> None
-        try:
-            copy_file_data(self.src_file, self.dst_file, chunk_size=1024 * 1024)
-        finally:
-            try:
-                self.src_file.close()
-            finally:
-                self.dst_file.close()
+        copy_file_internal(
+            self.src_fs,
+            self.src_path,
+            self.dst_fs,
+            self.dst_path,
+            preserve_time=self.preserve_time,
+        )
 
 
 class Copier(object):
@@ -88,7 +98,7 @@ def __init__(self, num_workers=4):
     def start(self):
         """Start the workers."""
         if self.num_workers:
-            self.queue = Queue(maxsize=self.num_workers)
+            self.queue = Queue()
             self.workers = [_Worker(self) for _ in range(self.num_workers)]
             for worker in self.workers:
                 worker.start()
@@ -133,12 +143,5 @@ def copy(self, src_fs, src_path, dst_fs, dst_path, preserve_time=False):
                 src_fs, src_path, dst_fs, dst_path, preserve_time=preserve_time
             )
         else:
-            # TODO(preserve_time)
-            src_file = src_fs.openbin(src_path, "r")
-            try:
-                dst_file = dst_fs.openbin(dst_path, "w")
-            except Exception:
-                src_file.close()
-                raise
-            task = _CopyTask(src_file, dst_file)
+            task = _CopyTask(src_fs, src_path, dst_fs, dst_path, preserve_time)
             self.queue.put(task)

fs/base.py

@@ -22,6 +22,7 @@
 import six
 
 from . import copy, errors, fsencode, iotools, move, tools, walk, wildcard
+from .copy import copy_metadata
 from .glob import BoundGlobber
 from .mode import validate_open_mode
 from .path import abspath, join, normpath
@@ -412,13 +413,13 @@ def copy(
                 ``dst_path`` does not exist.
 
         """
-        # TODO(preserve_time)
         with self._lock:
             if not overwrite and self.exists(dst_path):
                 raise errors.DestinationExists(dst_path)
             with closing(self.open(src_path, "rb")) as read_file:
                 # FIXME(@althonos): typing complains because open return IO
                 self.upload(dst_path, read_file)  # type: ignore
+            copy_metadata(self, src_path, self, dst_path)
 
     def copydir(
         self,
@@ -1030,8 +1031,8 @@ def movedir(self, src_path, dst_path, create=False, preserve_time=False):
             dst_path (str): Path to destination directory.
             create (bool): If `True`, then ``dst_path`` will be created
                 if it doesn't exist already (defaults to `False`).
-        preserve_time (bool): If `True`, try to preserve atime, ctime,
-            and mtime of the resources (defaults to `False`).
+            preserve_time (bool): If `True`, try to preserve atime, ctime,
+                and mtime of the resources (defaults to `False`).
 
         Raises:
             fs.errors.ResourceNotFound: if ``dst_path`` does not exist,
@@ -1086,8 +1087,8 @@ def makedirs(
                     raise
             return self.opendir(path)
 
-    def move(self, src_path, dst_path, overwrite=False):
-        # type: (Text, Text, bool) -> None
+    def move(self, src_path, dst_path, overwrite=False, preserve_time=False):
+        # type: (Text, Text, bool, bool) -> None
         """Move a file from ``src_path`` to ``dst_path``.
 
         Arguments:
@@ -1096,6 +1097,8 @@ def move(self, src_path, dst_path, overwrite=False):
                 file will be written to.
             overwrite (bool): If `True`, destination path will be
                 overwritten if it exists.
+            preserve_time (bool): If `True`, try to preserve atime, ctime,
+                and mtime of the resources (defaults to `False`).
 
         Raises:
             fs.errors.FileExpected: If ``src_path`` maps to a
@@ -1128,6 +1131,7 @@ def move(self, src_path, dst_path, overwrite=False):
                 # FIXME(@althonos): typing complains because open return IO
                 self.upload(dst_path, read_file)  # type: ignore
             self.remove(src_path)
+            copy_metadata(self, src_path, self, dst_path)
 
     def open(
         self,

fs/copy.py

@@ -151,7 +151,6 @@ def copy_file(
                     src_path, dst_path, overwrite=True, preserve_time=preserve_time
                 )
             else:
-                # TODO(preserve_time)
                 # Standard copy
                 with _src_fs.lock(), _dst_fs.lock():
                     if _dst_fs.hassyspath(dst_path):
@@ -160,6 +159,7 @@ def copy_file(
                     else:
                         with _src_fs.openbin(src_path) as read_file:
                             _dst_fs.upload(dst_path, read_file)
+                    copy_metadata(_src_fs, src_path, _dst_fs, dst_path)
 
 
 def copy_file_internal(
@@ -190,14 +190,17 @@ def copy_file_internal(
         # Same filesystem, so we can do a potentially optimized
         # copy
         src_fs.copy(src_path, dst_path, overwrite=True, preserve_time=preserve_time)
-        # TODO(preserve_time)
-    elif dst_fs.hassyspath(dst_path):
+        return
+
+    if dst_fs.hassyspath(dst_path):
         with dst_fs.openbin(dst_path, "w") as write_file:
             src_fs.download(src_path, write_file)
     else:
         with src_fs.openbin(src_path) as read_file:
             dst_fs.upload(dst_path, read_file)
 
+    copy_metadata(src_fs, src_path, dst_fs, dst_path)
+
 
 def copy_file_if_newer(
     src_fs,  # type: Union[FS, Text]
@@ -326,9 +329,10 @@ def dst():
     from ._bulk import Copier
 
     with src() as _src_fs, dst() as _dst_fs:
-        with _src_fs.lock(), _dst_fs.lock():
-            _thread_safe = is_thread_safe(_src_fs, _dst_fs)
-            with Copier(num_workers=workers if _thread_safe else 0) as copier:
+        _thread_safe = is_thread_safe(_src_fs, _dst_fs)
+        copier = Copier(num_workers=workers if _thread_safe else 0)
+        with copier:
+            with _src_fs.lock(), _dst_fs.lock():
                 _dst_fs.makedir(_dst_path, recreate=True)
                 for dir_path, dirs, files in walker.walk(_src_fs, _src_path):
                     copy_path = combine(_dst_path, frombase(_src_path, dir_path))
@@ -345,6 +349,7 @@ def dst():
                             preserve_time=preserve_time,
                         )
                         on_copy(_src_fs, src_path, _dst_fs, dst_path)
+            pass
 
 
 def copy_dir_if_newer(
@@ -438,3 +443,25 @@ def dst():
                                 preserve_time=preserve_time,
                             )
                             on_copy(_src_fs, dir_path, _dst_fs, copy_path)
+
+
+def copy_metadata(
+    src_fs,  # type: Union[FS, Text]
+    src_path,  # type: Text
+    dst_fs,  # type: Union[FS, Text]
+    dst_path,  # type: Text
+):
+    # type: (...) -> None
+    """Copies metadata from one file to another.
+
+    Arguments:
+        src_fs (FS or str): Source filesystem (instance or URL).
+        src_path (str): Path to a directory on the source filesystem.
+        dst_fs (FS or str): Destination filesystem (instance or URL).
+        dst_path (str): Path to a directory on the destination filesystem.
+
+    """
+    # TODO(preserve_time)
+    with manage_fs(src_fs, writeable=False) as _src_fs:
+        with manage_fs(dst_fs, create=True) as _dst_fs:
+            pass

fs/mirror.py

@@ -86,9 +86,9 @@ def dst():
         return manage_fs(dst_fs, create=True)
 
     with src() as _src_fs, dst() as _dst_fs:
-        with _src_fs.lock(), _dst_fs.lock():
-            _thread_safe = is_thread_safe(_src_fs, _dst_fs)
-            with Copier(num_workers=workers if _thread_safe else 0) as copier:
+        _thread_safe = is_thread_safe(_src_fs, _dst_fs)
+        with Copier(num_workers=workers if _thread_safe else 0) as copier:
+            with _src_fs.lock(), _dst_fs.lock():
                 _mirror(
                     _src_fs,
                     _dst_fs,

fs/move.py

@@ -55,12 +55,13 @@ def move_file(
             and mtime of the resources (defaults to `False`).
 
     """
-    # TODO(preserve_time)
     with manage_fs(src_fs) as _src_fs:
         with manage_fs(dst_fs, create=True) as _dst_fs:
             if _src_fs is _dst_fs:
                 # Same filesystem, may be optimized
-                _src_fs.move(src_path, dst_path, overwrite=True)
+                _src_fs.move(
+                    src_path, dst_path, overwrite=True, preserve_time=preserve_time
+                )
             else:
                 # Standard copy and delete
                 with _src_fs.lock(), _dst_fs.lock():

fs/osfs.py

@@ -433,7 +433,6 @@ def _check_copy(self, src_path, dst_path, overwrite=False):
 
         def copy(self, src_path, dst_path, overwrite=False, preserve_time=False):
             # type: (Text, Text, bool, bool) -> None
-            # TODO(preserve_time)
             with self._lock:
                 # validate and canonicalise paths
                 _src_path, _dst_path = self._check_copy(src_path, dst_path, overwrite)
@@ -464,7 +463,6 @@ def copy(self, src_path, dst_path, overwrite=False, preserve_time=False):
 
         def copy(self, src_path, dst_path, overwrite=False, preserve_time=False):
             # type: (Text, Text, bool, bool) -> None
-            # TODO(preserve_time)
             with self._lock:
                 _src_path, _dst_path = self._check_copy(src_path, dst_path, overwrite)
                 shutil.copy2(self.getsyspath(_src_path), self.getsyspath(_dst_path))

fs/wrap.py

@@ -181,8 +181,8 @@ def makedir(
         self.check()
         raise ResourceReadOnly(path)
 
-    def move(self, src_path, dst_path, overwrite=False):
-        # type: (Text, Text, bool) -> None
+    def move(self, src_path, dst_path, overwrite=False, preserve_time=False):
+        # type: (Text, Text, bool, bool) -> None
         self.check()
         raise ResourceReadOnly(dst_path)
 

fs/wrapfs.py

@@ -167,15 +167,15 @@ def makedir(
         with unwrap_errors(path):
             return _fs.makedir(_path, permissions=permissions, recreate=recreate)
 
-    def move(self, src_path, dst_path, overwrite=False):
-        # type: (Text, Text, bool) -> None
+    def move(self, src_path, dst_path, overwrite=False, preserve_time=False):
+        # type: (Text, Text, bool, bool) -> None
         # A custom move permits a potentially optimized code path
         src_fs, _src_path = self.delegate_path(src_path)
         dst_fs, _dst_path = self.delegate_path(dst_path)
         with unwrap_errors({_src_path: src_path, _dst_path: dst_path}):
             if not overwrite and dst_fs.exists(_dst_path):
                 raise errors.DestinationExists(_dst_path)
-            move_file(src_fs, _src_path, dst_fs, _dst_path)
+            move_file(src_fs, _src_path, dst_fs, _dst_path, preserve_time=preserve_time)
 
     def movedir(self, src_path, dst_path, create=False):
         # type: (Text, Text, bool) -> None