Enable upload/download ExtraArgs (#33)

* enable server-side bucket to bucket copies

The 'copy_object' method permits setting different source and target buckets. When different, a server-side copy takes place between the buckets.

Appropriate ACLs must exist in AWS, but this is a quick method for copying keys between buckets within the same account and region.

Copying across regions and/or accounts is more complex, and is not enabled with this change.

* Allow settable extra_args for uploads and downloads

Allow setting upload and download `extra_args` values for the lifetime of the filesystem instance.

Ticket #32 has details

* Moved args to constructor and removed bucket copy

* load unquoted json if value is a string

* Expose `acl` and `cache-control` directly and via opener

* Small cleanup

* (bugfix) Only set if value exists

* Boto3 changed from `extra_args` to `ExtraArgs` at some point

This has been tested and works.

I uploaded a file to a private bucket, setting the max-age and ACL.

Author: geoffjukes

README.md

@@ -59,6 +59,36 @@ source filesystem to the S3 filesystem.
 See [Moving and Copying](https://docs.pyfilesystem.org/en/latest/guide.html#moving-and-copying)
 for more information.
 
+## ExtraArgs
+
+S3 objects have additional properties, beyond a traditional
+filesystem. These options can be set using the ``upload_args``
+and ``download_args`` properties. which are handed to upload
+and download methods, as appropriate, for the lifetime of the
+filesystem instance.
+
+For example, to set the ``cache-control`` header of all objects
+uploaded to a bucket:
+
+```python
+import fs, fs.mirror
+s3fs = S3FS('example', upload_args={"CacheControl": "max-age=2592000", "ACL": "public-read"})
+fs.mirror.mirror('/path/to/mirror', s3fs)
+```
+
+see [the Boto3 docs](https://boto3.readthedocs.io/en/latest/reference/customizations/s3.html#boto3.s3.transfer.S3Transfer.ALLOWED_UPLOAD_ARGS)
+for more information.
+
+`acl` and `cache_control` are exposed explicitly for convenience, and can be used in URLs.
+It is important to URL-Escape the `cache_control` value in a URL, as it may contain special characters.
+
+```python
+import fs, fs.mirror
+with open fs.open_fs('s3://example?acl=public-read&cache_control=max-age%3D2592000%2Cpublic') as s3fs
+    fs.mirror.mirror('/path/to/mirror', s3fs)
+```
+
+
 ## S3 URLs
 
 You can get a public URL to a file on a S3 bucket as follows:

README.rst

@@ -64,6 +64,34 @@ filesystem to the S3 filesystem. See `Moving and
 Copying <https://docs.pyfilesystem.org/en/latest/guide.html#moving-and-copying>`__
 for more information.
 
+ExtraArgs
+---------
+
+S3 objects have additional properties, beyond a traditional
+filesystem. These options can be set using the ``upload_args``
+and ``download_args`` properties. which are handed to upload
+and download methods, as appropriate, for the lifetime of the
+filesystem instance.
+
+For example, to set the ``cache-control`` header of all objects
+uploaded to a bucket:
+
+.. code:: python
+    import fs, fs.mirror
+    s3fs = S3FS('example', upload_args={"CacheControl": "max-age=2592000", "ACL": "public-read"})
+    fs.mirror.mirror('/path/to/mirror', s3fs)
+
+see `the Boto3 docs <https://boto3.readthedocs.io/en/latest/reference/customizations/s3.html#boto3.s3.transfer.S3Transfer.ALLOWED_UPLOAD_ARGS>`__
+for more information.
+
+``acl`` and ``cache_control`` are exposed explicitly for convenience, and can be used in URLs.
+It is important to URL-Escape the ``cache_control`` value in a URL, as it may contain special characters.
+
+.. code:: python
+    import fs, fs.mirror
+    with open fs.open_fs('s3://example?acl=public-read&cache_control=max-age%3D2592000%2Cpublic') as s3fs
+        fs.mirror.mirror('/path/to/mirror', s3fs)
+
 S3 URLs
 -------
 

fs_s3fs/_s3fs.py

@@ -12,6 +12,7 @@
 from ssl import SSLError
 import tempfile
 import threading
+import json
 
 import boto3
 from botocore.exceptions import ClientError, EndpointConnectionError
@@ -269,7 +270,11 @@ def __init__(self,
                  endpoint_url=None,
                  region=None,
                  delimiter='/',
-                 strict=True):
+                 strict=True,
+                 cache_control=None,
+                 acl=None,
+                 upload_args=None,
+                 download_args=None):
         _creds = (aws_access_key_id, aws_secret_access_key)
         if any(_creds) and not all(_creds):
             raise ValueError(
@@ -287,6 +292,14 @@ def __init__(self,
         self.delimiter = delimiter
         self.strict = strict
         self._tlocal = threading.local()
+        if cache_control or acl:
+            upload_args = upload_args or {}
+            if cache_control:
+                upload_args['CacheControl'] = cache_control
+            if acl:
+                upload_args['ACL'] = acl
+        self.upload_args = upload_args
+        self.download_args = download_args
         super(S3FS, self).__init__()
 
     def __repr__(self):
@@ -540,7 +553,7 @@ def on_close_create(s3file):
                     s3file.raw.seek(0)
                     with s3errors(path):
                         self.client.upload_fileobj(
-                            s3file.raw, self._bucket_name, _key
+                            s3file.raw, self._bucket_name, _key, ExtraArgs=self.upload_args
                         )
                 finally:
                     s3file.raw.close()
@@ -568,7 +581,7 @@ def on_close_create(s3file):
                 try:
                     with s3errors(path):
                         self.client.download_fileobj(
-                            self._bucket_name, _key, s3file.raw
+                            self._bucket_name, _key, s3file.raw, ExtraArgs=self.download_args
                         )
                 except errors.ResourceNotFound:
                     pass
@@ -589,15 +602,15 @@ def on_close(s3file):
                     s3file.raw.seek(0, os.SEEK_SET)
                     with s3errors(path):
                         self.client.upload_fileobj(
-                            s3file.raw, self._bucket_name, _key
+                            s3file.raw, self._bucket_name, _key, ExtraArgs=self.upload_args
                         )
             finally:
                 s3file.raw.close()
 
         s3file = S3File.factory(path, _mode, on_close=on_close)
         with s3errors(path):
             self.client.download_fileobj(
-                self._bucket_name, _key, s3file.raw
+                self._bucket_name, _key, s3file.raw, ExtraArgs=self.download_args
             )
         s3file.seek(0, os.SEEK_SET)
         return s3file
@@ -660,7 +673,7 @@ def getbytes(self, path):
         bytes_file = io.BytesIO()
         with s3errors(path):
             self.client.download_fileobj(
-                self._bucket_name, _key, bytes_file
+                self._bucket_name, _key, bytes_file, ExtraArgs=self.download_args
             )
         return bytes_file.getvalue()
 
@@ -674,7 +687,7 @@ def getfile(self, path, file, chunk_size=None, **options):
         _key = self._path_to_key(_path)
         with s3errors(path):
             self.client.download_fileobj(
-                self._bucket_name, _key, file
+                self._bucket_name, _key, file, ExtraArgs=self.download_args
             )
 
     def exists(self, path):
@@ -758,7 +771,7 @@ def setbytes(self, path, contents):
         bytes_file = io.BytesIO(contents)
         with s3errors(path):
             self.client.upload_fileobj(
-                bytes_file, self._bucket_name, _key
+                bytes_file, self._bucket_name, _key, ExtraArgs=self.upload_args
             )
 
     def setbinfile(self, path, file):
@@ -776,7 +789,7 @@ def setbinfile(self, path, file):
                 pass
 
         with s3errors(path):
-            self.client.upload_fileobj(file, self._bucket_name, _key)
+            self.client.upload_fileobj(file, self._bucket_name, _key, ExtraArgs=self.upload_args)
 
     def copy(self, src_path, dst_path, overwrite=False):
         if not overwrite and self.exists(dst_path):

fs_s3fs/opener.py

@@ -33,6 +33,8 @@ def open_fs(self, fs_url, parse_result, writeable, create, cwd):
             aws_access_key_id=parse_result.username or None,
             aws_secret_access_key=parse_result.password or None,
             endpoint_url=parse_result.params.get('endpoint_url', None),
+            acl=parse_result.params.get('acl', None),
+            cache_control=parse_result.params.get('cache_control', None),
             strict=strict
         )
         return s3fs