Merge pull request #136 from HackSoftware/files/improvements

Files: Improvements

Author: RadoRado

config/settings/files_and_storages.py

@@ -7,13 +7,15 @@
 
 FILE_UPLOAD_STRATEGY = env_to_enum(
     FileUploadStrategy,
-    env("FILE_UPLOAD_STRATEGY", default="direct")
+    env("FILE_UPLOAD_STRATEGY", default="standard")
 )
 FILE_UPLOAD_STORAGE = env_to_enum(
     FileUploadStorage,
     env("FILE_UPLOAD_STORAGE", default="local")
 )
 
+FILE_MAX_SIZE = env.int("FILE_MAX_SIZE", default=10485760)  # 10 MiB
+
 if FILE_UPLOAD_STORAGE == FileUploadStorage.LOCAL:
     MEDIA_ROOT_NAME = "media"
     MEDIA_ROOT = os.path.join(BASE_DIR, MEDIA_ROOT_NAME)

styleguide_example/files/admin.py

@@ -5,7 +5,7 @@
 
 from styleguide_example.files.models import File
 from styleguide_example.files.services import (
-    FileDirectUploadService
+    FileStandardUploadService
 )
 
 
@@ -64,7 +64,7 @@ def save_model(self, request, obj, form, change):
         try:
             cleaned_data = form.cleaned_data
 
-            service = FileDirectUploadService(
+            service = FileStandardUploadService(
                 file_obj=cleaned_data["file"],
                 user=cleaned_data["uploaded_by"]
             )

styleguide_example/files/apis.py

@@ -6,16 +6,16 @@
 
 from styleguide_example.files.models import File
 from styleguide_example.files.services import (
-    FileDirectUploadService,
-    FilePassThruUploadService
+    FileStandardUploadService,
+    FileDirectUploadService
 )
 
 from styleguide_example.api.mixins import ApiAuthMixin
 
 
-class FileDirectUploadApi(ApiAuthMixin, APIView):
+class FileStandardUploadApi(ApiAuthMixin, APIView):
     def post(self, request):
-        service = FileDirectUploadService(
+        service = FileStandardUploadService(
             user=request.user,
             file_obj=request.FILES["file"]
         )
@@ -24,7 +24,7 @@ def post(self, request):
         return Response(data={"id": file.id}, status=status.HTTP_201_CREATED)
 
 
-class FilePassThruUploadStartApi(ApiAuthMixin, APIView):
+class FileDirectUploadStartApi(ApiAuthMixin, APIView):
     class InputSerializer(serializers.Serializer):
         file_name = serializers.CharField()
         file_type = serializers.CharField()
@@ -33,25 +33,25 @@ def post(self, request, *args, **kwargs):
         serializer = self.InputSerializer(data=request.data)
         serializer.is_valid(raise_exception=True)
 
-        service = FilePassThruUploadService(request.user)
+        service = FileDirectUploadService(request.user)
         presigned_data = service.start(**serializer.validated_data)
 
         return Response(data=presigned_data)
 
 
-class FilePassThruUploadLocalApi(ApiAuthMixin, APIView):
+class FileDirectUploadLocalApi(ApiAuthMixin, APIView):
     def post(self, request, file_id):
         file = get_object_or_404(File, id=file_id)
 
-        file_object = request.FILES["file"]
+        file_obj = request.FILES["file"]
 
-        service = FilePassThruUploadService(request.user)
-        file = service.upload_local(file=file, file_object=file_object)
+        service = FileDirectUploadService(request.user)
+        file = service.upload_local(file=file, file_obj=file_obj)
 
         return Response({"id": file.id})
 
 
-class FilePassThruUploadFinishApi(ApiAuthMixin, APIView):
+class FileDirectUploadFinishApi(ApiAuthMixin, APIView):
     class InputSerializer(serializers.Serializer):
         file_id = serializers.CharField()
 
@@ -63,7 +63,7 @@ def post(self, request):
 
         file = get_object_or_404(File, id=file_id)
 
-        service = FilePassThruUploadService(request.user)
+        service = FileDirectUploadService(request.user)
         service.finish(file=file)
 
         return Response({"id": file.id})

styleguide_example/files/enums.py

@@ -2,8 +2,8 @@
 
 
 class FileUploadStrategy(Enum):
+    STANDARD = "standard"
     DIRECT = "direct"
-    PASS_THRU = "pass-thru"
 
 
 class FileUploadStorage(Enum):

styleguide_example/files/models.py

@@ -1,4 +1,3 @@
-
 from django.db import models
 from django.conf import settings
 
@@ -13,7 +12,11 @@
 
 
 class File(BaseModel):
-    file = models.FileField(upload_to=file_generate_upload_path, null=True, blank=True)
+    file = models.FileField(
+        upload_to=file_generate_upload_path,
+        blank=True,
+        null=True
+    )
 
     original_file_name = models.TextField()
 

styleguide_example/files/services.py

@@ -5,12 +5,14 @@
 from django.conf import settings
 from django.db import transaction
 from django.utils import timezone
+from django.core.exceptions import ValidationError
 
 from styleguide_example.files.models import File
 from styleguide_example.files.utils import (
     file_generate_upload_path,
     file_generate_local_upload_url,
-    file_generate_name
+    file_generate_name,
+    bytes_to_mib
 )
 from styleguide_example.files.enums import FileUploadStorage
 
@@ -19,7 +21,14 @@
 from styleguide_example.users.models import BaseUser
 
 
-class FileDirectUploadService:
+def _validate_file_size(file_obj):
+    max_size = settings.FILE_MAX_SIZE
+
+    if file_obj.size > max_size:
+        raise ValidationError(f"File is too large. It should not exceed {bytes_to_mib(max_size)} MiB")
+
+
+class FileStandardUploadService:
     """
     This also serves as an example of a service class,
     which encapsulates 2 different behaviors (create & update) under a namespace.
@@ -49,6 +58,8 @@ def _infer_file_name_and_type(self, file_name: str = "", file_type: str = "") ->
 
     @transaction.atomic
     def create(self, file_name: str = "", file_type: str = "") -> File:
+        _validate_file_size(self.file_obj)
+
         file_name, file_type = self._infer_file_name_and_type(file_name, file_type)
 
         obj = File(
@@ -67,6 +78,8 @@ def create(self, file_name: str = "", file_type: str = "") -> File:
 
     @transaction.atomic
     def update(self, file: File, file_name: str = "", file_type: str = "") -> File:
+        _validate_file_size(self.file_obj)
+
         file_name, file_type = self._infer_file_name_and_type(file_name, file_type)
 
         file.file = self.file_obj
@@ -82,7 +95,7 @@ def update(self, file: File, file_name: str = "", file_type: str = "") -> File:
         return file
 
 
-class FilePassThruUploadService:
+class FileDirectUploadService:
     """
     This also serves as an example of a service class,
     which encapsulates a flow (start & finish) + one-off action (upload_local) into a namespace.
@@ -138,9 +151,11 @@ def finish(self, *, file: File) -> File:
         return file
 
     @transaction.atomic
-    def upload_local(self, *, file: File, file_object) -> File:
+    def upload_local(self, *, file: File, file_obj) -> File:
+        _validate_file_size(file_obj)
+
         # Potentially, check against user
-        file.file = file_object
+        file.file = file_obj
         file.full_clean()
         file.save()
 

styleguide_example/files/urls.py

@@ -1,11 +1,11 @@
 from django.urls import path, include
 
 from styleguide_example.files.apis import (
-    FileDirectUploadApi,
+    FileStandardUploadApi,
 
-    FilePassThruUploadStartApi,
-    FilePassThruUploadFinishApi,
-    FilePassThruUploadLocalApi,
+    FileDirectUploadStartApi,
+    FileDirectUploadFinishApi,
+    FileDirectUploadLocalApi,
 )
 
 
@@ -14,29 +14,29 @@
         "upload/",
         include(([
             path(
-                "direct/",
-                FileDirectUploadApi.as_view(),
-                name="direct"
+                "standard/",
+                FileStandardUploadApi.as_view(),
+                name="standard"
             ),
             path(
-                "pass-thru/",
+                "direct/",
                 include(([
                     path(
                         "start/",
-                        FilePassThruUploadStartApi.as_view(),
+                        FileDirectUploadStartApi.as_view(),
                         name="start"
                     ),
                     path(
                         "finish/",
-                        FilePassThruUploadFinishApi.as_view(),
+                        FileDirectUploadFinishApi.as_view(),
                         name="finish"
                     ),
                     path(
                         "local/<str:file_id>/",
-                        FilePassThruUploadLocalApi.as_view(),
+                        FileDirectUploadLocalApi.as_view(),
                         name="local"
                     )
-                ], "pass-thru"))
+                ], "direct"))
             )
         ], "upload"))
     )

styleguide_example/files/utils.py

@@ -18,8 +18,13 @@ def file_generate_upload_path(instance, filename):
 
 def file_generate_local_upload_url(*, file_id: str):
     url = reverse(
-        "api:files:upload:pass-thru:local",
+        "api:files:upload:direct:local",
         kwargs={"file_id": file_id}
     )
 
     return f"{settings.APP_DOMAIN}{url}"
+
+
+def bytes_to_mib(value: int) -> float:
+    # 1 bytes = 9.5367431640625E-7 mebibytes
+    return value * 9.5367431640625E-7

styleguide_example/integrations/aws/client.py

@@ -17,6 +17,7 @@ class S3Credentials:
     bucket_name: str
     default_acl: str
     presigned_expiry: int
+    max_size: int
 
 
 @lru_cache
@@ -28,7 +29,8 @@ def s3_get_credentials() -> S3Credentials:
             "AWS_S3_REGION_NAME",
             "AWS_STORAGE_BUCKET_NAME",
             "AWS_DEFAULT_ACL",
-            "AWS_PRESIGNED_EXPIRY"
+            "AWS_PRESIGNED_EXPIRY",
+            "FILE_MAX_SIZE"
         ],
         "S3 credentials not found."
     )
@@ -39,7 +41,8 @@ def s3_get_credentials() -> S3Credentials:
         region_name=required_config["AWS_S3_REGION_NAME"],
         bucket_name=required_config["AWS_STORAGE_BUCKET_NAME"],
         default_acl=required_config["AWS_DEFAULT_ACL"],
-        presigned_expiry=required_config["AWS_PRESIGNED_EXPIRY"]
+        presigned_expiry=required_config["AWS_PRESIGNED_EXPIRY"],
+        max_size=required_config["FILE_MAX_SIZE"]
     )
 
 
@@ -88,7 +91,11 @@ def s3_generate_presigned_post(*, file_path: str, file_type: str) -> Dict[str, A
         },
         Conditions=[
             {"acl": acl},
-            {"Content-Type": file_type}
+            {"Content-Type": file_type},
+            # As an example, allow file size up to 10 MiB
+            # More on conditions, here:
+            # https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-HTTPPOSTConstructPolicy.html
+            ["content-length-range", 1, credentials.max_size]
         ],
         ExpiresIn=expires_in,
     )