[Blob][BugFix]Incorrect handling of Short Reads in create_blob_from_s… (#583)

* [Blob][BugFix]Incorrect handling of Short Reads in create_blob_from_stream

Currently if the user want to upload a blob with short reads, only the
data read by the first short reads call is uploaded. This commit is to fix
this bug

#512

Author: xiafu-msft

azure-storage-blob/azure/storage/blob/blockblobservice.py

@@ -574,6 +574,20 @@ def create_blob_from_stream(
                 progress_callback(0, count)
 
             data = stream.read(count)
+            data_chunk = data  # to store the chunk of data read from stream each time
+
+            # keep reading from stream util length of data >= count or reaching the end of stream
+            while len(data) < count and len(data_chunk) is not 0:
+                data_chunk = stream.read(count)
+                data += data_chunk
+
+            if len(data) < count:
+                raise ValueError('Parameter:count is greater than the amount of data in the stream,'
+                                 'please specify a valid count')
+
+            if len(data) > count:
+                data = data[0:count]
+
             resp = self._put_blob(
                 container_name=container_name,
                 blob_name=blob_name,

tests/blob/test_block_blob.py

@@ -6,6 +6,7 @@
 # license information.
 # --------------------------------------------------------------------------
 import os
+import io
 import unittest
 
 from azure.common import AzureHttpError
@@ -15,7 +16,6 @@
     BlockBlobService,
     ContentSettings,
 )
-from azure.storage.blob.models import StandardBlobTier
 from tests.testcase import (
     StorageTestCase,
     TestMode,
@@ -520,6 +520,48 @@ def test_create_blob_from_path_with_properties(self):
         self.assertEqual(properties.content_settings.content_type, content_settings.content_type)
         self.assertEqual(properties.content_settings.content_language, content_settings.content_language)
 
+    @record
+    def test_createBlobFromStream_when_short_read_the_whole_stream(self):
+        # Arrange
+        blob_name = self._get_blob_reference()
+        content = b"Example blob content."
+        stream = ShortReader(io.BytesIO(content), 10)
+        upload_size = len(content)
+
+        # Act
+        self.bs.create_blob_from_stream(self.container_name, blob_name, stream, upload_size)
+
+        # Assert
+        uploaded_blob = self.bs.get_blob_to_bytes(self.container_name, blob_name)
+        self.assertEquals(content, uploaded_blob.content)
+
+    @record
+    def test_createBlobFromStream_when_short_read_designated_size_of_stream(self):
+        # Arrange
+        blob_name = self._get_blob_reference()
+        content = b"Example blob content."
+        stream = ShortReader(io.BytesIO(content), 10)
+        upload_size = 15  # designated size blob we want to upload
+
+        # Act
+        self.bs.create_blob_from_stream(self.container_name, blob_name, stream, upload_size)
+
+        # Assert
+        uploaded_blob = self.bs.get_blob_to_bytes(self.container_name, blob_name)
+        self.assertEqual(upload_size, len(uploaded_blob.content))
+
+    def test_createBlobFromStream_when_designated_size_larger_than_the_actual_stream_size(self):
+
+        # Arrange
+        blob_name = self._get_blob_reference()
+        content = b"Example blob content."
+        stream = ShortReader(io.BytesIO(content), 10)
+        upload_size = 100  # designated size blob we want to upload
+
+        # Assert
+        with self.assertRaises(ValueError):
+            self.bs.create_blob_from_stream(self.container_name, blob_name, stream, upload_size)
+
     def test_create_blob_from_stream_chunked_upload(self):
         # parallel tests introduce random order of requests, can only run live
         if TestMode.need_recording_file(self.test_mode):
@@ -771,6 +813,20 @@ def test_create_blob_with_md5_chunked(self):
 
         # Assert
 
+
+class ShortReader(io.RawIOBase):
+    """IOBase wrapper to limit read sizes, producing short reads"""
+
+    def __init__(self, reader, max_read_size):
+        self._reader = reader
+        self.max_read_size = max_read_size
+
+    def readinto(self, buf):
+        buf_size = len(buf)
+        read_size = min(buf_size, self.max_read_size)
+        data = self._reader.read(read_size)
+        buf[0:len(data)] = data
+        return len(data)
 #------------------------------------------------------------------------------
 if __name__ == '__main__':
     unittest.main()

tests/recordings/test_block_blob.test_createBlobFromStream_when_short_read_designated_size_of_stream.yaml

@@ -0,0 +1,106 @@
+interactions:
+- request:
+    body: Example blob co
+    headers:
+      Connection:
+      - keep-alive
+      Content-Length:
+      - '15'
+      User-Agent:
+      - Azure-Storage/1.4.0-1.5.0 (Python CPython 3.7.3; Windows 10)
+      x-ms-blob-type:
+      - BlockBlob
+      x-ms-client-request-id:
+      - 31e38124-8174-11e9-b09e-001a7dda7113
+      x-ms-date:
+      - Tue, 28 May 2019 18:12:46 GMT
+      x-ms-version:
+      - '2018-11-09'
+    method: PUT
+    uri: https://storagename.blob.core.windows.net/utcontainer7fa321b4/blob7fa321b4
+  response:
+    body:
+      string: ''
+    headers:
+      Content-Length:
+      - '0'
+      Content-MD5:
+      - 2bNSt3gr222I6yQwqlz12w==
+      Date:
+      - Tue, 28 May 2019 18:12:45 GMT
+      ETag:
+      - '"0x8D6E39816280908"'
+      Last-Modified:
+      - Tue, 28 May 2019 18:12:46 GMT
+      Server:
+      - Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0
+      x-ms-request-id:
+      - fa3febc7-601e-00ea-6c80-1574da000000
+      x-ms-request-server-encrypted:
+      - 'true'
+      x-ms-version:
+      - '2018-11-09'
+    status:
+      code: 201
+      message: Created
+- request:
+    body: null
+    headers:
+      Connection:
+      - keep-alive
+      User-Agent:
+      - Azure-Storage/1.4.0-1.5.0 (Python CPython 3.7.3; Windows 10)
+      x-ms-client-request-id:
+      - 31f05c0a-8174-11e9-a545-001a7dda7113
+      x-ms-date:
+      - Tue, 28 May 2019 18:12:46 GMT
+      x-ms-range:
+      - bytes=0-33554431
+      x-ms-version:
+      - '2018-11-09'
+    method: GET
+    uri: https://storagename.blob.core.windows.net/utcontainer7fa321b4/blob7fa321b4
+  response:
+    body:
+      string: Example blob co
+    headers:
+      Accept-Ranges:
+      - bytes
+      Content-Length:
+      - '15'
+      Content-Range:
+      - bytes 0-14/15
+      Content-Type:
+      - application/octet-stream
+      Date:
+      - Tue, 28 May 2019 18:12:45 GMT
+      ETag:
+      - '"0x8D6E39816280908"'
+      Last-Modified:
+      - Tue, 28 May 2019 18:12:46 GMT
+      Server:
+      - Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0
+      Vary:
+      - Origin
+      x-ms-blob-content-md5:
+      - 2bNSt3gr222I6yQwqlz12w==
+      x-ms-blob-type:
+      - BlockBlob
+      x-ms-creation-time:
+      - Tue, 28 May 2019 18:12:46 GMT
+      x-ms-lease-state:
+      - available
+      x-ms-lease-status:
+      - unlocked
+      x-ms-request-id:
+      - fa3febd8-601e-00ea-7a80-1574da000000
+      x-ms-server-encrypted:
+      - 'true'
+      x-ms-tag-count:
+      - '0'
+      x-ms-version:
+      - '2018-11-09'
+    status:
+      code: 206
+      message: Partial Content
+version: 1