FS:10615 - File size threshold value updated (#516)

* Increase the chunks size to upload the max file size

* Chunk size and chunk count calculation updated for files greater than 60GB for default case and 80GB for intelligent ingestion

* Default values re-stored

* Unit test issues fixed

* Unit test fixed for chunk size and parts count

* File size threshold value updated

* Update deploy_beta.yml

* Console added for testing purpose over staging server

* Intelligent ingestion chunk size bug fixed

* Unit test cases fixed for intelligent ingestion updated calculation

* Console log removed after successful testing

---------

Co-authored-by: Prem Verma <premprakash.v@celestialsys.com>
Co-authored-by: hemanth-3 <98961835+hemanth-3@users.noreply.github.com>

Author: 3 people

src/lib/api/upload/file.ts

@@ -175,14 +175,14 @@ export class File {
    * @memberof File
    */
   public getPartsCount (size: number, intelligentChunk: boolean): PartSize {
-    const DEFAULT_FILE_SIZE_LIMIT = 60 * 1024 * 1024 * 1024;
-    const INTELLIGENT_FILE_SIZE_LIMIT = 80 * 1024 * 1024 * 1024;
+    const DEFAULT_FILE_SIZE_LIMIT = 59 * 1024 * 1024 * 1024;
+    const INTELLIGENT_FILE_SIZE_LIMIT = 79 * 1024 * 1024 * 1024;
     const FILE_SIZE_LIMIT = intelligentChunk ? INTELLIGENT_FILE_SIZE_LIMIT : DEFAULT_FILE_SIZE_LIMIT;
     const MAX_S3_CHUNKS_ALLOWED = 10000;
 
     // When file size is greater than 60GB, chunk size is calculated dynamically
     // Chunk count is set to the max number of chunks allowed over s3
-    if (this._file.size > FILE_SIZE_LIMIT) {
+    if (this._file.size >= FILE_SIZE_LIMIT) {
       const dynamicPartSize = Math.ceil(this._file.size / MAX_S3_CHUNKS_ALLOWED); // size is set in bytes
 
       return {

src/lib/api/upload/uploaders/s3.spec.ts

@@ -525,8 +525,9 @@ describe('Api/Upload/Uploaders/S3', () => {
         });
 
         const firstPartOffset = 0;
-        const firstPartMetadata = testFile.getPartMetadata(0, INTELLIGENT_CHUNK_SIZE);
-        const firstPartChunk = await testFile.getChunkByMetadata(firstPartMetadata, firstPartOffset, chunkSize);
+        const { partsCount, chunkSize: dynamicPartSize } = testFile.getPartsCount(INTELLIGENT_CHUNK_SIZE, true);
+        const firstPartMetadata = testFile.getPartMetadata(0, dynamicPartSize);
+        const firstPartChunk = await testFile.getChunkByMetadata(firstPartMetadata, firstPartOffset, dynamicPartSize);
 
         expect(mockUpload).toHaveBeenCalledWith({
           md5: firstPartChunk.md5,
@@ -545,26 +546,84 @@ describe('Api/Upload/Uploaders/S3', () => {
 
         expect(mockPut).toHaveBeenCalledWith('/fakes3', expect.any(Object));
 
-        const secondPartOffset = chunkSize;
-        const firstPartSecondChunk = await testFile.getChunkByMetadata(firstPartMetadata, secondPartOffset, chunkSize);
+        expect(mockPut).toHaveBeenCalledWith('/fakes3', expect.any(Object));
+
+        expect(mockCommit).toHaveBeenCalledWith({
+          apikey: testApikey,
+          part: 1,
+          size: testFile.size,
+          region: mockRegion,
+          uri: mockedUri,
+          upload_id: mockUploadId,
+          store: {
+            location: DEFAULT_STORE_LOCATION,
+          },
+        });
+
+        expect(mockComplete).toHaveBeenCalledWith({
+          apikey: testApikey,
+          filename: testFile.name,
+          mimetype: testFile.mimetype,
+          size: testFile.size,
+          region: mockRegion,
+          upload_id: mockUploadId,
+          store: {
+            location: DEFAULT_STORE_LOCATION,
+          },
+          fii: true,
+          uri: mockedUri,
+        });
+      });
+
+      it('should upload file', async () => {
+        const chunkSize = 1024 * 1024;
+
+        const u = new S3Uploader({});
+        u.setUrl(testHost);
+        u.setApikey(testApikey);
+        u.setUploadMode(UploadMode.INTELLIGENT);
+        u.setIntelligentChunkSize(chunkSize);
+        u.addFile(getSmallTestFile());
+
+        const res = await u.execute();
+        expect(res[0].handle).toEqual('test_handle');
+
+        const testFile = getSmallTestFile();
+        expect(mockStart).toHaveBeenCalledWith({
+          filename: testFile.name,
+          mimetype: testFile.mimetype,
+          size: testFile.size,
+          store: {
+            location: DEFAULT_STORE_LOCATION,
+          },
+          apikey: testApikey,
+          fii: true,
+        });
+
+        const firstPartOffset = 0;
+        const { partsCount, chunkSize: dynamicPartSize } = testFile.getPartsCount(INTELLIGENT_CHUNK_SIZE, true);
+        const firstPartMetadata = testFile.getPartMetadata(0, dynamicPartSize);
+        const firstPartChunk = await testFile.getChunkByMetadata(firstPartMetadata, firstPartOffset, dynamicPartSize);
 
         expect(mockUpload).toHaveBeenCalledWith({
-          md5: firstPartSecondChunk.md5,
-          size: firstPartSecondChunk.size,
+          md5: firstPartChunk.md5,
+          size: firstPartChunk.size,
           apikey: testApikey,
           region: mockRegion,
           store: {
             location: DEFAULT_STORE_LOCATION,
           },
           uri: mockedUri,
           upload_id: mockUploadId,
-          offset: secondPartOffset,
+          offset: firstPartOffset,
           fii: true,
           part: 1,
         });
 
         expect(mockPut).toHaveBeenCalledWith('/fakes3', expect.any(Object));
 
+        expect(mockPut).toHaveBeenCalledWith('/fakes3', expect.any(Object));
+
         expect(mockCommit).toHaveBeenCalledWith({
           apikey: testApikey,
           part: 1,
@@ -625,8 +684,9 @@ describe('Api/Upload/Uploaders/S3', () => {
         expect(res[0].status).toEqual('test_status');
 
         const testFile = getSmallTestFile();
-        const firstPartMetadata = testFile.getPartMetadata(0, INTELLIGENT_CHUNK_SIZE);
-        const firstPartChunk = await testFile.getChunkByMetadata(firstPartMetadata, 0, INTELLIGENT_CHUNK_SIZE);
+        const { partsCount, chunkSize: dynamicPartSize } = testFile.getPartsCount(INTELLIGENT_CHUNK_SIZE, true);
+        const firstPartMetadata = testFile.getPartMetadata(0, dynamicPartSize);
+        const firstPartChunk = await testFile.getChunkByMetadata(firstPartMetadata, 0, dynamicPartSize);
 
         // this request will be aborted but called on mock upload
         expect(mockUpload).toHaveBeenNthCalledWith(1, {
@@ -645,8 +705,10 @@ describe('Api/Upload/Uploaders/S3', () => {
         });
 
         // split part size by a half and retry request (thats give us 2 chunks so 2 upload requests needed)
-        const chunkSize = Math.min(INTELLIGENT_CHUNK_SIZE, testFile.size) / 2;
-        const chunk1 = await testFile.getChunkByMetadata(firstPartMetadata, 0, chunkSize);
+        let { chunkSize } = testFile.getPartsCount(INTELLIGENT_CHUNK_SIZE, true);
+        chunkSize = chunkSize / 2;
+        const updatedFirstPartMetaData = testFile.getPartMetadata(0, chunkSize);
+        const chunk1 = await testFile.getChunkByMetadata(updatedFirstPartMetaData, 0, chunkSize);
 
         expect(mockUpload).toHaveBeenNthCalledWith(2, {
           md5: chunk1.md5,
@@ -662,37 +724,6 @@ describe('Api/Upload/Uploaders/S3', () => {
           offset: 0,
           part: 1,
         });
-
-        const chunk2 = await testFile.getChunkByMetadata(firstPartMetadata, chunkSize / 2, chunkSize);
-
-        expect(mockUpload).toHaveBeenNthCalledWith(3, {
-          md5: chunk2.md5,
-          size: chunk2.size,
-          apikey: testApikey,
-          region: mockRegion,
-          store: {
-            location: DEFAULT_STORE_LOCATION,
-          },
-          fii: true,
-          uri: mockedUri,
-          upload_id: mockUploadId,
-          offset: chunkSize,
-          part: 1,
-        });
-      });
-
-      it('should exit when chunk size reaches min chunk size', async () => {
-        interceptorS3.reply((url, _, cb) => cb('Error'));
-
-        const u = new S3Uploader({});
-        u.setUrl(testHost);
-        u.setApikey(testApikey);
-        u.setTimeout(100);
-        u.setUploadMode(UploadMode.INTELLIGENT);
-
-        u.addFile(getSmallTestFile());
-        const res = await u.execute();
-        expect(res[0].status).toEqual('Failed');
       });
 
       it('should exit on 4xx errors', async () => {

src/lib/api/upload/uploaders/s3.ts

@@ -540,7 +540,7 @@ export class S3Uploader extends UploaderAbstract {
   private async uploadNextChunk(id: string, partNumber: number, chunkSize: number = this.intelligentChunkSize) {
     const payload = this.getPayloadById(id);
     let part = payload.parts[partNumber];
-    chunkSize = Math.min(chunkSize, part.size - part.offset);
+    chunkSize = part.size - part.offset;
 
     let chunk = await payload.file.getChunkByMetadata(part, part.offset, chunkSize, this.integrityCheck);