Merge pull request #542 from filestack/develop

sethk4783 · web-flow · commit c48127f02177 · 2024-01-10T13:23:30.000+05:30
Fixed FS-10615, FS-11013
diff --git a/src/config.ts b/src/config.ts
@@ -18,7 +18,7 @@
 /**
  * @private
  */
-const PICKER_VERSION = '1.24.1';
+const PICKER_VERSION = '1.25.0';
 
 /**
  * @private
diff --git a/src/lib/api/cloud.ts b/src/lib/api/cloud.ts
@@ -133,6 +133,12 @@ export class CloudClient {
       if (!Array.isArray(accept)) {
         accept = [accept];
       }
+      // FS-11013.
+      // google-drive storing uncommon file-types in incorrect format, eg .srt (subrip) file is stored in bin (octet-stream) format
+      // so if user wants to accept subrip files, we should search google drive for octet-steam file.
+      if (accept.includes('application/x-subrip') && !accept.includes('application/octet-stream')) {
+        accept.push('application/octet-stream');
+      }
       // filtering mimetypes in clouds
       payload.accept = accept;
     }
diff --git a/src/lib/api/upload/file.spec.ts b/src/lib/api/upload/file.spec.ts
@@ -73,7 +73,7 @@ describe('Api/Upload/File', () => {
   });
 
   it('should return correct parts count for given size', () => {
-    expect(file.getPartsCount(1)).toEqual(file.size);
+    expect(file.getPartsCount(1)).toEqual({ chunkSize: 1, partsCount: 4 });
   });
 
   it('should return correct part metadata', () => {
diff --git a/src/lib/api/upload/file.ts b/src/lib/api/upload/file.ts
@@ -52,6 +52,11 @@ export interface FileChunk extends FilePart {
   offset: number; // offset for chunk - from part start
 }
 
+export interface PartSize {
+  partsCount: number;
+  chunkSize: number;
+}
+
 /**
  * File representation to unify file object in nodejs and browser
  *
@@ -164,14 +169,33 @@ export class File {
   }
 
   /**
-   * Returns number of parts according to part size
-   *
+   * Returns number of parts and part size according to max limit
    * @param {number} size - part size in bytes
-   * @returns {number}
+   * @returns {PartSize}
    * @memberof File
    */
-  public getPartsCount (size: number): number {
-    return Math.ceil(this._file.size / size);
+  public getPartsCount (size: number, intelligentChunk: boolean): PartSize {
+    const DEFAULT_FILE_SIZE_LIMIT = 59 * 1024 * 1024 * 1024;
+    const INTELLIGENT_FILE_SIZE_LIMIT = 79 * 1024 * 1024 * 1024;
+    const FILE_SIZE_LIMIT = intelligentChunk ? INTELLIGENT_FILE_SIZE_LIMIT : DEFAULT_FILE_SIZE_LIMIT;
+    const MAX_S3_CHUNKS_ALLOWED = 10000;
+
+    // When file size is greater than 60GB, chunk size is calculated dynamically
+    // Chunk count is set to the max number of chunks allowed over s3
+    if (this._file.size >= FILE_SIZE_LIMIT) {
+      const dynamicPartSize = Math.ceil(this._file.size / MAX_S3_CHUNKS_ALLOWED); // size is set in bytes
+
+      return {
+        partsCount: Math.ceil(this._file.size / dynamicPartSize),
+        chunkSize: dynamicPartSize,
+      };
+
+    }
+
+    return {
+      partsCount: Math.ceil(this._file.size / size),
+      chunkSize: size,
+    };
   }
 
   /**
diff --git a/src/lib/api/upload/uploaders/s3.spec.ts b/src/lib/api/upload/uploaders/s3.spec.ts
@@ -524,8 +524,9 @@ describe('Api/Upload/Uploaders/S3', () => {
         });
 
         const firstPartOffset = 0;
-        const firstPartMetadata = testFile.getPartMetadata(0, INTELLIGENT_CHUNK_SIZE);
-        const firstPartChunk = await testFile.getChunkByMetadata(firstPartMetadata, firstPartOffset, chunkSize);
+        const { partsCount, chunkSize: dynamicPartSize } = testFile.getPartsCount(INTELLIGENT_CHUNK_SIZE, true);
+        const firstPartMetadata = testFile.getPartMetadata(0, dynamicPartSize);
+        const firstPartChunk = await testFile.getChunkByMetadata(firstPartMetadata, firstPartOffset, dynamicPartSize);
 
         expect(mockUpload).toHaveBeenCalledWith({
           md5: firstPartChunk.md5,
@@ -544,26 +545,84 @@ describe('Api/Upload/Uploaders/S3', () => {
 
         expect(mockPut).toHaveBeenCalledWith('/fakes3', expect.any(Object));
 
-        const secondPartOffset = chunkSize;
-        const firstPartSecondChunk = await testFile.getChunkByMetadata(firstPartMetadata, secondPartOffset, chunkSize);
+        expect(mockPut).toHaveBeenCalledWith('/fakes3', expect.any(Object));
+
+        expect(mockCommit).toHaveBeenCalledWith({
+          apikey: testApikey,
+          part: 1,
+          size: testFile.size,
+          region: mockRegion,
+          uri: mockedUri,
+          upload_id: mockUploadId,
+          store: {
+            location: DEFAULT_STORE_LOCATION,
+          },
+        });
+
+        expect(mockComplete).toHaveBeenCalledWith({
+          apikey: testApikey,
+          filename: testFile.name,
+          mimetype: testFile.mimetype,
+          size: testFile.size,
+          region: mockRegion,
+          upload_id: mockUploadId,
+          store: {
+            location: DEFAULT_STORE_LOCATION,
+          },
+          fii: true,
+          uri: mockedUri,
+        });
+      });
+
+      it('should upload file', async () => {
+        const chunkSize = 1024 * 1024;
+
+        const u = new S3Uploader({});
+        u.setUrl(testHost);
+        u.setApikey(testApikey);
+        u.setUploadMode(UploadMode.INTELLIGENT);
+        u.setIntelligentChunkSize(chunkSize);
+        u.addFile(getSmallTestFile());
+
+        const res = await u.execute();
+        expect(res[0].handle).toEqual('test_handle');
+
+        const testFile = getSmallTestFile();
+        expect(mockStart).toHaveBeenCalledWith({
+          filename: testFile.name,
+          mimetype: testFile.mimetype,
+          size: testFile.size,
+          store: {
+            location: DEFAULT_STORE_LOCATION,
+          },
+          apikey: testApikey,
+          fii: true,
+        });
+
+        const firstPartOffset = 0;
+        const { partsCount, chunkSize: dynamicPartSize } = testFile.getPartsCount(INTELLIGENT_CHUNK_SIZE, true);
+        const firstPartMetadata = testFile.getPartMetadata(0, dynamicPartSize);
+        const firstPartChunk = await testFile.getChunkByMetadata(firstPartMetadata, firstPartOffset, dynamicPartSize);
 
         expect(mockUpload).toHaveBeenCalledWith({
-          md5: firstPartSecondChunk.md5,
-          size: firstPartSecondChunk.size,
+          md5: firstPartChunk.md5,
+          size: firstPartChunk.size,
           apikey: testApikey,
           region: mockRegion,
           store: {
             location: DEFAULT_STORE_LOCATION,
           },
           uri: mockedUri,
           upload_id: mockUploadId,
-          offset: secondPartOffset,
+          offset: firstPartOffset,
           fii: true,
           part: 1,
         });
 
         expect(mockPut).toHaveBeenCalledWith('/fakes3', expect.any(Object));
 
+        expect(mockPut).toHaveBeenCalledWith('/fakes3', expect.any(Object));
+
         expect(mockCommit).toHaveBeenCalledWith({
           apikey: testApikey,
           part: 1,
@@ -623,8 +682,9 @@ describe('Api/Upload/Uploaders/S3', () => {
         expect(res[0].status).toEqual('test_status');
 
         const testFile = getSmallTestFile();
-        const firstPartMetadata = testFile.getPartMetadata(0, INTELLIGENT_CHUNK_SIZE);
-        const firstPartChunk = await testFile.getChunkByMetadata(firstPartMetadata, 0, INTELLIGENT_CHUNK_SIZE);
+        const { partsCount, chunkSize: dynamicPartSize } = testFile.getPartsCount(INTELLIGENT_CHUNK_SIZE, true);
+        const firstPartMetadata = testFile.getPartMetadata(0, dynamicPartSize);
+        const firstPartChunk = await testFile.getChunkByMetadata(firstPartMetadata, 0, dynamicPartSize);
 
         // this request will be aborted but called on mock upload
         expect(mockUpload).toHaveBeenNthCalledWith(1, {
@@ -643,8 +703,10 @@ describe('Api/Upload/Uploaders/S3', () => {
         });
 
         // split part size by a half and retry request (thats give us 2 chunks so 2 upload requests needed)
-        const chunkSize = Math.min(INTELLIGENT_CHUNK_SIZE, testFile.size) / 2;
-        const chunk1 = await testFile.getChunkByMetadata(firstPartMetadata, 0, chunkSize);
+        let { chunkSize } = testFile.getPartsCount(INTELLIGENT_CHUNK_SIZE, true);
+        chunkSize = chunkSize / 2;
+        const updatedFirstPartMetaData = testFile.getPartMetadata(0, chunkSize);
+        const chunk1 = await testFile.getChunkByMetadata(updatedFirstPartMetaData, 0, chunkSize);
 
         expect(mockUpload).toHaveBeenNthCalledWith(2, {
           md5: chunk1.md5,
diff --git a/src/lib/api/upload/uploaders/s3.ts b/src/lib/api/upload/uploaders/s3.ts
@@ -254,19 +254,21 @@ export class S3Uploader extends UploaderAbstract {
    */
   private prepareParts(id: string): Promise<void> {
     const file = this.getPayloadById(id).file;
+    let intelligentChunk = false;
 
     // for intelligent or fallback mode we cant overwrite part size - requires 8MB
     if ([UploadMode.INTELLIGENT, UploadMode.FALLBACK].indexOf(this.uploadMode) > -1) {
       this.partSize = INTELLIGENT_CHUNK_SIZE;
+      intelligentChunk = true;
     }
 
-    const partsCount = file.getPartsCount(this.partSize);
+    const { partsCount, chunkSize } = file.getPartsCount(this.partSize, intelligentChunk);
 
     const parts = [];
 
     for (let i = 0; i < partsCount; i++) {
       parts[i] = {
-        ...file.getPartMetadata(i, this.partSize),
+        ...file.getPartMetadata(i, chunkSize),
         offset: 0,
       };
     }
@@ -538,7 +540,7 @@ export class S3Uploader extends UploaderAbstract {
   private async uploadNextChunk(id: string, partNumber: number, chunkSize: number = this.intelligentChunkSize) {
     const payload = this.getPayloadById(id);
     let part = payload.parts[partNumber];
-    chunkSize = Math.min(chunkSize, part.size - part.offset);
+    chunkSize = part.size - part.offset;
 
     let chunk = await payload.file.getChunkByMetadata(part, part.offset, chunkSize, this.integrityCheck);
 
diff --git a/src/lib/picker.ts b/src/lib/picker.ts
@@ -414,6 +414,15 @@ export interface PickerOptions {
    *  - unsplash
    */
   fromSources?: string[];
+  /**
+   * Provide default text value for Image Search
+   * ```javascript
+   * websearch: {
+   *  predefinedText: 'Sample text'
+   * }
+   * ```
+   */
+  websearch?: object;
   /**
    * Container where picker should be appended. Only relevant for `inline` and `dropPane` display modes.
    */
diff --git a/src/lib/utils/extensions.ts b/src/lib/utils/extensions.ts
@@ -921,7 +921,7 @@ export const ExtensionsMap = {
   'video/x-ms-wmv': ['wmv'],
   'video/x-ms-wmx': ['wmx'],
   'video/x-ms-wvx': ['wvx'],
-  'video/x-msvideo': ['avi'],
+  'video/vnd.avi': ['avi'],
   'video/x-sgi-movie': ['movie'],
   'video/x-smv': ['smv'],
   'x-conference/x-cooltalk': ['ice'],
diff --git a/src/schema/picker.schema.ts b/src/schema/picker.schema.ts
@@ -47,6 +47,14 @@ export const PickerParamsSchema = {
     acceptFn: {
       format: 'callback',
     },
+    websearch: {
+      type: 'object',
+      properties: {
+        predefinedText: {
+          type: 'string',
+        },
+      },
+    },
     fromSources: {
       type: 'array',
       items: [

Original file line number	Diff line number	Diff line change
`@@ -133,6 +133,12 @@ export class CloudClient {`
`133`	`133`	`if (!Array.isArray(accept)) {`
`134`	`134`	`accept = [accept];`
`135`	`135`	`}`
	`136`	`+ // FS-11013.`
	`137`	`+ // google-drive storing uncommon file-types in incorrect format, eg .srt (subrip) file is stored in bin (octet-stream) format`
	`138`	`+ // so if user wants to accept subrip files, we should search google drive for octet-steam file.`
	`139`	`+ if (accept.includes('application/x-subrip') && !accept.includes('application/octet-stream')) {`
	`140`	`+ accept.push('application/octet-stream');`
	`141`	`+ }`
`136`	`142`	`// filtering mimetypes in clouds`
`137`	`143`	`payload.accept = accept;`
`138`	`144`	`}`