feat: integrated to minio (#5748)

* feat: migrate to minio

* feat: migrate apps' and dataset's avatar to minio

* feat: migrate more avatars to minio

* fix: lock file

* feat: migrate copyright settings' logo to minio

* feat: integrate minio

* chore: improve code

* chore: rename variables

* refactor: s3 class

* fix: s3 and mongo operations

* chore: add session for avatar source

* fix: init s3 buckets

* fix: bugbot issues

* expired time code

* perf: avatar code

* union type

* export favouriteContract

* empty bucket check

---------

Co-authored-by: archer <545436317@qq.com>

Author: xqvvu

packages/global/common/file/s3TTL/type.d.ts

@@ -0,0 +1,6 @@
+export type S3TtlSchemaType = {
+  _id: string;
+  bucketName: string;
+  minioKey: string;
+  expiredTime: Date;
+};

packages/global/common/tsRest/fastgpt/contracts/core/chat/setting.ts

@@ -8,7 +8,7 @@ import { ObjectIdSchema } from '../../../../../type';
 import { initContract } from '@ts-rest/core';
 
 const c = initContract();
-const favouriteContract = c.router({
+export const favouriteContract = c.router({
   list: {
     path: '/proApi/core/chat/setting/favourite/list',
     method: 'GET',

packages/service/common/file/image/controller.ts

@@ -109,23 +109,6 @@ const getIdFromPath = (path?: string) => {
 
   return id;
 };
-// 删除旧的头像，新的头像去除过期时间
-export const refreshSourceAvatar = async (
-  path?: string,
-  oldPath?: string,
-  session?: ClientSession
-) => {
-  const newId = getIdFromPath(path);
-  const oldId = getIdFromPath(oldPath);
-
-  if (!newId || newId === oldId) return;
-
-  await MongoImage.updateOne({ _id: newId }, { $unset: { expiredTime: 1 } }, { session });
-
-  if (oldId) {
-    await MongoImage.deleteOne({ _id: oldId }, { session });
-  }
-};
 export const removeImageByPath = (path?: string, session?: ClientSession) => {
   if (!path) return;
 

packages/service/common/s3/buckets/base.ts

@@ -0,0 +1,132 @@
+import { Client, type RemoveOptions, type CopyConditions, type LifecycleConfig } from 'minio';
+import {
+  type ExtensionType,
+  type CreatePostPresignedUrlOptions,
+  type CreatePostPresignedUrlParams,
+  type CreatePostPresignedUrlResult,
+  type S3BucketName,
+  type S3OptionsType
+} from '../type';
+import { defaultS3Options, Mimes } from '../constants';
+import path from 'node:path';
+import { MongoS3TTL } from '../schema';
+import { UserError } from '@fastgpt/global/common/error/utils';
+import { getNanoid } from '@fastgpt/global/common/string/tools';
+import { addHours } from 'date-fns';
+
+export class S3BaseBucket {
+  private _client: Client;
+  private _externalClient: Client | undefined;
+
+  /**
+   *
+   * @param _bucket the bucket you want to operate
+   * @param options the options for the s3 client
+   */
+  constructor(
+    private readonly _bucket: S3BucketName,
+    public options: Partial<S3OptionsType> = defaultS3Options
+  ) {
+    options = { ...defaultS3Options, ...options };
+    this.options = options;
+    this._client = new Client(options as S3OptionsType);
+
+    if (this.options.externalBaseURL) {
+      const externalBaseURL = new URL(this.options.externalBaseURL);
+      const endpoint = externalBaseURL.hostname;
+      const useSSL = externalBaseURL.protocol === 'https';
+
+      this._externalClient = new Client({
+        useSSL: useSSL,
+        endPoint: endpoint,
+        port: options.port,
+        accessKey: options.accessKey,
+        secretKey: options.secretKey,
+        transportAgent: options.transportAgent
+      });
+    }
+
+    const init = async () => {
+      if (!(await this.exist())) {
+        await this.client.makeBucket(this._bucket);
+      }
+      await this.options.afterInit?.();
+    };
+    init();
+  }
+
+  get name(): string {
+    return this._bucket;
+  }
+
+  protected get client(): Client {
+    return this._externalClient ?? this._client;
+  }
+
+  move(src: string, dst: string, options?: CopyConditions): Promise<void> {
+    const bucket = this.name;
+    this.client.copyObject(bucket, dst, `/${bucket}/${src}`, options);
+    return this.delete(src);
+  }
+
+  copy(src: string, dst: string, options?: CopyConditions): ReturnType<Client['copyObject']> {
+    return this.client.copyObject(this.name, src, dst, options);
+  }
+
+  exist(): Promise<boolean> {
+    return this.client.bucketExists(this.name);
+  }
+
+  delete(objectKey: string, options?: RemoveOptions): Promise<void> {
+    return this.client.removeObject(this.name, objectKey, options);
+  }
+
+  async createPostPresignedUrl(
+    params: CreatePostPresignedUrlParams,
+    options: CreatePostPresignedUrlOptions = {}
+  ): Promise<CreatePostPresignedUrlResult> {
+    try {
+      const { expiredHours } = options;
+      const filename = params.filename;
+      const ext = path.extname(filename).toLowerCase() as ExtensionType;
+      const contentType = Mimes[ext] ?? 'application/octet-stream';
+      const maxFileSize = this.options.maxFileSize as number;
+
+      const key = (() => {
+        if ('rawKey' in params) return params.rawKey;
+
+        return `${params.source}/${params.teamId}/${getNanoid(6)}-${filename}`;
+      })();
+
+      const policy = this.client.newPostPolicy();
+      policy.setKey(key);
+      policy.setBucket(this.name);
+      policy.setContentType(contentType);
+      policy.setContentLengthRange(1, maxFileSize);
+      policy.setExpires(new Date(Date.now() + 10 * 60 * 1000));
+      policy.setUserMetaData({
+        'content-type': contentType,
+        'content-disposition': `attachment; filename="${encodeURIComponent(filename)}"`,
+        'origin-filename': encodeURIComponent(filename),
+        'upload-time': new Date().toISOString()
+      });
+
+      const { formData, postURL } = await this.client.presignedPostPolicy(policy);
+
+      if (expiredHours) {
+        await MongoS3TTL.create({
+          minioKey: key,
+          bucketName: this.name,
+          expiredTime: addHours(new Date(), expiredHours)
+        });
+      }
+
+      return {
+        url: postURL,
+        fields: formData
+      };
+    } catch (error) {
+      return Promise.reject(error);
+    }
+  }
+}

packages/service/common/s3/buckets/private.ts

@@ -0,0 +1,9 @@
+import { S3BaseBucket } from './base';
+import { S3Buckets } from '../constants';
+import { type S3OptionsType } from '../type';
+
+export class S3PrivateBucket extends S3BaseBucket {
+  constructor(options?: Partial<S3OptionsType>) {
+    super(S3Buckets.private, options);
+  }
+}

packages/service/common/s3/buckets/public.ts

@@ -0,0 +1,51 @@
+import { S3BaseBucket } from './base';
+import { S3Buckets } from '../constants';
+import { type S3OptionsType } from '../type';
+
+export class S3PublicBucket extends S3BaseBucket {
+  constructor(options?: Partial<S3OptionsType>) {
+    super(S3Buckets.public, {
+      ...options,
+      afterInit: async () => {
+        const bucket = this.name;
+        const policy = JSON.stringify({
+          Version: '2012-10-17',
+          Statement: [
+            {
+              Effect: 'Allow',
+              Principal: '*',
+              Action: 's3:GetObject',
+              Resource: `arn:aws:s3:::${bucket}/*`
+            }
+          ]
+        });
+        try {
+          await this.client.setBucketPolicy(bucket, policy);
+        } catch (error) {
+          // NOTE: maybe it was a cloud S3 that doesn't allow us to set the policy, so that cause the error,
+          // maybe we can ignore the error, or we have other plan to handle this.
+          console.error('Failed to set bucket policy:', error);
+        }
+      }
+    });
+  }
+
+  createPublicUrl(objectKey: string): string {
+    const protocol = this.options.useSSL ? 'https' : 'http';
+    const hostname = this.options.endPoint;
+    const port = this.options.port;
+    const bucket = this.name;
+
+    const url = new URL(`${protocol}://${hostname}:${port}/${bucket}/${objectKey}`);
+
+    if (this.options.externalBaseURL) {
+      const externalBaseURL = new URL(this.options.externalBaseURL);
+
+      url.port = externalBaseURL.port;
+      url.hostname = externalBaseURL.hostname;
+      url.protocol = externalBaseURL.protocol;
+    }
+
+    return url.toString();
+  }
+}

packages/service/common/s3/config.ts

@@ -0,0 +1,58 @@
+import type { S3PrivateBucket } from './buckets/private';
+import type { S3PublicBucket } from './buckets/public';
+import { HttpProxyAgent } from 'http-proxy-agent';
+import { HttpsProxyAgent } from 'https-proxy-agent';
+import type { ClientOptions } from 'minio';
+
+export const Mimes = {
+  '.gif': 'image/gif',
+  '.png': 'image/png',
+  '.jpg': 'image/jpeg',
+  '.jpeg': 'image/jpeg',
+  '.webp': 'image/webp',
+  '.svg': 'image/svg+xml',
+
+  '.csv': 'text/csv',
+  '.txt': 'text/plain',
+
+  '.pdf': 'application/pdf',
+  '.zip': 'application/zip',
+  '.json': 'application/json',
+  '.doc': 'application/msword',
+  '.js': 'application/javascript',
+  '.xls': 'application/vnd.ms-excel',
+  '.ppt': 'application/vnd.ms-powerpoint',
+  '.xlsx': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
+  '.docx': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
+  '.pptx': 'application/vnd.openxmlformats-officedocument.presentationml.presentation'
+} as const;
+
+export const defaultS3Options: {
+  externalBaseURL?: string;
+  maxFileSize?: number;
+  afterInit?: () => Promise<void> | void;
+} & ClientOptions = {
+  maxFileSize: 1024 ** 3, // 1GB
+
+  useSSL: process.env.S3_USE_SSL === 'true',
+  endPoint: process.env.S3_ENDPOINT || 'localhost',
+  externalBaseURL: process.env.S3_EXTERNAL_BASE_URL,
+  accessKey: process.env.S3_ACCESS_KEY || 'minioadmin',
+  secretKey: process.env.S3_SECRET_KEY || 'minioadmin',
+  port: process.env.S3_PORT ? parseInt(process.env.S3_PORT) : 9000,
+  transportAgent: process.env.HTTP_PROXY
+    ? new HttpProxyAgent(process.env.HTTP_PROXY)
+    : process.env.HTTPS_PROXY
+      ? new HttpsProxyAgent(process.env.HTTPS_PROXY)
+      : undefined
+};
+
+export const S3Buckets = {
+  public: process.env.S3_PUBLIC_BUCKET || 'fastgpt-public',
+  private: process.env.S3_PRIVATE_BUCKET || 'fastgpt-private'
+} as const;
+
+export const S3BucketMap = {
+  public: null as unknown as S3PublicBucket,
+  private: null as unknown as S3PrivateBucket
+};