feature: saving files as blob in database

+ files (e.g. branding pictues, attachments) can be saved as a blob in a database. The feature can be turned
  on and off (feature toggle) with the env var FILE_STORAGE_MODE

+ a new controller with the endpoints /file/branding etc. had to be
  created since the files are statically linked when fs is used.

+ the files plus metadata are stored in the table 'file'

Author: Daniel Auer

cmd/wire_gen.go

@@ -53,4 +53,5 @@ var ProviderSetController = wire.NewSet(
 	NewEmbedController,
 	NewBadgeController,
 	NewRenderController,
+	NewFileController,
 )

internal/controller/controller.go

@@ -0,0 +1,36 @@
+package controller
+
+import (
+	"strconv"
+
+	"github.com/apache/answer/internal/base/handler"
+	"github.com/apache/answer/internal/repo/file"
+	"github.com/gin-gonic/gin"
+)
+
+type FileController struct {
+	FileRepo file.FileRepo
+}
+
+func NewFileController(repo file.FileRepo) *FileController {
+	return &FileController{FileRepo: repo}
+}
+
+func (bc *FileController) GetFile(ctx *gin.Context) {
+	id := ctx.Param("id")
+	download := ctx.DefaultQuery("download", "")
+
+	blob, err := bc.FileRepo.GetByID(ctx.Request.Context(), id)
+	if err != nil || blob == nil {
+		handler.HandleResponse(ctx, err, "file not found")
+		return
+	}
+
+	ctx.Header("Content-Type", blob.MimeType)
+	ctx.Header("Content-Length", strconv.FormatInt(blob.Size, 10))
+	if download != "" {
+		ctx.Header("Content-Disposition", "attachment; filename=\""+download+"\"")
+	}
+
+	ctx.Data(200, blob.MimeType, blob.Content)
+}

internal/controller/file_controller.go

@@ -0,0 +1,18 @@
+package entity
+
+import (
+	"time"
+)
+
+type File struct {
+	ID        string    `xorm:"pk varchar(36)"`
+	FileName  string    `xorm:"varchar(255) not null"`
+	MimeType  string    `xorm:"varchar(100)"`
+	Size      int64     `xorm:"bigint"`
+	Content   []byte    `xorm:"blob"`
+	CreatedAt time.Time `xorm:"created"`
+}
+
+func (File) TableName() string {
+	return "file"
+}

internal/entity/file_entity.go

@@ -74,6 +74,7 @@ var (
 		&entity.Badge{},
 		&entity.BadgeGroup{},
 		&entity.BadgeAward{},
+		&entity.File{},
 	}
 
 	roles = []*entity.Role{

internal/migrations/init_data.go

@@ -0,0 +1,44 @@
+package file
+
+import (
+	"context"
+	"database/sql"
+
+	"github.com/apache/answer/internal/base/data"
+	"github.com/apache/answer/internal/base/reason"
+	"github.com/apache/answer/internal/entity"
+	"github.com/segmentfault/pacman/errors"
+)
+
+type FileRepo interface {
+	Save(ctx context.Context, file *entity.File) error
+	GetByID(ctx context.Context, id string) (*entity.File, error)
+}
+
+type fileRepo struct {
+	data *data.Data
+}
+
+func NewFileRepo(data *data.Data) FileRepo {
+	return &fileRepo{data: data}
+}
+
+func (r *fileRepo) Save(ctx context.Context, file *entity.File) error {
+	_, err := r.data.DB.Context(ctx).Insert(file)
+	if err != nil {
+		return errors.InternalServer(reason.DatabaseError).WithError(err).WithStack()
+	}
+	return nil
+}
+
+func (r *fileRepo) GetByID(ctx context.Context, id string) (*entity.File, error) {
+	var blob entity.File
+	ok, err := r.data.DB.Context(ctx).ID(id).Get(&blob)
+	if err != nil {
+		return nil, errors.InternalServer(reason.DatabaseError).WithError(err).WithStack()
+	}
+	if !ok {
+		return nil, sql.ErrNoRows
+	}
+	return &blob, nil
+}

internal/repo/file/file_repo.go

@@ -57,6 +57,7 @@ type AnswerAPIRouter struct {
 	metaController          *controller.MetaController
 	badgeController         *controller.BadgeController
 	adminBadgeController    *controller_admin.BadgeController
+	fileController          *controller.FileController
 }
 
 func NewAnswerAPIRouter(
@@ -90,6 +91,7 @@ func NewAnswerAPIRouter(
 	metaController *controller.MetaController,
 	badgeController *controller.BadgeController,
 	adminBadgeController *controller_admin.BadgeController,
+	fileController *controller.FileController,
 ) *AnswerAPIRouter {
 	return &AnswerAPIRouter{
 		langController:          langController,
@@ -122,6 +124,7 @@ func NewAnswerAPIRouter(
 		metaController:          metaController,
 		badgeController:         badgeController,
 		adminBadgeController:    adminBadgeController,
+		fileController:          fileController,
 	}
 }
 
@@ -148,6 +151,9 @@ func (a *AnswerAPIRouter) RegisterMustUnAuthAnswerAPIRouter(authUserMiddleware *
 
 	// plugins
 	r.GET("/plugin/status", a.pluginController.GetAllPluginStatus)
+
+	// file branding
+	r.GET("/file/branding/:id", a.fileController.GetFile)
 }
 
 func (a *AnswerAPIRouter) RegisterUnAuthAnswerAPIRouter(r *gin.RouterGroup) {
@@ -171,6 +177,10 @@ func (a *AnswerAPIRouter) RegisterUnAuthAnswerAPIRouter(r *gin.RouterGroup) {
 	r.GET("/personal/question/page", a.questionController.PersonalQuestionPage)
 	r.GET("/question/link", a.questionController.GetQuestionLink)
 
+	//file
+	r.GET("/file/post/:id", a.fileController.GetFile)
+	r.GET("/file/avatar/:id", a.fileController.GetFile)
+
 	// comment
 	r.GET("/comment/page", a.commentController.GetCommentWithPage)
 	r.GET("/personal/comment/page", a.commentController.GetCommentPersonalWithPage)
@@ -310,6 +320,7 @@ func (a *AnswerAPIRouter) RegisterAnswerAPIRouter(r *gin.RouterGroup) {
 
 	// meta
 	r.PUT("/meta/reaction", a.metaController.AddOrUpdateReaction)
+
 }
 
 func (a *AnswerAPIRouter) RegisterAnswerAdminAPIRouter(r *gin.RouterGroup) {

internal/router/answer_api_router.go

@@ -20,6 +20,7 @@
 package service
 
 import (
+	"github.com/apache/answer/internal/repo/file"
 	"github.com/apache/answer/internal/service/action"
 	"github.com/apache/answer/internal/service/activity"
 	"github.com/apache/answer/internal/service/activity_common"
@@ -40,7 +41,7 @@ import (
 	"github.com/apache/answer/internal/service/follow"
 	"github.com/apache/answer/internal/service/importer"
 	"github.com/apache/answer/internal/service/meta"
-	"github.com/apache/answer/internal/service/meta_common"
+	metacommon "github.com/apache/answer/internal/service/meta_common"
 	"github.com/apache/answer/internal/service/notice_queue"
 	"github.com/apache/answer/internal/service/notification"
 	notficationcommon "github.com/apache/answer/internal/service/notification_common"
@@ -128,4 +129,5 @@ var ProviderSetService = wire.NewSet(
 	badge.NewBadgeGroupService,
 	importer.NewImporterService,
 	file_record.NewFileRecordService,
+	file.NewFileRepo,
 )

internal/service/provider.go

@@ -30,8 +30,12 @@ import (
 	"path"
 	"path/filepath"
 	"strings"
+	"time"
 
+	"github.com/apache/answer/internal/entity"
+	"github.com/apache/answer/internal/repo/file"
 	"github.com/apache/answer/internal/service/file_record"
+	"github.com/google/uuid"
 
 	"github.com/apache/answer/internal/base/constant"
 	"github.com/apache/answer/internal/base/reason"
@@ -65,6 +69,10 @@ var (
 	}
 )
 
+var (
+	FileStorageMode = os.Getenv("FILE_STORAGE_MODE") // eg "fs" or "db"
+)
+
 type UploaderService interface {
 	UploadAvatarFile(ctx *gin.Context, userID string) (url string, err error)
 	UploadPostFile(ctx *gin.Context, userID string) (url string, err error)
@@ -78,13 +86,15 @@ type uploaderService struct {
 	serviceConfig     *service_config.ServiceConfig
 	siteInfoService   siteinfo_common.SiteInfoCommonService
 	fileRecordService *file_record.FileRecordService
+	fileRepo          file.FileRepo
 }
 
 // NewUploaderService new upload service
 func NewUploaderService(
 	serviceConfig *service_config.ServiceConfig,
 	siteInfoService siteinfo_common.SiteInfoCommonService,
 	fileRecordService *file_record.FileRecordService,
+	fileRepo file.FileRepo,
 ) UploaderService {
 	for _, subPath := range subPathList {
 		err := dir.CreateDirIfNotExist(filepath.Join(serviceConfig.UploadPath, subPath))
@@ -96,9 +106,14 @@ func NewUploaderService(
 		serviceConfig:     serviceConfig,
 		siteInfoService:   siteInfoService,
 		fileRecordService: fileRecordService,
+		fileRepo:          fileRepo,
 	}
 }
 
+func UseDbStorage() bool {
+	return FileStorageMode != "fs"
+}
+
 // UploadAvatarFile upload avatar file
 func (us *uploaderService) UploadAvatarFile(ctx *gin.Context, userID string) (url string, err error) {
 	url, err = us.tryToUploadByPlugin(ctx, plugin.UserAvatar)
@@ -126,8 +141,8 @@ func (us *uploaderService) UploadAvatarFile(ctx *gin.Context, userID string) (ur
 	}
 
 	newFilename := fmt.Sprintf("%s%s", uid.IDStr12(), fileExt)
-	avatarFilePath := path.Join(constant.AvatarSubPath, newFilename)
-	return us.uploadImageFile(ctx, fileHeader, avatarFilePath)
+	fileHeader.Filename = newFilename
+	return us.uploadImageFile(ctx, fileHeader, constant.AvatarSubPath)
 }
 
 func (us *uploaderService) AvatarThumbFile(ctx *gin.Context, fileName string, size int) (url string, err error) {
@@ -209,12 +224,14 @@ func (us *uploaderService) UploadPostFile(ctx *gin.Context, userID string) (
 
 	fileExt := strings.ToLower(path.Ext(fileHeader.Filename))
 	newFilename := fmt.Sprintf("%s%s", uid.IDStr12(), fileExt)
-	avatarFilePath := path.Join(constant.PostSubPath, newFilename)
-	url, err = us.uploadImageFile(ctx, fileHeader, avatarFilePath)
+	fileHeader.Filename = newFilename
+	url, err = us.uploadImageFile(ctx, fileHeader, constant.PostSubPath)
+	postFilePath := path.Join(constant.PostSubPath, newFilename)
+
 	if err != nil {
 		return "", err
 	}
-	us.fileRecordService.AddFileRecord(ctx, userID, avatarFilePath, url, string(plugin.UserPost))
+	us.fileRecordService.AddFileRecord(ctx, userID, postFilePath, url, string(plugin.UserPost))
 	return url, nil
 }
 
@@ -279,10 +296,9 @@ func (us *uploaderService) UploadBrandingFile(ctx *gin.Context, userID string) (
 	if _, ok := plugin.DefaultFileTypeCheckMapping[plugin.AdminBranding][fileExt]; !ok {
 		return "", errors.BadRequest(reason.RequestFormatError).WithError(err)
 	}
-
 	newFilename := fmt.Sprintf("%s%s", uid.IDStr12(), fileExt)
-	avatarFilePath := path.Join(constant.BrandingSubPath, newFilename)
-	return us.uploadImageFile(ctx, fileHeader, avatarFilePath)
+	fileHeader.Filename = newFilename
+	return us.uploadImageFile(ctx, fileHeader, constant.BrandingSubPath)
 }
 
 func (us *uploaderService) uploadImageFile(ctx *gin.Context, file *multipart.FileHeader, fileSubPath string) (
@@ -295,7 +311,37 @@ func (us *uploaderService) uploadImageFile(ctx *gin.Context, file *multipart.Fil
 	if err != nil {
 		return "", err
 	}
-	filePath := path.Join(us.serviceConfig.UploadPath, fileSubPath)
+	if UseDbStorage() {
+		src, err := file.Open()
+		if err != nil {
+			return "", errors.InternalServer(reason.UnknownError).WithError(err).WithStack()
+		}
+		defer src.Close()
+
+		buffer := new(bytes.Buffer)
+		if _, err = io.Copy(buffer, src); err != nil {
+			return "", errors.InternalServer(reason.UnknownError).WithError(err).WithStack()
+		}
+
+		file := &entity.File{
+			ID:        uuid.New().String(),
+			FileName:  file.Filename,
+			MimeType:  file.Header.Get("Content-Type"),
+			Size:      int64(len(buffer.Bytes())),
+			Content:   buffer.Bytes(),
+			CreatedAt: time.Now(),
+		}
+
+		err = us.fileRepo.Save(ctx, file)
+		if err != nil {
+			return "", errors.InternalServer(reason.UnknownError).WithError(err).WithStack()
+		}
+
+		return fmt.Sprintf("%s/answer/api/v1/file/%s/%s", siteGeneral.SiteUrl, fileSubPath, file.ID), nil
+		//TODO checks: DecodeAndCheckImageFile removeExif
+	}
+	filePath := path.Join(us.serviceConfig.UploadPath, fileSubPath, file.Filename)
+
 	if err := ctx.SaveUploadedFile(file, filePath); err != nil {
 		return "", errors.InternalServer(reason.UnknownError).WithError(err).WithStack()
 	}
@@ -324,6 +370,35 @@ func (us *uploaderService) uploadAttachmentFile(ctx *gin.Context, file *multipar
 	if err != nil {
 		return "", err
 	}
+	if UseDbStorage() {
+		src, err := file.Open()
+		if err != nil {
+			return "", errors.InternalServer(reason.UnknownError).WithError(err).WithStack()
+		}
+		defer src.Close()
+
+		buf := new(bytes.Buffer)
+		if _, err = io.Copy(buf, src); err != nil {
+			return "", errors.InternalServer(reason.UnknownError).WithError(err).WithStack()
+		}
+
+		blob := &entity.File{
+			ID:        uuid.New().String(),
+			FileName:  originalFilename,
+			MimeType:  file.Header.Get("Content-Type"),
+			Size:      int64(len(buf.Bytes())),
+			Content:   buf.Bytes(),
+			CreatedAt: time.Now(),
+		}
+
+		err = us.fileRepo.Save(ctx, blob)
+		if err != nil {
+			return "", errors.InternalServer(reason.UnknownError).WithError(err).WithStack()
+		}
+
+		downloadUrl = fmt.Sprintf("%s/answer/api/v1/file/%s?download=%s", siteGeneral.SiteUrl, blob.ID, url.QueryEscape(originalFilename))
+		return downloadUrl, nil
+	}
 	filePath := path.Join(us.serviceConfig.UploadPath, fileSubPath)
 	if err := ctx.SaveUploadedFile(file, filePath); err != nil {
 		return "", errors.InternalServer(reason.UnknownError).WithError(err).WithStack()