feat: add direct s3 upload flow

2026-05-13 11:24:13 -07:00
parent f6911965dc
commit 4fa9fcd232
16 changed files with 773 additions and 106 deletions
@@ -0,0 +1,148 @@
+package file
+
+import (
+	"context"
+	"crypto/sha1"
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
+	"path/filepath"
+	"regexp"
+	"strings"
+	"time"
+
+	"github.com/perfect-panel/server/internal/model/user"
+	"github.com/perfect-panel/server/internal/svc"
+	"github.com/perfect-panel/server/pkg/constant"
+	"github.com/perfect-panel/server/pkg/xerr"
+	"github.com/pkg/errors"
+)
+
+const (
+	fileUploadInitStatus     = "init"
+	fileUploadCompleteStatus = "completed"
+	fileUploadRedisPrefix    = "file:upload:"
+)
+
+var safeFileNameRegexp = regexp.MustCompile(`[^a-zA-Z0-9._-]+`)
+
+type uploadMeta struct {
+	FileID      string `json:"file_id"`
+	UserID      int64  `json:"user_id"`
+	BizType     string `json:"biz_type"`
+	FileName    string `json:"file_name"`
+	ContentType string `json:"content_type"`
+	ObjectKey   string `json:"object_key"`
+	Size        int64  `json:"size"`
+	Sha256      string `json:"sha256"`
+	Status      string `json:"status"`
+	Etag        string `json:"etag,omitempty"`
+	CreatedAt   int64  `json:"created_at"`
+	CompletedAt int64  `json:"completed_at,omitempty"`
+}
+
+func currentUserFromContext(ctx context.Context) (*user.User, error) {
+	u, ok := ctx.Value(constant.CtxKeyUser).(*user.User)
+	if !ok || u == nil {
+		return nil, errors.Wrapf(xerr.NewErrCode(xerr.InvalidAccess), "Invalid Access")
+	}
+	return u, nil
+}
+
+func allowedContentTypeSet(csv string) map[string]struct{} {
+	set := make(map[string]struct{})
+	for _, item := range strings.Split(csv, ",") {
+		item = strings.TrimSpace(strings.ToLower(item))
+		if item == "" {
+			continue
+		}
+		set[item] = struct{}{}
+	}
+	return set
+}
+
+func validateInitRequest(svcCtx *svc.ServiceContext, bizType, fileName, contentType string, size int64) error {
+	if svcCtx.S3Store == nil || !svcCtx.Config.S3.Enable {
+		return errors.Wrapf(xerr.NewErrCode(xerr.ERROR), "S3 upload is not enabled")
+	}
+	if strings.TrimSpace(bizType) == "" {
+		return errors.Wrapf(xerr.NewErrCode(xerr.InvalidParams), "biz_type is required")
+	}
+	if strings.TrimSpace(fileName) == "" {
+		return errors.Wrapf(xerr.NewErrCode(xerr.InvalidParams), "file_name is required")
+	}
+	if size <= 0 {
+		return errors.Wrapf(xerr.NewErrCode(xerr.InvalidParams), "size must be greater than 0")
+	}
+	if svcCtx.Config.S3.MaxUploadSize > 0 && size > svcCtx.Config.S3.MaxUploadSize {
+		return errors.Wrapf(xerr.NewErrCode(xerr.InvalidParams), "file size exceeds max limit")
+	}
+	allowed := allowedContentTypeSet(svcCtx.Config.S3.AllowedContentTypes)
+	if len(allowed) > 0 {
+		if _, ok := allowed[strings.ToLower(strings.TrimSpace(contentType))]; !ok {
+			return errors.Wrapf(xerr.NewErrCode(xerr.InvalidParams), "content_type is not allowed")
+		}
+	}
+	return nil
+}
+
+func buildFileID(userID int64, bizType, fileName string) string {
+	h := sha1.New()
+	_, _ = h.Write([]byte(fmt.Sprintf("%d:%s:%s:%d", userID, bizType, fileName, time.Now().UnixNano())))
+	return hex.EncodeToString(h.Sum(nil))[:24]
+}
+
+func safeFileName(name string) string {
+	base := filepath.Base(strings.TrimSpace(name))
+	if base == "." || base == "/" || base == "" {
+		return "file.bin"
+	}
+	safe := safeFileNameRegexp.ReplaceAllString(base, "_")
+	if safe == "" {
+		return "file.bin"
+	}
+	return safe
+}
+
+func buildObjectKey(prefix string, userID int64, bizType, fileID, fileName string, now time.Time) string {
+	prefix = strings.Trim(prefix, "/")
+	if prefix == "" {
+		prefix = "app-upload"
+	}
+	return fmt.Sprintf("%s/%s/%d/%04d/%02d/%s_%s",
+		prefix,
+		strings.Trim(safeFileNameRegexp.ReplaceAllString(strings.ToLower(strings.TrimSpace(bizType)), "-"), "-"),
+		userID,
+		now.Year(),
+		int(now.Month()),
+		fileID,
+		safeFileName(fileName),
+	)
+}
+
+func uploadRedisKey(fileID string) string {
+	return fileUploadRedisPrefix + fileID
+}
+
+func saveUploadMeta(ctx context.Context, svcCtx *svc.ServiceContext, meta *uploadMeta, ttl time.Duration) error {
+	data, err := json.Marshal(meta)
+	if err != nil {
+		return errors.Wrapf(xerr.NewErrCode(xerr.ERROR), "marshal upload meta failed")
+	}
+	if err := svcCtx.Redis.Set(ctx, uploadRedisKey(meta.FileID), data, ttl).Err(); err != nil {
+		return errors.Wrapf(xerr.NewErrCode(xerr.ERROR), "save upload meta failed")
+	}
+	return nil
+}
+
+func loadUploadMeta(ctx context.Context, svcCtx *svc.ServiceContext, fileID string) (*uploadMeta, error) {
+	val, err := svcCtx.Redis.Get(ctx, uploadRedisKey(fileID)).Result()
+	if err != nil {
+		return nil, errors.Wrapf(xerr.NewErrCode(xerr.DatabaseQueryError), "upload meta not found")
+	}
+	var meta uploadMeta
+	if err := json.Unmarshal([]byte(val), &meta); err != nil {
+		return nil, errors.Wrapf(xerr.NewErrCode(xerr.ERROR), "parse upload meta failed")
+	}
+	return &meta, nil
+}
@@ -0,0 +1,71 @@
+// Code scaffolded by goctl. Safe to edit.
+// goctl 1.9.2
+
+package file
+
+import (
+	"context"
+	"time"
+
+	"github.com/perfect-panel/server/internal/svc"
+	"github.com/perfect-panel/server/internal/types"
+	"github.com/perfect-panel/server/pkg/logger"
+	"github.com/perfect-panel/server/pkg/xerr"
+	"github.com/pkg/errors"
+)
+
+type FileUploadCompleteLogic struct {
+	logger.Logger
+	ctx    context.Context
+	svcCtx *svc.ServiceContext
+}
+
+// Complete file upload
+func NewFileUploadCompleteLogic(ctx context.Context, svcCtx *svc.ServiceContext) *FileUploadCompleteLogic {
+	return &FileUploadCompleteLogic{
+		Logger: logger.WithContext(ctx),
+		ctx:    ctx,
+		svcCtx: svcCtx,
+	}
+}
+
+func (l *FileUploadCompleteLogic) FileUploadComplete(req *types.FileUploadCompleteRequest) (resp *types.FileUploadCompleteResponse, err error) {
+	u, err := currentUserFromContext(l.ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	meta, err := loadUploadMeta(l.ctx, l.svcCtx, req.FileId)
+	if err != nil {
+		return nil, err
+	}
+	if meta.UserID != u.Id {
+		return nil, errors.Wrapf(xerr.NewErrCode(xerr.InvalidAccess), "Invalid Access")
+	}
+
+	head, err := l.svcCtx.S3Store.HeadObject(l.ctx, meta.ObjectKey)
+	if err != nil {
+		l.Errorw("head object failed", logger.Field("error", err.Error()), logger.Field("file_id", meta.FileID))
+		return nil, errors.Wrapf(xerr.NewErrCode(xerr.DatabaseQueryError), "uploaded file not found")
+	}
+	if meta.Size > 0 && head.ContentLength != meta.Size {
+		return nil, errors.Wrapf(xerr.NewErrCode(xerr.InvalidParams), "uploaded file size mismatch")
+	}
+
+	meta.Status = fileUploadCompleteStatus
+	meta.Etag = head.ETag
+	meta.ContentType = head.ContentType
+	meta.CompletedAt = time.Now().Unix()
+	if err := saveUploadMeta(l.ctx, l.svcCtx, meta, 7*24*time.Hour); err != nil {
+		return nil, err
+	}
+
+	return &types.FileUploadCompleteResponse{
+		FileId:      meta.FileID,
+		ObjectKey:   meta.ObjectKey,
+		Size:        head.ContentLength,
+		ContentType: head.ContentType,
+		Etag:        head.ETag,
+		Status:      meta.Status,
+	}, nil
+}
@@ -0,0 +1,80 @@
+// Code scaffolded by goctl. Safe to edit.
+// goctl 1.9.2
+
+package file
+
+import (
+	"context"
+	"time"
+
+	"github.com/perfect-panel/server/internal/svc"
+	"github.com/perfect-panel/server/internal/types"
+	"github.com/perfect-panel/server/pkg/logger"
+)
+
+type FileUploadInitLogic struct {
+	logger.Logger
+	ctx    context.Context
+	svcCtx *svc.ServiceContext
+}
+
+// Init file upload
+func NewFileUploadInitLogic(ctx context.Context, svcCtx *svc.ServiceContext) *FileUploadInitLogic {
+	return &FileUploadInitLogic{
+		Logger: logger.WithContext(ctx),
+		ctx:    ctx,
+		svcCtx: svcCtx,
+	}
+}
+
+func (l *FileUploadInitLogic) FileUploadInit(req *types.FileUploadInitRequest) (resp *types.FileUploadInitResponse, err error) {
+	u, err := currentUserFromContext(l.ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	if err := validateInitRequest(l.svcCtx, req.BizType, req.FileName, req.ContentType, req.Size); err != nil {
+		return nil, err
+	}
+
+	now := time.Now()
+	fileID := buildFileID(u.Id, req.BizType, req.FileName)
+	objectKey := buildObjectKey(l.svcCtx.Config.S3.Prefix, u.Id, req.BizType, fileID, req.FileName, now)
+	expireSeconds := l.svcCtx.Config.S3.PresignExpireSeconds
+	if expireSeconds <= 0 {
+		expireSeconds = 300
+	}
+
+	presigned, err := l.svcCtx.S3Store.PresignPutObject(l.ctx, objectKey, req.ContentType, time.Duration(expireSeconds)*time.Second)
+	if err != nil {
+		l.Errorw("presign put object failed", logger.Field("error", err.Error()), logger.Field("user_id", u.Id))
+		return nil, err
+	}
+
+	meta := &uploadMeta{
+		FileID:      fileID,
+		UserID:      u.Id,
+		BizType:     req.BizType,
+		FileName:    req.FileName,
+		ContentType: req.ContentType,
+		ObjectKey:   objectKey,
+		Size:        req.Size,
+		Sha256:      req.Sha256,
+		Status:      fileUploadInitStatus,
+		CreatedAt:   now.Unix(),
+	}
+	if err := saveUploadMeta(l.ctx, l.svcCtx, meta, 24*time.Hour); err != nil {
+		return nil, err
+	}
+
+	return &types.FileUploadInitResponse{
+		FileId:    fileID,
+		ObjectKey: objectKey,
+		UploadURL: presigned.URL,
+		Method:    presigned.Method,
+		Headers: map[string]string{
+			"Content-Type": req.ContentType,
+		},
+		ExpiredAt: presigned.ExpiresAt,
+	}, nil
+}