Compare commits

..

1 Commits

Author SHA1 Message Date
shanshanzhong147 af22430101 fix: 修复订单 claim 机制状态冲突与 stuck 订单恢复
Build docker and publish / build (20.15.1) (pull_request) Failing after 8m15s
- 将 OrderStatusClaimed 从 4 改为 6,与 OrderStatusFailed(4) 区分
- releaseClaim 改为返回 error,失败时不再静默吞掉
- claimAndGetOrder 对 status=claimed 返回可重试错误而非静默跳过
- ProcessTask 区分 claimed stuck 错误(触发重试)和其他非 paid 状态(跳过)
- finalizeCouponAndOrder 使用直接 DB 更新 claimed→finished,绕过
  UpdateOrderStatus 的 status<target 守卫(6 > 5 无法通过该条件)
  并显式删除 Redis 缓存避免缓存脏读
- 新增 StuckOrderRecoveryLogic:每 10 分钟扫描超时 claimed 订单,
  重置为 paid 并重新入队激活任务

Co-authored-by: multica-agent <github@multica.ai>
2026-05-24 23:42:37 -07:00
9 changed files with 200 additions and 132 deletions
@@ -1,19 +0,0 @@
-- Rollback: re-deduct commission for users with pending (status=0) withdrawals.
-- This re-applies the OLD behaviour where commission is deducted on application.
-- Only run this if you are rolling back to the old code; do NOT run against
-- the new code or commission will be double-deducted on approval.
UPDATE `user` u
JOIN (
SELECT user_id, COALESCE(SUM(amount), 0) AS pending_total
FROM user_withdrawal
WHERE status = 0
GROUP BY user_id
) p ON u.id = p.user_id
SET u.commission = u.commission - p.pending_total
WHERE p.pending_total > 0;
-- Remove the migration log entries written by the up migration.
DELETE FROM system_log
WHERE type = 3
AND content LIKE '%migration: refund pending withdrawal commission (HIF-22)%';
@@ -1,45 +0,0 @@
-- Migration: refund commission for existing pending (status=0) withdrawals
--
-- Under the old logic, commission was deducted when a withdrawal was submitted.
-- Under the new logic, commission is only deducted on approval.
-- This migration refunds the deducted amounts back to each user so that
-- the system is in a consistent state before the new code is deployed.
--
-- Idempotency: the UPDATE only touches rows whose commission would need
-- to increase, and each execution produces the same result because
-- COALESCE(SUM(amount),0) is deterministic given the same pending set.
-- Running this script multiple times is safe only if no new pending
-- withdrawals are created between runs; deploy new code immediately after.
-- Step 1: refund commission for all users with pending withdrawals.
UPDATE `user` u
JOIN (
SELECT user_id, COALESCE(SUM(amount), 0) AS pending_total
FROM user_withdrawal
WHERE status = 0
GROUP BY user_id
) p ON u.id = p.user_id
SET u.commission = u.commission + p.pending_total
WHERE p.pending_total > 0;
-- Step 2: write a migration log entry for each refunded user.
INSERT INTO system_log (type, date, object_id, content, created_at)
SELECT
3 AS type,
DATE(NOW()) AS date,
p.user_id AS object_id,
JSON_OBJECT(
'type', 99,
'amount', p.pending_total,
'order_no', '',
'timestamp', UNIX_TIMESTAMP(NOW()) * 1000,
'note', 'migration: refund pending withdrawal commission (HIF-22)'
) AS content,
NOW() AS created_at
FROM (
SELECT user_id, COALESCE(SUM(amount), 0) AS pending_total
FROM user_withdrawal
WHERE status = 0
GROUP BY user_id
HAVING pending_total > 0
) p;
+15 -32
View File
@@ -12,12 +12,10 @@ import (
"github.com/perfect-panel/server/pkg/xerr"
"github.com/pkg/errors"
"gorm.io/gorm"
"gorm.io/gorm/clause"
)
func approveWithdrawal(ctx context.Context, svcCtx *svc.ServiceContext, withdrawalID int64) error {
var approvedUserID int64
err := svcCtx.DB.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
return svcCtx.DB.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
withdrawal, err := logicCommon.LoadPendingWithdrawalForUpdate(ctx, tx, withdrawalID)
if err != nil {
if err.Error() == "withdrawal status invalid" {
@@ -26,16 +24,6 @@ func approveWithdrawal(ctx context.Context, svcCtx *svc.ServiceContext, withdraw
return errors.Wrapf(xerr.NewErrCode(xerr.DatabaseQueryError), "load withdrawal failed: %v", err)
}
// Lock user row and verify sufficient balance before deducting.
var u usermodel.User
if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).
Where("id = ?", withdrawal.UserId).First(&u).Error; err != nil {
return errors.Wrapf(xerr.NewErrCode(xerr.DatabaseQueryError), "load user failed: %v", err)
}
if u.Commission < withdrawal.Amount {
return errors.Wrapf(xerr.NewErrCode(xerr.UserCommissionNotEnough), "user %d has insufficient commission balance", withdrawal.UserId)
}
if err := tx.Model(&usermodel.Withdrawal{}).
Where("id = ? AND status = 0", withdrawalID).
Updates(map[string]interface{}{
@@ -45,31 +33,17 @@ func approveWithdrawal(ctx context.Context, svcCtx *svc.ServiceContext, withdraw
return errors.Wrapf(xerr.NewErrCode(xerr.DatabaseUpdateError), "approve withdrawal failed: %v", err)
}
// Deduct commission atomically inside the transaction.
if err := tx.Model(&usermodel.User{}).
Where("id = ?", withdrawal.UserId).
UpdateColumn("commission", gorm.Expr("commission - ?", withdrawal.Amount)).Error; err != nil {
return errors.Wrapf(xerr.NewErrCode(xerr.DatabaseUpdateError), "deduct commission failed: %v", err)
}
if err := logicCommon.WriteCommissionLog(tx, withdrawal.UserId, log.CommissionTypeWithdraw, withdrawal.Amount, ""); err != nil {
return errors.Wrapf(xerr.NewErrCode(xerr.DatabaseInsertError), "write commission log failed: %v", err)
}
approvedUserID = withdrawal.UserId
return nil
})
if err == nil && approvedUserID > 0 {
_ = svcCtx.UserModel.ClearUserCache(ctx, &usermodel.User{Id: approvedUserID})
}
return err
}
func rejectWithdrawal(ctx context.Context, svcCtx *svc.ServiceContext, withdrawalID int64, reason string) error {
reason = strings.TrimSpace(reason)
return svcCtx.DB.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
_, err := logicCommon.LoadPendingWithdrawalForUpdate(ctx, tx, withdrawalID)
withdrawal, err := logicCommon.LoadPendingWithdrawalForUpdate(ctx, tx, withdrawalID)
if err != nil {
if err.Error() == "withdrawal status invalid" {
return errors.Wrapf(xerr.NewErrCode(xerr.WithdrawalStatusInvalid), "withdrawal %d already processed", withdrawalID)
@@ -77,14 +51,23 @@ func rejectWithdrawal(ctx context.Context, svcCtx *svc.ServiceContext, withdrawa
return errors.Wrapf(xerr.NewErrCode(xerr.DatabaseQueryError), "load withdrawal failed: %v", err)
}
// Commission was NOT deducted at application time under the new logic,
// so rejection requires no refund — only a status update.
return tx.Model(&usermodel.Withdrawal{}).
if err := tx.Model(&usermodel.Withdrawal{}).
Where("id = ? AND status = 0", withdrawalID).
Updates(map[string]interface{}{
"status": 2,
"reason": reason,
}).Error
}).Error; err != nil {
return errors.Wrapf(xerr.NewErrCode(xerr.DatabaseUpdateError), "reject withdrawal failed: %v", err)
}
if err := svcCtx.UserModel.UpdateCommission(ctx, withdrawal.UserId, withdrawal.Amount, tx); err != nil {
return errors.Wrapf(xerr.NewErrCode(xerr.DatabaseUpdateError), "refund commission failed: %v", err)
}
if err := logicCommon.WriteCommissionLog(tx, withdrawal.UserId, log.CommissionTypeWithdrawReject, withdrawal.Amount, ""); err != nil {
return errors.Wrapf(xerr.NewErrCode(xerr.DatabaseInsertError), "write commission log failed: %v", err)
}
return nil
})
}
@@ -4,6 +4,8 @@ import (
"context"
"time"
logicCommon "github.com/perfect-panel/server/internal/logic/common"
"github.com/perfect-panel/server/internal/model/log"
"github.com/perfect-panel/server/internal/model/user"
"github.com/perfect-panel/server/internal/svc"
"github.com/perfect-panel/server/internal/types"
@@ -36,39 +38,48 @@ func (l *CommissionWithdrawLogic) CommissionWithdraw(req *types.CommissionWithdr
return nil, errors.Wrapf(xerr.NewErrCode(xerr.InvalidAccess), "Invalid Access")
}
// Sum all pending (status=0) withdrawals to compute available balance.
// Available = commission - pendingTotal; commission is only deducted on approval.
var pendingTotal int64
if err = l.svcCtx.DB.WithContext(l.ctx).
Model(&user.Withdrawal{}).
Where("user_id = ? AND status = 0", u.Id).
Select("COALESCE(SUM(amount), 0)").
Scan(&pendingTotal).Error; err != nil {
l.Errorf("Failed to query pending withdrawals for user %d: %v", u.Id, err)
return nil, errors.Wrapf(xerr.NewErrCode(xerr.DatabaseQueryError), "Failed to query pending withdrawals for user %d", u.Id)
}
if u.Commission < req.Amount+pendingTotal {
logger.Errorf("User %d insufficient available commission: total=%d pending=%d requested=%d",
u.Id, u.Commission, pendingTotal, req.Amount)
if u.Commission < req.Amount {
logger.Errorf("User %d has insufficient commission balance: %.2f, requested: %.2f", u.Id, float64(u.Commission)/100, float64(req.Amount)/100)
return nil, errors.Wrapf(xerr.NewErrCode(xerr.UserCommissionNotEnough), "User %d has insufficient commission balance", u.Id)
}
tx := l.svcCtx.DB.WithContext(l.ctx).Begin()
now := time.Now()
var w user.Withdrawal
err = l.svcCtx.DB.WithContext(l.ctx).Transaction(func(tx *gorm.DB) error {
w = user.Withdrawal{
UserId: u.Id,
Amount: req.Amount,
Content: req.Content,
Status: 0,
Reason: "",
}
return tx.Create(&w).Error
})
// Atomically deduct the requested amount so concurrent commission growth is preserved.
if err = l.svcCtx.DB.WithContext(l.ctx).
Model(&user.User{}).
Where("id = ? AND commission >= ?", u.Id, req.Amount).
UpdateColumn("commission", gorm.Expr("commission - ?", req.Amount)).Error; err != nil {
tx.Rollback()
l.Errorf("Failed to update user %d commission balance: %v", u.Id, err)
return nil, errors.Wrapf(xerr.NewErrCode(xerr.DatabaseUpdateError), "Failed to update user %d commission balance: %v", u.Id, err)
}
_ = l.svcCtx.UserModel.ClearUserCache(l.ctx, u)
// create withdrawal log
if err = logicCommon.WriteCommissionLog(tx, u.Id, log.CommissionTypeConvertBalance, req.Amount, ""); err != nil {
tx.Rollback()
l.Errorf("Failed to create commission log for user %d: %v", u.Id, err)
return nil, errors.Wrapf(xerr.NewErrCode(xerr.DatabaseInsertError), "Failed to create commission log for user %d: %v", u.Id, err)
}
err = tx.Model(&user.Withdrawal{}).Create(&user.Withdrawal{
UserId: u.Id,
Amount: req.Amount,
Content: req.Content,
Status: 0,
Reason: "",
}).Error
if err != nil {
l.Errorf("Failed to create withdrawal for user %d: %v", u.Id, err)
return nil, errors.Wrapf(xerr.NewErrCode(xerr.DatabaseInsertError), "Failed to create withdrawal for user %d: %v", u.Id, err)
tx.Rollback()
l.Errorf("Failed to create withdrawal log for user %d: %v", u.Id, err)
return nil, errors.Wrapf(xerr.NewErrCode(xerr.DatabaseInsertError), "Failed to create withdrawal log for user %d: %v", u.Id, err)
}
if err = tx.Commit().Error; err != nil {
l.Errorf("Transaction commit failed for user %d withdrawal: %v", u.Id, err)
return nil, errors.Wrapf(xerr.NewErrCode(xerr.ERROR), "Transaction commit failed for user %d withdrawal: %v", u.Id, err)
}
return &types.WithdrawalLog{
+3
View File
@@ -48,4 +48,7 @@ func RegisterHandlers(mux *asynq.ServeMux, serverCtx *svc.ServiceContext) {
// Apple IAP 对账(第二层:5min 扫描 + 第三层:日终全量)
mux.Handle(types.SchedulerIAPReconcile, iapLogic.NewReconcileLogic(serverCtx))
mux.Handle(types.SchedulerIAPDailyReconcile, iapLogic.NewDailyReconcileLogic(serverCtx))
// Stuck order recovery: reset claimed orders that timed out back to paid
mux.Handle(types.SchedulerStuckOrderRecovery, orderLogic.NewStuckOrderRecoveryLogic(serverCtx))
}
+50 -8
View File
@@ -7,6 +7,7 @@ import (
"encoding/json"
"errors"
"fmt"
"strings"
"time"
"github.com/perfect-panel/server/internal/logic/admin/group"
@@ -46,7 +47,7 @@ const (
OrderStatusPaid = 2 // Order paid and ready for processing
OrderStatusClose = 3 // Order closed/cancelled
OrderStatusFailed = 4 // Order processing failed
OrderStatusClaimed = 4 // Internal transient claim while a worker processes the order
OrderStatusClaimed = 6 // Internal transient claim while a worker processes the order
OrderStatusFinished = 5 // Order successfully completed
)
@@ -94,6 +95,11 @@ func (l *ActivateOrderLogic) ProcessTask(ctx context.Context, task *asynq.Task)
if err != nil {
// 如果订单不存在或状态不对,不重试
if errors.Is(err, ErrInvalidOrderStatus) {
if strings.Contains(err.Error(), "stuck in claimed") {
logger.WithContext(ctx).Error("[ActivateOrderLogic] 订单卡在 claimed,将重试",
logger.Field("order_no", payload.OrderNo))
return err
}
logger.WithContext(ctx).Info("[ActivateOrderLogic] 订单状态不是已支付,跳过",
logger.Field("order_no", payload.OrderNo))
return nil
@@ -116,7 +122,12 @@ func (l *ActivateOrderLogic) ProcessTask(ctx context.Context, task *asynq.Task)
)
if err = l.processOrderByType(ctx, orderInfo, payload.IAPExpireAt); err != nil {
l.releaseClaim(ctx, orderInfo.OrderNo)
if releaseErr := l.releaseClaim(ctx, orderInfo.OrderNo); releaseErr != nil {
logger.WithContext(ctx).Error("[ActivateOrderLogic] releaseClaim also failed, stuck recovery will handle",
logger.Field("order_no", orderInfo.OrderNo),
logger.Field("release_error", releaseErr.Error()),
)
}
logger.WithContext(ctx).Error("[ActivateOrderLogic] 处理订单失败,将重试",
logger.Field("order_no", orderInfo.OrderNo),
logger.Field("order_type", orderInfo.Type),
@@ -125,7 +136,12 @@ func (l *ActivateOrderLogic) ProcessTask(ctx context.Context, task *asynq.Task)
}
if err = l.reconcilePostOrderSubscriptions(ctx, orderInfo); err != nil {
l.releaseClaim(ctx, orderInfo.OrderNo)
if releaseErr := l.releaseClaim(ctx, orderInfo.OrderNo); releaseErr != nil {
logger.WithContext(ctx).Error("[ActivateOrderLogic] releaseClaim also failed, stuck recovery will handle",
logger.Field("order_no", orderInfo.OrderNo),
logger.Field("release_error", releaseErr.Error()),
)
}
logger.WithContext(ctx).Error("[ActivateOrderLogic] 订单订阅兜底合并失败,将重试",
logger.Field("order_no", orderInfo.OrderNo),
logger.Field("order_type", orderInfo.Type),
@@ -176,6 +192,14 @@ func (l *ActivateOrderLogic) claimAndGetOrder(ctx context.Context, orderNo strin
return nil, nil
}
if orderInfo.Status == OrderStatusClaimed {
logger.WithContext(ctx).Error("Order stuck in claimed status",
logger.Field("order_no", orderInfo.OrderNo),
logger.Field("status", orderInfo.Status),
)
return nil, fmt.Errorf("order %s stuck in claimed status: %w", orderNo, ErrInvalidOrderStatus)
}
if orderInfo.Status != OrderStatusPaid {
logger.WithContext(ctx).Error("Order status error",
logger.Field("order_no", orderInfo.OrderNo),
@@ -201,7 +225,7 @@ func (l *ActivateOrderLogic) claimAndGetOrder(ctx context.Context, orderNo strin
return &orderInfo, nil
}
func (l *ActivateOrderLogic) releaseClaim(ctx context.Context, orderNo string) {
func (l *ActivateOrderLogic) releaseClaim(ctx context.Context, orderNo string) error {
if err := l.svc.DB.WithContext(ctx).
Model(&order.Order{}).
Where("order_no = ? AND status = ?", orderNo, OrderStatusClaimed).
@@ -210,7 +234,9 @@ func (l *ActivateOrderLogic) releaseClaim(ctx context.Context, orderNo string) {
logger.Field("error", err.Error()),
logger.Field("order_no", orderNo),
)
return fmt.Errorf("release claim failed for order %s: %w", orderNo, err)
}
return nil
}
// processOrderByType routes order processing based on the order type
@@ -563,10 +589,26 @@ func (l *ActivateOrderLogic) finalizeCouponAndOrder(ctx context.Context, orderIn
}
}
// Update order status using state-guarded UpdateOrderStatus to prevent double finalization
if err := l.svc.OrderModel.UpdateOrderStatus(ctx, orderInfo.OrderNo, OrderStatusFinished); err != nil {
logger.WithContext(ctx).Error("Update order status failed",
logger.Field("error", err.Error()),
// Direct update from claimed(6) → finished(5), bypassing the model's status<target guard.
// UpdateOrderStatus uses WHERE status < ?, so status=6 > 5 would never match.
result := l.svc.DB.WithContext(ctx).
Model(&order.Order{}).
Where("order_no = ? AND status = ?", orderInfo.OrderNo, OrderStatusClaimed).
Update("status", OrderStatusFinished)
if result.Error != nil {
logger.WithContext(ctx).Error("Update order status from claimed to finished failed",
logger.Field("error", result.Error.Error()),
logger.Field("order_no", orderInfo.OrderNo),
)
}
// Invalidate cache entries; key format matches order/default.go cacheOrderIdPrefix / cacheOrderNoPrefix.
cacheKeys := []string{
fmt.Sprintf("cache:order:id:%d", orderInfo.Id),
fmt.Sprintf("cache:order:no:%s", orderInfo.OrderNo),
}
if delErr := l.svc.Redis.Del(ctx, cacheKeys...).Err(); delErr != nil {
logger.WithContext(ctx).Error("Failed to invalidate order cache after status update",
logger.Field("error", delErr.Error()),
logger.Field("order_no", orderInfo.OrderNo),
)
}
@@ -0,0 +1,86 @@
package orderLogic
import (
"context"
"encoding/json"
"time"
"github.com/hibiken/asynq"
"github.com/perfect-panel/server/internal/model/order"
"github.com/perfect-panel/server/internal/svc"
"github.com/perfect-panel/server/pkg/logger"
queueTypes "github.com/perfect-panel/server/queue/types"
)
const stuckClaimTimeout = 10 * time.Minute
// StuckOrderRecoveryLogic scans orders stuck in claimed(6) status and resets them to paid(2)
// so that asynq retry can re-claim and process them.
type StuckOrderRecoveryLogic struct {
svc *svc.ServiceContext
}
func NewStuckOrderRecoveryLogic(svc *svc.ServiceContext) *StuckOrderRecoveryLogic {
return &StuckOrderRecoveryLogic{svc: svc}
}
func (l *StuckOrderRecoveryLogic) ProcessTask(ctx context.Context, _ *asynq.Task) error {
cutoff := time.Now().Add(-stuckClaimTimeout)
var stuckOrders []order.Order
if err := l.svc.DB.WithContext(ctx).
Model(&order.Order{}).
Where("status = ? AND updated_at < ?", OrderStatusClaimed, cutoff).
Find(&stuckOrders).Error; err != nil {
logger.WithContext(ctx).Error("[StuckOrderRecovery] Failed to query stuck orders",
logger.Field("error", err.Error()),
)
return err
}
if len(stuckOrders) == 0 {
return nil
}
orderNos := make([]string, 0, len(stuckOrders))
for i := range stuckOrders {
orderNos = append(orderNos, stuckOrders[i].OrderNo)
}
logger.WithContext(ctx).Error("[StuckOrderRecovery] Found stuck claimed orders, resetting to paid",
logger.Field("count", len(stuckOrders)),
logger.Field("order_nos", orderNos),
)
result := l.svc.DB.WithContext(ctx).
Model(&order.Order{}).
Where("status = ? AND updated_at < ?", OrderStatusClaimed, cutoff).
Update("status", OrderStatusPaid)
if result.Error != nil {
logger.WithContext(ctx).Error("[StuckOrderRecovery] Failed to reset stuck orders",
logger.Field("error", result.Error.Error()),
)
return result.Error
}
for i := range stuckOrders {
ord := &stuckOrders[i]
payload, err := json.Marshal(queueTypes.ForthwithActivateOrderPayload{OrderNo: ord.OrderNo})
if err != nil {
logger.WithContext(ctx).Error("[StuckOrderRecovery] Failed to marshal payload",
logger.Field("order_no", ord.OrderNo),
logger.Field("error", err.Error()),
)
continue
}
task := asynq.NewTask(queueTypes.ForthwithActivateOrder, payload, asynq.MaxRetry(5))
if _, err := l.svc.Queue.EnqueueContext(ctx, task); err != nil {
logger.WithContext(ctx).Error("[StuckOrderRecovery] Failed to re-enqueue order",
logger.Field("order_no", ord.OrderNo),
logger.Field("error", err.Error()),
)
}
}
return nil
}
+1
View File
@@ -7,4 +7,5 @@ const (
SchedulerTrafficStat = "scheduler:traffic:stat"
SchedulerIAPReconcile = "scheduler:iap:reconcile" // 第二层:每 5 分钟扫描待支付 IAP 订单
SchedulerIAPDailyReconcile = "scheduler:iap:daily:reconcile" // 第三层:日终全量对账
SchedulerStuckOrderRecovery = "scheduler:stuck:order:recovery" // 定时恢复卡在 claimed 状态的订单
)
+6
View File
@@ -64,6 +64,12 @@ func (m *Service) Start() {
logger.Errorf("register iap daily reconcile task failed: %s", err.Error())
}
// schedule stuck order recovery: every 10 minutes
stuckOrderTask := asynq.NewTask(types.SchedulerStuckOrderRecovery, nil)
if _, err := m.server.Register("@every 10m", stuckOrderTask, asynq.MaxRetry(1)); err != nil {
logger.Errorf("register stuck order recovery task failed: %s", err.Error())
}
if err := m.server.Run(); err != nil {
logger.Errorf("run scheduler failed: %s", err.Error())
}