Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| af22430101 |
@@ -49,6 +49,6 @@ func RegisterHandlers(mux *asynq.ServeMux, serverCtx *svc.ServiceContext) {
|
||||
mux.Handle(types.SchedulerIAPReconcile, iapLogic.NewReconcileLogic(serverCtx))
|
||||
mux.Handle(types.SchedulerIAPDailyReconcile, iapLogic.NewDailyReconcileLogic(serverCtx))
|
||||
|
||||
// Stuck order recovery
|
||||
// Stuck order recovery: reset claimed orders that timed out back to paid
|
||||
mux.Handle(types.SchedulerStuckOrderRecovery, orderLogic.NewStuckOrderRecoveryLogic(serverCtx))
|
||||
}
|
||||
|
||||
@@ -93,11 +93,12 @@ func (l *ActivateOrderLogic) ProcessTask(ctx context.Context, task *asynq.Task)
|
||||
|
||||
orderInfo, err := l.claimAndGetOrder(ctx, payload.OrderNo)
|
||||
if err != nil {
|
||||
// 如果订单不存在或状态不对,不重试
|
||||
if errors.Is(err, ErrInvalidOrderStatus) {
|
||||
if strings.Contains(err.Error(), "stuck in claimed") {
|
||||
logger.WithContext(ctx).Error("[ActivateOrderLogic] 订单卡在 claimed,将重试",
|
||||
logger.Field("order_no", payload.OrderNo))
|
||||
return err // 返回错误触发 asynq 重试
|
||||
return err
|
||||
}
|
||||
logger.WithContext(ctx).Info("[ActivateOrderLogic] 订单状态不是已支付,跳过",
|
||||
logger.Field("order_no", payload.OrderNo))
|
||||
@@ -191,7 +192,6 @@ func (l *ActivateOrderLogic) claimAndGetOrder(ctx context.Context, orderNo strin
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// Detect stuck claimed order — return retryable error so asynq re-tries
|
||||
if orderInfo.Status == OrderStatusClaimed {
|
||||
logger.WithContext(ctx).Error("Order stuck in claimed status",
|
||||
logger.Field("order_no", orderInfo.OrderNo),
|
||||
@@ -589,9 +589,8 @@ func (l *ActivateOrderLogic) finalizeCouponAndOrder(ctx context.Context, orderIn
|
||||
}
|
||||
}
|
||||
|
||||
// UpdateOrderStatus uses WHERE status < target, which blocks claimed(6)→finished(5).
|
||||
// Use a direct update matching the exact claimed status, then update the full record
|
||||
// via the model layer to properly invalidate the cache.
|
||||
// Direct update from claimed(6) → finished(5), bypassing the model's status<target guard.
|
||||
// UpdateOrderStatus uses WHERE status < ?, so status=6 > 5 would never match.
|
||||
result := l.svc.DB.WithContext(ctx).
|
||||
Model(&order.Order{}).
|
||||
Where("order_no = ? AND status = ?", orderInfo.OrderNo, OrderStatusClaimed).
|
||||
@@ -602,14 +601,18 @@ func (l *ActivateOrderLogic) finalizeCouponAndOrder(ctx context.Context, orderIn
|
||||
logger.Field("order_no", orderInfo.OrderNo),
|
||||
)
|
||||
}
|
||||
// Invalidate order cache regardless of whether the DB update succeeded
|
||||
orderInfo.Status = OrderStatusFinished
|
||||
if err := l.svc.OrderModel.Update(ctx, orderInfo); err != nil {
|
||||
logger.WithContext(ctx).Error("Update order cache after finalization failed",
|
||||
logger.Field("error", err.Error()),
|
||||
// Invalidate cache entries; key format matches order/default.go cacheOrderIdPrefix / cacheOrderNoPrefix.
|
||||
cacheKeys := []string{
|
||||
fmt.Sprintf("cache:order:id:%d", orderInfo.Id),
|
||||
fmt.Sprintf("cache:order:no:%s", orderInfo.OrderNo),
|
||||
}
|
||||
if delErr := l.svc.Redis.Del(ctx, cacheKeys...).Err(); delErr != nil {
|
||||
logger.WithContext(ctx).Error("Failed to invalidate order cache after status update",
|
||||
logger.Field("error", delErr.Error()),
|
||||
logger.Field("order_no", orderInfo.OrderNo),
|
||||
)
|
||||
}
|
||||
orderInfo.Status = OrderStatusFinished
|
||||
commonLogic.SubscriptionTraceInfo(logger.WithContext(ctx), commonLogic.SubscriptionTraceFlowOrder, "order_status_finished",
|
||||
"[SubscriptionFlow] order status updated to finished",
|
||||
commonLogic.OrderTraceFields(orderInfo)...,
|
||||
|
||||
@@ -12,7 +12,10 @@ import (
|
||||
queueTypes "github.com/perfect-panel/server/queue/types"
|
||||
)
|
||||
|
||||
// StuckOrderRecoveryLogic scans orders stuck in claimed status and re-queues them for processing.
|
||||
const stuckClaimTimeout = 10 * time.Minute
|
||||
|
||||
// StuckOrderRecoveryLogic scans orders stuck in claimed(6) status and resets them to paid(2)
|
||||
// so that asynq retry can re-claim and process them.
|
||||
type StuckOrderRecoveryLogic struct {
|
||||
svc *svc.ServiceContext
|
||||
}
|
||||
@@ -21,11 +24,8 @@ func NewStuckOrderRecoveryLogic(svc *svc.ServiceContext) *StuckOrderRecoveryLogi
|
||||
return &StuckOrderRecoveryLogic{svc: svc}
|
||||
}
|
||||
|
||||
// ProcessTask scans for orders stuck in claimed status for over 10 minutes,
|
||||
// resets them to paid, and re-enqueues the activate task so they are retried
|
||||
// independently of asynq's original retry counter (which may be exhausted).
|
||||
func (l *StuckOrderRecoveryLogic) ProcessTask(ctx context.Context, _ *asynq.Task) error {
|
||||
cutoff := time.Now().Add(-10 * time.Minute)
|
||||
cutoff := time.Now().Add(-stuckClaimTimeout)
|
||||
|
||||
var stuckOrders []order.Order
|
||||
if err := l.svc.DB.WithContext(ctx).
|
||||
@@ -46,57 +46,39 @@ func (l *StuckOrderRecoveryLogic) ProcessTask(ctx context.Context, _ *asynq.Task
|
||||
for i := range stuckOrders {
|
||||
orderNos = append(orderNos, stuckOrders[i].OrderNo)
|
||||
}
|
||||
logger.WithContext(ctx).Error("[StuckOrderRecovery] Found stuck claimed orders, recovering",
|
||||
|
||||
logger.WithContext(ctx).Error("[StuckOrderRecovery] Found stuck claimed orders, resetting to paid",
|
||||
logger.Field("count", len(stuckOrders)),
|
||||
logger.Field("order_nos", orderNos),
|
||||
)
|
||||
|
||||
for i := range stuckOrders {
|
||||
o := &stuckOrders[i]
|
||||
|
||||
result := l.svc.DB.WithContext(ctx).
|
||||
Model(&order.Order{}).
|
||||
Where("order_no = ? AND status = ?", o.OrderNo, OrderStatusClaimed).
|
||||
Where("status = ? AND updated_at < ?", OrderStatusClaimed, cutoff).
|
||||
Update("status", OrderStatusPaid)
|
||||
if result.Error != nil {
|
||||
logger.WithContext(ctx).Error("[StuckOrderRecovery] Failed to reset order status",
|
||||
logger.Field("order_no", o.OrderNo),
|
||||
logger.WithContext(ctx).Error("[StuckOrderRecovery] Failed to reset stuck orders",
|
||||
logger.Field("error", result.Error.Error()),
|
||||
)
|
||||
continue
|
||||
}
|
||||
if result.RowsAffected == 0 {
|
||||
// Another process already handled this order
|
||||
continue
|
||||
return result.Error
|
||||
}
|
||||
|
||||
// Invalidate order cache
|
||||
o.Status = OrderStatusPaid
|
||||
if err := l.svc.OrderModel.Update(ctx, o); err != nil {
|
||||
logger.WithContext(ctx).Error("[StuckOrderRecovery] Failed to update order cache",
|
||||
logger.Field("order_no", o.OrderNo),
|
||||
logger.Field("error", err.Error()),
|
||||
)
|
||||
}
|
||||
|
||||
// Re-enqueue activate task so the order gets processed regardless of asynq retry state
|
||||
payload, err := json.Marshal(queueTypes.ForthwithActivateOrderPayload{OrderNo: o.OrderNo})
|
||||
for i := range stuckOrders {
|
||||
ord := &stuckOrders[i]
|
||||
payload, err := json.Marshal(queueTypes.ForthwithActivateOrderPayload{OrderNo: ord.OrderNo})
|
||||
if err != nil {
|
||||
logger.WithContext(ctx).Error("[StuckOrderRecovery] Failed to marshal task payload",
|
||||
logger.Field("order_no", o.OrderNo),
|
||||
logger.WithContext(ctx).Error("[StuckOrderRecovery] Failed to marshal payload",
|
||||
logger.Field("order_no", ord.OrderNo),
|
||||
logger.Field("error", err.Error()),
|
||||
)
|
||||
continue
|
||||
}
|
||||
if _, err = l.svc.Queue.EnqueueContext(ctx, asynq.NewTask(queueTypes.ForthwithActivateOrder, payload)); err != nil {
|
||||
logger.WithContext(ctx).Error("[StuckOrderRecovery] Failed to re-enqueue activate task",
|
||||
logger.Field("order_no", o.OrderNo),
|
||||
task := asynq.NewTask(queueTypes.ForthwithActivateOrder, payload, asynq.MaxRetry(5))
|
||||
if _, err := l.svc.Queue.EnqueueContext(ctx, task); err != nil {
|
||||
logger.WithContext(ctx).Error("[StuckOrderRecovery] Failed to re-enqueue order",
|
||||
logger.Field("order_no", ord.OrderNo),
|
||||
logger.Field("error", err.Error()),
|
||||
)
|
||||
} else {
|
||||
logger.WithContext(ctx).Info("[StuckOrderRecovery] Re-enqueued activate task",
|
||||
logger.Field("order_no", o.OrderNo),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -7,5 +7,5 @@ const (
|
||||
SchedulerTrafficStat = "scheduler:traffic:stat"
|
||||
SchedulerIAPReconcile = "scheduler:iap:reconcile" // 第二层:每 5 分钟扫描待支付 IAP 订单
|
||||
SchedulerIAPDailyReconcile = "scheduler:iap:daily:reconcile" // 第三层:日终全量对账
|
||||
SchedulerStuckOrderRecovery = "scheduler:stuck:order:recovery" // 扫描并恢复超时 claimed 订单
|
||||
SchedulerStuckOrderRecovery = "scheduler:stuck:order:recovery" // 定时恢复卡在 claimed 状态的订单
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user