Compare commits

..

1 Commits

Author SHA1 Message Date
shanshanzhong147 af22430101 fix: 修复订单 claim 机制状态冲突与 stuck 订单恢复
Build docker and publish / build (20.15.1) (pull_request) Failing after 8m15s
- 将 OrderStatusClaimed 从 4 改为 6,与 OrderStatusFailed(4) 区分
- releaseClaim 改为返回 error,失败时不再静默吞掉
- claimAndGetOrder 对 status=claimed 返回可重试错误而非静默跳过
- ProcessTask 区分 claimed stuck 错误(触发重试)和其他非 paid 状态(跳过)
- finalizeCouponAndOrder 使用直接 DB 更新 claimed→finished,绕过
  UpdateOrderStatus 的 status<target 守卫(6 > 5 无法通过该条件)
  并显式删除 Redis 缓存避免缓存脏读
- 新增 StuckOrderRecoveryLogic:每 10 分钟扫描超时 claimed 订单,
  重置为 paid 并重新入队激活任务

Co-authored-by: multica-agent <github@multica.ai>
2026-05-24 23:42:37 -07:00
4 changed files with 43 additions and 58 deletions
+1 -1
View File
@@ -49,6 +49,6 @@ func RegisterHandlers(mux *asynq.ServeMux, serverCtx *svc.ServiceContext) {
mux.Handle(types.SchedulerIAPReconcile, iapLogic.NewReconcileLogic(serverCtx))
mux.Handle(types.SchedulerIAPDailyReconcile, iapLogic.NewDailyReconcileLogic(serverCtx))
// Stuck order recovery
// Stuck order recovery: reset claimed orders that timed out back to paid
mux.Handle(types.SchedulerStuckOrderRecovery, orderLogic.NewStuckOrderRecoveryLogic(serverCtx))
}
+13 -10
View File
@@ -93,11 +93,12 @@ func (l *ActivateOrderLogic) ProcessTask(ctx context.Context, task *asynq.Task)
orderInfo, err := l.claimAndGetOrder(ctx, payload.OrderNo)
if err != nil {
// 如果订单不存在或状态不对,不重试
if errors.Is(err, ErrInvalidOrderStatus) {
if strings.Contains(err.Error(), "stuck in claimed") {
logger.WithContext(ctx).Error("[ActivateOrderLogic] 订单卡在 claimed,将重试",
logger.Field("order_no", payload.OrderNo))
return err // 返回错误触发 asynq 重试
return err
}
logger.WithContext(ctx).Info("[ActivateOrderLogic] 订单状态不是已支付,跳过",
logger.Field("order_no", payload.OrderNo))
@@ -191,7 +192,6 @@ func (l *ActivateOrderLogic) claimAndGetOrder(ctx context.Context, orderNo strin
return nil, nil
}
// Detect stuck claimed order — return retryable error so asynq re-tries
if orderInfo.Status == OrderStatusClaimed {
logger.WithContext(ctx).Error("Order stuck in claimed status",
logger.Field("order_no", orderInfo.OrderNo),
@@ -589,9 +589,8 @@ func (l *ActivateOrderLogic) finalizeCouponAndOrder(ctx context.Context, orderIn
}
}
// UpdateOrderStatus uses WHERE status < target, which blocks claimed(6)finished(5).
// Use a direct update matching the exact claimed status, then update the full record
// via the model layer to properly invalidate the cache.
// Direct update from claimed(6)finished(5), bypassing the model's status<target guard.
// UpdateOrderStatus uses WHERE status < ?, so status=6 > 5 would never match.
result := l.svc.DB.WithContext(ctx).
Model(&order.Order{}).
Where("order_no = ? AND status = ?", orderInfo.OrderNo, OrderStatusClaimed).
@@ -602,14 +601,18 @@ func (l *ActivateOrderLogic) finalizeCouponAndOrder(ctx context.Context, orderIn
logger.Field("order_no", orderInfo.OrderNo),
)
}
// Invalidate order cache regardless of whether the DB update succeeded
orderInfo.Status = OrderStatusFinished
if err := l.svc.OrderModel.Update(ctx, orderInfo); err != nil {
logger.WithContext(ctx).Error("Update order cache after finalization failed",
logger.Field("error", err.Error()),
// Invalidate cache entries; key format matches order/default.go cacheOrderIdPrefix / cacheOrderNoPrefix.
cacheKeys := []string{
fmt.Sprintf("cache:order:id:%d", orderInfo.Id),
fmt.Sprintf("cache:order:no:%s", orderInfo.OrderNo),
}
if delErr := l.svc.Redis.Del(ctx, cacheKeys...).Err(); delErr != nil {
logger.WithContext(ctx).Error("Failed to invalidate order cache after status update",
logger.Field("error", delErr.Error()),
logger.Field("order_no", orderInfo.OrderNo),
)
}
orderInfo.Status = OrderStatusFinished
commonLogic.SubscriptionTraceInfo(logger.WithContext(ctx), commonLogic.SubscriptionTraceFlowOrder, "order_status_finished",
"[SubscriptionFlow] order status updated to finished",
commonLogic.OrderTraceFields(orderInfo)...,
+19 -37
View File
@@ -12,7 +12,10 @@ import (
queueTypes "github.com/perfect-panel/server/queue/types"
)
// StuckOrderRecoveryLogic scans orders stuck in claimed status and re-queues them for processing.
const stuckClaimTimeout = 10 * time.Minute
// StuckOrderRecoveryLogic scans orders stuck in claimed(6) status and resets them to paid(2)
// so that asynq retry can re-claim and process them.
type StuckOrderRecoveryLogic struct {
svc *svc.ServiceContext
}
@@ -21,11 +24,8 @@ func NewStuckOrderRecoveryLogic(svc *svc.ServiceContext) *StuckOrderRecoveryLogi
return &StuckOrderRecoveryLogic{svc: svc}
}
// ProcessTask scans for orders stuck in claimed status for over 10 minutes,
// resets them to paid, and re-enqueues the activate task so they are retried
// independently of asynq's original retry counter (which may be exhausted).
func (l *StuckOrderRecoveryLogic) ProcessTask(ctx context.Context, _ *asynq.Task) error {
cutoff := time.Now().Add(-10 * time.Minute)
cutoff := time.Now().Add(-stuckClaimTimeout)
var stuckOrders []order.Order
if err := l.svc.DB.WithContext(ctx).
@@ -46,57 +46,39 @@ func (l *StuckOrderRecoveryLogic) ProcessTask(ctx context.Context, _ *asynq.Task
for i := range stuckOrders {
orderNos = append(orderNos, stuckOrders[i].OrderNo)
}
logger.WithContext(ctx).Error("[StuckOrderRecovery] Found stuck claimed orders, recovering",
logger.WithContext(ctx).Error("[StuckOrderRecovery] Found stuck claimed orders, resetting to paid",
logger.Field("count", len(stuckOrders)),
logger.Field("order_nos", orderNos),
)
for i := range stuckOrders {
o := &stuckOrders[i]
result := l.svc.DB.WithContext(ctx).
Model(&order.Order{}).
Where("order_no = ? AND status = ?", o.OrderNo, OrderStatusClaimed).
Where("status = ? AND updated_at < ?", OrderStatusClaimed, cutoff).
Update("status", OrderStatusPaid)
if result.Error != nil {
logger.WithContext(ctx).Error("[StuckOrderRecovery] Failed to reset order status",
logger.Field("order_no", o.OrderNo),
logger.WithContext(ctx).Error("[StuckOrderRecovery] Failed to reset stuck orders",
logger.Field("error", result.Error.Error()),
)
continue
}
if result.RowsAffected == 0 {
// Another process already handled this order
continue
return result.Error
}
// Invalidate order cache
o.Status = OrderStatusPaid
if err := l.svc.OrderModel.Update(ctx, o); err != nil {
logger.WithContext(ctx).Error("[StuckOrderRecovery] Failed to update order cache",
logger.Field("order_no", o.OrderNo),
logger.Field("error", err.Error()),
)
}
// Re-enqueue activate task so the order gets processed regardless of asynq retry state
payload, err := json.Marshal(queueTypes.ForthwithActivateOrderPayload{OrderNo: o.OrderNo})
for i := range stuckOrders {
ord := &stuckOrders[i]
payload, err := json.Marshal(queueTypes.ForthwithActivateOrderPayload{OrderNo: ord.OrderNo})
if err != nil {
logger.WithContext(ctx).Error("[StuckOrderRecovery] Failed to marshal task payload",
logger.Field("order_no", o.OrderNo),
logger.WithContext(ctx).Error("[StuckOrderRecovery] Failed to marshal payload",
logger.Field("order_no", ord.OrderNo),
logger.Field("error", err.Error()),
)
continue
}
if _, err = l.svc.Queue.EnqueueContext(ctx, asynq.NewTask(queueTypes.ForthwithActivateOrder, payload)); err != nil {
logger.WithContext(ctx).Error("[StuckOrderRecovery] Failed to re-enqueue activate task",
logger.Field("order_no", o.OrderNo),
task := asynq.NewTask(queueTypes.ForthwithActivateOrder, payload, asynq.MaxRetry(5))
if _, err := l.svc.Queue.EnqueueContext(ctx, task); err != nil {
logger.WithContext(ctx).Error("[StuckOrderRecovery] Failed to re-enqueue order",
logger.Field("order_no", ord.OrderNo),
logger.Field("error", err.Error()),
)
} else {
logger.WithContext(ctx).Info("[StuckOrderRecovery] Re-enqueued activate task",
logger.Field("order_no", o.OrderNo),
)
}
}
+1 -1
View File
@@ -7,5 +7,5 @@ const (
SchedulerTrafficStat = "scheduler:traffic:stat"
SchedulerIAPReconcile = "scheduler:iap:reconcile" // 第二层:每 5 分钟扫描待支付 IAP 订单
SchedulerIAPDailyReconcile = "scheduler:iap:daily:reconcile" // 第三层:日终全量对账
SchedulerStuckOrderRecovery = "scheduler:stuck:order:recovery" // 扫描并恢复超时 claimed 订单
SchedulerStuckOrderRecovery = "scheduler:stuck:order:recovery" // 定时恢复卡在 claimed 状态的订单
)