Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 79ab4460bc |
@@ -239,6 +239,30 @@ type (
|
||||
OccurredAt int64 `json:"occurred_at"`
|
||||
CreatedAt int64 `json:"created_at"`
|
||||
}
|
||||
GetLogMessageRawRequest {
|
||||
Id int64 `form:"id" validate:"required"`
|
||||
}
|
||||
GetLogMessageRawResponse {
|
||||
Id int64 `json:"id"`
|
||||
Platform string `json:"platform"`
|
||||
AppVersion string `json:"app_version"`
|
||||
OsName string `json:"os_name"`
|
||||
OsVersion string `json:"os_version"`
|
||||
DeviceId string `json:"device_id"`
|
||||
UserId *int64 `json:"user_id"`
|
||||
SessionId string `json:"session_id"`
|
||||
Level uint8 `json:"level"`
|
||||
ErrorCode string `json:"error_code"`
|
||||
Message string `json:"message"`
|
||||
Stack string `json:"stack"`
|
||||
Context interface{} `json:"context"`
|
||||
ClientIP string `json:"client_ip"`
|
||||
UserAgent string `json:"user_agent"`
|
||||
Locale string `json:"locale"`
|
||||
Digest string `json:"digest"`
|
||||
OccurredAt int64 `json:"occurred_at"`
|
||||
CreatedAt int64 `json:"created_at"`
|
||||
}
|
||||
)
|
||||
|
||||
@server (
|
||||
@@ -314,5 +338,9 @@ service ppanel {
|
||||
@doc "Get error log message detail"
|
||||
@handler GetErrorLogMessageDetail
|
||||
get /error_message/detail returns (GetErrorLogMessageDetailResponse)
|
||||
|
||||
@doc "Get log message raw detail (temporary)"
|
||||
@handler GetLogMessageRaw
|
||||
get /message/detail (GetLogMessageRawRequest) returns (GetLogMessageRawResponse)
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
package log
|
||||
|
||||
import (
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/perfect-panel/server/internal/logic/admin/log"
|
||||
"github.com/perfect-panel/server/internal/svc"
|
||||
"github.com/perfect-panel/server/internal/types"
|
||||
"github.com/perfect-panel/server/pkg/result"
|
||||
)
|
||||
|
||||
func GetLogMessageRawHandler(svcCtx *svc.ServiceContext) func(c *gin.Context) {
|
||||
return func(c *gin.Context) {
|
||||
var req types.GetLogMessageRawRequest
|
||||
_ = c.ShouldBind(&req)
|
||||
if err := svcCtx.Validate(&req); err != nil {
|
||||
result.ParamErrorResult(c, err)
|
||||
return
|
||||
}
|
||||
l := log.NewGetLogMessageRawLogic(c.Request.Context(), svcCtx)
|
||||
resp, err := l.GetLogMessageRaw(&req)
|
||||
result.HttpResult(c, resp, err)
|
||||
}
|
||||
}
|
||||
@@ -256,6 +256,9 @@ func RegisterHandlers(router *gin.Engine, serverCtx *svc.ServiceContext) {
|
||||
// Get error log message detail
|
||||
adminLogGroupRouter.GET("/error_message/detail", adminLog.GetErrorLogMessageDetailHandler(serverCtx))
|
||||
|
||||
// Get log message raw detail (temporary)
|
||||
adminLogGroupRouter.GET("/message/detail", adminLog.GetLogMessageRawHandler(serverCtx))
|
||||
|
||||
// Get error log message list
|
||||
adminLogGroupRouter.GET("/error_message/list", adminLog.GetErrorLogMessageListHandler(serverCtx))
|
||||
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
package log
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
|
||||
"github.com/perfect-panel/server/internal/svc"
|
||||
"github.com/perfect-panel/server/internal/types"
|
||||
"github.com/perfect-panel/server/pkg/logger"
|
||||
"github.com/perfect-panel/server/pkg/xerr"
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
type GetLogMessageRawLogic struct {
|
||||
logger.Logger
|
||||
ctx context.Context
|
||||
svcCtx *svc.ServiceContext
|
||||
}
|
||||
|
||||
func NewGetLogMessageRawLogic(ctx context.Context, svcCtx *svc.ServiceContext) *GetLogMessageRawLogic {
|
||||
return &GetLogMessageRawLogic{Logger: logger.WithContext(ctx), ctx: ctx, svcCtx: svcCtx}
|
||||
}
|
||||
|
||||
func (l *GetLogMessageRawLogic) GetLogMessageRaw(req *types.GetLogMessageRawRequest) (resp *types.GetLogMessageRawResponse, err error) {
|
||||
row, err := l.svcCtx.LogMessageModel.FindOne(l.ctx, req.Id)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(xerr.NewErrCode(xerr.DatabaseQueryError), "FindOne log_message id=%d: %v", req.Id, err)
|
||||
}
|
||||
|
||||
var contextJSON json.RawMessage
|
||||
if row.Context != "" {
|
||||
if json.Valid([]byte(row.Context)) {
|
||||
contextJSON = json.RawMessage(row.Context)
|
||||
} else {
|
||||
b, _ := json.Marshal(row.Context)
|
||||
contextJSON = b
|
||||
}
|
||||
}
|
||||
|
||||
var occurredAt int64
|
||||
if row.OccurredAt != nil {
|
||||
occurredAt = row.OccurredAt.UnixMilli()
|
||||
}
|
||||
|
||||
return &types.GetLogMessageRawResponse{
|
||||
Id: row.Id,
|
||||
Platform: row.Platform,
|
||||
AppVersion: row.AppVersion,
|
||||
OsName: row.OsName,
|
||||
OsVersion: row.OsVersion,
|
||||
DeviceId: row.DeviceId,
|
||||
UserId: row.UserId,
|
||||
SessionId: row.SessionId,
|
||||
Level: row.Level,
|
||||
ErrorCode: row.ErrorCode,
|
||||
Message: row.Message,
|
||||
Stack: row.Stack,
|
||||
Context: contextJSON,
|
||||
ClientIP: row.ClientIP,
|
||||
UserAgent: row.UserAgent,
|
||||
Locale: row.Locale,
|
||||
Digest: row.Digest,
|
||||
OccurredAt: occurredAt,
|
||||
CreatedAt: row.CreatedAt.UnixMilli(),
|
||||
}, nil
|
||||
}
|
||||
@@ -3,6 +3,8 @@
|
||||
|
||||
package types
|
||||
|
||||
import "encoding/json"
|
||||
|
||||
type ActivateOrderRequest struct {
|
||||
OrderNo string `json:"order_no" validate:"required"`
|
||||
}
|
||||
@@ -3641,3 +3643,29 @@ type GetAdminUserInviteListResponse struct {
|
||||
Total int64 `json:"total"`
|
||||
List []AdminInvitedUser `json:"list"`
|
||||
}
|
||||
|
||||
type GetLogMessageRawRequest struct {
|
||||
Id int64 `form:"id" validate:"required"`
|
||||
}
|
||||
|
||||
type GetLogMessageRawResponse struct {
|
||||
Id int64 `json:"id"`
|
||||
Platform string `json:"platform"`
|
||||
AppVersion string `json:"app_version"`
|
||||
OsName string `json:"os_name"`
|
||||
OsVersion string `json:"os_version"`
|
||||
DeviceId string `json:"device_id"`
|
||||
UserId *int64 `json:"user_id"`
|
||||
SessionId string `json:"session_id"`
|
||||
Level uint8 `json:"level"`
|
||||
ErrorCode string `json:"error_code"`
|
||||
Message string `json:"message"`
|
||||
Stack string `json:"stack"`
|
||||
Context json.RawMessage `json:"context"`
|
||||
ClientIP string `json:"client_ip"`
|
||||
UserAgent string `json:"user_agent"`
|
||||
Locale string `json:"locale"`
|
||||
Digest string `json:"digest"`
|
||||
OccurredAt int64 `json:"occurred_at"`
|
||||
CreatedAt int64 `json:"created_at"`
|
||||
}
|
||||
|
||||
@@ -48,7 +48,4 @@ func RegisterHandlers(mux *asynq.ServeMux, serverCtx *svc.ServiceContext) {
|
||||
// Apple IAP 对账(第二层:5min 扫描 + 第三层:日终全量)
|
||||
mux.Handle(types.SchedulerIAPReconcile, iapLogic.NewReconcileLogic(serverCtx))
|
||||
mux.Handle(types.SchedulerIAPDailyReconcile, iapLogic.NewDailyReconcileLogic(serverCtx))
|
||||
|
||||
// Stuck order recovery
|
||||
mux.Handle(types.SchedulerStuckOrderRecovery, orderLogic.NewStuckOrderRecoveryLogic(serverCtx))
|
||||
}
|
||||
|
||||
@@ -7,7 +7,6 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/perfect-panel/server/internal/logic/admin/group"
|
||||
@@ -47,7 +46,7 @@ const (
|
||||
OrderStatusPaid = 2 // Order paid and ready for processing
|
||||
OrderStatusClose = 3 // Order closed/cancelled
|
||||
OrderStatusFailed = 4 // Order processing failed
|
||||
OrderStatusClaimed = 6 // Internal transient claim while a worker processes the order
|
||||
OrderStatusClaimed = 4 // Internal transient claim while a worker processes the order
|
||||
OrderStatusFinished = 5 // Order successfully completed
|
||||
)
|
||||
|
||||
@@ -93,12 +92,8 @@ func (l *ActivateOrderLogic) ProcessTask(ctx context.Context, task *asynq.Task)
|
||||
|
||||
orderInfo, err := l.claimAndGetOrder(ctx, payload.OrderNo)
|
||||
if err != nil {
|
||||
// 如果订单不存在或状态不对,不重试
|
||||
if errors.Is(err, ErrInvalidOrderStatus) {
|
||||
if strings.Contains(err.Error(), "stuck in claimed") {
|
||||
logger.WithContext(ctx).Error("[ActivateOrderLogic] 订单卡在 claimed,将重试",
|
||||
logger.Field("order_no", payload.OrderNo))
|
||||
return err // 返回错误触发 asynq 重试
|
||||
}
|
||||
logger.WithContext(ctx).Info("[ActivateOrderLogic] 订单状态不是已支付,跳过",
|
||||
logger.Field("order_no", payload.OrderNo))
|
||||
return nil
|
||||
@@ -121,12 +116,7 @@ func (l *ActivateOrderLogic) ProcessTask(ctx context.Context, task *asynq.Task)
|
||||
)
|
||||
|
||||
if err = l.processOrderByType(ctx, orderInfo, payload.IAPExpireAt); err != nil {
|
||||
if releaseErr := l.releaseClaim(ctx, orderInfo.OrderNo); releaseErr != nil {
|
||||
logger.WithContext(ctx).Error("[ActivateOrderLogic] releaseClaim also failed, stuck recovery will handle",
|
||||
logger.Field("order_no", orderInfo.OrderNo),
|
||||
logger.Field("release_error", releaseErr.Error()),
|
||||
)
|
||||
}
|
||||
l.releaseClaim(ctx, orderInfo.OrderNo)
|
||||
logger.WithContext(ctx).Error("[ActivateOrderLogic] 处理订单失败,将重试",
|
||||
logger.Field("order_no", orderInfo.OrderNo),
|
||||
logger.Field("order_type", orderInfo.Type),
|
||||
@@ -135,12 +125,7 @@ func (l *ActivateOrderLogic) ProcessTask(ctx context.Context, task *asynq.Task)
|
||||
}
|
||||
|
||||
if err = l.reconcilePostOrderSubscriptions(ctx, orderInfo); err != nil {
|
||||
if releaseErr := l.releaseClaim(ctx, orderInfo.OrderNo); releaseErr != nil {
|
||||
logger.WithContext(ctx).Error("[ActivateOrderLogic] releaseClaim also failed, stuck recovery will handle",
|
||||
logger.Field("order_no", orderInfo.OrderNo),
|
||||
logger.Field("release_error", releaseErr.Error()),
|
||||
)
|
||||
}
|
||||
l.releaseClaim(ctx, orderInfo.OrderNo)
|
||||
logger.WithContext(ctx).Error("[ActivateOrderLogic] 订单订阅兜底合并失败,将重试",
|
||||
logger.Field("order_no", orderInfo.OrderNo),
|
||||
logger.Field("order_type", orderInfo.Type),
|
||||
@@ -191,15 +176,6 @@ func (l *ActivateOrderLogic) claimAndGetOrder(ctx context.Context, orderNo strin
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// Detect stuck claimed order — return retryable error so asynq re-tries
|
||||
if orderInfo.Status == OrderStatusClaimed {
|
||||
logger.WithContext(ctx).Error("Order stuck in claimed status",
|
||||
logger.Field("order_no", orderInfo.OrderNo),
|
||||
logger.Field("status", orderInfo.Status),
|
||||
)
|
||||
return nil, fmt.Errorf("order %s stuck in claimed status: %w", orderNo, ErrInvalidOrderStatus)
|
||||
}
|
||||
|
||||
if orderInfo.Status != OrderStatusPaid {
|
||||
logger.WithContext(ctx).Error("Order status error",
|
||||
logger.Field("order_no", orderInfo.OrderNo),
|
||||
@@ -225,7 +201,7 @@ func (l *ActivateOrderLogic) claimAndGetOrder(ctx context.Context, orderNo strin
|
||||
return &orderInfo, nil
|
||||
}
|
||||
|
||||
func (l *ActivateOrderLogic) releaseClaim(ctx context.Context, orderNo string) error {
|
||||
func (l *ActivateOrderLogic) releaseClaim(ctx context.Context, orderNo string) {
|
||||
if err := l.svc.DB.WithContext(ctx).
|
||||
Model(&order.Order{}).
|
||||
Where("order_no = ? AND status = ?", orderNo, OrderStatusClaimed).
|
||||
@@ -234,9 +210,7 @@ func (l *ActivateOrderLogic) releaseClaim(ctx context.Context, orderNo string) e
|
||||
logger.Field("error", err.Error()),
|
||||
logger.Field("order_no", orderNo),
|
||||
)
|
||||
return fmt.Errorf("release claim failed for order %s: %w", orderNo, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// processOrderByType routes order processing based on the order type
|
||||
@@ -589,27 +563,14 @@ func (l *ActivateOrderLogic) finalizeCouponAndOrder(ctx context.Context, orderIn
|
||||
}
|
||||
}
|
||||
|
||||
// UpdateOrderStatus uses WHERE status < target, which blocks claimed(6)→finished(5).
|
||||
// Use a direct update matching the exact claimed status, then update the full record
|
||||
// via the model layer to properly invalidate the cache.
|
||||
result := l.svc.DB.WithContext(ctx).
|
||||
Model(&order.Order{}).
|
||||
Where("order_no = ? AND status = ?", orderInfo.OrderNo, OrderStatusClaimed).
|
||||
Update("status", OrderStatusFinished)
|
||||
if result.Error != nil {
|
||||
logger.WithContext(ctx).Error("Update order status from claimed to finished failed",
|
||||
logger.Field("error", result.Error.Error()),
|
||||
logger.Field("order_no", orderInfo.OrderNo),
|
||||
)
|
||||
}
|
||||
// Invalidate order cache regardless of whether the DB update succeeded
|
||||
orderInfo.Status = OrderStatusFinished
|
||||
if err := l.svc.OrderModel.Update(ctx, orderInfo); err != nil {
|
||||
logger.WithContext(ctx).Error("Update order cache after finalization failed",
|
||||
// Update order status using state-guarded UpdateOrderStatus to prevent double finalization
|
||||
if err := l.svc.OrderModel.UpdateOrderStatus(ctx, orderInfo.OrderNo, OrderStatusFinished); err != nil {
|
||||
logger.WithContext(ctx).Error("Update order status failed",
|
||||
logger.Field("error", err.Error()),
|
||||
logger.Field("order_no", orderInfo.OrderNo),
|
||||
)
|
||||
}
|
||||
orderInfo.Status = OrderStatusFinished
|
||||
commonLogic.SubscriptionTraceInfo(logger.WithContext(ctx), commonLogic.SubscriptionTraceFlowOrder, "order_status_finished",
|
||||
"[SubscriptionFlow] order status updated to finished",
|
||||
commonLogic.OrderTraceFields(orderInfo)...,
|
||||
|
||||
@@ -1,104 +0,0 @@
|
||||
package orderLogic
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"time"
|
||||
|
||||
"github.com/hibiken/asynq"
|
||||
"github.com/perfect-panel/server/internal/model/order"
|
||||
"github.com/perfect-panel/server/internal/svc"
|
||||
"github.com/perfect-panel/server/pkg/logger"
|
||||
queueTypes "github.com/perfect-panel/server/queue/types"
|
||||
)
|
||||
|
||||
// StuckOrderRecoveryLogic scans orders stuck in claimed status and re-queues them for processing.
|
||||
type StuckOrderRecoveryLogic struct {
|
||||
svc *svc.ServiceContext
|
||||
}
|
||||
|
||||
func NewStuckOrderRecoveryLogic(svc *svc.ServiceContext) *StuckOrderRecoveryLogic {
|
||||
return &StuckOrderRecoveryLogic{svc: svc}
|
||||
}
|
||||
|
||||
// ProcessTask scans for orders stuck in claimed status for over 10 minutes,
|
||||
// resets them to paid, and re-enqueues the activate task so they are retried
|
||||
// independently of asynq's original retry counter (which may be exhausted).
|
||||
func (l *StuckOrderRecoveryLogic) ProcessTask(ctx context.Context, _ *asynq.Task) error {
|
||||
cutoff := time.Now().Add(-10 * time.Minute)
|
||||
|
||||
var stuckOrders []order.Order
|
||||
if err := l.svc.DB.WithContext(ctx).
|
||||
Model(&order.Order{}).
|
||||
Where("status = ? AND updated_at < ?", OrderStatusClaimed, cutoff).
|
||||
Find(&stuckOrders).Error; err != nil {
|
||||
logger.WithContext(ctx).Error("[StuckOrderRecovery] Failed to query stuck orders",
|
||||
logger.Field("error", err.Error()),
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
||||
if len(stuckOrders) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
orderNos := make([]string, 0, len(stuckOrders))
|
||||
for i := range stuckOrders {
|
||||
orderNos = append(orderNos, stuckOrders[i].OrderNo)
|
||||
}
|
||||
logger.WithContext(ctx).Error("[StuckOrderRecovery] Found stuck claimed orders, recovering",
|
||||
logger.Field("count", len(stuckOrders)),
|
||||
logger.Field("order_nos", orderNos),
|
||||
)
|
||||
|
||||
for i := range stuckOrders {
|
||||
o := &stuckOrders[i]
|
||||
|
||||
result := l.svc.DB.WithContext(ctx).
|
||||
Model(&order.Order{}).
|
||||
Where("order_no = ? AND status = ?", o.OrderNo, OrderStatusClaimed).
|
||||
Update("status", OrderStatusPaid)
|
||||
if result.Error != nil {
|
||||
logger.WithContext(ctx).Error("[StuckOrderRecovery] Failed to reset order status",
|
||||
logger.Field("order_no", o.OrderNo),
|
||||
logger.Field("error", result.Error.Error()),
|
||||
)
|
||||
continue
|
||||
}
|
||||
if result.RowsAffected == 0 {
|
||||
// Another process already handled this order
|
||||
continue
|
||||
}
|
||||
|
||||
// Invalidate order cache
|
||||
o.Status = OrderStatusPaid
|
||||
if err := l.svc.OrderModel.Update(ctx, o); err != nil {
|
||||
logger.WithContext(ctx).Error("[StuckOrderRecovery] Failed to update order cache",
|
||||
logger.Field("order_no", o.OrderNo),
|
||||
logger.Field("error", err.Error()),
|
||||
)
|
||||
}
|
||||
|
||||
// Re-enqueue activate task so the order gets processed regardless of asynq retry state
|
||||
payload, err := json.Marshal(queueTypes.ForthwithActivateOrderPayload{OrderNo: o.OrderNo})
|
||||
if err != nil {
|
||||
logger.WithContext(ctx).Error("[StuckOrderRecovery] Failed to marshal task payload",
|
||||
logger.Field("order_no", o.OrderNo),
|
||||
logger.Field("error", err.Error()),
|
||||
)
|
||||
continue
|
||||
}
|
||||
if _, err = l.svc.Queue.EnqueueContext(ctx, asynq.NewTask(queueTypes.ForthwithActivateOrder, payload)); err != nil {
|
||||
logger.WithContext(ctx).Error("[StuckOrderRecovery] Failed to re-enqueue activate task",
|
||||
logger.Field("order_no", o.OrderNo),
|
||||
logger.Field("error", err.Error()),
|
||||
)
|
||||
} else {
|
||||
logger.WithContext(ctx).Info("[StuckOrderRecovery] Re-enqueued activate task",
|
||||
logger.Field("order_no", o.OrderNo),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -5,7 +5,6 @@ const (
|
||||
SchedulerTotalServerData = "scheduler:total:server"
|
||||
SchedulerResetTraffic = "scheduler:reset:traffic"
|
||||
SchedulerTrafficStat = "scheduler:traffic:stat"
|
||||
SchedulerIAPReconcile = "scheduler:iap:reconcile" // 第二层:每 5 分钟扫描待支付 IAP 订单
|
||||
SchedulerIAPDailyReconcile = "scheduler:iap:daily:reconcile" // 第三层:日终全量对账
|
||||
SchedulerStuckOrderRecovery = "scheduler:stuck:order:recovery" // 扫描并恢复超时 claimed 订单
|
||||
SchedulerIAPReconcile = "scheduler:iap:reconcile" // 第二层:每 5 分钟扫描待支付 IAP 订单
|
||||
SchedulerIAPDailyReconcile = "scheduler:iap:daily:reconcile" // 第三层:日终全量对账
|
||||
)
|
||||
|
||||
@@ -64,12 +64,6 @@ func (m *Service) Start() {
|
||||
logger.Errorf("register iap daily reconcile task failed: %s", err.Error())
|
||||
}
|
||||
|
||||
// schedule stuck order recovery: every 10 minutes
|
||||
stuckOrderTask := asynq.NewTask(types.SchedulerStuckOrderRecovery, nil)
|
||||
if _, err := m.server.Register("@every 10m", stuckOrderTask, asynq.MaxRetry(1)); err != nil {
|
||||
logger.Errorf("register stuck order recovery task failed: %s", err.Error())
|
||||
}
|
||||
|
||||
if err := m.server.Run(); err != nil {
|
||||
logger.Errorf("run scheduler failed: %s", err.Error())
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user