* refactor: rename queryannouncementhandler.go to queryAnnouncementLogic.go for clarity

* feat(panDomain): update subscription logic to use V2 handler for improved functionality

* refactor(subscribe): replace V2 handler with a unified Handler method for subscription logic

* feat(subscribe): implement user agent limit feature with configurable list

* fix(subscribe): improve error handling and logging for subscription requests

* feat(subscribe): add user agent limit configuration to system settings

* refactor(api): remove deprecated application-related endpoints and types

* refactor(swagger): remove deprecated app.json generation from swagger configuration

* refactor(swagger): remove deprecated app.json check from swagger configuration

* fix(subscribe): update delete method to use Where clause for improved query accuracy

* fix(subscribe): update Id field tag to use primaryKey and improve save method query

* fix(subscribe): update Id field tag to use primaryKey and improve model queries

* fix(subscribe): rename variable for clarity and add special handling for Stash user agent

* fix(email): convert RegisterStartTime and RegisterEndTime to time.Time for accurate query filtering

* refactor(log): consolidate logging models and update related logic for improved clarity and functionality

* fix(types): change Content field type in MessageLog to interface{} for improved flexibility

* fix(log): change MessageLog list to use value type for improved performance and memory efficiency

* fix(email): set EmailTypeVerify in task payload and update content type conversion for verification email

* fix(log): remove unused Id field from SystemLog during login log insertion

* fix(login): remove debug logs and error logging during user login process

* fix(log): add traffic reset logging for subscription resets

* fix(log): insert reset traffic log during subscription activation

* feat(log): add endpoints for retrieving and resetting subscribe traffic logs

* refactor(log): remove Reset Subscribe Traffic Log endpoint and related types

* feat(traffic): add traffic statistics logging and scheduling

* fix(subscribe): ensure active status and reset timestamps during traffic resets

* feat(api): enhance server and node management with new request/response structures

* refactor(api): rename OnlineUser to ServerOnlineUser for clarity

* feat(api): define OnlineUser type with SID and IP fields

* feat(server): implement server management handlers and database schema

* feat(api): add traffic log details filtering and enhance traffic log structures

* feat(api): migrate server and node data handling, update related structures and logic

* feat(server): implement server deletion logic with error handling

* feat(api): update log filtering to use ResetSubscribe type for subscription logs

* feat(api): standardize timestamp field across log structures

* feat(api): refactor cache key handling for server and user lists

* feat(api): enhance server status handling with protocol support and refactor related logic

* fix(traffic): adjust start date for traffic statistics and improve log deletion comment

* feat(api): implement daily traffic ranking for users and servers with error handling

* feat(api): update server total data response to use 'OnlineUsers' and implement daily traffic statistics logging

* feat(api): add log settings management with auto-clear and clear days configuration

* fix(log): correct category in log settings update query

* feat(routes): add handler for scheduled traffic statistics

* feat(model): add user counts struct and update queries for new and renewal users

* feat(api): add referral percentage and only first purchase fields to user model and requests

* feat(database): update user table to add referral percentage and only first purchase fields

* feat(api): add reset sort endpoints for server and node

* feat(api): add sort field to server model

* feat(api): implement sorting functionality for nodes and servers

* fix(database): add sort column to nodes table

* fix(model): enhance user statistics queries with new order and renewal order counts

* fix(log): update timestamp handling in login and registration logs

* fix(log): update sorting logic for server and user subscribe traffic logs

* fix(server): add server status handling based on last reported time

* fix(model): correct filter condition to use 'date' instead of 'data'

* fix(migration): add index for traffic log on timestamp, user_id, and subscribe_id

* fix(log): optimize user traffic rank data handling by using append instead of index assignment

* fix(filter): refactor node list creation to use append and remove duplicates from tags

* fix(node): add ServerId and Enabled fields to node update logic

* feat(tags): add endpoint to query all node tags

* fix(preview): add Preload parameter to FilterNodeList for improved data retrieval

* fix(log): date is empty

* feat(subscribe): add Language field to subscription models and update query logic

* feat(subscription): add Language parameter to GetSubscription request and update query logic

* fix(server): encode ServerKey in base64 and update last reported time for nodes

* feat: delete common GetSubscription

* feat(subscription): implement FilterList method for subscription queries and update related logic

* fix(subscribe): remove duplicate user agents in SubscribeHandler

* fix(push): initialize onlineUsers as a map in pushOnlineUsersLogic

* fix(reset): initialize subs as a map in clearCache method

* refactor(query): simplify node and tag filtering using InSet function

* feat(userlist): enhance GetServerUserListLogic with improved node and tag handling

* fix(userlist): correct node ID assignment and update query logic for tag filtering

* fix(userlist): correct node ID assignment in getServerUserListLogic

* refactor(query): streamline query construction for tag filtering

* fix(statistics): optimize server ranking data handling in QueryServerTotalDataLogic

* refactor(statistics): simplify server ranking data construction in QueryServerTotalDataLogic

* fix(statistics): correct server traffic data assignment in QueryServerTotalDataLogic

* fix(statistics): optimize yesterday's top 10 server traffic data assignment in QueryServerTotalDataLogic

* fix(middleware): remove duplicate elements from user agent list in PanDomainMiddleware

* feat(middleware): enhance user agent handling by querying client list in PanDomainMiddleware

* feat(client): subscribe_template

* feat(oauth): add user agent and IP logging to registration and login processes

* fix(balance): add timestamp to balance logs for payment, refund, and recharge transactions

* fix(log): correct comment for CommissionTypeRefund to improve clarity

* fix(log): replace magic number with constant for gift type in purchase checkout logic

* fix(log): rename OrderId to OrderNo for consistency in balance logging

* feat(log): add logging for balance, gift amount, and commission adjustments

* fix(user): correct placement of DeepCopy for user info update logic

* feat(log): add UserSubscribeId to FilterSubscribeLogRequest for enhanced filtering

* fix(purchase): streamline error handling and improve JSON marshaling for temporary orders

* fix(order): simplify commission handling and improve payload parsing logic

* fix(order): update commission calculation to actual payment amount minus gateway handling fee

* feat(payment): add support for CryptoSaaS payment platform and enhance configuration handling

* fix(balance): update QueryUserBalanceLog response structure to include balance log list

* fix(email): update task progress handling to use specific task ID for updates

* feat(quota): add quota task creation and querying endpoints with updated data structures

* fix(email): update task handling to use generic task model and improve error logging

* fix(order): improve error logging for database transaction and user cache updates

* feat(quota): enhance quota task management with new request structures and processing logic

* fix(quota): remove redundant quota task status endpoint from admin marketing routes

* fix(worker): update task completion status handling in worker logic

* fix(quota): update taskInfo to include current subscription count in quota logic

* doc(log): rename function for clarity and add cache cleanup comment

* fix(quota): update time handling in quota logic and correct subscriber ID query

* fix(quota): update time handling to use UnixMilli for start time in quota logic

* feat(protocol): add server protocol configuration query and enhance protocol options

* fix(quota): correct time range queries for start and expire times in quota logic

* fix(types): update plugin options to include 'none' in the plugin field

---------

Co-authored-by: Chang lue Tsen <tension@ppanel.dev>
This commit is contained in:
Leif Draven
2025-09-14 22:50:22 +09:00
committed by GitHub
parent b5e244d83d
commit e895180388
359 changed files with 9213 additions and 12125 deletions
@@ -6,12 +6,15 @@ import (
"strings"
"time"
"github.com/perfect-panel/server/pkg/xerr"
"github.com/perfect-panel/server/internal/model/log"
"github.com/perfect-panel/server/internal/model/node"
"github.com/perfect-panel/server/internal/model/traffic"
"github.com/perfect-panel/server/internal/svc"
"github.com/perfect-panel/server/internal/types"
"github.com/perfect-panel/server/pkg/logger"
"github.com/perfect-panel/server/pkg/xerr"
"github.com/pkg/errors"
"gorm.io/gorm"
)
type QueryServerTotalDataLogic struct {
@@ -35,127 +38,194 @@ func (l *QueryServerTotalDataLogic) QueryServerTotalData() (resp *types.ServerTo
return l.mockRevenueStatistics(), nil
}
resp = &types.ServerTotalDataResponse{
ServerTrafficRankingToday: make([]types.ServerTrafficData, 0),
ServerTrafficRankingYesterday: make([]types.ServerTrafficData, 0),
UserTrafficRankingToday: make([]types.UserTrafficData, 0),
UserTrafficRankingYesterday: make([]types.UserTrafficData, 0),
now := time.Now()
todayStart := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, now.Location())
todayEnd := todayStart.Add(24 * time.Hour).Add(-time.Second)
query := l.svcCtx.DB.WithContext(l.ctx)
var todayTop10User []log.UserTraffic
err = query.Model(&traffic.TrafficLog{}).
Select("user_id, subscribe_id, SUM(download + upload) AS total, SUM(download) AS download, SUM(upload) AS upload").
Where("timestamp BETWEEN ? AND ?", todayStart, todayEnd).
Group("user_id, subscribe_id").
Order("total DESC").
Limit(10).
Scan(&todayTop10User).Error
if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
logger.Errorf("[Traffic Stat Queue] Query user traffic failed: %v", err.Error())
return nil, errors.Wrapf(xerr.NewErrCode(xerr.DatabaseQueryError), " Query user traffic failed: %v", err.Error())
}
var userTodayTrafficRanking []types.UserTrafficData
for _, item := range todayTop10User {
userTodayTrafficRanking = append(userTodayTrafficRanking, types.UserTrafficData{
SID: item.SubscribeId,
Upload: item.Upload,
Download: item.Download,
})
}
// Query node server status
servers, err := l.svcCtx.ServerModel.FindAllServer(l.ctx)
if err != nil {
l.Errorw("[QueryServerTotalDataLogic] FindAllServer error", logger.Field("error", err.Error()))
return nil, errors.Wrapf(err, "FindAllServer error: %v", err)
}
onlineServers, err := l.svcCtx.NodeCache.GetOnlineNodeStatusCount(l.ctx)
if err != nil {
l.Errorw("[QueryServerTotalDataLogic] GetOnlineNodeStatusCount error", logger.Field("error", err.Error()))
return nil, errors.Wrapf(err, "GetOnlineNodeStatusCount error: %v", err)
}
resp.OnlineServers = onlineServers
resp.OfflineServers = int64(len(servers) - int(onlineServers))
// query yesterday user traffic rank log
yesterday := todayStart.Add(-24 * time.Hour).Format(time.DateOnly)
// 获取所有节点在线用户
allNodeOnlineUser, err := l.svcCtx.NodeCache.GetAllNodeOnlineUser(l.ctx)
if err != nil {
l.Errorw("[QueryServerTotalDataLogic] Get all node online user failed", logger.Field("error", err.Error()))
var yesterdayLog log.SystemLog
err = query.Model(&log.SystemLog{}).Where("`date` = ? AND `type` = ?", yesterday, log.TypeUserTrafficRank).First(&yesterdayLog).Error
if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
l.Errorw("[QueryServerTotalDataLogic] Query yesterday user traffic rank log error", logger.Field("error", err.Error()))
return nil, errors.Wrapf(xerr.NewErrCode(xerr.DatabaseQueryError), "Query yesterday user traffic rank log error: %v", err)
}
resp.OnlineUserIPs = int64(len(allNodeOnlineUser))
// 获取所有节点今日上传下载流量
allNodeUploadTraffic, err := l.svcCtx.NodeCache.GetAllNodeUploadTraffic(l.ctx)
if err != nil {
l.Errorw("[QueryServerTotalDataLogic] Get all node upload traffic failed", logger.Field("error", err.Error()))
var yesterdayUserRankData []types.UserTrafficData
if yesterdayLog.Id > 0 {
var rank log.UserTrafficRank
err = rank.Unmarshal([]byte(yesterdayLog.Content))
if err != nil {
l.Errorw("[QueryServerTotalDataLogic] Unmarshal yesterday user traffic rank log error", logger.Field("error", err.Error()))
}
for _, v := range rank.Rank {
yesterdayUserRankData = append(yesterdayUserRankData, types.UserTrafficData{
SID: v.SubscribeId,
Upload: v.Upload,
Download: v.Download,
})
}
}
resp.TodayUpload = allNodeUploadTraffic
allNodeDownloadTraffic, err := l.svcCtx.NodeCache.GetAllNodeDownloadTraffic(l.ctx)
if err != nil {
l.Errorw("[QueryServerTotalDataLogic] Get all node download traffic failed", logger.Field("error", err.Error()))
// query server traffic rank today
var todayTop10Server []log.ServerTraffic
err = query.Model(&traffic.TrafficLog{}).Select("server_id, SUM(download + upload) AS total, SUM(download) AS download, SUM(upload) AS upload").
Where("timestamp BETWEEN ? AND ?", todayStart, todayEnd).
Group("server_id").
Order("total DESC").
Limit(10).
Scan(&todayTop10Server).Error
if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
logger.Errorf("[Traffic Stat Queue] Query server traffic failed: %v", err.Error())
return nil, errors.Wrapf(xerr.NewErrCode(xerr.DatabaseQueryError), " Query server traffic failed: %v", err.Error())
}
resp.TodayDownload = allNodeDownloadTraffic
// 获取节点流量排行榜 前10
nodeTrafficRankingToday, err := l.svcCtx.NodeCache.GetNodeTodayTotalTrafficRank(l.ctx, 10)
if err != nil {
l.Errorw("[QueryServerTotalDataLogic] Get node today total traffic rank failed", logger.Field("error", err.Error()))
var todayServerRanking []types.ServerTrafficData
for _, item := range todayTop10Server {
info, err := l.svcCtx.NodeModel.FindOneServer(l.ctx, item.ServerId)
if err != nil {
l.Errorw("[QueryServerTotalDataLogic] FindOneServer error", logger.Field("error", err.Error()), logger.Field("server_id", item.ServerId))
continue
}
todayServerRanking = append(todayServerRanking, types.ServerTrafficData{
ServerId: item.ServerId,
Name: info.Name,
Upload: item.Upload,
Download: item.Download,
})
}
if len(nodeTrafficRankingToday) > 0 {
var serverTrafficData []types.ServerTrafficData
for _, rank := range nodeTrafficRankingToday {
serverInfo, err := l.svcCtx.ServerModel.FindOne(l.ctx, rank.ID)
// query server traffic rank yesterday
var yesterdayTop10Server []types.ServerTrafficData
var yesterdayServerTrafficLog log.SystemLog
err = query.Model(&log.SystemLog{}).Where("`date` = ? AND `type` = ?", yesterday, log.TypeServerTrafficRank).First(&yesterdayServerTrafficLog).Error
if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
l.Errorw("[QueryServerTotalDataLogic] Query yesterday server traffic rank log error", logger.Field("error", err.Error()))
return nil, errors.Wrapf(xerr.NewErrCode(xerr.DatabaseQueryError), "Query yesterday server traffic rank log error: %v", err)
}
if yesterdayServerTrafficLog.Id > 0 {
var rank log.ServerTrafficRank
err = rank.Unmarshal([]byte(yesterdayServerTrafficLog.Content))
if err != nil {
l.Errorw("[QueryServerTotalDataLogic] Unmarshal yesterday server traffic rank log error", logger.Field("error", err.Error()))
}
for _, v := range rank.Rank {
info, err := l.svcCtx.NodeModel.FindOneServer(l.ctx, v.ServerId)
if err != nil {
l.Errorw("[QueryServerTotalDataLogic] FindOne error", logger.Field("error", err))
l.Errorw("[QueryServerTotalDataLogic] FindOneServer error", logger.Field("error", err.Error()), logger.Field("server_id", v.ServerId))
continue
}
serverTrafficData = append(serverTrafficData, types.ServerTrafficData{
ServerId: rank.ID,
Name: serverInfo.Name,
Upload: rank.Upload,
Download: rank.Download,
yesterdayTop10Server = append(yesterdayTop10Server, types.ServerTrafficData{
ServerId: v.ServerId,
Name: info.Name,
Upload: v.Upload,
Download: v.Download,
})
}
resp.ServerTrafficRankingToday = serverTrafficData
}
// 获取用户流量排行榜 前10
userTrafficRankingToday, err := l.svcCtx.NodeCache.GetUserTodayTotalTrafficRank(l.ctx, 10)
if err != nil {
l.Errorw("[QueryServerTotalDataLogic] Get user today total traffic rank failed", logger.Field("error", err.Error()))
}
if len(userTrafficRankingToday) > 0 {
var userTrafficData []types.UserTrafficData
for _, rank := range userTrafficRankingToday {
userTrafficData = append(userTrafficData, types.UserTrafficData{
SID: rank.SID,
Upload: rank.Upload,
Download: rank.Download,
})
}
resp.UserTrafficRankingToday = userTrafficData
}
// 获取昨日节点流量排行榜 前10
nodeTrafficRankingYesterday, err := l.svcCtx.NodeCache.GetYesterdayNodeTotalTrafficRank(l.ctx)
// query online user count
onlineUsers, err := l.svcCtx.NodeModel.OnlineUserSubscribeGlobal(l.ctx)
if err != nil {
l.Errorw("[QueryServerTotalDataLogic] Get yesterday node total traffic rank failed", logger.Field("error", err.Error()))
}
if len(nodeTrafficRankingYesterday) > 0 {
var serverTrafficData []types.ServerTrafficData
for _, rank := range nodeTrafficRankingYesterday {
serverTrafficData = append(serverTrafficData, types.ServerTrafficData{
ServerId: rank.ID,
Name: rank.Name,
Upload: rank.Upload,
Download: rank.Download,
})
}
resp.ServerTrafficRankingYesterday = serverTrafficData
}
// 获取昨日用户流量排行榜 前10
userTrafficRankingYesterday, err := l.svcCtx.NodeCache.GetYesterdayUserTotalTrafficRank(l.ctx)
if err != nil {
l.Errorw("[QueryServerTotalDataLogic] Get yesterday user total traffic rank failed", logger.Field("error", err.Error()))
}
if len(userTrafficRankingYesterday) > 0 {
var userTrafficData []types.UserTrafficData
for _, rank := range userTrafficRankingYesterday {
userTrafficData = append(userTrafficData, types.UserTrafficData{
SID: rank.SID,
Upload: rank.Upload,
Download: rank.Download,
})
}
resp.UserTrafficRankingYesterday = userTrafficData
l.Errorw("[QueryServerTotalDataLogic] OnlineUserSubscribeGlobal error", logger.Field("error", err.Error()))
return nil, errors.Wrapf(xerr.NewErrCode(xerr.DatabaseQueryError), "OnlineUserSubscribeGlobal error: %v", err)
}
// Query node traffic by monthly
nodeTraffic, err := l.svcCtx.TrafficLogModel.QueryTrafficByMonthly(l.ctx, time.Now())
if err != nil {
l.Errorw("[QueryServerTotalDataLogic] QueryTrafficByMonthly error", logger.Field("error", err.Error()))
return nil, errors.Wrapf(xerr.NewErrCode(xerr.DatabaseQueryError), "QueryTrafficByMonthly error: %v", err.Error())
// query online/offline server count
var onlineServers, offlineServers int64
err = query.Model(&node.Server{}).Where("`last_reported_at` > ?", now.Add(-5*time.Minute)).Count(&onlineServers).Error
if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
l.Errorw("[QueryServerTotalDataLogic] Count online servers error", logger.Field("error", err.Error()))
return nil, errors.Wrapf(xerr.NewErrCode(xerr.DatabaseQueryError), "Count online servers error: %v", err)
}
err = query.Model(&node.Server{}).Where("`last_reported_at` <= ? OR `last_reported_at` IS NULL", now.Add(-5*time.Minute)).Count(&offlineServers).Error
if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
l.Errorw("[QueryServerTotalDataLogic] Count offline servers error", logger.Field("error", err.Error()))
return nil, errors.Wrapf(xerr.NewErrCode(xerr.DatabaseQueryError), "Count offline servers error: %v", err)
}
// TodayUpload, TodayDownload, MonthlyUpload, MonthlyDownload
var todayUpload, todayDownload, monthlyUpload, monthlyDownload int64
type trafficSum struct {
Upload int64
Download int64
}
var todayTraffic trafficSum
// Today
err = query.Model(&traffic.TrafficLog{}).Select("SUM(upload) AS upload, SUM(download) AS download").
Where("timestamp BETWEEN ? AND ?", todayStart, todayEnd).
Scan(&todayTraffic).Error
if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
l.Errorw("[QueryServerTotalDataLogic] Sum today traffic error", logger.Field("error", err.Error()))
return nil, errors.Wrapf(xerr.NewErrCode(xerr.DatabaseQueryError), "Sum today traffic error: %v", err)
}
todayUpload = todayTraffic.Upload
todayDownload = todayTraffic.Download
// Monthly
monthlyUpload += todayUpload
monthlyDownload += todayDownload
for i := now.Day() - 1; i >= 1; i-- {
var logInfo log.SystemLog
date := time.Date(now.Year(), now.Month(), i, 0, 0, 0, 0, now.Location()).Format(time.DateOnly)
err = query.Model(&log.SystemLog{}).Where("`date` = ? AND `type` = ?", date, log.TypeTrafficStat).First(&logInfo).Error
if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
l.Errorw("[QueryServerTotalDataLogic] Query daily traffic stat log error", logger.Field("error", err.Error()), logger.Field("date", date))
return nil, errors.Wrapf(xerr.NewErrCode(xerr.DatabaseQueryError), "Query daily traffic stat log error: %v", err)
}
if logInfo.Id > 0 {
var stat log.TrafficStat
err = stat.Unmarshal([]byte(logInfo.Content))
if err != nil {
l.Errorw("[QueryServerTotalDataLogic] Unmarshal daily traffic stat log error", logger.Field("error", err.Error()), logger.Field("date", date))
continue
}
monthlyUpload += stat.Upload
monthlyDownload += stat.Download
}
}
resp = &types.ServerTotalDataResponse{
OnlineUsers: onlineUsers,
OnlineServers: onlineServers,
OfflineServers: offlineServers,
TodayUpload: todayUpload,
TodayDownload: todayDownload,
MonthlyUpload: monthlyUpload,
MonthlyDownload: monthlyDownload,
UpdatedAt: now.Unix(),
ServerTrafficRankingToday: todayServerRanking,
ServerTrafficRankingYesterday: yesterdayTop10Server,
UserTrafficRankingToday: userTodayTrafficRanking,
UserTrafficRankingYesterday: yesterdayUserRankData,
}
resp.MonthlyUpload = nodeTraffic.Upload
resp.MonthlyDownload = nodeTraffic.Download
return resp, nil
}
@@ -215,7 +285,7 @@ func (l *QueryServerTotalDataLogic) mockRevenueStatistics() *types.ServerTotalDa
//}
//
return &types.ServerTotalDataResponse{
OnlineUserIPs: 1688,
OnlineUsers: 1688,
OnlineServers: 8,
OfflineServers: 2,
TodayUpload: 8888888888, // ~8.3GB