Develop (#76)

* refactor: rename queryannouncementhandler.go to queryAnnouncementLogic.go for clarity

* feat(panDomain): update subscription logic to use V2 handler for improved functionality

* refactor(subscribe): replace V2 handler with a unified Handler method for subscription logic

* feat(subscribe): implement user agent limit feature with configurable list

* fix(subscribe): improve error handling and logging for subscription requests

* feat(subscribe): add user agent limit configuration to system settings

* refactor(api): remove deprecated application-related endpoints and types

* refactor(swagger): remove deprecated app.json generation from swagger configuration

* refactor(swagger): remove deprecated app.json check from swagger configuration

* fix(subscribe): update delete method to use Where clause for improved query accuracy

* fix(subscribe): update Id field tag to use primaryKey and improve save method query

* fix(subscribe): update Id field tag to use primaryKey and improve model queries

* fix(subscribe): rename variable for clarity and add special handling for Stash user agent

* fix(email): convert RegisterStartTime and RegisterEndTime to time.Time for accurate query filtering

* refactor(log): consolidate logging models and update related logic for improved clarity and functionality

* fix(types): change Content field type in MessageLog to interface{} for improved flexibility

* fix(log): change MessageLog list to use value type for improved performance and memory efficiency

* fix(email): set EmailTypeVerify in task payload and update content type conversion for verification email

* fix(log): remove unused Id field from SystemLog during login log insertion

* fix(login): remove debug logs and error logging during user login process

* fix(log): add traffic reset logging for subscription resets

* fix(log): insert reset traffic log during subscription activation

* feat(log): add endpoints for retrieving and resetting subscribe traffic logs

* refactor(log): remove Reset Subscribe Traffic Log endpoint and related types

* feat(traffic): add traffic statistics logging and scheduling

* fix(subscribe): ensure active status and reset timestamps during traffic resets

* feat(api): enhance server and node management with new request/response structures

* refactor(api): rename OnlineUser to ServerOnlineUser for clarity

* feat(api): define OnlineUser type with SID and IP fields

* feat(server): implement server management handlers and database schema

* feat(api): add traffic log details filtering and enhance traffic log structures

* feat(api): migrate server and node data handling, update related structures and logic

* feat(server): implement server deletion logic with error handling

* feat(api): update log filtering to use ResetSubscribe type for subscription logs

* feat(api): standardize timestamp field across log structures

* feat(api): refactor cache key handling for server and user lists

* feat(api): enhance server status handling with protocol support and refactor related logic

* fix(traffic): adjust start date for traffic statistics and improve log deletion comment

* feat(api): implement daily traffic ranking for users and servers with error handling

* feat(api): update server total data response to use 'OnlineUsers' and implement daily traffic statistics logging

* feat(api): add log settings management with auto-clear and clear days configuration

* fix(log): correct category in log settings update query

* feat(routes): add handler for scheduled traffic statistics

* feat(model): add user counts struct and update queries for new and renewal users

* feat(api): add referral percentage and only first purchase fields to user model and requests

* feat(database): update user table to add referral percentage and only first purchase fields

* feat(api): add reset sort endpoints for server and node

* feat(api): add sort field to server model

* feat(api): implement sorting functionality for nodes and servers

* fix(database): add sort column to nodes table

* fix(model): enhance user statistics queries with new order and renewal order counts

* fix(log): update timestamp handling in login and registration logs

* fix(log): update sorting logic for server and user subscribe traffic logs

* fix(server): add server status handling based on last reported time

* fix(model): correct filter condition to use 'date' instead of 'data'

* fix(migration): add index for traffic log on timestamp, user_id, and subscribe_id

* fix(log): optimize user traffic rank data handling by using append instead of index assignment

* fix(filter): refactor node list creation to use append and remove duplicates from tags

* fix(node): add ServerId and Enabled fields to node update logic

* feat(tags): add endpoint to query all node tags

* fix(preview): add Preload parameter to FilterNodeList for improved data retrieval

* fix(log): date is empty

* feat(subscribe): add Language field to subscription models and update query logic

* feat(subscription): add Language parameter to GetSubscription request and update query logic

* fix(server): encode ServerKey in base64 and update last reported time for nodes

* feat: delete common GetSubscription

* feat(subscription): implement FilterList method for subscription queries and update related logic

* fix(subscribe): remove duplicate user agents in SubscribeHandler

* fix(push): initialize onlineUsers as a map in pushOnlineUsersLogic

* fix(reset): initialize subs as a map in clearCache method

* refactor(query): simplify node and tag filtering using InSet function

* feat(userlist): enhance GetServerUserListLogic with improved node and tag handling

* fix(userlist): correct node ID assignment and update query logic for tag filtering

* fix(userlist): correct node ID assignment in getServerUserListLogic

* refactor(query): streamline query construction for tag filtering

* fix(statistics): optimize server ranking data handling in QueryServerTotalDataLogic

* refactor(statistics): simplify server ranking data construction in QueryServerTotalDataLogic

* fix(statistics): correct server traffic data assignment in QueryServerTotalDataLogic

* fix(statistics): optimize yesterday's top 10 server traffic data assignment in QueryServerTotalDataLogic

* fix(middleware): remove duplicate elements from user agent list in PanDomainMiddleware

* feat(middleware): enhance user agent handling by querying client list in PanDomainMiddleware

* feat(client): subscribe_template

* feat(oauth): add user agent and IP logging to registration and login processes

* fix(balance): add timestamp to balance logs for payment, refund, and recharge transactions

* fix(log): correct comment for CommissionTypeRefund to improve clarity

* fix(log): replace magic number with constant for gift type in purchase checkout logic

* fix(log): rename OrderId to OrderNo for consistency in balance logging

* feat(log): add logging for balance, gift amount, and commission adjustments

* fix(user): correct placement of DeepCopy for user info update logic

* feat(log): add UserSubscribeId to FilterSubscribeLogRequest for enhanced filtering

* fix(purchase): streamline error handling and improve JSON marshaling for temporary orders

* fix(order): simplify commission handling and improve payload parsing logic

* fix(order): update commission calculation to actual payment amount minus gateway handling fee

* feat(payment): add support for CryptoSaaS payment platform and enhance configuration handling

* fix(balance): update QueryUserBalanceLog response structure to include balance log list

* fix(email): update task progress handling to use specific task ID for updates

* feat(quota): add quota task creation and querying endpoints with updated data structures

* fix(email): update task handling to use generic task model and improve error logging

* fix(order): improve error logging for database transaction and user cache updates

* feat(quota): enhance quota task management with new request structures and processing logic

* fix(quota): remove redundant quota task status endpoint from admin marketing routes

* fix(worker): update task completion status handling in worker logic

* fix(quota): update taskInfo to include current subscription count in quota logic

* doc(log): rename function for clarity and add cache cleanup comment

* fix(quota): update time handling in quota logic and correct subscriber ID query

* fix(quota): update time handling to use UnixMilli for start time in quota logic

* feat(protocol): add server protocol configuration query and enhance protocol options

* fix(quota): correct time range queries for start and expire times in quota logic

* fix(types): update plugin options to include 'none' in the plugin field

---------

Co-authored-by: Chang lue Tsen <tension@ppanel.dev>

internal/middleware/appMiddleware.go

@@ -1,282 +0,0 @@
-package middleware
-
-import (
-	"bufio"
-	"bytes"
-	"encoding/json"
-	"fmt"
-	"io"
-	"net"
-	"net/http"
-	"strings"
-
-	"github.com/perfect-panel/server/pkg/logger"
-	"github.com/perfect-panel/server/pkg/result"
-	"github.com/perfect-panel/server/pkg/xerr"
-	"github.com/pkg/errors"
-
-	"github.com/gin-gonic/gin"
-	"github.com/perfect-panel/server/internal/svc"
-	pkgaes "github.com/perfect-panel/server/pkg/aes"
-)
-
-const (
-	noWritten     = -1
-	defaultStatus = http.StatusOK
-	key           = "123456"
-)
-
-func AppMiddleware(svc *svc.ServiceContext) func(c *gin.Context) {
-	return func(c *gin.Context) {
-		if !strings.Contains(c.Request.URL.Path, "/v1/app") {
-			c.Next()
-			return
-		}
-		rw := NewResponseWriter(c, svc)
-		if !rw.Decrypt() {
-			result.HttpResult(c, nil, errors.Wrapf(xerr.NewErrCode(xerr.InvalidCiphertext), "Invalid ciphertext"))
-			c.Abort()
-			return
-		}
-		c.Writer = rw
-		c.Next()
-		rw.FlushAbort()
-	}
-}
-
-func NewResponseWriter(c *gin.Context, srvCtx *svc.ServiceContext) (rw *ResponseWriter) {
-	rw = &ResponseWriter{
-		c:              c,
-		body:           new(bytes.Buffer),
-		ResponseWriter: c.Writer,
-	}
-	applicationConfig, err := srvCtx.ApplicationModel.FindOneConfig(c, 1)
-	if err != nil {
-		logger.Errorf("[AppMiddleware] find application config error: %v", err.Error())
-		return
-	}
-	if strings.ToUpper(applicationConfig.EncryptionMethod) == "AES" && applicationConfig.EncryptionKey != "" {
-		rw.encryptionKey = applicationConfig.EncryptionKey
-		rw.encryptionMethod = applicationConfig.EncryptionMethod
-		rw.encryption = true
-	}
-	return
-}
-
-func (rw *ResponseWriter) Encrypt() {
-	if !rw.encryption {
-		return
-	}
-	buf := rw.body.Bytes()
-	params := map[string]interface{}{}
-	err := json.Unmarshal(buf, &params)
-	if err != nil {
-		return
-	}
-	data := params["data"]
-	if data != nil {
-		var jsonData []byte
-		str, ok := data.(string)
-		if ok {
-			jsonData = []byte(str)
-		} else {
-			jsonData, _ = json.Marshal(data)
-		}
-		encrypt, iv, err := pkgaes.Encrypt(jsonData, rw.encryptionKey)
-		if err != nil {
-			return
-		}
-		params["data"] = map[string]interface{}{
-			"data": encrypt,
-			"time": iv,
-		}
-
-	}
-	marshal, _ := json.Marshal(params)
-	rw.body.Reset()
-	rw.body.Write(marshal)
-}
-
-func (rw *ResponseWriter) Decrypt() bool {
-	if !rw.encryption {
-		return true
-	}
-
-	//判断url链接中是否存在data和iv数据，存在就进行解密并设置回去
-	query := rw.c.Request.URL.Query()
-	dataStr := query.Get("data")
-	timeStr := query.Get("time")
-	if dataStr != "" && timeStr != "" {
-		decrypt, err := pkgaes.Decrypt(dataStr, rw.encryptionKey, timeStr)
-		if err == nil {
-			params := map[string]interface{}{}
-			err = json.Unmarshal([]byte(decrypt), &params)
-			if err == nil {
-				for k, v := range params {
-					query.Set(k, fmt.Sprintf("%v", v))
-				}
-				query.Del("data")
-				query.Del("time")
-				rw.c.Request.RequestURI = fmt.Sprintf("%s?%s", rw.c.Request.RequestURI[:strings.Index(rw.c.Request.RequestURI, "?")], query.Encode())
-				rw.c.Request.URL.RawQuery = query.Encode()
-			}
-		}
-	}
-
-	//判断body是否存在数据，存在就尝试解密，并设置回去
-	body, err := io.ReadAll(rw.c.Request.Body)
-	if err != nil {
-		return true
-	}
-
-	if len(body) == 0 {
-		return true
-	}
-
-	params := map[string]interface{}{}
-	err = json.Unmarshal(body, &params)
-	data := params["data"]
-	nonce := params["time"]
-	if err != nil || data == nil {
-		return false
-	}
-
-	str, ok := data.(string)
-	if !ok {
-		return false
-	}
-	iv, ok := nonce.(string)
-	if !ok {
-		return false
-	}
-
-	decrypt, err := pkgaes.Decrypt(str, rw.encryptionKey, iv)
-	if err != nil {
-		return false
-	}
-	rw.c.Request.Body = io.NopCloser(bytes.NewBuffer([]byte(decrypt)))
-	return true
-}
-
-func (rw *ResponseWriter) FlushAbort() {
-	defer rw.c.Abort()
-	responseBody := rw.body.String()
-	fmt.Println("Original Response Body:", responseBody)
-	rw.flush = true
-	if rw.encryption {
-		rw.Encrypt()
-	}
-	_, err := rw.Write(rw.body.Bytes())
-	if err != nil {
-		return
-	}
-}
-
-type ResponseWriter struct {
-	http.ResponseWriter
-	size             int
-	status           int
-	flush            bool
-	body             *bytes.Buffer
-	c                *gin.Context
-	encryption       bool
-	encryptionKey    string
-	encryptionMethod string
-}
-
-func (rw *ResponseWriter) Unwrap() http.ResponseWriter {
-	return rw.ResponseWriter
-}
-
-//nolint:unused
-func (rw *ResponseWriter) reset(writer http.ResponseWriter) {
-	rw.ResponseWriter = writer
-	rw.size = noWritten
-	rw.status = defaultStatus
-}
-
-func (rw *ResponseWriter) WriteHeader(code int) {
-	if code > 0 && rw.status != code {
-		if rw.Written() {
-			return
-		}
-		rw.status = code
-	}
-}
-
-func (rw *ResponseWriter) WriteHeaderNow() {
-	if !rw.Written() {
-		rw.size = 0
-		rw.ResponseWriter.WriteHeader(rw.status)
-	}
-}
-
-func (rw *ResponseWriter) Write(data []byte) (n int, err error) {
-	if rw.flush {
-		rw.WriteHeaderNow()
-		n, err = rw.ResponseWriter.Write(data)
-		rw.size += n
-	} else {
-		rw.body.Write(data)
-	}
-	return
-}
-
-func (rw *ResponseWriter) WriteString(s string) (n int, err error) {
-	if rw.flush {
-		rw.WriteHeaderNow()
-		n, err = rw.ResponseWriter.Write([]byte(s))
-		rw.size += n
-	} else {
-		rw.body.Write([]byte(s))
-	}
-	return
-}
-
-func (rw *ResponseWriter) Status() int {
-	return rw.status
-}
-
-func (rw *ResponseWriter) Size() int {
-	return rw.size
-}
-
-func (rw *ResponseWriter) Written() bool {
-	return rw.size != noWritten
-}
-
-// Hijack implements the http.Hijacker interface.
-func (rw *ResponseWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) {
-	if rw.size < 0 {
-		rw.size = 0
-	}
-	return rw.ResponseWriter.(http.Hijacker).Hijack()
-}
-
-// CloseNotify implements the http.CloseNotifier interface.
-func (rw *ResponseWriter) CloseNotify() <-chan bool {
-	// 通过 r.Context().Done() 来监听请求的取消
-	done := rw.c.Request.Context().Done()
-	closed := make(chan bool)
-
-	// 当上下文被取消时，通过 closed channel 发送通知
-	go func() {
-		<-done
-		closed <- true
-	}()
-
-	return closed
-}
-
-// Flush implements the http.Flusher interface.
-func (rw *ResponseWriter) Flush() {
-	rw.WriteHeaderNow()
-	rw.ResponseWriter.(http.Flusher).Flush()
-}
-
-func (rw *ResponseWriter) Pusher() (pusher http.Pusher) {
-	if pusher, ok := rw.ResponseWriter.(http.Pusher); ok {
-		return pusher
-	}
-	return nil
-}

internal/middleware/loggerMiddleware.go

@@ -44,6 +44,9 @@ func LoggerMiddleware(svc *svc.ServiceContext) func(c *gin.Context) {
 		// Start recording logs
 		cost := time.Since(start)
 		responseStatus := c.Writer.Status()
+
+		host := c.Request.Host
+
 		logs := []logger.LogField{
 			{
 				Key:   "status",
@@ -51,7 +54,7 @@ func LoggerMiddleware(svc *svc.ServiceContext) func(c *gin.Context) {
 			},
 			{
 				Key:   "request",
-				Value: c.Request.Method + " " + c.Request.URL.String(),
+				Value: c.Request.Method + " " + host + c.Request.URL.String(),
 			},
 			{
 				Key:   "query",
@@ -88,6 +91,10 @@ func LoggerMiddleware(svc *svc.ServiceContext) func(c *gin.Context) {
 		} else {
 			logger.WithContext(c.Request.Context()).Infow("HTTP Request", logs...)
 		}
+
+		if responseStatus == 404 {
+			logger.WithContext(c.Request.Context()).Debugf("404 Not Found: Host:%s Path:%s IsPanDomain:%v", host, c.Request.URL.Path, svc.Config.Subscribe.PanDomain)
+		}
 	}
 }
 

internal/middleware/panDomainMiddleware.go

@@ -8,24 +8,47 @@ import (
 	"github.com/perfect-panel/server/internal/logic/subscribe"
 	"github.com/perfect-panel/server/internal/svc"
 	"github.com/perfect-panel/server/internal/types"
+	"github.com/perfect-panel/server/pkg/logger"
+	"github.com/perfect-panel/server/pkg/tool"
 )
 
 func PanDomainMiddleware(svc *svc.ServiceContext) func(c *gin.Context) {
 	return func(c *gin.Context) {
 
 		if svc.Config.Subscribe.PanDomain && c.Request.URL.Path == "/" {
-
 			// intercept browser
 			ua := c.GetHeader("User-Agent")
-			if ua == "" {
-				c.String(http.StatusForbidden, "Access denied")
-				c.Abort()
-				return
-			}
-			browserKeywords := []string{"chrome", "firefox", "safari", "edge", "opera", "micromessenger"}
-			for _, keyword := range browserKeywords {
-				lcUA := strings.ToLower(ua)
-				if strings.Contains(lcUA, keyword) {
+
+			if svc.Config.Subscribe.UserAgentLimit {
+				if ua == "" {
+					c.String(http.StatusForbidden, "Access denied")
+					c.Abort()
+					return
+				}
+				browserKeywords := tool.RemoveDuplicateElements(strings.Split(svc.Config.Subscribe.UserAgentList, "\n")...)
+				var allow = false
+
+				// query client list
+				clients, err := svc.ClientModel.List(c.Request.Context())
+				if err != nil {
+					logger.Errorw("[PanDomainMiddleware] Query client list failed", logger.Field("error", err.Error()))
+				}
+				for _, item := range clients {
+					u := strings.ToLower(item.UserAgent)
+					u = strings.Trim(u, " ")
+					browserKeywords = append(browserKeywords, u)
+				}
+
+				for _, keyword := range browserKeywords {
+					keyword = strings.ToLower(strings.Trim(keyword, " "))
+					if keyword == "" {
+						continue
+					}
+					if strings.Contains(strings.ToLower(ua), keyword) {
+						allow = true
+					}
+				}
+				if !allow {
 					c.String(http.StatusForbidden, "Access denied")
 					c.Abort()
 					return
@@ -41,7 +64,7 @@ func PanDomainMiddleware(svc *svc.ServiceContext) func(c *gin.Context) {
 				UA:    c.Request.Header.Get("User-Agent"),
 			}
 			l := subscribe.NewSubscribeLogic(c, svc)
-			resp, err := l.Generate(&request)
+			resp, err := l.Handler(&request)
 			if err != nil {
 				return
 			}

internal/middleware/traceMiddleware.go

@@ -9,14 +9,12 @@ import (
 	"github.com/perfect-panel/server/pkg/constant"
 
 	"github.com/gin-gonic/gin"
-	"github.com/google/uuid"
 	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/codes"
 	semconv "go.opentelemetry.io/otel/semconv/v1.24.0"
 	oteltrace "go.opentelemetry.io/otel/trace"
 
 	"github.com/perfect-panel/server/internal/svc"
-	"github.com/perfect-panel/server/pkg/logger"
 	"github.com/perfect-panel/server/pkg/trace"
 )
 
@@ -71,19 +69,13 @@ func TraceMiddleware(_ *svc.ServiceContext) func(ctx *gin.Context) {
 		)
 		defer span.End()
 
-		requestId, err := uuid.NewV7()
-		if err != nil {
-			logger.Errorw(
-				"failed to generate request id in uuid v7 format, fallback to uuid v4",
-				logger.Field("error", err),
-			)
-			requestId = uuid.New()
-		}
-		c.Header(trace.RequestIdKey, requestId.String())
+		requestId := trace.TraceIDFromContext(ctx)
+
+		c.Header(trace.RequestIdKey, requestId)
 
 		span.SetAttributes(requestAttributes(c.Request)...)
 		span.SetAttributes(
-			attribute.String("http.request_id", requestId.String()),
+			attribute.String("http.request_id", requestId),
 			semconv.HTTPRouteKey.String(c.FullPath()),
 		)
 		// context with request host