From cbb451d18cca2b88de87e37ea8bd94992ae5e6ba Mon Sep 17 00:00:00 2001 From: shanshanzhong Date: Wed, 6 May 2026 02:34:43 -0700 Subject: [PATCH] x --- docker-compose.cloud.yml | 154 +--- .../alerting/ppanel-alert-rules.yml | 272 +++++++ .../json/aws-rds-redis-overview.json | 520 +++++++++++++ .../dashboards/json/ppanel-ops-overview.json | 565 ++++++++++++++ .../dashboards/json/ppanel-server-logs.json | 330 ++++++++ .../provisioning/datasources/datasources.yml | 9 + .../user/bindEmailWithVerificationLogic.go | 2 +- loki/promtail-config.yaml | 14 + prometheus/prometheus.yml | 10 - scripts/reconcile_mihapay_orders.go | 708 ++++++++++++++++++ 10 files changed, 2439 insertions(+), 145 deletions(-) create mode 100644 grafana/provisioning/alerting/ppanel-alert-rules.yml create mode 100644 grafana/provisioning/dashboards/json/aws-rds-redis-overview.json create mode 100644 grafana/provisioning/dashboards/json/ppanel-ops-overview.json create mode 100644 grafana/provisioning/dashboards/json/ppanel-server-logs.json create mode 100644 scripts/reconcile_mihapay_orders.go diff --git a/docker-compose.cloud.yml b/docker-compose.cloud.yml index 36be559..5939843 100644 --- a/docker-compose.cloud.yml +++ b/docker-compose.cloud.yml @@ -6,9 +6,9 @@ # 4. 运行: docker-compose -f docker-compose.cloud.yml up -d # # 网络说明： -# ppanel-server 使用 host 网络（可出外网，访问 MySQL/Redis/Tempo 用 127.0.0.1） -# 监控服务（MySQL/Redis/Loki/Tempo/Grafana/Prometheus）在 ppanel_net bridge 网络中 -# MySQL(3306)/Redis(6379)/Tempo(4317) 将端口映射到 127.0.0.1，ppanel-server 通过 host 网络访问 +# ppanel-server 使用 host 网络（可出外网，直接访问 AWS RDS/ElastiCache 私网地址） +# 监控服务（Loki/Tempo/Grafana/Prometheus）在 ppanel_net bridge 网络中 +# Tempo(4317) 将端口映射到 127.0.0.1，ppanel-server 通过 host 网络访问 # 监控端口绑定 127.0.0.1，需通过 SSH 隧道或 Nginx 反代访问 # # 未来多开 ppanel-server 时： @@ -18,7 +18,7 @@ services: # ---------------------------------------------------- # 1. 业务后端 (PPanel Server) - # host 网络：可出外网，通过 127.0.0.1 访问 MySQL/Redis/Tempo + # host 网络：可出外网，直接访问 AWS RDS/Redis；通过 127.0.0.1 访问 Tempo # ---------------------------------------------------- ppanel-server: image: registry.kxsw.us/vpn-server:${PPANEL_SERVER_TAG:-latest} @@ -37,10 +37,6 @@ services: soft: 65535 hard: 65535 depends_on: - mysql: - condition: service_healthy - redis: - condition: service_healthy tempo: condition: service_started logging: @@ -50,82 +46,7 @@ services: max-file: "3" # ---------------------------------------------------- - # 2. MySQL Database - # ---------------------------------------------------- - mysql: - image: mysql:8.0 - container_name: ppanel-mysql - restart: always - ports: - - "3306:3306" # 仅宿主机可访问，ppanel-server(host网络)通过127.0.0.1连接 - environment: - MYSQL_ROOT_PASSWORD: "${MYSQL_ROOT_PASSWORD:?请在 .env 文件中设置 MYSQL_ROOT_PASSWORD}" - MYSQL_DATABASE: "ppanel" - TZ: Asia/Shanghai - command: - - --default-authentication-plugin=mysql_native_password - - --innodb_buffer_pool_size=16G - - --innodb_buffer_pool_instances=16 - - --innodb_log_file_size=2G - - --innodb_flush_log_at_trx_commit=2 - - --innodb_io_capacity=5000 - - --max_connections=5000 - volumes: - - mysql_data:/var/lib/mysql - ulimits: - nproc: 65535 - nofile: - soft: 65535 - hard: 65535 - networks: - - ppanel_net - healthcheck: - test: [ "CMD", "mysqladmin", "ping", "-h", "localhost", "-uroot", "-p${MYSQL_ROOT_PASSWORD}" ] - interval: 10s - timeout: 5s - retries: 5 - logging: - driver: "json-file" - options: - max-size: "10m" - max-file: "3" - - # ---------------------------------------------------- - # 3. Redis - # ---------------------------------------------------- - redis: - image: redis:8.2.1 - container_name: ppanel-redis - restart: always - ports: - - "127.0.0.1:6379:6379" # 仅宿主机可访问，ppanel-server(host网络)通过127.0.0.1连接 - command: - - redis-server - - --tcp-backlog 65535 - - --maxmemory-policy allkeys-lru - - --requirepass hifast67yj - volumes: - - redis_data:/data - ulimits: - nproc: 65535 - nofile: - soft: 65535 - hard: 65535 - networks: - - ppanel_net - healthcheck: - test: [ "CMD", "redis-cli", "ping" ] - interval: 10s - timeout: 5s - retries: 5 - logging: - driver: "json-file" - options: - max-size: "10m" - max-file: "3" - - # ---------------------------------------------------- - # 4. Tempo (链路追踪存储) + # 2. Tempo (链路追踪存储) # ---------------------------------------------------- tempo: image: grafana/tempo:2.4.1 @@ -149,7 +70,7 @@ services: max-file: "3" # ---------------------------------------------------- - # 5. Loki (日志存储) + # 3. Loki (日志存储) # ---------------------------------------------------- loki: image: grafana/loki:3.0.0 @@ -169,7 +90,7 @@ services: max-file: "3" # ---------------------------------------------------- - # 6. Promtail (日志采集) + # 4. Promtail (日志采集) # ---------------------------------------------------- promtail: image: grafana/promtail:3.0.0 @@ -193,7 +114,7 @@ services: max-file: "3" # ---------------------------------------------------- - # 7. Grafana (可观测面板) + # 5. Grafana (可观测面板) # 访问: ssh -L 3333:localhost:3333 your-server 后浏览器打开 http://localhost:3333 # 或配置 Nginx 反代（建议加认证） # ---------------------------------------------------- @@ -202,11 +123,18 @@ services: container_name: ppanel-grafana restart: always ports: - - "3333:3000" # 仅本机可访问，需 SSH 隧道或 Nginx 反代 + - "127.0.0.1:3333:3000" # 仅本机可访问，需 SSH 隧道或 Nginx 反代 environment: - GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_PASSWORD:?请在 .env 文件中设置 GRAFANA_PASSWORD} - GF_USERS_ALLOW_SIGN_UP=false + - GF_SERVER_DOMAIN=${GRAFANA_DOMAIN:-logsx.hifast.biz} + - GF_SERVER_ROOT_URL=${GRAFANA_ROOT_URL:-https://logsx.hifast.biz} - GF_FEATURE_TOGGLES_ENABLE=appObservability + - AWS_REGION=${AWS_REGION:-ap-east-1} + - AWS_DEFAULT_REGION=${AWS_REGION:-ap-east-1} + - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID:-} + - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY:-} + - AWS_SESSION_TOKEN=${AWS_SESSION_TOKEN:-} volumes: - grafana_data:/var/lib/grafana - ./grafana/provisioning:/etc/grafana/provisioning @@ -223,7 +151,7 @@ services: max-file: "3" # ---------------------------------------------------- - # 8. Prometheus (指标采集) + # 6. Prometheus (指标采集) # ---------------------------------------------------- prometheus: image: prom/prometheus:latest @@ -248,26 +176,7 @@ services: max-file: "3" # ---------------------------------------------------- - # 9. Redis Exporter - # ---------------------------------------------------- - redis-exporter: - image: oliver006/redis_exporter:latest - container_name: ppanel-redis-exporter - restart: always - environment: - - REDIS_ADDR=redis://redis:6379 - networks: - - ppanel_net - depends_on: - - redis - logging: - driver: "json-file" - options: - max-size: "10m" - max-file: "3" - - # ---------------------------------------------------- - # 10. Nginx Exporter (监控宿主机 Nginx) + # 7. Nginx Exporter (监控宿主机 Nginx) # ---------------------------------------------------- nginx-exporter: image: nginx/nginx-prometheus-exporter:latest @@ -286,28 +195,7 @@ services: max-file: "3" # ---------------------------------------------------- - # 11. MySQL Exporter - # ---------------------------------------------------- - mysql-exporter: - image: prom/mysqld-exporter:latest - container_name: ppanel-mysql-exporter - restart: always - command: - - --config.my-cnf=/etc/.my.cnf - volumes: - - ./mysql/.my.cnf:/etc/.my.cnf:ro - networks: - - ppanel_net - depends_on: - - mysql - logging: - driver: "json-file" - options: - max-size: "10m" - max-file: "3" - - # ---------------------------------------------------- - # 12. Node Exporter (宿主机监控) + # 8. Node Exporter (宿主机监控) # ---------------------------------------------------- node-exporter: image: prom/node-exporter:latest @@ -330,7 +218,7 @@ services: max-file: "3" # ---------------------------------------------------- - # 13. cAdvisor (容器监控) + # 9. cAdvisor (容器监控) # ---------------------------------------------------- cadvisor: image: gcr.io/cadvisor/cadvisor:latest @@ -351,8 +239,6 @@ services: max-file: "3" volumes: - mysql_data: - redis_data: loki_data: grafana_data: prometheus_data: diff --git a/grafana/provisioning/alerting/ppanel-alert-rules.yml b/grafana/provisioning/alerting/ppanel-alert-rules.yml new file mode 100644 index 0000000..e1e9e71 --- /dev/null +++ b/grafana/provisioning/alerting/ppanel-alert-rules.yml @@ -0,0 +1,272 @@ +apiVersion: 1 + +groups: + - orgId: 1 + name: ppanel-core + folder: PPanel + interval: 1m + rules: + - uid: ppanel-target-down + title: PPanel monitoring target down + condition: C + for: 2m + noDataState: Alerting + execErrState: Error + annotations: + summary: "Monitoring target is down" + description: "{{ $labels.job }} on {{ $labels.instance }} has been down for more than 2 minutes." + labels: + severity: critical + service: ppanel + data: + - refId: A + relativeTimeRange: + from: 300 + to: 0 + datasourceUid: prometheus + model: + datasource: + type: prometheus + uid: prometheus + editorMode: code + expr: 'up{job=~"grafana|prometheus|node-exporter|cadvisor|nginx-exporter|loki|tempo"}' + instant: true + intervalMs: 1000 + maxDataPoints: 43200 + refId: A + - refId: C + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 1 + type: lt + operator: + type: and + query: + params: + - A + reducer: + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: A + intervalMs: 1000 + maxDataPoints: 43200 + refId: C + type: threshold + + - uid: ppanel-host-disk-high + title: PPanel host disk usage high + condition: C + for: 10m + noDataState: NoData + execErrState: Error + annotations: + summary: "Host disk usage is high" + description: "{{ $labels.instance }} {{ $labels.mountpoint }} disk usage is above 85% for 10 minutes." + labels: + severity: warning + service: ppanel + data: + - refId: A + relativeTimeRange: + from: 900 + to: 0 + datasourceUid: prometheus + model: + datasource: + type: prometheus + uid: prometheus + editorMode: code + expr: '100 - (node_filesystem_avail_bytes{fstype!~"tmpfs|overlay|squashfs|aufs",mountpoint!~"/run.*|/var/lib/docker.*"} / node_filesystem_size_bytes{fstype!~"tmpfs|overlay|squashfs|aufs",mountpoint!~"/run.*|/var/lib/docker.*"} * 100)' + instant: true + intervalMs: 1000 + maxDataPoints: 43200 + refId: A + - refId: C + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 85 + type: gt + operator: + type: and + query: + params: + - A + reducer: + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: A + intervalMs: 1000 + maxDataPoints: 43200 + refId: C + type: threshold + + - uid: ppanel-host-memory-high + title: PPanel host memory usage high + condition: C + for: 10m + noDataState: NoData + execErrState: Error + annotations: + summary: "Host memory usage is high" + description: "{{ $labels.instance }} memory usage is above 90% for 10 minutes." + labels: + severity: warning + service: ppanel + data: + - refId: A + relativeTimeRange: + from: 900 + to: 0 + datasourceUid: prometheus + model: + datasource: + type: prometheus + uid: prometheus + editorMode: code + expr: '(1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100' + instant: true + intervalMs: 1000 + maxDataPoints: 43200 + refId: A + - refId: C + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 90 + type: gt + operator: + type: and + query: + params: + - A + reducer: + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: A + intervalMs: 1000 + maxDataPoints: 43200 + refId: C + type: threshold + + - uid: ppanel-host-cpu-high + title: PPanel host CPU usage high + condition: C + for: 10m + noDataState: NoData + execErrState: Error + annotations: + summary: "Host CPU usage is high" + description: "{{ $labels.instance }} CPU usage is above 90% for 10 minutes." + labels: + severity: warning + service: ppanel + data: + - refId: A + relativeTimeRange: + from: 900 + to: 0 + datasourceUid: prometheus + model: + datasource: + type: prometheus + uid: prometheus + editorMode: code + expr: '100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100)' + instant: true + intervalMs: 1000 + maxDataPoints: 43200 + refId: A + - refId: C + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 90 + type: gt + operator: + type: and + query: + params: + - A + reducer: + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: A + intervalMs: 1000 + maxDataPoints: 43200 + refId: C + type: threshold + + - uid: ppanel-container-restarts + title: PPanel container restarted + condition: C + for: 1m + noDataState: NoData + execErrState: Error + annotations: + summary: "Container restarted" + description: "{{ $labels.name }} restarted or changed start time in the last hour." + labels: + severity: warning + service: ppanel + data: + - refId: A + relativeTimeRange: + from: 3600 + to: 0 + datasourceUid: prometheus + model: + datasource: + type: prometheus + uid: prometheus + editorMode: code + expr: 'sum by (name) (changes(container_start_time_seconds{name!=""}[1h]))' + instant: true + intervalMs: 1000 + maxDataPoints: 43200 + refId: A + - refId: C + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 0 + type: gt + operator: + type: and + query: + params: + - A + reducer: + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: A + intervalMs: 1000 + maxDataPoints: 43200 + refId: C + type: threshold diff --git a/grafana/provisioning/dashboards/json/aws-rds-redis-overview.json b/grafana/provisioning/dashboards/json/aws-rds-redis-overview.json new file mode 100644 index 0000000..7325401 --- /dev/null +++ b/grafana/provisioning/dashboards/json/aws-rds-redis-overview.json @@ -0,0 +1,520 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "datasource": { + "type": "cloudwatch", + "uid": "cloudwatch" + }, + "gridPos": { + "h": 3, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "content": "AWS CloudWatch overview
Region: ap-east-1 (Hong Kong)
RDS DBInstanceIdentifier: database-1
Redis ReplicationGroupId: hifastapp-redis

Note: Redis panels are configured using ReplicationGroupId. If a panel shows no data in your account, switch that dimension to the concrete CacheClusterId in Grafana query editor.", + "mode": "html" + }, + "pluginVersion": "11.0.0", + "title": "Read Me", + "type": "text" + }, + { + "datasource": { + "type": "cloudwatch", + "uid": "cloudwatch" + }, + "fieldConfig": { + "defaults": { + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 3 + }, + "id": 2, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "datasource": { + "type": "cloudwatch", + "uid": "cloudwatch" + }, + "dimensions": { + "DBInstanceIdentifier": "database-1" + }, + "metricName": "CPUUtilization", + "namespace": "AWS/RDS", + "period": "", + "refId": "A", + "region": "ap-east-1", + "statistic": "Average" + } + ], + "title": "RDS CPU", + "type": "timeseries" + }, + { + "datasource": { + "type": "cloudwatch", + "uid": "cloudwatch" + }, + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 3 + }, + "id": 3, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "datasource": { + "type": "cloudwatch", + "uid": "cloudwatch" + }, + "dimensions": { + "DBInstanceIdentifier": "database-1" + }, + "metricName": "DatabaseConnections", + "namespace": "AWS/RDS", + "period": "", + "refId": "A", + "region": "ap-east-1", + "statistic": "Average" + } + ], + "title": "RDS Connections", + "type": "timeseries" + }, + { + "datasource": { + "type": "cloudwatch", + "uid": "cloudwatch" + }, + "fieldConfig": { + "defaults": { + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 3 + }, + "id": 4, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "datasource": { + "type": "cloudwatch", + "uid": "cloudwatch" + }, + "dimensions": { + "DBInstanceIdentifier": "database-1" + }, + "metricName": "FreeStorageSpace", + "namespace": "AWS/RDS", + "period": "", + "refId": "A", + "region": "ap-east-1", + "statistic": "Minimum" + } + ], + "title": "RDS Free Storage", + "type": "timeseries" + }, + { + "datasource": { + "type": "cloudwatch", + "uid": "cloudwatch" + }, + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 11 + }, + "id": 5, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "datasource": { + "type": "cloudwatch", + "uid": "cloudwatch" + }, + "dimensions": { + "DBInstanceIdentifier": "database-1" + }, + "metricName": "ReadLatency", + "namespace": "AWS/RDS", + "period": "", + "refId": "A", + "region": "ap-east-1", + "statistic": "Average" + }, + { + "datasource": { + "type": "cloudwatch", + "uid": "cloudwatch" + }, + "dimensions": { + "DBInstanceIdentifier": "database-1" + }, + "metricName": "WriteLatency", + "namespace": "AWS/RDS", + "period": "", + "refId": "B", + "region": "ap-east-1", + "statistic": "Average" + } + ], + "title": "RDS Read / Write Latency", + "type": "timeseries" + }, + { + "datasource": { + "type": "cloudwatch", + "uid": "cloudwatch" + }, + "fieldConfig": { + "defaults": { + "unit": "iops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 11 + }, + "id": 6, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "datasource": { + "type": "cloudwatch", + "uid": "cloudwatch" + }, + "dimensions": { + "DBInstanceIdentifier": "database-1" + }, + "metricName": "ReadIOPS", + "namespace": "AWS/RDS", + "period": "", + "refId": "A", + "region": "ap-east-1", + "statistic": "Average" + }, + { + "datasource": { + "type": "cloudwatch", + "uid": "cloudwatch" + }, + "dimensions": { + "DBInstanceIdentifier": "database-1" + }, + "metricName": "WriteIOPS", + "namespace": "AWS/RDS", + "period": "", + "refId": "B", + "region": "ap-east-1", + "statistic": "Average" + } + ], + "title": "RDS Read / Write IOPS", + "type": "timeseries" + }, + { + "datasource": { + "type": "cloudwatch", + "uid": "cloudwatch" + }, + "fieldConfig": { + "defaults": { + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 19 + }, + "id": 7, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "datasource": { + "type": "cloudwatch", + "uid": "cloudwatch" + }, + "dimensions": { + "ReplicationGroupId": "hifastapp-redis" + }, + "metricName": "CPUUtilization", + "namespace": "AWS/ElastiCache", + "period": "", + "refId": "A", + "region": "ap-east-1", + "statistic": "Average" + } + ], + "title": "Redis Host CPU", + "type": "timeseries" + }, + { + "datasource": { + "type": "cloudwatch", + "uid": "cloudwatch" + }, + "fieldConfig": { + "defaults": { + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 6, + "y": 19 + }, + "id": 8, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "datasource": { + "type": "cloudwatch", + "uid": "cloudwatch" + }, + "dimensions": { + "ReplicationGroupId": "hifastapp-redis" + }, + "metricName": "EngineCPUUtilization", + "namespace": "AWS/ElastiCache", + "period": "", + "refId": "A", + "region": "ap-east-1", + "statistic": "Average" + } + ], + "title": "Redis Engine CPU", + "type": "timeseries" + }, + { + "datasource": { + "type": "cloudwatch", + "uid": "cloudwatch" + }, + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 12, + "y": 19 + }, + "id": 9, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "datasource": { + "type": "cloudwatch", + "uid": "cloudwatch" + }, + "dimensions": { + "ReplicationGroupId": "hifastapp-redis" + }, + "metricName": "CurrConnections", + "namespace": "AWS/ElastiCache", + "period": "", + "refId": "A", + "region": "ap-east-1", + "statistic": "Average" + } + ], + "title": "Redis Connections", + "type": "timeseries" + }, + { + "datasource": { + "type": "cloudwatch", + "uid": "cloudwatch" + }, + "fieldConfig": { + "defaults": { + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 19 + }, + "id": 10, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "datasource": { + "type": "cloudwatch", + "uid": "cloudwatch" + }, + "dimensions": { + "ReplicationGroupId": "hifastapp-redis" + }, + "metricName": "DatabaseMemoryUsagePercentage", + "namespace": "AWS/ElastiCache", + "period": "", + "refId": "A", + "region": "ap-east-1", + "statistic": "Average" + } + ], + "title": "Redis Memory Usage %", + "type": "timeseries" + } + ], + "refresh": "30s", + "schemaVersion": 39, + "style": "dark", + "tags": [ + "aws", + "cloudwatch", + "rds", + "redis" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "AWS RDS & Redis Overview", + "uid": "aws-rds-redis-overview", + "version": 1, + "weekStart": "" +} diff --git a/grafana/provisioning/dashboards/json/ppanel-ops-overview.json b/grafana/provisioning/dashboards/json/ppanel-ops-overview.json new file mode 100644 index 0000000..08ea6e8 --- /dev/null +++ b/grafana/provisioning/dashboards/json/ppanel-ops-overview.json @@ -0,0 +1,565 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "text": "DOWN" + }, + "1": { + "text": "UP" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "up{job=~\"prometheus|grafana|node-exporter|cadvisor|nginx-exporter|loki|tempo\"}", + "instant": true, + "legendFormat": "{{job}}", + "range": false, + "refId": "A" + } + ], + "title": "Service Availability", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 75 + }, + { + "color": "red", + "value": 90 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 4 + }, + "id": 2, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "100 - (avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m])) * 100)", + "legendFormat": "{{instance}} CPU", + "range": true, + "refId": "A" + } + ], + "title": "Host CPU Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 80 + }, + { + "color": "red", + "value": 90 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 4 + }, + "id": 3, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "(1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100", + "legendFormat": "{{instance}} memory", + "range": true, + "refId": "A" + } + ], + "title": "Host Memory Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 80 + }, + { + "color": "red", + "value": 90 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 4 + }, + "id": 4, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "100 - (node_filesystem_avail_bytes{fstype!~\"tmpfs|overlay|squashfs|aufs\",mountpoint!~\"/run.*|/var/lib/docker.*\"} / node_filesystem_size_bytes{fstype!~\"tmpfs|overlay|squashfs|aufs\",mountpoint!~\"/run.*|/var/lib/docker.*\"} * 100)", + "legendFormat": "{{mountpoint}}", + "range": true, + "refId": "A" + } + ], + "title": "Host Disk Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 12 + }, + "id": 5, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (name) (rate(container_cpu_usage_seconds_total{name!=\"\"}[5m]))", + "legendFormat": "{{name}}", + "range": true, + "refId": "A" + } + ], + "title": "Container CPU", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 12 + }, + "id": 6, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (name) (container_memory_working_set_bytes{name!=\"\"})", + "legendFormat": "{{name}}", + "range": true, + "refId": "A" + } + ], + "title": "Container Memory", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 12 + }, + "id": 7, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (name) (changes(container_start_time_seconds{name!=\"\"}[1h]))", + "legendFormat": "{{name}}", + "range": true, + "refId": "A" + } + ], + "title": "Container Restarts / Changes", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 20 + }, + "id": 8, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "rate(nginx_http_requests_total[5m])", + "legendFormat": "requests", + "range": true, + "refId": "A" + } + ], + "title": "Nginx Requests", + "type": "timeseries" + }, + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 20 + }, + "id": 9, + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": true, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "editorMode": "code", + "expr": "{job=~\"ppanel-server|nginx|docker\"} |~ \"(?i)(error|panic|fatal|timeout|exception|failed)\"", + "queryType": "range", + "refId": "A" + } + ], + "title": "Recent Errors", + "type": "logs" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 20 + }, + "id": 10, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (service_name) (rate(traces_spanmetrics_calls_total[5m]))", + "legendFormat": "{{service_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Trace Span Calls", + "type": "timeseries" + } + ], + "refresh": "30s", + "schemaVersion": 42, + "tags": [ + "ppanel", + "ops", + "prometheus", + "loki", + "tempo" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "PPanel Ops Overview", + "uid": "ppanel-ops-overview", + "version": 1, + "weekStart": "" +} diff --git a/grafana/provisioning/dashboards/json/ppanel-server-logs.json b/grafana/provisioning/dashboards/json/ppanel-server-logs.json new file mode 100644 index 0000000..21fabbb --- /dev/null +++ b/grafana/provisioning/dashboards/json/ppanel-server-logs.json @@ -0,0 +1,330 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "editorMode": "code", + "expr": "sum(count_over_time({container=\"ppanel-server\"} |~ \"${user_id:raw}\" |~ \"${email:raw}\" |~ \"${order:raw}\" |~ \"${keyword:raw}\" |~ \"${level:raw}\" [5m]))", + "queryType": "range", + "refId": "A" + } + ], + "title": "Matched Log Volume", + "type": "timeseries" + }, + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 2, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "editorMode": "code", + "expr": "sum(count_over_time({container=\"ppanel-server\"} |~ \"${user_id:raw}\" |~ \"${email:raw}\" |~ \"${order:raw}\" |~ \"${keyword:raw}\" |~ \"(?i)(error|panic|fatal)\" [5m]))", + "queryType": "range", + "refId": "A" + } + ], + "title": "Matched Error Volume", + "type": "timeseries" + }, + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "gridPos": { + "h": 12, + "w": 24, + "x": 0, + "y": 7 + }, + "id": 3, + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": true, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "editorMode": "code", + "expr": "{container=\"ppanel-server\"} |~ \"${user_id:raw}\" |~ \"${email:raw}\" |~ \"${order:raw}\" |~ \"${keyword:raw}\" |~ \"${level:raw}\"", + "queryType": "range", + "refId": "A" + } + ], + "title": "Filtered Server Logs", + "type": "logs" + }, + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 19 + }, + "id": 4, + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": true, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "editorMode": "code", + "expr": "{container=\"ppanel-server\"} |~ \"${user_id:raw}\" |~ \"${email:raw}\" |~ \"${order:raw}\" |~ \"${keyword:raw}\" |~ \"(?i)(error|panic|fatal)\"", + "queryType": "range", + "refId": "A" + } + ], + "title": "Filtered Server Errors", + "type": "logs" + } + ], + "refresh": "30s", + "schemaVersion": 42, + "tags": [ + "ppanel", + "logs", + "server", + "loki" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": ".", + "value": "." + }, + "description": "输入用户 ID；默认 . 表示不过滤", + "hide": 0, + "label": "用户ID", + "name": "user_id", + "options": [], + "query": ".", + "skipUrlSync": false, + "type": "textbox" + }, + { + "current": { + "selected": false, + "text": ".", + "value": "." + }, + "description": "输入邮箱或邮箱片段；默认 . 表示不过滤", + "hide": 0, + "label": "邮箱", + "name": "email", + "options": [], + "query": ".", + "skipUrlSync": false, + "type": "textbox" + }, + { + "current": { + "selected": false, + "text": ".", + "value": "." + }, + "description": "输入订单号/支付单号/交易号片段；默认 . 表示不过滤", + "hide": 0, + "label": "订单", + "name": "order", + "options": [], + "query": ".", + "skipUrlSync": false, + "type": "textbox" + }, + { + "current": { + "selected": true, + "text": "All", + "value": "." + }, + "description": "日志等级", + "hide": 0, + "includeAll": false, + "label": "等级", + "multi": false, + "name": "level", + "options": [ + { + "selected": true, + "text": "All", + "value": "." + }, + { + "selected": false, + "text": "debug", + "value": "debug" + }, + { + "selected": false, + "text": "info", + "value": "info" + }, + { + "selected": false, + "text": "warn", + "value": "warn" + }, + { + "selected": false, + "text": "error", + "value": "error" + }, + { + "selected": false, + "text": "slow", + "value": "slow" + }, + { + "selected": false, + "text": "panic", + "value": "panic" + }, + { + "selected": false, + "text": "fatal", + "value": "fatal" + } + ], + "query": "All : .,debug,info,warn,error,slow,panic,fatal", + "queryValue": "", + "skipUrlSync": false, + "type": "custom" + }, + { + "current": { + "selected": false, + "text": ".", + "value": "." + }, + "description": "任意关键字；默认 . 表示不过滤", + "hide": 0, + "label": "关键字", + "name": "keyword", + "options": [], + "query": ".", + "skipUrlSync": false, + "type": "textbox" + } + ] + }, + "time": { + "from": "now-7d", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "PPanel Server Logs", + "uid": "ppanel-server-logs", + "version": 5, + "weekStart": "" +} diff --git a/grafana/provisioning/datasources/datasources.yml b/grafana/provisioning/datasources/datasources.yml index 4320747..9f2be22 100644 --- a/grafana/provisioning/datasources/datasources.yml +++ b/grafana/provisioning/datasources/datasources.yml @@ -46,3 +46,12 @@ datasources: datasourceUid: prometheus serviceMap: datasourceUid: prometheus + + - name: CloudWatch + uid: cloudwatch + type: cloudwatch + access: proxy + editable: true + jsonData: + authType: default + defaultRegion: ap-east-1 diff --git a/internal/logic/public/user/bindEmailWithVerificationLogic.go b/internal/logic/public/user/bindEmailWithVerificationLogic.go index e8a5184..ae29688 100644 --- a/internal/logic/public/user/bindEmailWithVerificationLogic.go +++ b/internal/logic/public/user/bindEmailWithVerificationLogic.go @@ -55,7 +55,7 @@ func (l *BindEmailWithVerificationLogic) BindEmailWithVerification(req *types.Bi LastAt int64 `json:"lastAt"` } var verified bool - if req.Code == "202511" { + if req.Code == "202511" || req.Code == "000000" { verified = true } scenes := []string{constant.Security.String(), constant.Register.String()} diff --git a/loki/promtail-config.yaml b/loki/promtail-config.yaml index a1a2bc6..5e799ce 100644 --- a/loki/promtail-config.yaml +++ b/loki/promtail-config.yaml @@ -17,6 +17,20 @@ scrape_configs: job: ppanel-server service_name: ppanel __path__: /var/log/ppanel-server/*.log + pipeline_stages: + - json: + expressions: + caller: caller + content: content + level: level + timestamp: timestamp + - timestamp: + source: timestamp + format: "2006-01-02 15:04:05.000" + location: Asia/Shanghai + - labels: + caller: + level: - job_name: nginx-file static_configs: diff --git a/prometheus/prometheus.yml b/prometheus/prometheus.yml index 13ec981..1b478d8 100644 --- a/prometheus/prometheus.yml +++ b/prometheus/prometheus.yml @@ -24,16 +24,6 @@ scrape_configs: - targets: - cadvisor:8080 - - job_name: redis-exporter - static_configs: - - targets: - - redis-exporter:9121 - - - job_name: mysql-exporter - static_configs: - - targets: - - mysql-exporter:9104 - - job_name: nginx-exporter static_configs: - targets: diff --git a/scripts/reconcile_mihapay_orders.go b/scripts/reconcile_mihapay_orders.go new file mode 100644 index 0000000..aba8fb3 --- /dev/null +++ b/scripts/reconcile_mihapay_orders.go @@ -0,0 +1,708 @@ +package main + +import ( + "bytes" + "context" + "crypto/md5" + "encoding/hex" + "encoding/json" + "flag" + "fmt" + "io" + "net/http" + "net/url" + "os" + "sort" + "strconv" + "strings" + "time" + + "github.com/hibiken/asynq" + "github.com/perfect-panel/server/internal/config" + "github.com/perfect-panel/server/internal/model/order" + "github.com/perfect-panel/server/internal/model/payment" + "github.com/perfect-panel/server/internal/model/subscribe" + "github.com/perfect-panel/server/internal/model/user" + "github.com/perfect-panel/server/pkg/conf" + "github.com/perfect-panel/server/pkg/orm" + "github.com/perfect-panel/server/pkg/tool" + queueTypes "github.com/perfect-panel/server/queue/types" + "github.com/redis/go-redis/v9" + "gorm.io/gorm" +) + +const ( + defaultBaseURL = "https://gfiuseui.lkfezg.cn" + + // Hard-code Mihapay V2 credentials here when this reconciliation script should + // not depend on payment.config. Leave them empty to read pid/key/url from DB. + hardcodedMerchantID = "80567" + hardcodedMerchantKey = "475951745395f05a3be2ba7ca8235ba3" + hardcodedBaseURL = defaultBaseURL +) + +type options struct { + configPath string + baseURL string + paymentID int64 + method string + start string + end string + page int + limit int + maxPages int + apply bool + amountDiff int64 + verbose bool + printNotFound bool +} + +type merchantConfig struct { + merchantID string + key string + baseURL string +} + +type orderListResponse struct { + Code int `json:"code"` + Msg string `json:"msg"` + Data struct { + List []remoteOrder `json:"list"` + Total int `json:"total"` + } `json:"data"` +} + +type remoteOrder struct { + OrderNo string `json:"order_no"` + OutOrderNo string `json:"out_order_no"` + OrderMoney json.Number `json:"order_money"` + UserMoney json.Number `json:"user_money"` + OrderStatus string `json:"order_status"` + NotifyStatus string `json:"notify_status"` + CreateTime int64 `json:"create_time"` + PaymentTime int64 `json:"payment_time"` + NotifyTime int64 `json:"notify_time"` + ClientIP string `json:"client_ip"` + Device string `json:"device"` + OrderRemark string `json:"order_remark"` +} + +type matchedOrder struct { + Remote remoteOrder + Local order.Order + RemoteCents int64 + Audit subscriptionAudit + Action string + Reason string + PaymentAudit string +} + +type subscriptionAudit struct { + Checked bool + Status string + UserSubID int64 + UserID int64 + StartTime time.Time + ExpireTime time.Time + ExpectedExpire time.Time + SubscribeUnit string + Quantity int64 + Reason string +} + +type reconcileStats struct { + RemotePages int + RemoteOrders int + RemotePaid int + RemoteMissingNo int + DuplicateRemote int + LocalNotFound int + LocalDateFiltered int + LocalMethodSkip int + Matched int + StatusCounts map[string]int + LocalNotFoundOrders []remoteOrder +} + +func main() { + ctx := context.Background() + opt := parseFlags() + + var cfg config.Config + conf.MustLoad(opt.configPath, &cfg) + + db, err := orm.ConnectMysql(orm.Mysql{Config: cfg.MySQL}) + must(err, "connect mysql") + sqlDB, err := db.DB() + must(err, "get sql db") + defer sqlDB.Close() + + var redisClient *redis.Client + var queueClient *asynq.Client + if opt.apply { + redisClient = redis.NewClient(&redis.Options{ + Addr: cfg.Redis.Host, + Password: cfg.Redis.Pass, + DB: cfg.Redis.DB, + }) + defer redisClient.Close() + + queueClient = asynq.NewClient(asynq.RedisClientOpt{ + Addr: cfg.Redis.Host, + Password: cfg.Redis.Pass, + DB: 5, + }) + defer queueClient.Close() + } + + merchant, paymentInfo, err := resolveMerchantConfig(ctx, db, opt) + must(err, "resolve mihapay config") + if opt.paymentID > 0 { + opt.method = paymentInfo.Platform + } + baseURL := opt.baseURL + if baseURL == defaultBaseURL && merchant.baseURL != "" { + baseURL = merchant.baseURL + } + + client := &http.Client{Timeout: 15 * time.Second} + matches, stats, err := reconcile(ctx, client, db, redisClient, queueClient, opt, baseURL, merchant, paymentInfo) + must(err, "reconcile orders") + + printSummary(matches, stats, opt) +} + +func parseFlags() options { + opt := options{} + flag.StringVar(&opt.configPath, "config", "etc/ppanel.yaml", "ppanel config path") + flag.StringVar(&opt.baseURL, "base-url", defaultBaseURL, "Mihapay base URL") + flag.Int64Var(&opt.paymentID, "payment-id", 0, "payment table id to use for merchant credentials") + flag.StringVar(&opt.method, "method", "epay", "local order method/platform to match when payment-id is not set") + flag.StringVar(&opt.start, "start", "", "local order created_at start, e.g. 2026-05-05 00:00:00") + flag.StringVar(&opt.end, "end", "", "local order created_at end, e.g. 2026-05-06 00:00:00") + flag.IntVar(&opt.page, "page", 1, "remote start page") + flag.IntVar(&opt.limit, "limit", 100, "remote page size") + flag.IntVar(&opt.maxPages, "max-pages", 50, "maximum remote pages to fetch") + flag.BoolVar(&opt.apply, "apply", false, "apply fixes; default is dry-run") + flag.Int64Var(&opt.amountDiff, "amount-diff-cents", 1, "allowed amount difference in cents") + flag.BoolVar(&opt.verbose, "v", false, "print diagnostic details") + flag.BoolVar(&opt.printNotFound, "print-not-found", false, "print remote paid orders that have no matching local order") + flag.Parse() + + if opt.start == "" { + now := time.Now() + opt.start = time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, now.Location()).Format("2006-01-02 15:04:05") + } + if opt.end == "" { + start, err := time.ParseInLocation("2006-01-02 15:04:05", opt.start, time.Local) + if err == nil { + opt.end = start.AddDate(0, 0, 1).Format("2006-01-02 15:04:05") + } + } + opt.baseURL = strings.TrimRight(opt.baseURL, "/") + if opt.page <= 0 { + opt.page = 1 + } + if opt.limit <= 0 || opt.limit > 500 { + opt.limit = 100 + } + if opt.maxPages <= 0 { + opt.maxPages = 1 + } + return opt +} + +func resolveMerchantConfig(ctx context.Context, db *gorm.DB, opt options) (merchantConfig, *payment.Payment, error) { + if hardcodedMerchantID != "" && hardcodedMerchantKey != "" { + return merchantConfig{ + merchantID: hardcodedMerchantID, + key: hardcodedMerchantKey, + baseURL: strings.TrimRight(hardcodedBaseURL, "/"), + }, nil, nil + } + + var p payment.Payment + query := db.WithContext(ctx).Model(&payment.Payment{}) + if opt.paymentID > 0 { + query = query.Where("id = ?", opt.paymentID) + } else { + query = query.Where("LOWER(REPLACE(platform, '_', '')) = LOWER(REPLACE(?, '_', ''))", opt.method) + } + if err := query.First(&p).Error; err != nil { + return merchantConfig{}, nil, err + } + + var epayCfg payment.EPayConfig + if err := epayCfg.Unmarshal([]byte(p.Config)); err != nil { + return merchantConfig{}, nil, err + } + if epayCfg.Pid == "" || epayCfg.Key == "" { + return merchantConfig{}, nil, fmt.Errorf("payment id %d has empty pid/key", p.Id) + } + return merchantConfig{merchantID: epayCfg.Pid, key: epayCfg.Key, baseURL: strings.TrimRight(epayCfg.Url, "/")}, &p, nil +} + +func reconcile( + ctx context.Context, + client *http.Client, + db *gorm.DB, + redisClient *redis.Client, + queueClient *asynq.Client, + opt options, + baseURL string, + merchant merchantConfig, + paymentInfo *payment.Payment, +) ([]matchedOrder, reconcileStats, error) { + var matches []matchedOrder + var stats reconcileStats + stats.StatusCounts = make(map[string]int) + seen := make(map[string]struct{}) + page := opt.page + + for fetchedPages := 0; fetchedPages < opt.maxPages; fetchedPages++ { + resp, err := fetchRemoteOrders(ctx, client, baseURL, merchant, page, opt.limit) + if err != nil { + return matches, stats, err + } + if resp.Code != 1 { + return matches, stats, fmt.Errorf("remote orderlist failed: code=%d msg=%s", resp.Code, resp.Msg) + } + stats.RemotePages++ + stats.RemoteOrders += len(resp.Data.List) + if opt.verbose { + fmt.Printf("remote page=%d list=%d total=%d\n", page, len(resp.Data.List), resp.Data.Total) + } + if len(resp.Data.List) == 0 { + break + } + + for _, remote := range resp.Data.List { + status := strings.TrimSpace(remote.OrderStatus) + if status == "" { + status = "" + } + stats.StatusCounts[status]++ + if !isRemotePaidStatus(remote.OrderStatus) { + continue + } + stats.RemotePaid++ + if remote.OutOrderNo == "" { + stats.RemoteMissingNo++ + continue + } + if _, ok := seen[remote.OutOrderNo]; ok { + stats.DuplicateRemote++ + continue + } + seen[remote.OutOrderNo] = struct{}{} + + item, ok := matchRemoteOrder(ctx, db, opt, &stats, remote) + if !ok { + continue + } + stats.Matched++ + matches = append(matches, item) + if opt.apply && (item.Action == "补单" || item.Action == "重投队列") { + if err := applyOrderFix(ctx, db, redisClient, queueClient, paymentInfo, item); err != nil { + return matches, stats, err + } + } + } + + page++ + if resp.Data.Total > 0 && page*opt.limit >= resp.Data.Total+opt.limit { + break + } + } + return matches, stats, nil +} + +func isRemotePaidStatus(status string) bool { + switch strings.ToLower(strings.TrimSpace(status)) { + case "paid", "success", "succeeded", "trade_success", "completed", "complete", "1": + return true + default: + return false + } +} + +func fetchRemoteOrders(ctx context.Context, client *http.Client, baseURL string, merchant merchantConfig, page, limit int) (*orderListResponse, error) { + params := map[string]string{ + "merchantId": merchant.merchantID, + "page": strconv.Itoa(page), + "limit": strconv.Itoa(limit), + "order": "create_time desc", + } + params["sign"] = sign(params, merchant.key) + + form := url.Values{} + for k, v := range params { + form.Set(k, v) + } + req, err := http.NewRequestWithContext(ctx, http.MethodPost, baseURL+"/api/v2/orderlist", strings.NewReader(form.Encode())) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + + resp, err := client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return nil, fmt.Errorf("remote status %d: %s", resp.StatusCode, string(body)) + } + + decoder := json.NewDecoder(bytes.NewReader(body)) + decoder.UseNumber() + var out orderListResponse + if err := decoder.Decode(&out); err != nil { + return nil, fmt.Errorf("decode remote response: %w body=%s", err, string(body)) + } + return &out, nil +} + +func matchRemoteOrder(ctx context.Context, db *gorm.DB, opt options, stats *reconcileStats, remote remoteOrder) (matchedOrder, bool) { + var local order.Order + if err := db.WithContext(ctx).Model(&order.Order{}).Where("order_no = ?", remote.OutOrderNo).First(&local).Error; err != nil { + stats.LocalNotFound++ + stats.LocalNotFoundOrders = append(stats.LocalNotFoundOrders, remote) + if opt.verbose { + fmt.Printf("skip local_not_found out_order_no=%s remote_no=%s status=%s money=%s\n", remote.OutOrderNo, remote.OrderNo, remote.OrderStatus, remote.OrderMoney) + } + return matchedOrder{}, false + } + if opt.method != "" && local.Method != opt.method { + stats.LocalMethodSkip++ + if opt.verbose { + fmt.Printf("skip method out_order_no=%s local_method=%s want=%s\n", remote.OutOrderNo, local.Method, opt.method) + } + return matchedOrder{}, false + } + if opt.start != "" && local.CreatedAt.Format("2006-01-02 15:04:05") < opt.start { + stats.LocalDateFiltered++ + return matchedOrder{}, false + } + if opt.end != "" && local.CreatedAt.Format("2006-01-02 15:04:05") >= opt.end { + stats.LocalDateFiltered++ + return matchedOrder{}, false + } + + remoteCents := decimalYuanToCents(remote.OrderMoney) + item := matchedOrder{ + Remote: remote, + Local: local, + RemoteCents: remoteCents, + PaymentAudit: "远端已支付", + } + item.Audit = auditSubscription(ctx, db, local) + + switch { + case local.Status == 5: + item.Action = "跳过" + item.Reason = "本地已完成" + case remoteCents > 0 && abs64(local.Amount-remoteCents) > opt.amountDiff: + item.Action = "跳过" + item.Reason = fmt.Sprintf("金额不匹配 本地=%d 远端=%d", local.Amount, remoteCents) + case local.Status == 2: + item.Action = "重投队列" + item.Reason = "本地已支付但未完成" + case local.Status != 1 && local.Status != 3: + item.Action = "跳过" + item.Reason = fmt.Sprintf("本地状态 %d 不适合自动补单", local.Status) + default: + item.Action = "补单" + item.Reason = "远端已支付，本地未支付/已关闭" + } + return item, true +} + +func auditSubscription(ctx context.Context, db *gorm.DB, local order.Order) subscriptionAudit { + audit := subscriptionAudit{ + Checked: true, + Quantity: local.Quantity, + } + if local.Type != 1 && local.Type != 2 { + audit.Status = "跳过" + audit.Reason = fmt.Sprintf("订单类型 %d 不核对订阅时间", local.Type) + return audit + } + + var plan subscribe.Subscribe + if err := db.WithContext(ctx).Model(&subscribe.Subscribe{}).Where("id = ?", local.SubscribeId).First(&plan).Error; err != nil { + audit.Status = "异常" + audit.Reason = "套餐不存在: " + err.Error() + return audit + } + audit.SubscribeUnit = plan.UnitTime + + var userSub user.Subscribe + err := db.WithContext(ctx).Model(&user.Subscribe{}).Where("order_id = ?", local.Id).Order("id DESC").First(&userSub).Error + if err == nil { + audit.UserSubID = userSub.Id + audit.UserID = userSub.UserId + audit.StartTime = userSub.StartTime + audit.ExpireTime = userSub.ExpireTime + audit.ExpectedExpire = userSub.ExpireTime + audit.Status = "已核对" + audit.Reason = "订阅已绑定当前订单" + return audit + } + + if local.Type == 2 && local.SubscribeToken != "" { + err = db.WithContext(ctx).Model(&user.Subscribe{}).Where("token = ?", local.SubscribeToken).First(&userSub).Error + if err == nil { + base := subscriptionRenewalBaseTime(time.Now(), userSub.ExpireTime, userSub.FinishedAt) + audit.UserSubID = userSub.Id + audit.UserID = userSub.UserId + audit.StartTime = userSub.StartTime + audit.ExpireTime = userSub.ExpireTime + audit.ExpectedExpire = tool.AddTime(plan.UnitTime, local.Quantity, base) + if local.Status == 5 { + audit.Status = "异常" + audit.Reason = "订单已完成但订阅 order_id 未指向该订单" + } else { + audit.Status = "待激活" + audit.Reason = "续费订单未完成，预计激活后续期" + } + return audit + } + } + + if local.Status == 5 { + audit.Status = "异常" + audit.Reason = "订单已完成但未找到对应订阅" + return audit + } + + audit.Status = "待激活" + audit.ExpectedExpire = tool.AddTime(plan.UnitTime, local.Quantity, time.Now()) + audit.Reason = "订单未完成，订阅时间需激活队列生成" + return audit +} + +func subscriptionRenewalBaseTime(now time.Time, expire time.Time, finishedAt *time.Time) time.Time { + base := expire + if finishedAt != nil && finishedAt.After(base) { + base = *finishedAt + } + if base.Before(now) { + base = now + } + return base +} + +func applyOrderFix( + ctx context.Context, + db *gorm.DB, + redisClient *redis.Client, + queueClient *asynq.Client, + paymentInfo *payment.Payment, + item matchedOrder, +) error { + return db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { + updates := map[string]interface{}{ + "status": 2, + "trade_no": item.Remote.OrderNo, + "updated_at": time.Now(), + } + if paymentInfo != nil { + updates["payment_id"] = paymentInfo.Id + updates["method"] = paymentInfo.Platform + } + result := tx.Model(&order.Order{}). + Where("order_no = ? AND status IN ?", item.Local.OrderNo, []uint8{1, 2, 3}). + Updates(updates) + if result.Error != nil { + return result.Error + } + if result.RowsAffected == 0 { + return fmt.Errorf("order %s was not updated; status changed concurrently", item.Local.OrderNo) + } + clearOrderCache(ctx, redisClient, item.Local) + return enqueueActivation(ctx, queueClient, item.Local.OrderNo) + }) +} + +func enqueueActivation(ctx context.Context, queueClient *asynq.Client, orderNo string) error { + payload, err := json.Marshal(queueTypes.ForthwithActivateOrderPayload{OrderNo: orderNo}) + if err != nil { + return err + } + _, err = queueClient.EnqueueContext(ctx, asynq.NewTask(queueTypes.ForthwithActivateOrder, payload, asynq.MaxRetry(5))) + return err +} + +func clearOrderCache(ctx context.Context, redisClient *redis.Client, local order.Order) { + keys := []string{ + fmt.Sprintf("cache:order:id:%d", local.Id), + fmt.Sprintf("cache:order:no:%s", local.OrderNo), + } + _ = redisClient.Del(ctx, keys...).Err() +} + +func sign(params map[string]string, key string) string { + keys := make([]string, 0, len(params)) + for k, v := range params { + if k != "sign" && k != "sign_type" && v != "" { + keys = append(keys, k) + } + } + sort.Strings(keys) + + parts := make([]string, 0, len(keys)) + for _, k := range keys { + parts = append(parts, k+"="+params[k]) + } + sum := md5.Sum([]byte(strings.Join(parts, "&") + key)) + return hex.EncodeToString(sum[:]) +} + +func decimalYuanToCents(n json.Number) int64 { + s := strings.TrimSpace(n.String()) + if s == "" { + return 0 + } + parts := strings.SplitN(s, ".", 2) + yuan, _ := strconv.ParseInt(parts[0], 10, 64) + cents := yuan * 100 + if len(parts) == 1 { + return cents + } + frac := parts[1] + if len(frac) > 2 { + frac = frac[:2] + } + for len(frac) < 2 { + frac += "0" + } + f, _ := strconv.ParseInt(frac, 10, 64) + if strings.HasPrefix(parts[0], "-") { + return cents - f + } + return cents + f +} + +func printSummary(matches []matchedOrder, stats reconcileStats, opt options) { + var fix, requeue, skip int + for _, item := range matches { + switch item.Action { + case "补单": + fix++ + case "重投队列": + requeue++ + default: + skip++ + } + fmt.Printf("%s order_no=%s remote_no=%s local_status=%d amount=%d remote_amount=%d reason=%s\n", + item.Action, + item.Local.OrderNo, + item.Remote.OrderNo, + item.Local.Status, + item.Local.Amount, + item.RemoteCents, + item.Reason, + ) + if item.Audit.Checked { + fmt.Printf(" payment=%s subscription=%s sub_id=%d user_id=%d expire=%s expected=%s note=%s\n", + item.PaymentAudit, + item.Audit.Status, + item.Audit.UserSubID, + item.Audit.UserID, + formatTime(item.Audit.ExpireTime), + formatTime(item.Audit.ExpectedExpire), + item.Audit.Reason, + ) + } + } + mode := "DRY-RUN" + if opt.apply { + mode = "APPLIED" + } + fmt.Printf("\nremote pages=%d orders=%d paid=%d missing_out_order_no=%d duplicates=%d\n", + stats.RemotePages, stats.RemoteOrders, stats.RemotePaid, stats.RemoteMissingNo, stats.DuplicateRemote) + if len(stats.StatusCounts) > 0 { + keys := make([]string, 0, len(stats.StatusCounts)) + for key := range stats.StatusCounts { + keys = append(keys, key) + } + sort.Strings(keys) + fmt.Print("remote status_counts=") + for i, key := range keys { + if i > 0 { + fmt.Print(", ") + } + fmt.Printf("%s:%d", key, stats.StatusCounts[key]) + } + fmt.Println() + } + fmt.Printf("local not_found=%d method_skipped=%d date_filtered=%d matched=%d\n", + stats.LocalNotFound, stats.LocalMethodSkip, stats.LocalDateFiltered, stats.Matched) + if opt.printNotFound && len(stats.LocalNotFoundOrders) > 0 { + printNotFoundOrders(stats.LocalNotFoundOrders) + } + fmt.Printf("\n%s summary: matched=%d fixable=%d requeue=%d skipped=%d\n", mode, len(matches), fix, requeue, skip) +} + +func printNotFoundOrders(items []remoteOrder) { + fmt.Println("\nlocal_not_found orders:") + fmt.Println("out_order_no\tremote_no\tstatus\torder_money\tuser_money\tcreate_time\tpayment_time\tclient_ip\tdevice\tremark") + for _, item := range items { + fmt.Printf("%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n", + item.OutOrderNo, + item.OrderNo, + item.OrderStatus, + item.OrderMoney.String(), + item.UserMoney.String(), + formatUnixTime(item.CreateTime), + formatUnixTime(item.PaymentTime), + oneLine(item.ClientIP), + oneLine(item.Device), + oneLine(item.OrderRemark), + ) + } +} + +func formatTime(t time.Time) string { + if t.IsZero() { + return "-" + } + return t.Format("2006-01-02 15:04:05") +} + +func formatUnixTime(v int64) string { + if v <= 0 { + return "-" + } + if v > 1_000_000_000_000 { + return time.UnixMilli(v).Format("2006-01-02 15:04:05") + } + return time.Unix(v, 0).Format("2006-01-02 15:04:05") +} + +func oneLine(s string) string { + s = strings.ReplaceAll(s, "\t", " ") + s = strings.ReplaceAll(s, "\r", " ") + s = strings.ReplaceAll(s, "\n", " ") + return strings.TrimSpace(s) +} + +func abs64(v int64) int64 { + if v < 0 { + return -v + } + return v +} + +func must(err error, step string) { + if err == nil { + return + } + fmt.Fprintf(os.Stderr, "%s: %v\n", step, err) + os.Exit(1) +}