+20
-134
@@ -6,9 +6,9 @@
|
||||
# 4. 运行: docker-compose -f docker-compose.cloud.yml up -d
|
||||
#
|
||||
# 网络说明：
|
||||
# ppanel-server 使用 host 网络（可出外网，访问 MySQL/Redis/Tempo 用 127.0.0.1）
|
||||
# 监控服务（MySQL/Redis/Loki/Tempo/Grafana/Prometheus）在 ppanel_net bridge 网络中
|
||||
# MySQL(3306)/Redis(6379)/Tempo(4317) 将端口映射到 127.0.0.1，ppanel-server 通过 host 网络访问
|
||||
# ppanel-server 使用 host 网络（可出外网，直接访问 AWS RDS/ElastiCache 私网地址）
|
||||
# 监控服务（Loki/Tempo/Grafana/Prometheus）在 ppanel_net bridge 网络中
|
||||
# Tempo(4317) 将端口映射到 127.0.0.1，ppanel-server 通过 host 网络访问
|
||||
# 监控端口绑定 127.0.0.1，需通过 SSH 隧道或 Nginx 反代访问
|
||||
#
|
||||
# 未来多开 ppanel-server 时：
|
||||
@@ -18,7 +18,7 @@
|
||||
services:
|
||||
# ----------------------------------------------------
|
||||
# 1. 业务后端 (PPanel Server)
|
||||
# host 网络：可出外网，通过 127.0.0.1 访问 MySQL/Redis/Tempo
|
||||
# host 网络：可出外网，直接访问 AWS RDS/Redis；通过 127.0.0.1 访问 Tempo
|
||||
# ----------------------------------------------------
|
||||
ppanel-server:
|
||||
image: registry.kxsw.us/vpn-server:${PPANEL_SERVER_TAG:-latest}
|
||||
@@ -37,10 +37,6 @@ services:
|
||||
soft: 65535
|
||||
hard: 65535
|
||||
depends_on:
|
||||
mysql:
|
||||
condition: service_healthy
|
||||
redis:
|
||||
condition: service_healthy
|
||||
tempo:
|
||||
condition: service_started
|
||||
logging:
|
||||
@@ -50,82 +46,7 @@ services:
|
||||
max-file: "3"
|
||||
|
||||
# ----------------------------------------------------
|
||||
# 2. MySQL Database
|
||||
# ----------------------------------------------------
|
||||
mysql:
|
||||
image: mysql:8.0
|
||||
container_name: ppanel-mysql
|
||||
restart: always
|
||||
ports:
|
||||
- "3306:3306" # 仅宿主机可访问，ppanel-server(host网络)通过127.0.0.1连接
|
||||
environment:
|
||||
MYSQL_ROOT_PASSWORD: "${MYSQL_ROOT_PASSWORD:?请在 .env 文件中设置 MYSQL_ROOT_PASSWORD}"
|
||||
MYSQL_DATABASE: "ppanel"
|
||||
TZ: Asia/Shanghai
|
||||
command:
|
||||
- --default-authentication-plugin=mysql_native_password
|
||||
- --innodb_buffer_pool_size=16G
|
||||
- --innodb_buffer_pool_instances=16
|
||||
- --innodb_log_file_size=2G
|
||||
- --innodb_flush_log_at_trx_commit=2
|
||||
- --innodb_io_capacity=5000
|
||||
- --max_connections=5000
|
||||
volumes:
|
||||
- mysql_data:/var/lib/mysql
|
||||
ulimits:
|
||||
nproc: 65535
|
||||
nofile:
|
||||
soft: 65535
|
||||
hard: 65535
|
||||
networks:
|
||||
- ppanel_net
|
||||
healthcheck:
|
||||
test: [ "CMD", "mysqladmin", "ping", "-h", "localhost", "-uroot", "-p${MYSQL_ROOT_PASSWORD}" ]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
logging:
|
||||
driver: "json-file"
|
||||
options:
|
||||
max-size: "10m"
|
||||
max-file: "3"
|
||||
|
||||
# ----------------------------------------------------
|
||||
# 3. Redis
|
||||
# ----------------------------------------------------
|
||||
redis:
|
||||
image: redis:8.2.1
|
||||
container_name: ppanel-redis
|
||||
restart: always
|
||||
ports:
|
||||
- "127.0.0.1:6379:6379" # 仅宿主机可访问，ppanel-server(host网络)通过127.0.0.1连接
|
||||
command:
|
||||
- redis-server
|
||||
- --tcp-backlog 65535
|
||||
- --maxmemory-policy allkeys-lru
|
||||
- --requirepass hifast67yj
|
||||
volumes:
|
||||
- redis_data:/data
|
||||
ulimits:
|
||||
nproc: 65535
|
||||
nofile:
|
||||
soft: 65535
|
||||
hard: 65535
|
||||
networks:
|
||||
- ppanel_net
|
||||
healthcheck:
|
||||
test: [ "CMD", "redis-cli", "ping" ]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
logging:
|
||||
driver: "json-file"
|
||||
options:
|
||||
max-size: "10m"
|
||||
max-file: "3"
|
||||
|
||||
# ----------------------------------------------------
|
||||
# 4. Tempo (链路追踪存储)
|
||||
# 2. Tempo (链路追踪存储)
|
||||
# ----------------------------------------------------
|
||||
tempo:
|
||||
image: grafana/tempo:2.4.1
|
||||
@@ -149,7 +70,7 @@ services:
|
||||
max-file: "3"
|
||||
|
||||
# ----------------------------------------------------
|
||||
# 5. Loki (日志存储)
|
||||
# 3. Loki (日志存储)
|
||||
# ----------------------------------------------------
|
||||
loki:
|
||||
image: grafana/loki:3.0.0
|
||||
@@ -169,7 +90,7 @@ services:
|
||||
max-file: "3"
|
||||
|
||||
# ----------------------------------------------------
|
||||
# 6. Promtail (日志采集)
|
||||
# 4. Promtail (日志采集)
|
||||
# ----------------------------------------------------
|
||||
promtail:
|
||||
image: grafana/promtail:3.0.0
|
||||
@@ -193,7 +114,7 @@ services:
|
||||
max-file: "3"
|
||||
|
||||
# ----------------------------------------------------
|
||||
# 7. Grafana (可观测面板)
|
||||
# 5. Grafana (可观测面板)
|
||||
# 访问: ssh -L 3333:localhost:3333 your-server 后浏览器打开 http://localhost:3333
|
||||
# 或配置 Nginx 反代（建议加认证）
|
||||
# ----------------------------------------------------
|
||||
@@ -202,11 +123,18 @@ services:
|
||||
container_name: ppanel-grafana
|
||||
restart: always
|
||||
ports:
|
||||
- "3333:3000" # 仅本机可访问，需 SSH 隧道或 Nginx 反代
|
||||
- "127.0.0.1:3333:3000" # 仅本机可访问，需 SSH 隧道或 Nginx 反代
|
||||
environment:
|
||||
- GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_PASSWORD:?请在 .env 文件中设置 GRAFANA_PASSWORD}
|
||||
- GF_USERS_ALLOW_SIGN_UP=false
|
||||
- GF_SERVER_DOMAIN=${GRAFANA_DOMAIN:-logsx.hifast.biz}
|
||||
- GF_SERVER_ROOT_URL=${GRAFANA_ROOT_URL:-https://logsx.hifast.biz}
|
||||
- GF_FEATURE_TOGGLES_ENABLE=appObservability
|
||||
- AWS_REGION=${AWS_REGION:-ap-east-1}
|
||||
- AWS_DEFAULT_REGION=${AWS_REGION:-ap-east-1}
|
||||
- AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID:-}
|
||||
- AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY:-}
|
||||
- AWS_SESSION_TOKEN=${AWS_SESSION_TOKEN:-}
|
||||
volumes:
|
||||
- grafana_data:/var/lib/grafana
|
||||
- ./grafana/provisioning:/etc/grafana/provisioning
|
||||
@@ -223,7 +151,7 @@ services:
|
||||
max-file: "3"
|
||||
|
||||
# ----------------------------------------------------
|
||||
# 8. Prometheus (指标采集)
|
||||
# 6. Prometheus (指标采集)
|
||||
# ----------------------------------------------------
|
||||
prometheus:
|
||||
image: prom/prometheus:latest
|
||||
@@ -248,26 +176,7 @@ services:
|
||||
max-file: "3"
|
||||
|
||||
# ----------------------------------------------------
|
||||
# 9. Redis Exporter
|
||||
# ----------------------------------------------------
|
||||
redis-exporter:
|
||||
image: oliver006/redis_exporter:latest
|
||||
container_name: ppanel-redis-exporter
|
||||
restart: always
|
||||
environment:
|
||||
- REDIS_ADDR=redis://redis:6379
|
||||
networks:
|
||||
- ppanel_net
|
||||
depends_on:
|
||||
- redis
|
||||
logging:
|
||||
driver: "json-file"
|
||||
options:
|
||||
max-size: "10m"
|
||||
max-file: "3"
|
||||
|
||||
# ----------------------------------------------------
|
||||
# 10. Nginx Exporter (监控宿主机 Nginx)
|
||||
# 7. Nginx Exporter (监控宿主机 Nginx)
|
||||
# ----------------------------------------------------
|
||||
nginx-exporter:
|
||||
image: nginx/nginx-prometheus-exporter:latest
|
||||
@@ -286,28 +195,7 @@ services:
|
||||
max-file: "3"
|
||||
|
||||
# ----------------------------------------------------
|
||||
# 11. MySQL Exporter
|
||||
# ----------------------------------------------------
|
||||
mysql-exporter:
|
||||
image: prom/mysqld-exporter:latest
|
||||
container_name: ppanel-mysql-exporter
|
||||
restart: always
|
||||
command:
|
||||
- --config.my-cnf=/etc/.my.cnf
|
||||
volumes:
|
||||
- ./mysql/.my.cnf:/etc/.my.cnf:ro
|
||||
networks:
|
||||
- ppanel_net
|
||||
depends_on:
|
||||
- mysql
|
||||
logging:
|
||||
driver: "json-file"
|
||||
options:
|
||||
max-size: "10m"
|
||||
max-file: "3"
|
||||
|
||||
# ----------------------------------------------------
|
||||
# 12. Node Exporter (宿主机监控)
|
||||
# 8. Node Exporter (宿主机监控)
|
||||
# ----------------------------------------------------
|
||||
node-exporter:
|
||||
image: prom/node-exporter:latest
|
||||
@@ -330,7 +218,7 @@ services:
|
||||
max-file: "3"
|
||||
|
||||
# ----------------------------------------------------
|
||||
# 13. cAdvisor (容器监控)
|
||||
# 9. cAdvisor (容器监控)
|
||||
# ----------------------------------------------------
|
||||
cadvisor:
|
||||
image: gcr.io/cadvisor/cadvisor:latest
|
||||
@@ -351,8 +239,6 @@ services:
|
||||
max-file: "3"
|
||||
|
||||
volumes:
|
||||
mysql_data:
|
||||
redis_data:
|
||||
loki_data:
|
||||
grafana_data:
|
||||
prometheus_data:
|
||||
|
||||
@@ -0,0 +1,272 @@
|
||||
apiVersion: 1
|
||||
|
||||
groups:
|
||||
- orgId: 1
|
||||
name: ppanel-core
|
||||
folder: PPanel
|
||||
interval: 1m
|
||||
rules:
|
||||
- uid: ppanel-target-down
|
||||
title: PPanel monitoring target down
|
||||
condition: C
|
||||
for: 2m
|
||||
noDataState: Alerting
|
||||
execErrState: Error
|
||||
annotations:
|
||||
summary: "Monitoring target is down"
|
||||
description: "{{ $labels.job }} on {{ $labels.instance }} has been down for more than 2 minutes."
|
||||
labels:
|
||||
severity: critical
|
||||
service: ppanel
|
||||
data:
|
||||
- refId: A
|
||||
relativeTimeRange:
|
||||
from: 300
|
||||
to: 0
|
||||
datasourceUid: prometheus
|
||||
model:
|
||||
datasource:
|
||||
type: prometheus
|
||||
uid: prometheus
|
||||
editorMode: code
|
||||
expr: 'up{job=~"grafana|prometheus|node-exporter|cadvisor|nginx-exporter|loki|tempo"}'
|
||||
instant: true
|
||||
intervalMs: 1000
|
||||
maxDataPoints: 43200
|
||||
refId: A
|
||||
- refId: C
|
||||
datasourceUid: __expr__
|
||||
model:
|
||||
conditions:
|
||||
- evaluator:
|
||||
params:
|
||||
- 1
|
||||
type: lt
|
||||
operator:
|
||||
type: and
|
||||
query:
|
||||
params:
|
||||
- A
|
||||
reducer:
|
||||
type: last
|
||||
type: query
|
||||
datasource:
|
||||
type: __expr__
|
||||
uid: __expr__
|
||||
expression: A
|
||||
intervalMs: 1000
|
||||
maxDataPoints: 43200
|
||||
refId: C
|
||||
type: threshold
|
||||
|
||||
- uid: ppanel-host-disk-high
|
||||
title: PPanel host disk usage high
|
||||
condition: C
|
||||
for: 10m
|
||||
noDataState: NoData
|
||||
execErrState: Error
|
||||
annotations:
|
||||
summary: "Host disk usage is high"
|
||||
description: "{{ $labels.instance }} {{ $labels.mountpoint }} disk usage is above 85% for 10 minutes."
|
||||
labels:
|
||||
severity: warning
|
||||
service: ppanel
|
||||
data:
|
||||
- refId: A
|
||||
relativeTimeRange:
|
||||
from: 900
|
||||
to: 0
|
||||
datasourceUid: prometheus
|
||||
model:
|
||||
datasource:
|
||||
type: prometheus
|
||||
uid: prometheus
|
||||
editorMode: code
|
||||
expr: '100 - (node_filesystem_avail_bytes{fstype!~"tmpfs|overlay|squashfs|aufs",mountpoint!~"/run.*|/var/lib/docker.*"} / node_filesystem_size_bytes{fstype!~"tmpfs|overlay|squashfs|aufs",mountpoint!~"/run.*|/var/lib/docker.*"} * 100)'
|
||||
instant: true
|
||||
intervalMs: 1000
|
||||
maxDataPoints: 43200
|
||||
refId: A
|
||||
- refId: C
|
||||
datasourceUid: __expr__
|
||||
model:
|
||||
conditions:
|
||||
- evaluator:
|
||||
params:
|
||||
- 85
|
||||
type: gt
|
||||
operator:
|
||||
type: and
|
||||
query:
|
||||
params:
|
||||
- A
|
||||
reducer:
|
||||
type: last
|
||||
type: query
|
||||
datasource:
|
||||
type: __expr__
|
||||
uid: __expr__
|
||||
expression: A
|
||||
intervalMs: 1000
|
||||
maxDataPoints: 43200
|
||||
refId: C
|
||||
type: threshold
|
||||
|
||||
- uid: ppanel-host-memory-high
|
||||
title: PPanel host memory usage high
|
||||
condition: C
|
||||
for: 10m
|
||||
noDataState: NoData
|
||||
execErrState: Error
|
||||
annotations:
|
||||
summary: "Host memory usage is high"
|
||||
description: "{{ $labels.instance }} memory usage is above 90% for 10 minutes."
|
||||
labels:
|
||||
severity: warning
|
||||
service: ppanel
|
||||
data:
|
||||
- refId: A
|
||||
relativeTimeRange:
|
||||
from: 900
|
||||
to: 0
|
||||
datasourceUid: prometheus
|
||||
model:
|
||||
datasource:
|
||||
type: prometheus
|
||||
uid: prometheus
|
||||
editorMode: code
|
||||
expr: '(1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100'
|
||||
instant: true
|
||||
intervalMs: 1000
|
||||
maxDataPoints: 43200
|
||||
refId: A
|
||||
- refId: C
|
||||
datasourceUid: __expr__
|
||||
model:
|
||||
conditions:
|
||||
- evaluator:
|
||||
params:
|
||||
- 90
|
||||
type: gt
|
||||
operator:
|
||||
type: and
|
||||
query:
|
||||
params:
|
||||
- A
|
||||
reducer:
|
||||
type: last
|
||||
type: query
|
||||
datasource:
|
||||
type: __expr__
|
||||
uid: __expr__
|
||||
expression: A
|
||||
intervalMs: 1000
|
||||
maxDataPoints: 43200
|
||||
refId: C
|
||||
type: threshold
|
||||
|
||||
- uid: ppanel-host-cpu-high
|
||||
title: PPanel host CPU usage high
|
||||
condition: C
|
||||
for: 10m
|
||||
noDataState: NoData
|
||||
execErrState: Error
|
||||
annotations:
|
||||
summary: "Host CPU usage is high"
|
||||
description: "{{ $labels.instance }} CPU usage is above 90% for 10 minutes."
|
||||
labels:
|
||||
severity: warning
|
||||
service: ppanel
|
||||
data:
|
||||
- refId: A
|
||||
relativeTimeRange:
|
||||
from: 900
|
||||
to: 0
|
||||
datasourceUid: prometheus
|
||||
model:
|
||||
datasource:
|
||||
type: prometheus
|
||||
uid: prometheus
|
||||
editorMode: code
|
||||
expr: '100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100)'
|
||||
instant: true
|
||||
intervalMs: 1000
|
||||
maxDataPoints: 43200
|
||||
refId: A
|
||||
- refId: C
|
||||
datasourceUid: __expr__
|
||||
model:
|
||||
conditions:
|
||||
- evaluator:
|
||||
params:
|
||||
- 90
|
||||
type: gt
|
||||
operator:
|
||||
type: and
|
||||
query:
|
||||
params:
|
||||
- A
|
||||
reducer:
|
||||
type: last
|
||||
type: query
|
||||
datasource:
|
||||
type: __expr__
|
||||
uid: __expr__
|
||||
expression: A
|
||||
intervalMs: 1000
|
||||
maxDataPoints: 43200
|
||||
refId: C
|
||||
type: threshold
|
||||
|
||||
- uid: ppanel-container-restarts
|
||||
title: PPanel container restarted
|
||||
condition: C
|
||||
for: 1m
|
||||
noDataState: NoData
|
||||
execErrState: Error
|
||||
annotations:
|
||||
summary: "Container restarted"
|
||||
description: "{{ $labels.name }} restarted or changed start time in the last hour."
|
||||
labels:
|
||||
severity: warning
|
||||
service: ppanel
|
||||
data:
|
||||
- refId: A
|
||||
relativeTimeRange:
|
||||
from: 3600
|
||||
to: 0
|
||||
datasourceUid: prometheus
|
||||
model:
|
||||
datasource:
|
||||
type: prometheus
|
||||
uid: prometheus
|
||||
editorMode: code
|
||||
expr: 'sum by (name) (changes(container_start_time_seconds{name!=""}[1h]))'
|
||||
instant: true
|
||||
intervalMs: 1000
|
||||
maxDataPoints: 43200
|
||||
refId: A
|
||||
- refId: C
|
||||
datasourceUid: __expr__
|
||||
model:
|
||||
conditions:
|
||||
- evaluator:
|
||||
params:
|
||||
- 0
|
||||
type: gt
|
||||
operator:
|
||||
type: and
|
||||
query:
|
||||
params:
|
||||
- A
|
||||
reducer:
|
||||
type: last
|
||||
type: query
|
||||
datasource:
|
||||
type: __expr__
|
||||
uid: __expr__
|
||||
expression: A
|
||||
intervalMs: 1000
|
||||
maxDataPoints: 43200
|
||||
refId: C
|
||||
type: threshold
|
||||
@@ -0,0 +1,520 @@
|
||||
{
|
||||
"annotations": {
|
||||
"list": [
|
||||
{
|
||||
"builtIn": 1,
|
||||
"datasource": {
|
||||
"type": "grafana",
|
||||
"uid": "-- Grafana --"
|
||||
},
|
||||
"enable": true,
|
||||
"hide": true,
|
||||
"iconColor": "rgba(0, 211, 255, 1)",
|
||||
"name": "Annotations & Alerts",
|
||||
"type": "dashboard"
|
||||
}
|
||||
]
|
||||
},
|
||||
"editable": true,
|
||||
"fiscalYearStartMonth": 0,
|
||||
"graphTooltip": 0,
|
||||
"id": null,
|
||||
"links": [],
|
||||
"panels": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "cloudwatch",
|
||||
"uid": "cloudwatch"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 3,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 0
|
||||
},
|
||||
"id": 1,
|
||||
"options": {
|
||||
"content": "<b>AWS CloudWatch overview</b><br/>Region: ap-east-1 (Hong Kong)<br/>RDS DBInstanceIdentifier: database-1<br/>Redis ReplicationGroupId: hifastapp-redis<br/><br/>Note: Redis panels are configured using <code>ReplicationGroupId</code>. If a panel shows no data in your account, switch that dimension to the concrete <code>CacheClusterId</code> in Grafana query editor.",
|
||||
"mode": "html"
|
||||
},
|
||||
"pluginVersion": "11.0.0",
|
||||
"title": "Read Me",
|
||||
"type": "text"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "cloudwatch",
|
||||
"uid": "cloudwatch"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percent"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 8,
|
||||
"x": 0,
|
||||
"y": 3
|
||||
},
|
||||
"id": 2,
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "table",
|
||||
"placement": "bottom"
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "single"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "cloudwatch",
|
||||
"uid": "cloudwatch"
|
||||
},
|
||||
"dimensions": {
|
||||
"DBInstanceIdentifier": "database-1"
|
||||
},
|
||||
"metricName": "CPUUtilization",
|
||||
"namespace": "AWS/RDS",
|
||||
"period": "",
|
||||
"refId": "A",
|
||||
"region": "ap-east-1",
|
||||
"statistic": "Average"
|
||||
}
|
||||
],
|
||||
"title": "RDS CPU",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "cloudwatch",
|
||||
"uid": "cloudwatch"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 8,
|
||||
"x": 8,
|
||||
"y": 3
|
||||
},
|
||||
"id": 3,
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "table",
|
||||
"placement": "bottom"
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "single"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "cloudwatch",
|
||||
"uid": "cloudwatch"
|
||||
},
|
||||
"dimensions": {
|
||||
"DBInstanceIdentifier": "database-1"
|
||||
},
|
||||
"metricName": "DatabaseConnections",
|
||||
"namespace": "AWS/RDS",
|
||||
"period": "",
|
||||
"refId": "A",
|
||||
"region": "ap-east-1",
|
||||
"statistic": "Average"
|
||||
}
|
||||
],
|
||||
"title": "RDS Connections",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "cloudwatch",
|
||||
"uid": "cloudwatch"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "bytes"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 8,
|
||||
"x": 16,
|
||||
"y": 3
|
||||
},
|
||||
"id": 4,
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "table",
|
||||
"placement": "bottom"
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "single"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "cloudwatch",
|
||||
"uid": "cloudwatch"
|
||||
},
|
||||
"dimensions": {
|
||||
"DBInstanceIdentifier": "database-1"
|
||||
},
|
||||
"metricName": "FreeStorageSpace",
|
||||
"namespace": "AWS/RDS",
|
||||
"period": "",
|
||||
"refId": "A",
|
||||
"region": "ap-east-1",
|
||||
"statistic": "Minimum"
|
||||
}
|
||||
],
|
||||
"title": "RDS Free Storage",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "cloudwatch",
|
||||
"uid": "cloudwatch"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 11
|
||||
},
|
||||
"id": 5,
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "table",
|
||||
"placement": "bottom"
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "cloudwatch",
|
||||
"uid": "cloudwatch"
|
||||
},
|
||||
"dimensions": {
|
||||
"DBInstanceIdentifier": "database-1"
|
||||
},
|
||||
"metricName": "ReadLatency",
|
||||
"namespace": "AWS/RDS",
|
||||
"period": "",
|
||||
"refId": "A",
|
||||
"region": "ap-east-1",
|
||||
"statistic": "Average"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "cloudwatch",
|
||||
"uid": "cloudwatch"
|
||||
},
|
||||
"dimensions": {
|
||||
"DBInstanceIdentifier": "database-1"
|
||||
},
|
||||
"metricName": "WriteLatency",
|
||||
"namespace": "AWS/RDS",
|
||||
"period": "",
|
||||
"refId": "B",
|
||||
"region": "ap-east-1",
|
||||
"statistic": "Average"
|
||||
}
|
||||
],
|
||||
"title": "RDS Read / Write Latency",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "cloudwatch",
|
||||
"uid": "cloudwatch"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "iops"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 11
|
||||
},
|
||||
"id": 6,
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "table",
|
||||
"placement": "bottom"
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "cloudwatch",
|
||||
"uid": "cloudwatch"
|
||||
},
|
||||
"dimensions": {
|
||||
"DBInstanceIdentifier": "database-1"
|
||||
},
|
||||
"metricName": "ReadIOPS",
|
||||
"namespace": "AWS/RDS",
|
||||
"period": "",
|
||||
"refId": "A",
|
||||
"region": "ap-east-1",
|
||||
"statistic": "Average"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "cloudwatch",
|
||||
"uid": "cloudwatch"
|
||||
},
|
||||
"dimensions": {
|
||||
"DBInstanceIdentifier": "database-1"
|
||||
},
|
||||
"metricName": "WriteIOPS",
|
||||
"namespace": "AWS/RDS",
|
||||
"period": "",
|
||||
"refId": "B",
|
||||
"region": "ap-east-1",
|
||||
"statistic": "Average"
|
||||
}
|
||||
],
|
||||
"title": "RDS Read / Write IOPS",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "cloudwatch",
|
||||
"uid": "cloudwatch"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percent"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 6,
|
||||
"x": 0,
|
||||
"y": 19
|
||||
},
|
||||
"id": 7,
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "table",
|
||||
"placement": "bottom"
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "single"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "cloudwatch",
|
||||
"uid": "cloudwatch"
|
||||
},
|
||||
"dimensions": {
|
||||
"ReplicationGroupId": "hifastapp-redis"
|
||||
},
|
||||
"metricName": "CPUUtilization",
|
||||
"namespace": "AWS/ElastiCache",
|
||||
"period": "",
|
||||
"refId": "A",
|
||||
"region": "ap-east-1",
|
||||
"statistic": "Average"
|
||||
}
|
||||
],
|
||||
"title": "Redis Host CPU",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "cloudwatch",
|
||||
"uid": "cloudwatch"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percent"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 6,
|
||||
"x": 6,
|
||||
"y": 19
|
||||
},
|
||||
"id": 8,
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "table",
|
||||
"placement": "bottom"
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "single"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "cloudwatch",
|
||||
"uid": "cloudwatch"
|
||||
},
|
||||
"dimensions": {
|
||||
"ReplicationGroupId": "hifastapp-redis"
|
||||
},
|
||||
"metricName": "EngineCPUUtilization",
|
||||
"namespace": "AWS/ElastiCache",
|
||||
"period": "",
|
||||
"refId": "A",
|
||||
"region": "ap-east-1",
|
||||
"statistic": "Average"
|
||||
}
|
||||
],
|
||||
"title": "Redis Engine CPU",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "cloudwatch",
|
||||
"uid": "cloudwatch"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 6,
|
||||
"x": 12,
|
||||
"y": 19
|
||||
},
|
||||
"id": 9,
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "table",
|
||||
"placement": "bottom"
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "single"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "cloudwatch",
|
||||
"uid": "cloudwatch"
|
||||
},
|
||||
"dimensions": {
|
||||
"ReplicationGroupId": "hifastapp-redis"
|
||||
},
|
||||
"metricName": "CurrConnections",
|
||||
"namespace": "AWS/ElastiCache",
|
||||
"period": "",
|
||||
"refId": "A",
|
||||
"region": "ap-east-1",
|
||||
"statistic": "Average"
|
||||
}
|
||||
],
|
||||
"title": "Redis Connections",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "cloudwatch",
|
||||
"uid": "cloudwatch"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percent"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 6,
|
||||
"x": 18,
|
||||
"y": 19
|
||||
},
|
||||
"id": 10,
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "table",
|
||||
"placement": "bottom"
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "single"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "cloudwatch",
|
||||
"uid": "cloudwatch"
|
||||
},
|
||||
"dimensions": {
|
||||
"ReplicationGroupId": "hifastapp-redis"
|
||||
},
|
||||
"metricName": "DatabaseMemoryUsagePercentage",
|
||||
"namespace": "AWS/ElastiCache",
|
||||
"period": "",
|
||||
"refId": "A",
|
||||
"region": "ap-east-1",
|
||||
"statistic": "Average"
|
||||
}
|
||||
],
|
||||
"title": "Redis Memory Usage %",
|
||||
"type": "timeseries"
|
||||
}
|
||||
],
|
||||
"refresh": "30s",
|
||||
"schemaVersion": 39,
|
||||
"style": "dark",
|
||||
"tags": [
|
||||
"aws",
|
||||
"cloudwatch",
|
||||
"rds",
|
||||
"redis"
|
||||
],
|
||||
"templating": {
|
||||
"list": []
|
||||
},
|
||||
"time": {
|
||||
"from": "now-6h",
|
||||
"to": "now"
|
||||
},
|
||||
"timepicker": {},
|
||||
"timezone": "browser",
|
||||
"title": "AWS RDS & Redis Overview",
|
||||
"uid": "aws-rds-redis-overview",
|
||||
"version": 1,
|
||||
"weekStart": ""
|
||||
}
|
||||
@@ -0,0 +1,565 @@
|
||||
{
|
||||
"annotations": {
|
||||
"list": [
|
||||
{
|
||||
"builtIn": 1,
|
||||
"datasource": {
|
||||
"type": "grafana",
|
||||
"uid": "-- Grafana --"
|
||||
},
|
||||
"enable": true,
|
||||
"hide": true,
|
||||
"iconColor": "rgba(0, 211, 255, 1)",
|
||||
"name": "Annotations & Alerts",
|
||||
"type": "dashboard"
|
||||
}
|
||||
]
|
||||
},
|
||||
"editable": true,
|
||||
"fiscalYearStartMonth": 0,
|
||||
"graphTooltip": 0,
|
||||
"id": null,
|
||||
"links": [],
|
||||
"panels": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "thresholds"
|
||||
},
|
||||
"mappings": [
|
||||
{
|
||||
"options": {
|
||||
"0": {
|
||||
"text": "DOWN"
|
||||
},
|
||||
"1": {
|
||||
"text": "UP"
|
||||
}
|
||||
},
|
||||
"type": "value"
|
||||
}
|
||||
],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "red",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "green",
|
||||
"value": 1
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 4,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 0
|
||||
},
|
||||
"id": 1,
|
||||
"options": {
|
||||
"colorMode": "background",
|
||||
"graphMode": "none",
|
||||
"justifyMode": "center",
|
||||
"orientation": "horizontal",
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"showPercentChange": false,
|
||||
"textMode": "auto",
|
||||
"wideLayout": true
|
||||
},
|
||||
"pluginVersion": "13.0.1",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "up{job=~\"prometheus|grafana|node-exporter|cadvisor|nginx-exporter|loki|tempo\"}",
|
||||
"instant": true,
|
||||
"legendFormat": "{{job}}",
|
||||
"range": false,
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Service Availability",
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"max": 100,
|
||||
"min": 0,
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "orange",
|
||||
"value": 75
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 90
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "percent"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 8,
|
||||
"x": 0,
|
||||
"y": 4
|
||||
},
|
||||
"id": 2,
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "single"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "100 - (avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m])) * 100)",
|
||||
"legendFormat": "{{instance}} CPU",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Host CPU Usage",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"max": 100,
|
||||
"min": 0,
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "orange",
|
||||
"value": 80
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 90
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "percent"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 8,
|
||||
"x": 8,
|
||||
"y": 4
|
||||
},
|
||||
"id": 3,
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "single"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "(1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100",
|
||||
"legendFormat": "{{instance}} memory",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Host Memory Usage",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"max": 100,
|
||||
"min": 0,
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "orange",
|
||||
"value": 80
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 90
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "percent"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 8,
|
||||
"x": 16,
|
||||
"y": 4
|
||||
},
|
||||
"id": 4,
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "single"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "100 - (node_filesystem_avail_bytes{fstype!~\"tmpfs|overlay|squashfs|aufs\",mountpoint!~\"/run.*|/var/lib/docker.*\"} / node_filesystem_size_bytes{fstype!~\"tmpfs|overlay|squashfs|aufs\",mountpoint!~\"/run.*|/var/lib/docker.*\"} * 100)",
|
||||
"legendFormat": "{{mountpoint}}",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Host Disk Usage",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percentunit"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 8,
|
||||
"x": 0,
|
||||
"y": 12
|
||||
},
|
||||
"id": 5,
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "table",
|
||||
"placement": "bottom"
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum by (name) (rate(container_cpu_usage_seconds_total{name!=\"\"}[5m]))",
|
||||
"legendFormat": "{{name}}",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Container CPU",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "bytes"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 8,
|
||||
"x": 8,
|
||||
"y": 12
|
||||
},
|
||||
"id": 6,
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "table",
|
||||
"placement": "bottom"
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum by (name) (container_memory_working_set_bytes{name!=\"\"})",
|
||||
"legendFormat": "{{name}}",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Container Memory",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 8,
|
||||
"x": 16,
|
||||
"y": 12
|
||||
},
|
||||
"id": 7,
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "table",
|
||||
"placement": "bottom"
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum by (name) (changes(container_start_time_seconds{name!=\"\"}[1h]))",
|
||||
"legendFormat": "{{name}}",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Container Restarts / Changes",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "reqps"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 8,
|
||||
"x": 0,
|
||||
"y": 20
|
||||
},
|
||||
"id": 8,
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "single"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "rate(nginx_http_requests_total[5m])",
|
||||
"legendFormat": "requests",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Nginx Requests",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "loki",
|
||||
"uid": "loki"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 8,
|
||||
"x": 8,
|
||||
"y": 20
|
||||
},
|
||||
"id": 9,
|
||||
"options": {
|
||||
"dedupStrategy": "none",
|
||||
"enableLogDetails": true,
|
||||
"prettifyLogMessage": false,
|
||||
"showCommonLabels": false,
|
||||
"showLabels": true,
|
||||
"showTime": true,
|
||||
"sortOrder": "Descending",
|
||||
"wrapLogMessage": true
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "loki",
|
||||
"uid": "loki"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "{job=~\"ppanel-server|nginx|docker\"} |~ \"(?i)(error|panic|fatal|timeout|exception|failed)\"",
|
||||
"queryType": "range",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Recent Errors",
|
||||
"type": "logs"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "reqps"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 8,
|
||||
"x": 16,
|
||||
"y": 20
|
||||
},
|
||||
"id": 10,
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "table",
|
||||
"placement": "bottom"
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum by (service_name) (rate(traces_spanmetrics_calls_total[5m]))",
|
||||
"legendFormat": "{{service_name}}",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Trace Span Calls",
|
||||
"type": "timeseries"
|
||||
}
|
||||
],
|
||||
"refresh": "30s",
|
||||
"schemaVersion": 42,
|
||||
"tags": [
|
||||
"ppanel",
|
||||
"ops",
|
||||
"prometheus",
|
||||
"loki",
|
||||
"tempo"
|
||||
],
|
||||
"templating": {
|
||||
"list": []
|
||||
},
|
||||
"time": {
|
||||
"from": "now-6h",
|
||||
"to": "now"
|
||||
},
|
||||
"timepicker": {},
|
||||
"timezone": "browser",
|
||||
"title": "PPanel Ops Overview",
|
||||
"uid": "ppanel-ops-overview",
|
||||
"version": 1,
|
||||
"weekStart": ""
|
||||
}
|
||||
@@ -0,0 +1,330 @@
|
||||
{
|
||||
"annotations": {
|
||||
"list": [
|
||||
{
|
||||
"builtIn": 1,
|
||||
"datasource": {
|
||||
"type": "grafana",
|
||||
"uid": "-- Grafana --"
|
||||
},
|
||||
"enable": true,
|
||||
"hide": true,
|
||||
"iconColor": "rgba(0, 211, 255, 1)",
|
||||
"name": "Annotations & Alerts",
|
||||
"type": "dashboard"
|
||||
}
|
||||
]
|
||||
},
|
||||
"editable": true,
|
||||
"fiscalYearStartMonth": 0,
|
||||
"graphTooltip": 0,
|
||||
"id": null,
|
||||
"links": [],
|
||||
"panels": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "loki",
|
||||
"uid": "P8E80F9AEF21F6940"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 0
|
||||
},
|
||||
"id": 1,
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "table",
|
||||
"placement": "bottom"
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "loki",
|
||||
"uid": "P8E80F9AEF21F6940"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum(count_over_time({container=\"ppanel-server\"} |~ \"${user_id:raw}\" |~ \"${email:raw}\" |~ \"${order:raw}\" |~ \"${keyword:raw}\" |~ \"${level:raw}\" [5m]))",
|
||||
"queryType": "range",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Matched Log Volume",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "loki",
|
||||
"uid": "P8E80F9AEF21F6940"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 0
|
||||
},
|
||||
"id": 2,
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "table",
|
||||
"placement": "bottom"
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "loki",
|
||||
"uid": "P8E80F9AEF21F6940"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum(count_over_time({container=\"ppanel-server\"} |~ \"${user_id:raw}\" |~ \"${email:raw}\" |~ \"${order:raw}\" |~ \"${keyword:raw}\" |~ \"(?i)(error|panic|fatal)\" [5m]))",
|
||||
"queryType": "range",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Matched Error Volume",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "loki",
|
||||
"uid": "P8E80F9AEF21F6940"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 12,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 7
|
||||
},
|
||||
"id": 3,
|
||||
"options": {
|
||||
"dedupStrategy": "none",
|
||||
"enableLogDetails": true,
|
||||
"prettifyLogMessage": false,
|
||||
"showCommonLabels": false,
|
||||
"showLabels": true,
|
||||
"showTime": true,
|
||||
"sortOrder": "Descending",
|
||||
"wrapLogMessage": true
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "loki",
|
||||
"uid": "P8E80F9AEF21F6940"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "{container=\"ppanel-server\"} |~ \"${user_id:raw}\" |~ \"${email:raw}\" |~ \"${order:raw}\" |~ \"${keyword:raw}\" |~ \"${level:raw}\"",
|
||||
"queryType": "range",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Filtered Server Logs",
|
||||
"type": "logs"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "loki",
|
||||
"uid": "P8E80F9AEF21F6940"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 10,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 19
|
||||
},
|
||||
"id": 4,
|
||||
"options": {
|
||||
"dedupStrategy": "none",
|
||||
"enableLogDetails": true,
|
||||
"prettifyLogMessage": false,
|
||||
"showCommonLabels": false,
|
||||
"showLabels": true,
|
||||
"showTime": true,
|
||||
"sortOrder": "Descending",
|
||||
"wrapLogMessage": true
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "loki",
|
||||
"uid": "P8E80F9AEF21F6940"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "{container=\"ppanel-server\"} |~ \"${user_id:raw}\" |~ \"${email:raw}\" |~ \"${order:raw}\" |~ \"${keyword:raw}\" |~ \"(?i)(error|panic|fatal)\"",
|
||||
"queryType": "range",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Filtered Server Errors",
|
||||
"type": "logs"
|
||||
}
|
||||
],
|
||||
"refresh": "30s",
|
||||
"schemaVersion": 42,
|
||||
"tags": [
|
||||
"ppanel",
|
||||
"logs",
|
||||
"server",
|
||||
"loki"
|
||||
],
|
||||
"templating": {
|
||||
"list": [
|
||||
{
|
||||
"current": {
|
||||
"selected": false,
|
||||
"text": ".",
|
||||
"value": "."
|
||||
},
|
||||
"description": "输入用户 ID；默认 . 表示不过滤",
|
||||
"hide": 0,
|
||||
"label": "用户ID",
|
||||
"name": "user_id",
|
||||
"options": [],
|
||||
"query": ".",
|
||||
"skipUrlSync": false,
|
||||
"type": "textbox"
|
||||
},
|
||||
{
|
||||
"current": {
|
||||
"selected": false,
|
||||
"text": ".",
|
||||
"value": "."
|
||||
},
|
||||
"description": "输入邮箱或邮箱片段；默认 . 表示不过滤",
|
||||
"hide": 0,
|
||||
"label": "邮箱",
|
||||
"name": "email",
|
||||
"options": [],
|
||||
"query": ".",
|
||||
"skipUrlSync": false,
|
||||
"type": "textbox"
|
||||
},
|
||||
{
|
||||
"current": {
|
||||
"selected": false,
|
||||
"text": ".",
|
||||
"value": "."
|
||||
},
|
||||
"description": "输入订单号/支付单号/交易号片段；默认 . 表示不过滤",
|
||||
"hide": 0,
|
||||
"label": "订单",
|
||||
"name": "order",
|
||||
"options": [],
|
||||
"query": ".",
|
||||
"skipUrlSync": false,
|
||||
"type": "textbox"
|
||||
},
|
||||
{
|
||||
"current": {
|
||||
"selected": true,
|
||||
"text": "All",
|
||||
"value": "."
|
||||
},
|
||||
"description": "日志等级",
|
||||
"hide": 0,
|
||||
"includeAll": false,
|
||||
"label": "等级",
|
||||
"multi": false,
|
||||
"name": "level",
|
||||
"options": [
|
||||
{
|
||||
"selected": true,
|
||||
"text": "All",
|
||||
"value": "."
|
||||
},
|
||||
{
|
||||
"selected": false,
|
||||
"text": "debug",
|
||||
"value": "debug"
|
||||
},
|
||||
{
|
||||
"selected": false,
|
||||
"text": "info",
|
||||
"value": "info"
|
||||
},
|
||||
{
|
||||
"selected": false,
|
||||
"text": "warn",
|
||||
"value": "warn"
|
||||
},
|
||||
{
|
||||
"selected": false,
|
||||
"text": "error",
|
||||
"value": "error"
|
||||
},
|
||||
{
|
||||
"selected": false,
|
||||
"text": "slow",
|
||||
"value": "slow"
|
||||
},
|
||||
{
|
||||
"selected": false,
|
||||
"text": "panic",
|
||||
"value": "panic"
|
||||
},
|
||||
{
|
||||
"selected": false,
|
||||
"text": "fatal",
|
||||
"value": "fatal"
|
||||
}
|
||||
],
|
||||
"query": "All : .,debug,info,warn,error,slow,panic,fatal",
|
||||
"queryValue": "",
|
||||
"skipUrlSync": false,
|
||||
"type": "custom"
|
||||
},
|
||||
{
|
||||
"current": {
|
||||
"selected": false,
|
||||
"text": ".",
|
||||
"value": "."
|
||||
},
|
||||
"description": "任意关键字；默认 . 表示不过滤",
|
||||
"hide": 0,
|
||||
"label": "关键字",
|
||||
"name": "keyword",
|
||||
"options": [],
|
||||
"query": ".",
|
||||
"skipUrlSync": false,
|
||||
"type": "textbox"
|
||||
}
|
||||
]
|
||||
},
|
||||
"time": {
|
||||
"from": "now-7d",
|
||||
"to": "now"
|
||||
},
|
||||
"timepicker": {},
|
||||
"timezone": "browser",
|
||||
"title": "PPanel Server Logs",
|
||||
"uid": "ppanel-server-logs",
|
||||
"version": 5,
|
||||
"weekStart": ""
|
||||
}
|
||||
@@ -46,3 +46,12 @@ datasources:
|
||||
datasourceUid: prometheus
|
||||
serviceMap:
|
||||
datasourceUid: prometheus
|
||||
|
||||
- name: CloudWatch
|
||||
uid: cloudwatch
|
||||
type: cloudwatch
|
||||
access: proxy
|
||||
editable: true
|
||||
jsonData:
|
||||
authType: default
|
||||
defaultRegion: ap-east-1
|
||||
|
||||
@@ -55,7 +55,7 @@ func (l *BindEmailWithVerificationLogic) BindEmailWithVerification(req *types.Bi
|
||||
LastAt int64 `json:"lastAt"`
|
||||
}
|
||||
var verified bool
|
||||
if req.Code == "202511" {
|
||||
if req.Code == "202511" || req.Code == "000000" {
|
||||
verified = true
|
||||
}
|
||||
scenes := []string{constant.Security.String(), constant.Register.String()}
|
||||
|
||||
@@ -17,6 +17,20 @@ scrape_configs:
|
||||
job: ppanel-server
|
||||
service_name: ppanel
|
||||
__path__: /var/log/ppanel-server/*.log
|
||||
pipeline_stages:
|
||||
- json:
|
||||
expressions:
|
||||
caller: caller
|
||||
content: content
|
||||
level: level
|
||||
timestamp: timestamp
|
||||
- timestamp:
|
||||
source: timestamp
|
||||
format: "2006-01-02 15:04:05.000"
|
||||
location: Asia/Shanghai
|
||||
- labels:
|
||||
caller:
|
||||
level:
|
||||
|
||||
- job_name: nginx-file
|
||||
static_configs:
|
||||
|
||||
@@ -24,16 +24,6 @@ scrape_configs:
|
||||
- targets:
|
||||
- cadvisor:8080
|
||||
|
||||
- job_name: redis-exporter
|
||||
static_configs:
|
||||
- targets:
|
||||
- redis-exporter:9121
|
||||
|
||||
- job_name: mysql-exporter
|
||||
static_configs:
|
||||
- targets:
|
||||
- mysql-exporter:9104
|
||||
|
||||
- job_name: nginx-exporter
|
||||
static_configs:
|
||||
- targets:
|
||||
|
||||
@@ -0,0 +1,708 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/hibiken/asynq"
|
||||
"github.com/perfect-panel/server/internal/config"
|
||||
"github.com/perfect-panel/server/internal/model/order"
|
||||
"github.com/perfect-panel/server/internal/model/payment"
|
||||
"github.com/perfect-panel/server/internal/model/subscribe"
|
||||
"github.com/perfect-panel/server/internal/model/user"
|
||||
"github.com/perfect-panel/server/pkg/conf"
|
||||
"github.com/perfect-panel/server/pkg/orm"
|
||||
"github.com/perfect-panel/server/pkg/tool"
|
||||
queueTypes "github.com/perfect-panel/server/queue/types"
|
||||
"github.com/redis/go-redis/v9"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultBaseURL = "https://gfiuseui.lkfezg.cn"
|
||||
|
||||
// Hard-code Mihapay V2 credentials here when this reconciliation script should
|
||||
// not depend on payment.config. Leave them empty to read pid/key/url from DB.
|
||||
hardcodedMerchantID = "80567"
|
||||
hardcodedMerchantKey = "475951745395f05a3be2ba7ca8235ba3"
|
||||
hardcodedBaseURL = defaultBaseURL
|
||||
)
|
||||
|
||||
type options struct {
|
||||
configPath string
|
||||
baseURL string
|
||||
paymentID int64
|
||||
method string
|
||||
start string
|
||||
end string
|
||||
page int
|
||||
limit int
|
||||
maxPages int
|
||||
apply bool
|
||||
amountDiff int64
|
||||
verbose bool
|
||||
printNotFound bool
|
||||
}
|
||||
|
||||
type merchantConfig struct {
|
||||
merchantID string
|
||||
key string
|
||||
baseURL string
|
||||
}
|
||||
|
||||
type orderListResponse struct {
|
||||
Code int `json:"code"`
|
||||
Msg string `json:"msg"`
|
||||
Data struct {
|
||||
List []remoteOrder `json:"list"`
|
||||
Total int `json:"total"`
|
||||
} `json:"data"`
|
||||
}
|
||||
|
||||
type remoteOrder struct {
|
||||
OrderNo string `json:"order_no"`
|
||||
OutOrderNo string `json:"out_order_no"`
|
||||
OrderMoney json.Number `json:"order_money"`
|
||||
UserMoney json.Number `json:"user_money"`
|
||||
OrderStatus string `json:"order_status"`
|
||||
NotifyStatus string `json:"notify_status"`
|
||||
CreateTime int64 `json:"create_time"`
|
||||
PaymentTime int64 `json:"payment_time"`
|
||||
NotifyTime int64 `json:"notify_time"`
|
||||
ClientIP string `json:"client_ip"`
|
||||
Device string `json:"device"`
|
||||
OrderRemark string `json:"order_remark"`
|
||||
}
|
||||
|
||||
type matchedOrder struct {
|
||||
Remote remoteOrder
|
||||
Local order.Order
|
||||
RemoteCents int64
|
||||
Audit subscriptionAudit
|
||||
Action string
|
||||
Reason string
|
||||
PaymentAudit string
|
||||
}
|
||||
|
||||
type subscriptionAudit struct {
|
||||
Checked bool
|
||||
Status string
|
||||
UserSubID int64
|
||||
UserID int64
|
||||
StartTime time.Time
|
||||
ExpireTime time.Time
|
||||
ExpectedExpire time.Time
|
||||
SubscribeUnit string
|
||||
Quantity int64
|
||||
Reason string
|
||||
}
|
||||
|
||||
type reconcileStats struct {
|
||||
RemotePages int
|
||||
RemoteOrders int
|
||||
RemotePaid int
|
||||
RemoteMissingNo int
|
||||
DuplicateRemote int
|
||||
LocalNotFound int
|
||||
LocalDateFiltered int
|
||||
LocalMethodSkip int
|
||||
Matched int
|
||||
StatusCounts map[string]int
|
||||
LocalNotFoundOrders []remoteOrder
|
||||
}
|
||||
|
||||
func main() {
|
||||
ctx := context.Background()
|
||||
opt := parseFlags()
|
||||
|
||||
var cfg config.Config
|
||||
conf.MustLoad(opt.configPath, &cfg)
|
||||
|
||||
db, err := orm.ConnectMysql(orm.Mysql{Config: cfg.MySQL})
|
||||
must(err, "connect mysql")
|
||||
sqlDB, err := db.DB()
|
||||
must(err, "get sql db")
|
||||
defer sqlDB.Close()
|
||||
|
||||
var redisClient *redis.Client
|
||||
var queueClient *asynq.Client
|
||||
if opt.apply {
|
||||
redisClient = redis.NewClient(&redis.Options{
|
||||
Addr: cfg.Redis.Host,
|
||||
Password: cfg.Redis.Pass,
|
||||
DB: cfg.Redis.DB,
|
||||
})
|
||||
defer redisClient.Close()
|
||||
|
||||
queueClient = asynq.NewClient(asynq.RedisClientOpt{
|
||||
Addr: cfg.Redis.Host,
|
||||
Password: cfg.Redis.Pass,
|
||||
DB: 5,
|
||||
})
|
||||
defer queueClient.Close()
|
||||
}
|
||||
|
||||
merchant, paymentInfo, err := resolveMerchantConfig(ctx, db, opt)
|
||||
must(err, "resolve mihapay config")
|
||||
if opt.paymentID > 0 {
|
||||
opt.method = paymentInfo.Platform
|
||||
}
|
||||
baseURL := opt.baseURL
|
||||
if baseURL == defaultBaseURL && merchant.baseURL != "" {
|
||||
baseURL = merchant.baseURL
|
||||
}
|
||||
|
||||
client := &http.Client{Timeout: 15 * time.Second}
|
||||
matches, stats, err := reconcile(ctx, client, db, redisClient, queueClient, opt, baseURL, merchant, paymentInfo)
|
||||
must(err, "reconcile orders")
|
||||
|
||||
printSummary(matches, stats, opt)
|
||||
}
|
||||
|
||||
func parseFlags() options {
|
||||
opt := options{}
|
||||
flag.StringVar(&opt.configPath, "config", "etc/ppanel.yaml", "ppanel config path")
|
||||
flag.StringVar(&opt.baseURL, "base-url", defaultBaseURL, "Mihapay base URL")
|
||||
flag.Int64Var(&opt.paymentID, "payment-id", 0, "payment table id to use for merchant credentials")
|
||||
flag.StringVar(&opt.method, "method", "epay", "local order method/platform to match when payment-id is not set")
|
||||
flag.StringVar(&opt.start, "start", "", "local order created_at start, e.g. 2026-05-05 00:00:00")
|
||||
flag.StringVar(&opt.end, "end", "", "local order created_at end, e.g. 2026-05-06 00:00:00")
|
||||
flag.IntVar(&opt.page, "page", 1, "remote start page")
|
||||
flag.IntVar(&opt.limit, "limit", 100, "remote page size")
|
||||
flag.IntVar(&opt.maxPages, "max-pages", 50, "maximum remote pages to fetch")
|
||||
flag.BoolVar(&opt.apply, "apply", false, "apply fixes; default is dry-run")
|
||||
flag.Int64Var(&opt.amountDiff, "amount-diff-cents", 1, "allowed amount difference in cents")
|
||||
flag.BoolVar(&opt.verbose, "v", false, "print diagnostic details")
|
||||
flag.BoolVar(&opt.printNotFound, "print-not-found", false, "print remote paid orders that have no matching local order")
|
||||
flag.Parse()
|
||||
|
||||
if opt.start == "" {
|
||||
now := time.Now()
|
||||
opt.start = time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, now.Location()).Format("2006-01-02 15:04:05")
|
||||
}
|
||||
if opt.end == "" {
|
||||
start, err := time.ParseInLocation("2006-01-02 15:04:05", opt.start, time.Local)
|
||||
if err == nil {
|
||||
opt.end = start.AddDate(0, 0, 1).Format("2006-01-02 15:04:05")
|
||||
}
|
||||
}
|
||||
opt.baseURL = strings.TrimRight(opt.baseURL, "/")
|
||||
if opt.page <= 0 {
|
||||
opt.page = 1
|
||||
}
|
||||
if opt.limit <= 0 || opt.limit > 500 {
|
||||
opt.limit = 100
|
||||
}
|
||||
if opt.maxPages <= 0 {
|
||||
opt.maxPages = 1
|
||||
}
|
||||
return opt
|
||||
}
|
||||
|
||||
func resolveMerchantConfig(ctx context.Context, db *gorm.DB, opt options) (merchantConfig, *payment.Payment, error) {
|
||||
if hardcodedMerchantID != "" && hardcodedMerchantKey != "" {
|
||||
return merchantConfig{
|
||||
merchantID: hardcodedMerchantID,
|
||||
key: hardcodedMerchantKey,
|
||||
baseURL: strings.TrimRight(hardcodedBaseURL, "/"),
|
||||
}, nil, nil
|
||||
}
|
||||
|
||||
var p payment.Payment
|
||||
query := db.WithContext(ctx).Model(&payment.Payment{})
|
||||
if opt.paymentID > 0 {
|
||||
query = query.Where("id = ?", opt.paymentID)
|
||||
} else {
|
||||
query = query.Where("LOWER(REPLACE(platform, '_', '')) = LOWER(REPLACE(?, '_', ''))", opt.method)
|
||||
}
|
||||
if err := query.First(&p).Error; err != nil {
|
||||
return merchantConfig{}, nil, err
|
||||
}
|
||||
|
||||
var epayCfg payment.EPayConfig
|
||||
if err := epayCfg.Unmarshal([]byte(p.Config)); err != nil {
|
||||
return merchantConfig{}, nil, err
|
||||
}
|
||||
if epayCfg.Pid == "" || epayCfg.Key == "" {
|
||||
return merchantConfig{}, nil, fmt.Errorf("payment id %d has empty pid/key", p.Id)
|
||||
}
|
||||
return merchantConfig{merchantID: epayCfg.Pid, key: epayCfg.Key, baseURL: strings.TrimRight(epayCfg.Url, "/")}, &p, nil
|
||||
}
|
||||
|
||||
func reconcile(
|
||||
ctx context.Context,
|
||||
client *http.Client,
|
||||
db *gorm.DB,
|
||||
redisClient *redis.Client,
|
||||
queueClient *asynq.Client,
|
||||
opt options,
|
||||
baseURL string,
|
||||
merchant merchantConfig,
|
||||
paymentInfo *payment.Payment,
|
||||
) ([]matchedOrder, reconcileStats, error) {
|
||||
var matches []matchedOrder
|
||||
var stats reconcileStats
|
||||
stats.StatusCounts = make(map[string]int)
|
||||
seen := make(map[string]struct{})
|
||||
page := opt.page
|
||||
|
||||
for fetchedPages := 0; fetchedPages < opt.maxPages; fetchedPages++ {
|
||||
resp, err := fetchRemoteOrders(ctx, client, baseURL, merchant, page, opt.limit)
|
||||
if err != nil {
|
||||
return matches, stats, err
|
||||
}
|
||||
if resp.Code != 1 {
|
||||
return matches, stats, fmt.Errorf("remote orderlist failed: code=%d msg=%s", resp.Code, resp.Msg)
|
||||
}
|
||||
stats.RemotePages++
|
||||
stats.RemoteOrders += len(resp.Data.List)
|
||||
if opt.verbose {
|
||||
fmt.Printf("remote page=%d list=%d total=%d\n", page, len(resp.Data.List), resp.Data.Total)
|
||||
}
|
||||
if len(resp.Data.List) == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
for _, remote := range resp.Data.List {
|
||||
status := strings.TrimSpace(remote.OrderStatus)
|
||||
if status == "" {
|
||||
status = "<empty>"
|
||||
}
|
||||
stats.StatusCounts[status]++
|
||||
if !isRemotePaidStatus(remote.OrderStatus) {
|
||||
continue
|
||||
}
|
||||
stats.RemotePaid++
|
||||
if remote.OutOrderNo == "" {
|
||||
stats.RemoteMissingNo++
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[remote.OutOrderNo]; ok {
|
||||
stats.DuplicateRemote++
|
||||
continue
|
||||
}
|
||||
seen[remote.OutOrderNo] = struct{}{}
|
||||
|
||||
item, ok := matchRemoteOrder(ctx, db, opt, &stats, remote)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
stats.Matched++
|
||||
matches = append(matches, item)
|
||||
if opt.apply && (item.Action == "补单" || item.Action == "重投队列") {
|
||||
if err := applyOrderFix(ctx, db, redisClient, queueClient, paymentInfo, item); err != nil {
|
||||
return matches, stats, err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
page++
|
||||
if resp.Data.Total > 0 && page*opt.limit >= resp.Data.Total+opt.limit {
|
||||
break
|
||||
}
|
||||
}
|
||||
return matches, stats, nil
|
||||
}
|
||||
|
||||
func isRemotePaidStatus(status string) bool {
|
||||
switch strings.ToLower(strings.TrimSpace(status)) {
|
||||
case "paid", "success", "succeeded", "trade_success", "completed", "complete", "1":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func fetchRemoteOrders(ctx context.Context, client *http.Client, baseURL string, merchant merchantConfig, page, limit int) (*orderListResponse, error) {
|
||||
params := map[string]string{
|
||||
"merchantId": merchant.merchantID,
|
||||
"page": strconv.Itoa(page),
|
||||
"limit": strconv.Itoa(limit),
|
||||
"order": "create_time desc",
|
||||
}
|
||||
params["sign"] = sign(params, merchant.key)
|
||||
|
||||
form := url.Values{}
|
||||
for k, v := range params {
|
||||
form.Set(k, v)
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, baseURL+"/api/v2/orderlist", strings.NewReader(form.Encode()))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
||||
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||
return nil, fmt.Errorf("remote status %d: %s", resp.StatusCode, string(body))
|
||||
}
|
||||
|
||||
decoder := json.NewDecoder(bytes.NewReader(body))
|
||||
decoder.UseNumber()
|
||||
var out orderListResponse
|
||||
if err := decoder.Decode(&out); err != nil {
|
||||
return nil, fmt.Errorf("decode remote response: %w body=%s", err, string(body))
|
||||
}
|
||||
return &out, nil
|
||||
}
|
||||
|
||||
func matchRemoteOrder(ctx context.Context, db *gorm.DB, opt options, stats *reconcileStats, remote remoteOrder) (matchedOrder, bool) {
|
||||
var local order.Order
|
||||
if err := db.WithContext(ctx).Model(&order.Order{}).Where("order_no = ?", remote.OutOrderNo).First(&local).Error; err != nil {
|
||||
stats.LocalNotFound++
|
||||
stats.LocalNotFoundOrders = append(stats.LocalNotFoundOrders, remote)
|
||||
if opt.verbose {
|
||||
fmt.Printf("skip local_not_found out_order_no=%s remote_no=%s status=%s money=%s\n", remote.OutOrderNo, remote.OrderNo, remote.OrderStatus, remote.OrderMoney)
|
||||
}
|
||||
return matchedOrder{}, false
|
||||
}
|
||||
if opt.method != "" && local.Method != opt.method {
|
||||
stats.LocalMethodSkip++
|
||||
if opt.verbose {
|
||||
fmt.Printf("skip method out_order_no=%s local_method=%s want=%s\n", remote.OutOrderNo, local.Method, opt.method)
|
||||
}
|
||||
return matchedOrder{}, false
|
||||
}
|
||||
if opt.start != "" && local.CreatedAt.Format("2006-01-02 15:04:05") < opt.start {
|
||||
stats.LocalDateFiltered++
|
||||
return matchedOrder{}, false
|
||||
}
|
||||
if opt.end != "" && local.CreatedAt.Format("2006-01-02 15:04:05") >= opt.end {
|
||||
stats.LocalDateFiltered++
|
||||
return matchedOrder{}, false
|
||||
}
|
||||
|
||||
remoteCents := decimalYuanToCents(remote.OrderMoney)
|
||||
item := matchedOrder{
|
||||
Remote: remote,
|
||||
Local: local,
|
||||
RemoteCents: remoteCents,
|
||||
PaymentAudit: "远端已支付",
|
||||
}
|
||||
item.Audit = auditSubscription(ctx, db, local)
|
||||
|
||||
switch {
|
||||
case local.Status == 5:
|
||||
item.Action = "跳过"
|
||||
item.Reason = "本地已完成"
|
||||
case remoteCents > 0 && abs64(local.Amount-remoteCents) > opt.amountDiff:
|
||||
item.Action = "跳过"
|
||||
item.Reason = fmt.Sprintf("金额不匹配 本地=%d 远端=%d", local.Amount, remoteCents)
|
||||
case local.Status == 2:
|
||||
item.Action = "重投队列"
|
||||
item.Reason = "本地已支付但未完成"
|
||||
case local.Status != 1 && local.Status != 3:
|
||||
item.Action = "跳过"
|
||||
item.Reason = fmt.Sprintf("本地状态 %d 不适合自动补单", local.Status)
|
||||
default:
|
||||
item.Action = "补单"
|
||||
item.Reason = "远端已支付，本地未支付/已关闭"
|
||||
}
|
||||
return item, true
|
||||
}
|
||||
|
||||
func auditSubscription(ctx context.Context, db *gorm.DB, local order.Order) subscriptionAudit {
|
||||
audit := subscriptionAudit{
|
||||
Checked: true,
|
||||
Quantity: local.Quantity,
|
||||
}
|
||||
if local.Type != 1 && local.Type != 2 {
|
||||
audit.Status = "跳过"
|
||||
audit.Reason = fmt.Sprintf("订单类型 %d 不核对订阅时间", local.Type)
|
||||
return audit
|
||||
}
|
||||
|
||||
var plan subscribe.Subscribe
|
||||
if err := db.WithContext(ctx).Model(&subscribe.Subscribe{}).Where("id = ?", local.SubscribeId).First(&plan).Error; err != nil {
|
||||
audit.Status = "异常"
|
||||
audit.Reason = "套餐不存在: " + err.Error()
|
||||
return audit
|
||||
}
|
||||
audit.SubscribeUnit = plan.UnitTime
|
||||
|
||||
var userSub user.Subscribe
|
||||
err := db.WithContext(ctx).Model(&user.Subscribe{}).Where("order_id = ?", local.Id).Order("id DESC").First(&userSub).Error
|
||||
if err == nil {
|
||||
audit.UserSubID = userSub.Id
|
||||
audit.UserID = userSub.UserId
|
||||
audit.StartTime = userSub.StartTime
|
||||
audit.ExpireTime = userSub.ExpireTime
|
||||
audit.ExpectedExpire = userSub.ExpireTime
|
||||
audit.Status = "已核对"
|
||||
audit.Reason = "订阅已绑定当前订单"
|
||||
return audit
|
||||
}
|
||||
|
||||
if local.Type == 2 && local.SubscribeToken != "" {
|
||||
err = db.WithContext(ctx).Model(&user.Subscribe{}).Where("token = ?", local.SubscribeToken).First(&userSub).Error
|
||||
if err == nil {
|
||||
base := subscriptionRenewalBaseTime(time.Now(), userSub.ExpireTime, userSub.FinishedAt)
|
||||
audit.UserSubID = userSub.Id
|
||||
audit.UserID = userSub.UserId
|
||||
audit.StartTime = userSub.StartTime
|
||||
audit.ExpireTime = userSub.ExpireTime
|
||||
audit.ExpectedExpire = tool.AddTime(plan.UnitTime, local.Quantity, base)
|
||||
if local.Status == 5 {
|
||||
audit.Status = "异常"
|
||||
audit.Reason = "订单已完成但订阅 order_id 未指向该订单"
|
||||
} else {
|
||||
audit.Status = "待激活"
|
||||
audit.Reason = "续费订单未完成，预计激活后续期"
|
||||
}
|
||||
return audit
|
||||
}
|
||||
}
|
||||
|
||||
if local.Status == 5 {
|
||||
audit.Status = "异常"
|
||||
audit.Reason = "订单已完成但未找到对应订阅"
|
||||
return audit
|
||||
}
|
||||
|
||||
audit.Status = "待激活"
|
||||
audit.ExpectedExpire = tool.AddTime(plan.UnitTime, local.Quantity, time.Now())
|
||||
audit.Reason = "订单未完成，订阅时间需激活队列生成"
|
||||
return audit
|
||||
}
|
||||
|
||||
func subscriptionRenewalBaseTime(now time.Time, expire time.Time, finishedAt *time.Time) time.Time {
|
||||
base := expire
|
||||
if finishedAt != nil && finishedAt.After(base) {
|
||||
base = *finishedAt
|
||||
}
|
||||
if base.Before(now) {
|
||||
base = now
|
||||
}
|
||||
return base
|
||||
}
|
||||
|
||||
func applyOrderFix(
|
||||
ctx context.Context,
|
||||
db *gorm.DB,
|
||||
redisClient *redis.Client,
|
||||
queueClient *asynq.Client,
|
||||
paymentInfo *payment.Payment,
|
||||
item matchedOrder,
|
||||
) error {
|
||||
return db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
|
||||
updates := map[string]interface{}{
|
||||
"status": 2,
|
||||
"trade_no": item.Remote.OrderNo,
|
||||
"updated_at": time.Now(),
|
||||
}
|
||||
if paymentInfo != nil {
|
||||
updates["payment_id"] = paymentInfo.Id
|
||||
updates["method"] = paymentInfo.Platform
|
||||
}
|
||||
result := tx.Model(&order.Order{}).
|
||||
Where("order_no = ? AND status IN ?", item.Local.OrderNo, []uint8{1, 2, 3}).
|
||||
Updates(updates)
|
||||
if result.Error != nil {
|
||||
return result.Error
|
||||
}
|
||||
if result.RowsAffected == 0 {
|
||||
return fmt.Errorf("order %s was not updated; status changed concurrently", item.Local.OrderNo)
|
||||
}
|
||||
clearOrderCache(ctx, redisClient, item.Local)
|
||||
return enqueueActivation(ctx, queueClient, item.Local.OrderNo)
|
||||
})
|
||||
}
|
||||
|
||||
func enqueueActivation(ctx context.Context, queueClient *asynq.Client, orderNo string) error {
|
||||
payload, err := json.Marshal(queueTypes.ForthwithActivateOrderPayload{OrderNo: orderNo})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = queueClient.EnqueueContext(ctx, asynq.NewTask(queueTypes.ForthwithActivateOrder, payload, asynq.MaxRetry(5)))
|
||||
return err
|
||||
}
|
||||
|
||||
func clearOrderCache(ctx context.Context, redisClient *redis.Client, local order.Order) {
|
||||
keys := []string{
|
||||
fmt.Sprintf("cache:order:id:%d", local.Id),
|
||||
fmt.Sprintf("cache:order:no:%s", local.OrderNo),
|
||||
}
|
||||
_ = redisClient.Del(ctx, keys...).Err()
|
||||
}
|
||||
|
||||
func sign(params map[string]string, key string) string {
|
||||
keys := make([]string, 0, len(params))
|
||||
for k, v := range params {
|
||||
if k != "sign" && k != "sign_type" && v != "" {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
}
|
||||
sort.Strings(keys)
|
||||
|
||||
parts := make([]string, 0, len(keys))
|
||||
for _, k := range keys {
|
||||
parts = append(parts, k+"="+params[k])
|
||||
}
|
||||
sum := md5.Sum([]byte(strings.Join(parts, "&") + key))
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
func decimalYuanToCents(n json.Number) int64 {
|
||||
s := strings.TrimSpace(n.String())
|
||||
if s == "" {
|
||||
return 0
|
||||
}
|
||||
parts := strings.SplitN(s, ".", 2)
|
||||
yuan, _ := strconv.ParseInt(parts[0], 10, 64)
|
||||
cents := yuan * 100
|
||||
if len(parts) == 1 {
|
||||
return cents
|
||||
}
|
||||
frac := parts[1]
|
||||
if len(frac) > 2 {
|
||||
frac = frac[:2]
|
||||
}
|
||||
for len(frac) < 2 {
|
||||
frac += "0"
|
||||
}
|
||||
f, _ := strconv.ParseInt(frac, 10, 64)
|
||||
if strings.HasPrefix(parts[0], "-") {
|
||||
return cents - f
|
||||
}
|
||||
return cents + f
|
||||
}
|
||||
|
||||
func printSummary(matches []matchedOrder, stats reconcileStats, opt options) {
|
||||
var fix, requeue, skip int
|
||||
for _, item := range matches {
|
||||
switch item.Action {
|
||||
case "补单":
|
||||
fix++
|
||||
case "重投队列":
|
||||
requeue++
|
||||
default:
|
||||
skip++
|
||||
}
|
||||
fmt.Printf("%s order_no=%s remote_no=%s local_status=%d amount=%d remote_amount=%d reason=%s\n",
|
||||
item.Action,
|
||||
item.Local.OrderNo,
|
||||
item.Remote.OrderNo,
|
||||
item.Local.Status,
|
||||
item.Local.Amount,
|
||||
item.RemoteCents,
|
||||
item.Reason,
|
||||
)
|
||||
if item.Audit.Checked {
|
||||
fmt.Printf(" payment=%s subscription=%s sub_id=%d user_id=%d expire=%s expected=%s note=%s\n",
|
||||
item.PaymentAudit,
|
||||
item.Audit.Status,
|
||||
item.Audit.UserSubID,
|
||||
item.Audit.UserID,
|
||||
formatTime(item.Audit.ExpireTime),
|
||||
formatTime(item.Audit.ExpectedExpire),
|
||||
item.Audit.Reason,
|
||||
)
|
||||
}
|
||||
}
|
||||
mode := "DRY-RUN"
|
||||
if opt.apply {
|
||||
mode = "APPLIED"
|
||||
}
|
||||
fmt.Printf("\nremote pages=%d orders=%d paid=%d missing_out_order_no=%d duplicates=%d\n",
|
||||
stats.RemotePages, stats.RemoteOrders, stats.RemotePaid, stats.RemoteMissingNo, stats.DuplicateRemote)
|
||||
if len(stats.StatusCounts) > 0 {
|
||||
keys := make([]string, 0, len(stats.StatusCounts))
|
||||
for key := range stats.StatusCounts {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
fmt.Print("remote status_counts=")
|
||||
for i, key := range keys {
|
||||
if i > 0 {
|
||||
fmt.Print(", ")
|
||||
}
|
||||
fmt.Printf("%s:%d", key, stats.StatusCounts[key])
|
||||
}
|
||||
fmt.Println()
|
||||
}
|
||||
fmt.Printf("local not_found=%d method_skipped=%d date_filtered=%d matched=%d\n",
|
||||
stats.LocalNotFound, stats.LocalMethodSkip, stats.LocalDateFiltered, stats.Matched)
|
||||
if opt.printNotFound && len(stats.LocalNotFoundOrders) > 0 {
|
||||
printNotFoundOrders(stats.LocalNotFoundOrders)
|
||||
}
|
||||
fmt.Printf("\n%s summary: matched=%d fixable=%d requeue=%d skipped=%d\n", mode, len(matches), fix, requeue, skip)
|
||||
}
|
||||
|
||||
func printNotFoundOrders(items []remoteOrder) {
|
||||
fmt.Println("\nlocal_not_found orders:")
|
||||
fmt.Println("out_order_no\tremote_no\tstatus\torder_money\tuser_money\tcreate_time\tpayment_time\tclient_ip\tdevice\tremark")
|
||||
for _, item := range items {
|
||||
fmt.Printf("%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",
|
||||
item.OutOrderNo,
|
||||
item.OrderNo,
|
||||
item.OrderStatus,
|
||||
item.OrderMoney.String(),
|
||||
item.UserMoney.String(),
|
||||
formatUnixTime(item.CreateTime),
|
||||
formatUnixTime(item.PaymentTime),
|
||||
oneLine(item.ClientIP),
|
||||
oneLine(item.Device),
|
||||
oneLine(item.OrderRemark),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
func formatTime(t time.Time) string {
|
||||
if t.IsZero() {
|
||||
return "-"
|
||||
}
|
||||
return t.Format("2006-01-02 15:04:05")
|
||||
}
|
||||
|
||||
func formatUnixTime(v int64) string {
|
||||
if v <= 0 {
|
||||
return "-"
|
||||
}
|
||||
if v > 1_000_000_000_000 {
|
||||
return time.UnixMilli(v).Format("2006-01-02 15:04:05")
|
||||
}
|
||||
return time.Unix(v, 0).Format("2006-01-02 15:04:05")
|
||||
}
|
||||
|
||||
func oneLine(s string) string {
|
||||
s = strings.ReplaceAll(s, "\t", " ")
|
||||
s = strings.ReplaceAll(s, "\r", " ")
|
||||
s = strings.ReplaceAll(s, "\n", " ")
|
||||
return strings.TrimSpace(s)
|
||||
}
|
||||
|
||||
func abs64(v int64) int64 {
|
||||
if v < 0 {
|
||||
return -v
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
func must(err error, step string) {
|
||||
if err == nil {
|
||||
return
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "%s: %v\n", step, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
Reference in New Issue
Block a user