From 9ddd8257c5d081e8612b448426eab251adc8d261 Mon Sep 17 00:00:00 2001 From: shanshanzhong Date: Tue, 2 Jun 2026 21:01:55 -0700 Subject: [PATCH] Remove unused deployment and observability assets --- .env.example | 13 - deploy/aws/ap-east-1/README.md | 363 --------- .../configs/backup-to-s3.env.example | 18 - ...mysql-seed-primary-and-replica.env.example | 20 - .../aws/ap-east-1/configs/ppanel.yaml.example | 111 --- .../ap-east-1/configs/replica-ops.env.example | 23 - deploy/aws/ap-east-1/nginx/ppanel-api.conf | 33 - deploy/aws/ap-northeast-1/README.md | 358 --------- .../configs/aws-jp-infra.env.example | 55 -- .../configs/backup-to-s3.env.example | 19 - ...mysql-seed-primary-and-replica.env.example | 21 - .../configs/ppanel.yaml.example | 112 --- .../configs/replica-ops.env.example | 23 - .../configs/resource-inventory.current.md | 186 ----- .../configs/resource-inventory.example.md | 69 -- .../aws/ap-northeast-1/nginx/ppanel-api.conf | 34 - deploy/systemd/hifast-mysql-backup.service | 17 - deploy/systemd/hifast-mysql-backup.timer | 11 - docker-compose.cloud.yml | 231 +----- etc/ppanel.yaml | 16 +- .../alerting/ppanel-alert-rules.yml | 272 ------- .../provisioning/dashboards/dashboards.yml | 14 - .../json/aws-rds-redis-overview.json | 520 ------------- .../dashboards/json/ppanel-ops-overview.json | 565 -------------- .../dashboards/json/ppanel-server-logs.json | 330 -------- .../provisioning/datasources/datasources.yml | 57 -- loki/loki-config.yaml | 36 - loki/promtail-config.yaml | 61 -- ops/aws-rds-external-replica-runbook.md | 652 ---------------- ops/docker-image-version-pins.md | 33 - ops/github-staging-deploy.md | 26 - ops/hifast-aws-jp-cutover-checklist-zh.md | 116 --- ops/hifast-aws-jp-migration-runbook-zh.md | 286 ------- ops/hifast-aws-standby-architecture-zh.md | 696 ----------------- ops/hifast-current-architecture-zh.md | 246 ------ ops/hifast-rds-v2-cutover-checklist-zh.md | 286 ------- ops/hifast-s3-backup-runbook-zh.md | 311 -------- prometheus/prometheus.yml | 40 - scripts/convert_recovery_orders/main.go | 229 ------ scripts/reconcile_mihapay_orders/main.go | 710 ------------------ tempo/tempo-config.yaml | 37 - 41 files changed, 9 insertions(+), 7247 deletions(-) delete mode 100644 deploy/aws/ap-east-1/README.md delete mode 100644 deploy/aws/ap-east-1/configs/backup-to-s3.env.example delete mode 100644 deploy/aws/ap-east-1/configs/mysql-seed-primary-and-replica.env.example delete mode 100644 deploy/aws/ap-east-1/configs/ppanel.yaml.example delete mode 100644 deploy/aws/ap-east-1/configs/replica-ops.env.example delete mode 100644 deploy/aws/ap-east-1/nginx/ppanel-api.conf delete mode 100644 deploy/aws/ap-northeast-1/README.md delete mode 100644 deploy/aws/ap-northeast-1/configs/aws-jp-infra.env.example delete mode 100644 deploy/aws/ap-northeast-1/configs/backup-to-s3.env.example delete mode 100644 deploy/aws/ap-northeast-1/configs/mysql-seed-primary-and-replica.env.example delete mode 100644 deploy/aws/ap-northeast-1/configs/ppanel.yaml.example delete mode 100644 deploy/aws/ap-northeast-1/configs/replica-ops.env.example delete mode 100644 deploy/aws/ap-northeast-1/configs/resource-inventory.current.md delete mode 100644 deploy/aws/ap-northeast-1/configs/resource-inventory.example.md delete mode 100644 deploy/aws/ap-northeast-1/nginx/ppanel-api.conf delete mode 100644 deploy/systemd/hifast-mysql-backup.service delete mode 100644 deploy/systemd/hifast-mysql-backup.timer delete mode 100644 grafana/provisioning/alerting/ppanel-alert-rules.yml delete mode 100644 grafana/provisioning/dashboards/dashboards.yml delete mode 100644 grafana/provisioning/dashboards/json/aws-rds-redis-overview.json delete mode 100644 grafana/provisioning/dashboards/json/ppanel-ops-overview.json delete mode 100644 grafana/provisioning/dashboards/json/ppanel-server-logs.json delete mode 100644 grafana/provisioning/datasources/datasources.yml delete mode 100644 loki/loki-config.yaml delete mode 100644 loki/promtail-config.yaml delete mode 100644 ops/aws-rds-external-replica-runbook.md delete mode 100644 ops/docker-image-version-pins.md delete mode 100644 ops/github-staging-deploy.md delete mode 100644 ops/hifast-aws-jp-cutover-checklist-zh.md delete mode 100644 ops/hifast-aws-jp-migration-runbook-zh.md delete mode 100644 ops/hifast-aws-standby-architecture-zh.md delete mode 100644 ops/hifast-current-architecture-zh.md delete mode 100644 ops/hifast-rds-v2-cutover-checklist-zh.md delete mode 100644 ops/hifast-s3-backup-runbook-zh.md delete mode 100644 prometheus/prometheus.yml delete mode 100644 scripts/convert_recovery_orders/main.go delete mode 100644 scripts/reconcile_mihapay_orders/main.go delete mode 100644 tempo/tempo-config.yaml diff --git a/.env.example b/.env.example index 6951e3f..0d1de2b 100644 --- a/.env.example +++ b/.env.example @@ -1,18 +1,5 @@ # 复制此文件为 .env 并填写真实值 # cp .env.example .env -# MySQL root 密码（同时需要在 configs/ppanel.yaml 的 MySQL.Password 中填写相同的值） -MYSQL_ROOT_PASSWORD=CHANGE_ME_TO_STRONG_PASSWORD - -# Grafana 管理员密码 -GRAFANA_PASSWORD=CHANGE_ME_TO_STRONG_PASSWORD - # PPanel Server 镜像标签（由 CI/CD 传入不可变 tag，如 git SHA） PPANEL_SERVER_TAG=CHANGE_ME_TO_GIT_SHA - -# AWS 区域（香港） -AWS_REGION=ap-east-1 - -# Grafana 公开域名（如需反代） -GRAFANA_DOMAIN=logs-new.hifast.biz -GRAFANA_ROOT_URL=https://logs-new.hifast.biz diff --git a/deploy/aws/ap-east-1/README.md b/deploy/aws/ap-east-1/README.md deleted file mode 100644 index cc77a0b..0000000 --- a/deploy/aws/ap-east-1/README.md +++ /dev/null @@ -1,363 +0,0 @@ -# PPanel 香港区新 AWS 账号部署说明 - -本目录用于在 **新 AWS 账号** 中按 **香港区 `ap-east-1`** 重建一套全新空环境。 - -目标架构： - -`DNS -> ALB -> WAF -> EC2(Nginx + ppanel-server + Redis + observability) -> RDS MySQL` - -## 1. 资源清单 - -按下面顺序创建资源： - -1. VPC -2. 2 个公有子网 + 2 个私有子网 -3. Internet Gateway -4. 公有 / 私有路由表 -5. 安全组 -6. RDS MySQL -7. EC2 本机 Redis Docker -8. EC2 -9. ACM 证书 -10. ALB + Target Group -11. WAF Web ACL -12. 平行环境域名 - -建议命名： - -- VPC: `ppanel-hk-prod` -- EC2: `ppanel-app-hk-01` -- RDS: `ppanel-mysql-hk` -- Redis container: `hifast-redis` -- ALB: `ppanel-alb-hk` -- WAF: `ppanel-waf-hk` - -## 2. 默认规格 - -### EC2 - -- Region: `ap-east-1` -- OS: Ubuntu 24.04 LTS -- Instance type: `t4g.large` 起步 -- Disk: `gp3 80GB` -- Public subnet: 是 -- IAM Role: 允许读取 CloudWatch / SSM（如使用） -- 如果要在 AWS EC2 本机执行 S3 备份：额外允许写入专用备份桶 - -### RDS MySQL - -- Engine: MySQL 8.0 -- Class: `db.r7g.xlarge` -- Storage: `gp3 100GB` -- DB name: `hifast` -- Username: `admin` -- Public access: `No` -- Charset: `utf8mb4` -- Backup: `7-14 days` - -### Redis - -- 部署位置：业务 EC2 本机 -- 部署方式：Docker -- 版本：`redis:8.2.1` -- 监听：`0.0.0.0:6379` -- 应用连接：`127.0.0.1:6379` -- 安全组：仅对白名单备用节点或同机应用开放 - -## 3. 网络与安全组 - -### 子网布局 - -- `public-a`, `public-b`: ALB / EC2 -- `private-a`, `private-b`: RDS - -### 安全组建议 - -#### `sg-alb` - -- Inbound - - `80/tcp` from `0.0.0.0/0` - - `443/tcp` from `0.0.0.0/0` -- Outbound - - `80/tcp` to `sg-ec2` - -#### `sg-ec2` - -- Inbound - - `80/tcp` from `sg-alb` - - `22/tcp` from `你的固定运维 IP` -- Outbound - - all - -说明： - -- 应用容器监听 `127.0.0.1:8080` -- EC2 对外只让 Nginx 监听 `80` -- Grafana / Prometheus / Tempo 仅监听 `127.0.0.1` - -#### `sg-rds` - -- Inbound - - `3306/tcp` from `sg-ec2` - -#### `sg-ec2` 额外说明 - -- 如果需要外部备用节点复制 Redis，再额外放行： - - `6379/tcp` from `104.238.220.230/32` - -## 4. ALB / Target Group / 健康检查 - -### Target Group - -- Type: `Instance` -- Protocol: `HTTP` -- Port: `80` -- Health check path: `/v1/common/heartbeat` -- Success code: `200` - -这个路径已由项目现有接口提供，无需额外改代码。 - -### ALB 监听器 - -- `80` -> redirect to `443` -- `443` -> forward 到 target group - -### ACM - -- 在 `ap-east-1` 申请证书 -- 先给平行环境域名，例如： - - `api-new.hifast.biz` - - `logs-new.hifast.biz` - -## 5. WAF 规则 - -首版至少启用： - -1. `AWSManagedRulesCommonRuleSet` -2. `AWSManagedRulesKnownBadInputsRuleSet` -3. `AWSManagedRulesAmazonIpReputationList` -4. 全站 rate-based rule -5. 针对高风险路径的 rate-based rule - -建议的第一版限流： - -- 全站：每 IP `2000 / 5 分钟` -- `/v1/public/user/subscribe`：每 IP `300 / 5 分钟` -- 登录 / 注册 / 验证码接口：每 IP `100 / 5 分钟` - -节点上报接口建议后续补： - -- `/v1/server/status` -- `/v1/server/online` -- `/v1/server/traffic` - -优先用节点出口 IP 白名单；没有固定出口 IP 的节点暂时保留 `secret_key`，但不要把它当成唯一防线。 - -## 6. EC2 文件落地 - -在 EC2 上建议使用： - -- 应用目录：`/opt/ppanel` -- Nginx 配置：`/etc/nginx/sites-available/ppanel-api.conf` - -需要上传这些文件 / 目录： - -- `docker-compose.cloud.yml` -- `deploy/aws/ap-east-1/configs/ppanel.yaml.example` -> 重命名为 `configs/ppanel.yaml` -- `deploy/aws/ap-east-1/nginx/ppanel-api.conf` -- `grafana/` -- `loki/` -- `prometheus/` -- `tempo/` -- `.env.example` -> 重命名为 `.env` - -目标目录示例： - -```text -/opt/ppanel/ - docker-compose.cloud.yml - .env - configs/ppanel.yaml - grafana/ - loki/ - prometheus/ - tempo/ - logs/ - cache/ - tempo_data/ -``` - -## 7. 应用配置 - -基线模板见： - -- [`configs/ppanel.yaml.example`](./configs/ppanel.yaml.example) -- [`nginx/ppanel-api.conf`](./nginx/ppanel-api.conf) - -关键值必须替换： - -- `MySQL.Addr` -- `MySQL.Password` -- `Redis.Host` -- `Redis.Pass` -- `JwtAuth.AccessSecret` -- `Administrator.Email` -- `Administrator.Password` -- `AppSignature.AppSecrets.*` -- `device.security_secret` -- `Site.Host` -- `Site.SiteName` - -Redis 约定保持不变： - -- 业务缓存：DB `0` -- Asynq：DB `5`（代码内部已固定使用） - -## 8. 部署步骤 - -### 8.1 初始化 EC2 - -把脚本上传到 EC2 后执行： - -## 9. 104 灾备节点常用运维脚本 - -如果你要在 `104.238.220.230` 上执行数据迁移、主从重拉、主库提升，可以直接复用仓库里的这几份脚本： - -- 数据导出 / 导入交互工具： - - [`deploy/scripts/hifast_data_sync_tool.sh`](/Users/Apple/code_vpn/vpn/ppanel-server/deploy/scripts/hifast_data_sync_tool.sh) -- MySQL 主从运维工具： - - [`deploy/scripts/mysql_replica_ops.sh`](/Users/Apple/code_vpn/vpn/ppanel-server/deploy/scripts/mysql_replica_ops.sh) -- Redis 主从运维工具： - - [`deploy/scripts/redis_replica_ops.sh`](/Users/Apple/code_vpn/vpn/ppanel-server/deploy/scripts/redis_replica_ops.sh) -- 统一总入口： - - [`deploy/scripts/hifast_data_sync_tool.sh`](/Users/Apple/code_vpn/vpn/ppanel-server/deploy/scripts/hifast_data_sync_tool.sh) -- 主从运维环境模板： - - [`deploy/aws/ap-east-1/configs/replica-ops.env.example`](/Users/Apple/code_vpn/vpn/ppanel-server/deploy/aws/ap-east-1/configs/replica-ops.env.example) - -### 9.1 数据迁移工具 - -支持： - -- 备份 MySQL 到 S3 -- 备份 Redis 到 S3 -- 从正式库导出 MySQL `sql.gz` -- 把 `sql.gz` 导入 AWS RDS -- 从正式 Redis 导出 `RDB` -- 将 `RDB` 导入 Docker Redis 或宿主机 Redis -- 查看 MySQL / Redis 当前主从状态 -- 强制重拉 MySQL / Redis 主从 -- 把 MySQL / Redis 从库提升为可写主库 - -示例： - -```bash -bash deploy/scripts/hifast_data_sync_tool.sh /root/replica-ops.env -``` - -```bash -chmod +x deploy/scripts/bootstrap_aws_ec2.sh -sudo APP_DIR=/opt/ppanel deploy/scripts/bootstrap_aws_ec2.sh -``` - -### 8.2 安装 Nginx 配置 - -```bash -sudo cp deploy/aws/ap-east-1/nginx/ppanel-api.conf /etc/nginx/sites-available/ppanel-api.conf -sudo ln -sf /etc/nginx/sites-available/ppanel-api.conf /etc/nginx/sites-enabled/ppanel-api.conf -sudo nginx -t -sudo systemctl reload nginx -``` - -### 8.3 启动容器 - -```bash -cd /opt/ppanel -docker compose -f docker-compose.cloud.yml up -d -``` - -### 8.4 预检 - -```bash -chmod +x deploy/scripts/preflight_aws_hk.sh -APP_DIR=/opt/ppanel \ -RDS_HOST= \ -REDIS_HOST=127.0.0.1 \ -deploy/scripts/preflight_aws_hk.sh -``` - -## 9. 平行环境验证 - -先验证 `api-new.hifast.biz`，不要直接切正式域名。 - -必测项： - -1. `ALB target` 为 healthy -2. `GET /v1/common/heartbeat` 返回 200 -3. 管理员登录 -4. 用户注册 / 登录 -5. 订阅查询 -6. 节点上报 `/v1/server/status` -7. 本机 Redis 可写缓存 -8. Asynq 可入队并消费 - -## 10. 正式切换 - -切换前检查： - -1. ALB 5xx 为 0 -2. EC2 CPU / Memory 正常 -3. RDS CPU / Connections 正常 -4. 本机 Redis CPU / Connections / Memory 正常 -5. WAF 已挂到 ALB -6. EC2 安全组没有对公网放 `8080/3333/9090/4317` - -切换方式： - -1. 保持新环境先跑平行域名 -2. 正式域名切到新 ALB -3. 观察至少 1 小时 -4. 确认无误后再处理旧环境 - -## 11. 监控建议 - -至少建这些 CloudWatch / Grafana 观测项： - -- ALB `RequestCount`, `HTTPCode_ELB_5XX_Count`, `TargetResponseTime` -- EC2 `CPUUtilization`, `NetworkIn`, `NetworkOut`, `StatusCheckFailed` -- RDS `CPUUtilization`, `DatabaseConnections`, `ReadLatency`, `WriteLatency` -- Redis 容器 CPU / Memory / restart count - -## 12. 这次方案的边界 - -本目录交付的是： - -- 香港区新账号的部署模板 -- 新空环境启动与验证流程 -- ALB / WAF / EC2 / RDS / 本机 Redis 的落地约定 - -不包含： - -- 旧数据迁移 -- Terraform / CloudFormation 自动建资源 -- Redis 托管版改造 -- 多活 / 自动扩缩容 - -## 13. S3 备份补强 - -当前已落地的 S3 备份桶： - -- `hifast-prod-backups-200810848252-ap-east-1` - -建议与现网结合方式： - -1. `RDS automated backup` 继续保留，作为第一层恢复能力 -2. `104` 外部 MySQL 从库执行逻辑备份并上传到 S3，作为第二层可下载备份 -3. `104` 外部 Redis 从库按需导出 `RDB` 到 S3，补齐缓存类灾备材料 - -仓库中已补充： - -- 环境变量模板：[`configs/backup-to-s3.env.example`](./configs/backup-to-s3.env.example) -- MySQL 备份脚本：[`../../scripts/mysql_backup_to_s3.sh`](../../scripts/mysql_backup_to_s3.sh) -- Redis 备份脚本：[`../../scripts/redis_rdb_backup_to_s3.sh`](../../scripts/redis_rdb_backup_to_s3.sh) - -建议把 MySQL 备份脚本优先部署到 `104`，因为它直接连接本地只读从库，对 AWS 主库扰动最小。 diff --git a/deploy/aws/ap-east-1/configs/backup-to-s3.env.example b/deploy/aws/ap-east-1/configs/backup-to-s3.env.example deleted file mode 100644 index a45fc7e..0000000 --- a/deploy/aws/ap-east-1/configs/backup-to-s3.env.example +++ /dev/null @@ -1,18 +0,0 @@ -AWS_REGION=ap-east-1 -S3_BUCKET=hifast-prod-backups-200810848252-ap-east-1 -S3_PREFIX=mysql -BACKUP_DIR=/var/backups/hifast -HOST_TAG=104-standby -KEEP_LOCAL_DAYS=3 -CHECK_REPLICA=1 - -MYSQL_HOST=127.0.0.1 -MYSQL_PORT=3306 -MYSQL_USER=backup_reader -MYSQL_PASSWORD=CHANGE_ME -MYSQL_SOCKET= -MYSQL_DATABASE=hifast - -REDIS_HOST=127.0.0.1 -REDIS_PORT=6379 -REDIS_PASSWORD=CHANGE_ME diff --git a/deploy/aws/ap-east-1/configs/mysql-seed-primary-and-replica.env.example b/deploy/aws/ap-east-1/configs/mysql-seed-primary-and-replica.env.example deleted file mode 100644 index 38382f4..0000000 --- a/deploy/aws/ap-east-1/configs/mysql-seed-primary-and-replica.env.example +++ /dev/null @@ -1,20 +0,0 @@ -PRIMARY_HOST=hifast-mysql-prod-v2.cd6aey40m6ag.ap-east-1.rds.amazonaws.com -PRIMARY_PORT=3306 -PRIMARY_USER=admin -PRIMARY_PASSWORD=CHANGE_ME -PRIMARY_DB=hifast - -PRIMARY_REPL_USER=repl -PRIMARY_REPL_PASSWORD=CHANGE_ME -PRIMARY_REPL_HOST=104.238.220.230 -PRIMARY_BINLOG_RETENTION_HOURS=24 - -REPLICA_HOST=127.0.0.1 -REPLICA_PORT=3306 -REPLICA_USER=root -REPLICA_PASSWORD= -REPLICA_SOCKET=/var/run/mysqld/mysqld.sock -REPLICA_DB=hifast -REPLICA_SOURCE_SSL=1 - -DUMP_FILE= diff --git a/deploy/aws/ap-east-1/configs/ppanel.yaml.example b/deploy/aws/ap-east-1/configs/ppanel.yaml.example deleted file mode 100644 index 8d16e5d..0000000 --- a/deploy/aws/ap-east-1/configs/ppanel.yaml.example +++ /dev/null @@ -1,111 +0,0 @@ -Host: 0.0.0.0 -Port: 8080 -Debug: false - -JwtAuth: - AccessSecret: CHANGE_ME_TO_A_LONG_RANDOM_SECRET - AccessExpire: 604800 - -Logger: - ServiceName: PPanel - Mode: console - Encoding: plain - TimeFormat: "2006-01-02 15:04:05.000" - Path: logs - Level: info - MaxContentLength: 0 - Compress: false - Stat: true - KeepDays: 7 - StackCooldownMillis: 100 - MaxBackups: 7 - MaxSize: 100 - Rotation: daily - FileTimeFormat: "2006-01-02T15:04:05.000Z07:00" - -MySQL: - Addr: YOUR_RDS_ENDPOINT:3306 - Dbname: hifast - Username: admin - Password: CHANGE_ME_TO_RDS_PASSWORD - Config: charset=utf8mb4&parseTime=true&loc=Asia%2FShanghai - MaxIdleConns: 10 - MaxOpenConns: 100 - SlowThreshold: 1000 - -Redis: - Host: 127.0.0.1:6379 - Pass: CHANGE_ME_TO_REDIS_PASSWORD - DB: 0 - PoolSize: 100 - MinIdleConns: 10 - MaxRetries: 3 - PoolTimeout: 4 - IdleTimeout: 300 - MaxConnAge: 0 - DialTimeout: 5 - ReadTimeout: 3 - WriteTimeout: 3 - -Trace: - Name: ppanel-server - Endpoint: 127.0.0.1:4317 - Sampler: 0.1 - Batcher: otlpgrpc - -Site: - Host: api-new.hifast.biz - SiteName: HiFastVPN - -Administrator: - Email: admin@example.com - Password: CHANGE_ME_TO_STRONG_ADMIN_PASSWORD - -Telegram: - Enable: false - BotID: 0 - BotName: "" - BotToken: "" - GroupChatID: "" - EnableNotify: false - WebHookDomain: "" - -Kutt: - Enable: false - ApiURL: "" - ApiKey: "" - TargetURL: "" - Domain: "" - -OpenInstall: - Enable: false - AppKey: "" - ApiKey: "" - -Loki: - Enable: true - URL: "http://localhost:3100" - -AppSignature: - AppSecrets: - android-client: CHANGE_ME_ANDROID_SIGNATURE_SECRET - ios-client: CHANGE_ME_IOS_SIGNATURE_SECRET - web-client: CHANGE_ME_WEB_SIGNATURE_SECRET - ValidWindowSeconds: 300 - SkipPrefixes: - - /v1/notify/ - - /v1/iap/notifications - - /v1/telegram/webhook - - /v1/subscribe/config - -Signature: - EnableSignature: false - -device: - enable: true - security_secret: CHANGE_ME_DEVICE_SECURITY_SECRET - -Register: - EnableTrial: true - EnableTrialEmailWhitelist: true - TrialEmailDomainWhitelist: "gmail.com,outlook.com,icloud.com,qq.com,163.com" diff --git a/deploy/aws/ap-east-1/configs/replica-ops.env.example b/deploy/aws/ap-east-1/configs/replica-ops.env.example deleted file mode 100644 index f6d7e59..0000000 --- a/deploy/aws/ap-east-1/configs/replica-ops.env.example +++ /dev/null @@ -1,23 +0,0 @@ -MYSQL_HOST=127.0.0.1 -MYSQL_PORT=3306 -MYSQL_USER=root -MYSQL_PASSWORD=CHANGE_ME -MYSQL_SOCKET= - -REPL_SOURCE_HOST=hifast-mysql-prod-v2.cd6aey40m6ag.ap-east-1.rds.amazonaws.com -REPL_SOURCE_PORT=3306 -REPL_SOURCE_USER=repl -REPL_SOURCE_PASSWORD=CHANGE_ME -REPL_SOURCE_SSL=1 -REPL_SOURCE_LOG_FILE= -REPL_SOURCE_LOG_POS= -REPL_SOURCE_AUTO_POSITION=1 - -REDIS_HOST=127.0.0.1 -REDIS_PORT=6379 -REDIS_PASSWORD=CHANGE_ME - -REDIS_SOURCE_HOST=18.163.33.75 -REDIS_SOURCE_PORT=6379 -REDIS_SOURCE_USER= -REDIS_SOURCE_PASSWORD=CHANGE_ME diff --git a/deploy/aws/ap-east-1/nginx/ppanel-api.conf b/deploy/aws/ap-east-1/nginx/ppanel-api.conf deleted file mode 100644 index 868f4ee..0000000 --- a/deploy/aws/ap-east-1/nginx/ppanel-api.conf +++ /dev/null @@ -1,33 +0,0 @@ -server { - listen 80 default_server; - listen [::]:80 default_server; - server_name _; - - client_max_body_size 20m; - - access_log /var/log/nginx/ppanel-access.log; - error_log /var/log/nginx/ppanel-error.log warn; - - location / { - proxy_http_version 1.1; - proxy_pass http://127.0.0.1:8080; - - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header X-Forwarded-Host $host; - proxy_set_header X-Forwarded-Port $server_port; - - proxy_connect_timeout 10s; - proxy_send_timeout 60s; - proxy_read_timeout 60s; - } - - location = /nginx_status { - stub_status; - access_log off; - allow 127.0.0.1; - deny all; - } -} diff --git a/deploy/aws/ap-northeast-1/README.md b/deploy/aws/ap-northeast-1/README.md deleted file mode 100644 index c4272ed..0000000 --- a/deploy/aws/ap-northeast-1/README.md +++ /dev/null @@ -1,358 +0,0 @@ -# PPanel 日本东京区 AWS 部署说明 - -本目录用于在 **AWS 日本东京区 `ap-northeast-1`** 重建一套全新生产环境，并承接当前香港区 `ap-east-1` 的正式迁移。 - -如果你要看“当前已经真实跑起来的东京架构”，优先看： - -- [`ops/hifast-current-architecture-zh.md`](/Users/Apple/code_vpn/vpn/ppanel-server/ops/hifast-current-architecture-zh.md) -- [`configs/resource-inventory.current.md`](./configs/resource-inventory.current.md) - -这份 README 更偏向： - -- 目标架构 -- 资源规划 -- 部署方法 -- 后续待完成项 - -目标架构： - -`DNS -> ALB -> WAF -> EC2(Nginx + ppanel-server + Redis + observability) -> RDS MySQL` - -灾备链路： - -`RDS MySQL / EC2 Redis -> 104.238.220.230 外部灾备` - -当前仓库内已补充： - -- 东京基础设施参数模板：[`configs/aws-jp-infra.env.example`](./configs/aws-jp-infra.env.example) -- 东京真实实施状态登记：[`configs/resource-inventory.current.md`](./configs/resource-inventory.current.md) -- 东京底座资源创建脚本：[`../../scripts/aws_jp_create_base_infra.sh`](../../scripts/aws_jp_create_base_infra.sh) -- 东京资源状态检查脚本：[`../../scripts/aws_jp_describe_state.sh`](../../scripts/aws_jp_describe_state.sh) -- MySQL 备份脚本：[`../../scripts/mysql_backup_to_s3.sh`](../../scripts/mysql_backup_to_s3.sh) -- 10 分钟 MySQL 备份定时器安装脚本：[`../../scripts/install_mysql_backup_timer.sh`](../../scripts/install_mysql_backup_timer.sh) - -## 1. 资源清单 - -按下面顺序创建资源： - -1. VPC -2. 2 个公有子网 + 2 个私有子网 -3. Internet Gateway -4. 公有 / 私有路由表 -5. 安全组 -6. RDS MySQL -7. EC2 本机 Redis Docker -8. EC2 -9. ACM 证书 -10. ALB + Target Group -11. WAF Web ACL -12. 东京平行环境域名 -13. 东京 S3 备份桶 - -建议命名： - -- VPC: `ppanel-jp-prod` -- EC2: `ppanel-app-jp-01` -- RDS: `ppanel-mysql-jp` -- Redis container: `hifast-redis` -- ALB: `ppanel-alb-jp` -- WAF: `ppanel-waf-jp` -- S3: `hifast-prod-backups-200810848252-ap-northeast-1` - -## 2. 默认规格 - -### EC2 - -- Region: `ap-northeast-1` -- OS: Ubuntu 24.04 LTS -- Instance type: `t4g.large` -- Disk: `gp3 80GB` -- Public subnet: 是 -- IAM Role: - - 允许读取 CloudWatch / SSM（如使用） - - 如果要在东京 EC2 上执行 S3 备份：额外允许写入东京备份桶 - -### RDS MySQL - -- Engine: `MySQL 8.4` -- Class: `db.r7g.xlarge` -- Storage: `gp3 100GB` -- DB name: `hifast` -- Username: `admin` -- Public access: `Yes` -- Charset: `utf8mb4` -- Backup retention: `7-14 days` -- Deletion protection: `On` -- Multi-AZ: `Yes`（当前按 2 实例 Multi-AZ 创建） - -说明： - -- 当前东京 RDS 需要允许 `104.238.220.230` 从公网直连 `3306`，用于外部 MySQL 从库复制 -- 因此本阶段 RDS 使用 `public subnet group + Publicly accessible = Yes` -- 访问面只通过 `sg-rds` 严格限制到业务 EC2 安全组和 `104.238.220.230/32` - -### Redis - -- 部署位置：业务 EC2 本机 -- 部署方式：Docker -- 版本：`redis:8.2.1` -- 监听：`0.0.0.0:6379` -- 应用连接：`127.0.0.1:6379` -- 安全组：仅对白名单备用节点 `104.238.220.230/32` 或同机应用开放 - -## 3. 网络与安全组 - -### 子网布局 - -- `public-a`, `public-c`: ALB / EC2 -- `private-a`, `private-c`: RDS - -说明： - -- 东京优先使用 `ap-northeast-1a` 和 `ap-northeast-1c` -- 如果账户映射不同，也可以用任意 2 个可用区，但公私网必须各 2 个子网 - -### 安全组建议 - -#### `sg-alb` - -- Inbound - - `80/tcp` from `0.0.0.0/0` - - `443/tcp` from `0.0.0.0/0` -- Outbound - - `80/tcp` to `sg-ec2` - -#### `sg-ec2` - -- Inbound - - `80/tcp` from `sg-alb` - - `22/tcp` from `你的固定运维 IP` - - `6379/tcp` from `104.238.220.230/32` -- Outbound - - all - -说明： - -- 应用容器监听 `127.0.0.1:8080` -- EC2 对外只让 Nginx 监听 `80` -- Grafana / Prometheus / Tempo 仅监听 `127.0.0.1` - -#### `sg-rds` - -- Inbound - - `3306/tcp` from `sg-ec2` - - `3306/tcp` from `104.238.220.230/32` - -## 4. ALB / Target Group / 健康检查 - -### Target Group - -- Type: `Instance` -- Protocol: `HTTP` -- Port: `80` -- Health check path: `/v1/common/heartbeat` -- Success code: `200` - -### ALB 监听器 - -- `80` -> redirect to `443` -- `443` -> forward 到 target group - -### ACM - -- 在 `ap-northeast-1` 重新申请证书 -- 先给平行环境域名，例如： - - `api-jp.hifast.biz` - - `logs-jp.hifast.biz` - -## 5. WAF 规则 - -首版至少启用： - -1. `AWSManagedRulesCommonRuleSet` -2. `AWSManagedRulesKnownBadInputsRuleSet` -3. `AWSManagedRulesAmazonIpReputationList` -4. 全站 rate-based rule -5. 针对高风险路径的 rate-based rule - -建议的第一版限流： - -- 全站：每 IP `2000 / 5 分钟` -- `/v1/public/user/subscribe`：每 IP `300 / 5 分钟` -- 登录 / 注册 / 验证码接口：每 IP `100 / 5 分钟` - -节点上报接口建议后续补： - -- `/v1/server/status` -- `/v1/server/online` -- `/v1/server/traffic` - -## 6. EC2 文件落地 - -在 EC2 上建议使用： - -- 应用目录：`/opt/ppanel` -- Nginx 配置：`/etc/nginx/sites-available/ppanel-api.conf` - -需要上传这些文件 / 目录： - -- `docker-compose.cloud.yml` -- `deploy/aws/ap-northeast-1/configs/ppanel.yaml.example` -> 重命名为 `configs/ppanel.yaml` -- `deploy/aws/ap-northeast-1/nginx/ppanel-api.conf` -- `grafana/` -- `loki/` -- `prometheus/` -- `tempo/` -- `.env.example` -> 重命名为 `.env` - -目标目录示例： - -```text -/opt/ppanel/ - docker-compose.cloud.yml - .env - configs/ppanel.yaml - grafana/ - loki/ - prometheus/ - tempo/ - logs/ - cache/ - tempo_data/ -``` - -## 7. 应用配置 - -基线模板见： - -- [`configs/ppanel.yaml.example`](./configs/ppanel.yaml.example) -- [`nginx/ppanel-api.conf`](./nginx/ppanel-api.conf) - -关键值必须替换： - -- `MySQL.Addr` -- `MySQL.Password` -- `Redis.Host` -- `Redis.Pass` -- `JwtAuth.AccessSecret` -- `Administrator.Email` -- `Administrator.Password` -- `AppSignature.AppSecrets.*` -- `device.security_secret` -- `Site.Host` -- `Site.SiteName` - -Redis 约定保持不变： - -- 业务缓存：DB `0` -- Asynq：DB `5` - -## 8. 部署步骤 - -### 8.0 创建东京基础设施 - -如果本机或跳板机已经配置好 AWS CLI 凭据，可以先直接执行： - -```bash -cp deploy/aws/ap-northeast-1/configs/aws-jp-infra.env.example /root/aws-jp-infra.env -chmod 600 /root/aws-jp-infra.env -vim /root/aws-jp-infra.env - -chmod +x deploy/scripts/aws_jp_create_base_infra.sh -bash deploy/scripts/aws_jp_create_base_infra.sh /root/aws-jp-infra.env -``` - -执行后可用下面命令随时核对东京底座状态： - -```bash -chmod +x deploy/scripts/aws_jp_describe_state.sh -bash deploy/scripts/aws_jp_describe_state.sh /root/aws-jp-infra.env -``` - -### 8.1 初始化 EC2 - -把脚本上传到东京 EC2 后执行： - -```bash -chmod +x deploy/scripts/bootstrap_aws_ec2.sh -sudo APP_DIR=/opt/ppanel APP_USER=ubuntu deploy/scripts/bootstrap_aws_ec2.sh -``` - -### 8.2 安装 Nginx 配置 - -```bash -sudo cp deploy/aws/ap-northeast-1/nginx/ppanel-api.conf /etc/nginx/sites-available/ppanel-api.conf -sudo ln -sf /etc/nginx/sites-available/ppanel-api.conf /etc/nginx/sites-enabled/ppanel-api.conf -sudo nginx -t -sudo systemctl reload nginx -``` - -### 8.3 启动容器 - -```bash -cd /opt/ppanel -docker compose -f docker-compose.cloud.yml up -d -``` - -### 8.4 预检 - -```bash -chmod +x deploy/scripts/preflight_aws_jp.sh -APP_DIR=/opt/ppanel \ -RDS_HOST= \ -REDIS_HOST=127.0.0.1 \ -deploy/scripts/preflight_aws_jp.sh -``` - -## 9. 数据迁移与切换 - -正式迁移请按： - -- [`ops/hifast-aws-jp-migration-runbook-zh.md`](/Users/Apple/code_vpn/vpn/ppanel-server/ops/hifast-aws-jp-migration-runbook-zh.md) - -执行。 - -核心原则： - -- 先搭平行环境 -- 停机后再导出香港主数据 -- 东京验收通过后再切正式域名 -- 切换后再重挂 `104` 灾备 - -## 10. 104 灾备节点常用模板 - -如果迁移完成后要把 `104.238.220.230` 重挂为东京主站从库，可复用： - -- [`deploy/scripts/hifast_mysql_seed_primary_and_replica.sh`](/Users/Apple/code_vpn/vpn/ppanel-server/deploy/scripts/hifast_mysql_seed_primary_and_replica.sh) -- [`deploy/scripts/hifast_mysql_attach_replica.sh`](/Users/Apple/code_vpn/vpn/ppanel-server/deploy/scripts/hifast_mysql_attach_replica.sh) -- [`deploy/scripts/hifast_redis_attach_replica.sh`](/Users/Apple/code_vpn/vpn/ppanel-server/deploy/scripts/hifast_redis_attach_replica.sh) -- [`deploy/scripts/hifast_data_sync_tool.sh`](/Users/Apple/code_vpn/vpn/ppanel-server/deploy/scripts/hifast_data_sync_tool.sh) -- [`configs/replica-ops.env.example`](./configs/replica-ops.env.example) - -## 11. 东京资源创建前置检查 - -在 AWS 控制台里至少先确认： - -- 东京区已启用 -- `ap-northeast-1` 可创建 `t4g.large` -- `ap-northeast-1` RDS 可创建 `db.r7g.xlarge` -- ACM / ALB / WAF / S3 服务在东京区可正常使用 -- Tokyo 对应配额满足： - - On-Demand Standard vCPU - - ALB 数量 - - Elastic IP（如需） - - RDS 实例数 - -## 12. 当前已知真实进度 - -截至 `2026-05-20`，已知状态如下： - -- 东京 VPC `ppanel-jp-prod` 已创建 -- VPC ID: `vpc-0846b23b4a7d64eac` -- VPC CIDR: `10.20.0.0/16` -- 4 个子网在 AWS 控制台里曾填写完成，但提交时控制台 session 失效 -- 因此： - - 子网是否真正创建成功，需要重新核实 - - IGW / 路由表 / 安全组 / RDS / EC2 / ALB / WAF 都应按“未完成”处理，重新复核 - -实时状态请以后续更新的 [`configs/resource-inventory.current.md`](./configs/resource-inventory.current.md) 为准。 diff --git a/deploy/aws/ap-northeast-1/configs/aws-jp-infra.env.example b/deploy/aws/ap-northeast-1/configs/aws-jp-infra.env.example deleted file mode 100644 index 583b37d..0000000 --- a/deploy/aws/ap-northeast-1/configs/aws-jp-infra.env.example +++ /dev/null @@ -1,55 +0,0 @@ -AWS_REGION=ap-northeast-1 -AWS_ACCOUNT_ID=200810848252 - -VPC_NAME=ppanel-jp-prod -VPC_ID= -VPC_CIDR=10.20.0.0/16 - -PUBLIC_SUBNET_A_NAME=ppanel-jp-public-a -PUBLIC_SUBNET_A_AZ=ap-northeast-1a -PUBLIC_SUBNET_A_CIDR=10.20.0.0/24 - -PUBLIC_SUBNET_C_NAME=ppanel-jp-public-c -PUBLIC_SUBNET_C_AZ=ap-northeast-1c -PUBLIC_SUBNET_C_CIDR=10.20.1.0/24 - -PRIVATE_SUBNET_A_NAME=ppanel-jp-private-a -PRIVATE_SUBNET_A_AZ=ap-northeast-1a -PRIVATE_SUBNET_A_CIDR=10.20.10.0/24 - -PRIVATE_SUBNET_C_NAME=ppanel-jp-private-c -PRIVATE_SUBNET_C_AZ=ap-northeast-1c -PRIVATE_SUBNET_C_CIDR=10.20.11.0/24 - -IGW_NAME=ppanel-jp-igw -PUBLIC_ROUTE_TABLE_NAME=ppanel-jp-public-rt -PRIVATE_ROUTE_TABLE_NAME=ppanel-jp-private-rt - -SG_ALB_NAME=ppanel-jp-sg-alb -SG_EC2_NAME=ppanel-jp-sg-ec2 -SG_RDS_NAME=ppanel-jp-sg-rds - -OPS_SSH_CIDR=CHANGE_ME_TO_YOUR_FIXED_PUBLIC_IP_OR_CIDR -DR_REPLICA_IP=104.238.220.230/32 - -EC2_NAME=ppanel-app-jp-01 -EC2_AMI_FAMILY=ubuntu-24.04 -EC2_INSTANCE_TYPE=t4g.large -EC2_DISK_GB=80 -EC2_KEY_PAIR=CHANGE_ME - -RDS_IDENTIFIER=ppanel-mysql-jp -RDS_DB_NAME=hifast -RDS_ADMIN_USER=admin -RDS_INSTANCE_CLASS=db.r7g.xlarge -RDS_STORAGE_GB=100 - -ALB_NAME=ppanel-alb-jp -TARGET_GROUP_NAME=ppanel-tg-jp -WAF_NAME=ppanel-waf-jp - -PARALLEL_API_DOMAIN=api-jp.hifast.biz -PARALLEL_LOGS_DOMAIN=logs-jp.hifast.biz -PRODUCTION_API_DOMAIN=CHANGE_ME - -S3_BACKUP_BUCKET=hifast-prod-backups-200810848252-ap-northeast-1 diff --git a/deploy/aws/ap-northeast-1/configs/backup-to-s3.env.example b/deploy/aws/ap-northeast-1/configs/backup-to-s3.env.example deleted file mode 100644 index 1a72193..0000000 --- a/deploy/aws/ap-northeast-1/configs/backup-to-s3.env.example +++ /dev/null @@ -1,19 +0,0 @@ -AWS_REGION=ap-northeast-1 -S3_BUCKET=hifast-prod-backups-200810848252-ap-northeast-1 -S3_PREFIX=mysql -BACKUP_DIR=/var/backups/hifast -HOST_TAG=104-standby-for-jp -KEEP_LOCAL_DAYS=3 -CHECK_REPLICA=1 - -MYSQL_HOST=127.0.0.1 -MYSQL_PORT=3306 -MYSQL_USER=backup_reader -MYSQL_PASSWORD=CHANGE_ME -MYSQL_SOCKET= -MYSQL_DATABASE=hifast - -REDIS_HOST=127.0.0.1 -REDIS_PORT=6379 -REDIS_PASSWORD=CHANGE_ME - diff --git a/deploy/aws/ap-northeast-1/configs/mysql-seed-primary-and-replica.env.example b/deploy/aws/ap-northeast-1/configs/mysql-seed-primary-and-replica.env.example deleted file mode 100644 index 93535fc..0000000 --- a/deploy/aws/ap-northeast-1/configs/mysql-seed-primary-and-replica.env.example +++ /dev/null @@ -1,21 +0,0 @@ -PRIMARY_HOST=ppanel-mysql-jp..ap-northeast-1.rds.amazonaws.com -PRIMARY_PORT=3306 -PRIMARY_USER=admin -PRIMARY_PASSWORD=CHANGE_ME -PRIMARY_DB=hifast - -PRIMARY_REPL_USER=repl -PRIMARY_REPL_PASSWORD=CHANGE_ME -PRIMARY_REPL_HOST=104.238.220.230 -PRIMARY_BINLOG_RETENTION_HOURS=24 - -REPLICA_HOST=127.0.0.1 -REPLICA_PORT=3306 -REPLICA_USER=root -REPLICA_PASSWORD= -REPLICA_SOCKET=/var/run/mysqld/mysqld.sock -REPLICA_DB=hifast -REPLICA_SOURCE_SSL=1 - -DUMP_FILE= - diff --git a/deploy/aws/ap-northeast-1/configs/ppanel.yaml.example b/deploy/aws/ap-northeast-1/configs/ppanel.yaml.example deleted file mode 100644 index 2173937..0000000 --- a/deploy/aws/ap-northeast-1/configs/ppanel.yaml.example +++ /dev/null @@ -1,112 +0,0 @@ -Host: 0.0.0.0 -Port: 8080 -Debug: false - -JwtAuth: - AccessSecret: CHANGE_ME_TO_A_LONG_RANDOM_SECRET - AccessExpire: 604800 - -Logger: - ServiceName: PPanel - Mode: console - Encoding: plain - TimeFormat: "2006-01-02 15:04:05.000" - Path: logs - Level: info - MaxContentLength: 0 - Compress: false - Stat: true - KeepDays: 7 - StackCooldownMillis: 100 - MaxBackups: 7 - MaxSize: 100 - Rotation: daily - FileTimeFormat: "2006-01-02T15:04:05.000Z07:00" - -MySQL: - Addr: YOUR_TOKYO_RDS_ENDPOINT:3306 - Dbname: hifast - Username: admin - Password: CHANGE_ME_TO_TOKYO_RDS_PASSWORD - Config: charset=utf8mb4&parseTime=true&loc=Asia%2FTokyo - MaxIdleConns: 10 - MaxOpenConns: 100 - SlowThreshold: 1000 - -Redis: - Host: 127.0.0.1:6379 - Pass: CHANGE_ME_TO_TOKYO_REDIS_PASSWORD - DB: 0 - PoolSize: 100 - MinIdleConns: 10 - MaxRetries: 3 - PoolTimeout: 4 - IdleTimeout: 300 - MaxConnAge: 0 - DialTimeout: 5 - ReadTimeout: 3 - WriteTimeout: 3 - -Trace: - Name: ppanel-server - Endpoint: 127.0.0.1:4317 - Sampler: 0.1 - Batcher: otlpgrpc - -Site: - Host: api-jp.hifast.biz - SiteName: HiFastVPN - -Administrator: - Email: admin@example.com - Password: CHANGE_ME_TO_STRONG_ADMIN_PASSWORD - -Telegram: - Enable: false - BotID: 0 - BotName: "" - BotToken: "" - GroupChatID: "" - EnableNotify: false - WebHookDomain: "" - -Kutt: - Enable: false - ApiURL: "" - ApiKey: "" - TargetURL: "" - Domain: "" - -OpenInstall: - Enable: false - AppKey: "" - ApiKey: "" - -Loki: - Enable: true - URL: "http://localhost:3100" - -AppSignature: - AppSecrets: - android-client: CHANGE_ME_ANDROID_SIGNATURE_SECRET - ios-client: CHANGE_ME_IOS_SIGNATURE_SECRET - web-client: CHANGE_ME_WEB_SIGNATURE_SECRET - ValidWindowSeconds: 300 - SkipPrefixes: - - /v1/notify/ - - /v1/iap/notifications - - /v1/telegram/webhook - - /v1/subscribe/config - -Signature: - EnableSignature: false - -device: - enable: true - security_secret: CHANGE_ME_DEVICE_SECURITY_SECRET - -Register: - EnableTrial: true - EnableTrialEmailWhitelist: true - TrialEmailDomainWhitelist: "gmail.com,outlook.com,icloud.com,qq.com,163.com" - diff --git a/deploy/aws/ap-northeast-1/configs/replica-ops.env.example b/deploy/aws/ap-northeast-1/configs/replica-ops.env.example deleted file mode 100644 index 8f4b19b..0000000 --- a/deploy/aws/ap-northeast-1/configs/replica-ops.env.example +++ /dev/null @@ -1,23 +0,0 @@ -MYSQL_HOST=127.0.0.1 -MYSQL_PORT=3306 -MYSQL_USER=root -MYSQL_PASSWORD=CHANGE_ME -MYSQL_SOCKET= - -REPL_SOURCE_HOST=ppanel-mysql-jp.cpo0keikgh80.ap-northeast-1.rds.amazonaws.com -REPL_SOURCE_PORT=3306 -REPL_SOURCE_USER=repl -REPL_SOURCE_PASSWORD=XwWrQGVWtxmXJ3etHmkFvnRSD54MKYer -REPL_SOURCE_SSL=1 -REPL_SOURCE_LOG_FILE=mysql-bin-changelog.000189 -REPL_SOURCE_LOG_POS=185053 -REPL_SOURCE_AUTO_POSITION=1 - -REDIS_HOST=127.0.0.1 -REDIS_PORT=6379 -REDIS_PASSWORD=CHANGE_ME - -REDIS_SOURCE_HOST=3.114.29.208 -REDIS_SOURCE_PORT=6379 -REDIS_SOURCE_USER= -REDIS_SOURCE_PASSWORD=hifast67yj diff --git a/deploy/aws/ap-northeast-1/configs/resource-inventory.current.md b/deploy/aws/ap-northeast-1/configs/resource-inventory.current.md deleted file mode 100644 index 1c21dff..0000000 --- a/deploy/aws/ap-northeast-1/configs/resource-inventory.current.md +++ /dev/null @@ -1,186 +0,0 @@ -# Tokyo Resource Inventory - -最后更新：`2026-05-21` - -这个文件记录当前东京迁移的真实实施状态，不是示例。 - -## Region - -- AWS account: `hifastvpn (200810848252)` -- Region: `ap-northeast-1` - -## Current Status - -- 东京迁移方案已在仓库内落地为执行资产 -- 东京 VPC 已创建 -- 东京子网、IGW、路由表已在 AWS 控制台创建并复核 -- 东京三层安全组已在 AWS 控制台创建并复核 -- 东京 VPC DNS 开关已开启，可支持公网可访问 RDS -- 东京 S3 备份桶已在 AWS 控制台创建并复核 -- 东京 ACM 证书请求已创建，等待 DNS 验证 -- 东京 RDS MySQL 已创建完成并可用 -- 东京业务 EC2 已创建完成并绑定固定 EIP -- 因当前本机没有可用 AWS CLI 凭据，云上资源状态仍需在 AWS 控制台或已登录环境中复查 - -## Networking - -- VPC - - Name: `ppanel-jp-prod` - - VPC ID: `vpc-0846b23b4a7d64eac` - - CIDR: `10.20.0.0/16` - - Status: `created` -- Public subnet A - - Name: `ppanel-jp-public-a` - - AZ: `ap-northeast-1a` - - CIDR: `10.20.0.0/24` - - Subnet ID: `subnet-091232bdb53e71490` - - Status: `created` -- Public subnet C - - Name: `ppanel-jp-public-c` - - AZ: `ap-northeast-1c` - - CIDR: `10.20.1.0/24` - - Subnet ID: `subnet-01ba0975c525ce8cf` - - Status: `created` -- Private subnet A - - Name: `ppanel-jp-private-a` - - AZ: `ap-northeast-1a` - - CIDR: `10.20.10.0/24` - - Subnet ID: `subnet-0bd13111c02f0edbe` - - Status: `created` -- Private subnet C - - Name: `ppanel-jp-private-c` - - AZ: `ap-northeast-1c` - - CIDR: `10.20.11.0/24` - - Subnet ID: `subnet-0d86c5c756dbc84b2` - - Status: `created` -- Internet Gateway - - Name: `ppanel-jp-igw` - - IGW ID: `igw-028041bcbf63b672c` - - Status: `created` -- Public route table - - Name: `ppanel-jp-public-rt` - - Route Table ID: `rtb-061b101080e4800e5` - - Default route: `0.0.0.0/0 -> igw-028041bcbf63b672c` - - Status: `created` -- Private route table - - Name: `ppanel-jp-private-rt` - - Route Table ID: `rtb-0d7a191a515031c45` - - Status: `created` - -## Security - -- `sg-alb` - - Name: `ppanel-jp-sg-alb` - - Security Group ID: `sg-0b3a23c31041a5a5a` - - Inbound: - - `80/tcp <- 0.0.0.0/0` - - `443/tcp <- 0.0.0.0/0` - - Status: `created` -- `sg-ec2` - - Name: `ppanel-jp-sg-ec2` - - Security Group ID: `sg-01f2a5a81e7505c91` - - Inbound: - - `80/tcp <- sg-0b3a23c31041a5a5a` - - `22/tcp <- 64.118.144.142/32` - - `6379/tcp <- 104.238.220.230/32` - - Status: `created` -- `sg-rds` - - Name: `ppanel-jp-sg-rds` - - Security Group ID: `sg-0b71db1e2c18b57c0` - - Inbound: - - `3306/tcp <- sg-01f2a5a81e7505c91` - - `3306/tcp <- 104.238.220.230/32` - - Status: `created` - -## Compute / Database / Edge - -- EC2 `ppanel-app-jp-01`: - - Instance ID: `i-07839130074cd7ed9` - - Type: `c7i.xlarge` - - Platform: `Ubuntu 26.04 / Linux` - - AZ: `ap-northeast-1c` - - VPC: `ppanel-jp-prod (vpc-0846b23b4a7d64eac)` - - Subnet: `ppanel-jp-public-c (subnet-01ba0975c525ce8cf)` - - Private IP: `10.20.1.168` - - Public IP / Elastic IP: `3.114.29.208` - - Public DNS: `ec2-3-114-29-208.ap-northeast-1.compute.amazonaws.com` - - Security group: `ppanel-jp-sg-ec2 (sg-01f2a5a81e7505c91)` - - Key pair: `ppanel-jp-key-20260521` - - Root volume: `gp3 100GiB` - - ENI: `eni-08accb427a470c9f7` - - EIP allocation ID: `eipalloc-038b32d5c0119accf` - - EIP association ID: `eipassoc-09184aa9161b6a4d9` - - Status: `running` - - SSH recovery key material: not stored in this repository; keep any recovery keys in an approved secret manager or other secure channel. -- RDS subnet group: - - Name: `ppanel-jp-rds-subnet-group` - - VPC: `vpc-0846b23b4a7d64eac` - - Subnets: - - `subnet-0bd13111c02f0edbe` / `ppanel-jp-private-a` - - `subnet-0d86c5c756dbc84b2` / `ppanel-jp-private-c` - - Status: `created` -- RDS public subnet group: - - Name: `ppanel-jp-rds-public-subnet-group` - - VPC: `vpc-0846b23b4a7d64eac` - - Subnets: - - `subnet-091232bdb53e71490` / `ppanel-jp-public-a` - - `subnet-01ba0975c525ce8cf` / `ppanel-jp-public-c` - - Status: `created` -- RDS `ppanel-mysql-jp`: - - Engine: `MySQL Community 8.4.8` - - Class: `db.r7g.xlarge` - - Storage: `gp3 100GiB` - - Deployment: `Single instance (current actual state)` - - VPC: `ppanel-jp-prod (vpc-0846b23b4a7d64eac)` - - Subnet group: `ppanel-jp-rds-public-subnet-group` - - Security group: `ppanel-jp-sg-rds (sg-0b71db1e2c18b57c0)` - - Master username: `admin` - - Credential management: `self-managed` - - Secrets Manager managed password: `disabled` - - Current master password visibility: `not retrievable from AWS console; reset only` - - Public access: `enabled (set at creation time for external replication)` - - Status: `available` - - Endpoint: `ppanel-mysql-jp.cpo0keikgh80.ap-northeast-1.rds.amazonaws.com` - - Public IP (resolved via public DNS): `52.196.204.186` - - Connection test from Tokyo EC2: - - `mysql -h ppanel-mysql-jp.cpo0keikgh80.ap-northeast-1.rds.amazonaws.com -u admin -e "select 1"` - - Result: `ERROR 1045 (28000): Access denied for user 'admin'@'ip-10-20-1-168.ap-northeast-1.compute.internal' (using password: NO)` - - Meaning: `network path and security group are working; only the password is missing` - - Current admin password: `TkyRds20260521!N9mQ8sKe2vLp7Xa` - - External replica prep for `104.238.220.230`: - - binlog retention hours: `24` - - replication user: `repl@104.238.220.230` - - replication password: `XwWrQGVWtxmXJ3etHmkFvnRSD54MKYer` - - current binlog file: `mysql-bin-changelog.000189` - - current binlog position: `185053` -- ALB `ppanel-alb-jp`: `not created` -- WAF `ppanel-waf-jp`: `not created` -- ACM certificate in `ap-northeast-1`: - - Certificate ID: `29d0b9b6-ab37-44d9-ad9e-18fa7e9aae3a` - - Domains: - - `api-jp.hifast.biz` - - `logs-jp.hifast.biz` - - Status: `pending_validation` - - Route 53 hosted zone in current AWS account: `not found` -- S3 backup bucket `hifast-prod-backups-200810848252-ap-northeast-1`: `created` - -## Domains - -- Parallel API domain: `api-jp.hifast.biz` -- Parallel logs domain: `logs-jp.hifast.biz` -- Production API domain: `pending user final confirmation` -- ACM DNS validation records pending external DNS add: - - `api-jp.hifast.biz` - - Name: `_0de5970dfbadaf46759447b2ea627a10.api-jp.hifast.biz.` - - Type: `CNAME` - - Value: `_6a632a5b85c5b3f304cc492090b741b3.jkddzztszm.acm-validations.aws.` - - `logs-jp.hifast.biz` - - Name: `_349a2b2bc4678d76c3ab341ccf73db61.logs-jp.hifast.biz.` - - Type: `CNAME` - - Value: `_74ced20dc96aa39070188605cf0ced18.jkddzztszm.acm-validations.aws.` - -## DR - -- DR host: `104.238.220.230` -- Planned MySQL upstream after cutover: `Tokyo RDS` -- Planned Redis upstream after cutover: `Tokyo EC2 public IP` diff --git a/deploy/aws/ap-northeast-1/configs/resource-inventory.example.md b/deploy/aws/ap-northeast-1/configs/resource-inventory.example.md deleted file mode 100644 index c58e1ce..0000000 --- a/deploy/aws/ap-northeast-1/configs/resource-inventory.example.md +++ /dev/null @@ -1,69 +0,0 @@ -# Tokyo Resource Inventory Example - -Use this file as the single source of truth while building the Tokyo environment. - -## Region - -- AWS account: `hifastvpn (200810848252)` -- Region: `ap-northeast-1` - -## DNS - -- Production API domain: `CHANGE_ME` -- Parallel API domain: `api-jp.hifast.biz` -- Parallel logs domain: `logs-jp.hifast.biz` - -## Networking - -- VPC name: `ppanel-jp-prod` -- VPC CIDR: `10.20.0.0/16` -- Public subnet A: `10.20.0.0/24` -- Public subnet C: `10.20.1.0/24` -- Private subnet A: `10.20.10.0/24` -- Private subnet C: `10.20.11.0/24` -- Ops CIDR for SSH: `CHANGE_ME` - -## Compute - -- EC2 name: `ppanel-app-jp-01` -- EC2 type: `t4g.large` -- EC2 disk: `gp3 80GB` -- SSH key pair: `CHANGE_ME` - -## Database - -- RDS identifier: `ppanel-mysql-jp` -- RDS engine: `MySQL 8.4` -- RDS class: `db.r7g.xlarge` -- RDS storage: `gp3 100GB` -- DB name: `hifast` -- DB admin user: `admin` - -## Cache - -- Redis container: `hifast-redis` -- Redis port: `6379` -- Redis password: `CHANGE_ME` - -## Security / Secrets - -- JWT secret: `CHANGE_ME` -- Admin email: `CHANGE_ME` -- Admin password: `CHANGE_ME` -- Android app signature secret: `CHANGE_ME` -- iOS app signature secret: `CHANGE_ME` -- Web app signature secret: `CHANGE_ME` -- Device security secret: `CHANGE_ME` - -## Backup - -- S3 backup bucket: `hifast-prod-backups-200810848252-ap-northeast-1` -- Versioning: `Enabled` - -## DR - -- DR host: `104.238.220.230` -- MySQL repl user: `repl` -- MySQL repl password: `CHANGE_ME` -- Redis source password: `CHANGE_ME` - diff --git a/deploy/aws/ap-northeast-1/nginx/ppanel-api.conf b/deploy/aws/ap-northeast-1/nginx/ppanel-api.conf deleted file mode 100644 index a103e03..0000000 --- a/deploy/aws/ap-northeast-1/nginx/ppanel-api.conf +++ /dev/null @@ -1,34 +0,0 @@ -server { - listen 80 default_server; - listen [::]:80 default_server; - server_name _; - - client_max_body_size 20m; - - access_log /var/log/nginx/ppanel-access.log; - error_log /var/log/nginx/ppanel-error.log warn; - - location / { - proxy_http_version 1.1; - proxy_pass http://127.0.0.1:8080; - - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header X-Forwarded-Host $host; - proxy_set_header X-Forwarded-Port $server_port; - - proxy_connect_timeout 10s; - proxy_send_timeout 60s; - proxy_read_timeout 60s; - } - - location = /nginx_status { - stub_status; - access_log off; - allow 127.0.0.1; - deny all; - } -} - diff --git a/deploy/systemd/hifast-mysql-backup.service b/deploy/systemd/hifast-mysql-backup.service deleted file mode 100644 index ac2dbb9..0000000 --- a/deploy/systemd/hifast-mysql-backup.service +++ /dev/null @@ -1,17 +0,0 @@ -[Unit] -Description=Hifast MySQL backup to S3 -Wants=network-online.target -After=network-online.target - -[Service] -Type=oneshot -User=root -Group=root -EnvironmentFile=/root/backup-to-s3.env -ExecStart=/usr/bin/env bash -lc 'exec /opt/ppanel/deploy/scripts/mysql_backup_to_s3.sh' -Nice=10 -IOSchedulingClass=best-effort -IOSchedulingPriority=7 - -[Install] -WantedBy=multi-user.target diff --git a/deploy/systemd/hifast-mysql-backup.timer b/deploy/systemd/hifast-mysql-backup.timer deleted file mode 100644 index aa623d9..0000000 --- a/deploy/systemd/hifast-mysql-backup.timer +++ /dev/null @@ -1,11 +0,0 @@ -[Unit] -Description=Run Hifast MySQL backup to S3 every 10 minutes - -[Timer] -OnCalendar=*:0/10 -Persistent=true -RandomizedDelaySec=30 -Unit=hifast-mysql-backup.service - -[Install] -WantedBy=timers.target diff --git a/docker-compose.cloud.yml b/docker-compose.cloud.yml index 6c04d54..8cf2833 100644 --- a/docker-compose.cloud.yml +++ b/docker-compose.cloud.yml @@ -1,26 +1,4 @@ -# PPanel 服务部署 (云端/无源码版) -# 使用方法: -# 1. 确保已将 docker-compose.cloud.yml, configs/, loki/, grafana/, prometheus/, tempo/ 目录上传到服务器同一目录 -# 2. 确保 configs/ 目录下有 ppanel.yaml 配置文件（参考 etc/ppanel.yaml） -# 3. 确保 logs/ cache/ tempo_data/ 目录存在 (mkdir -p logs cache tempo_data) -# 4. 运行: docker-compose -f docker-compose.cloud.yml up -d -# -# 网络说明： -# ppanel-server 使用 host 网络（可出外网，直接访问 AWS RDS / 本机 Redis） -# 监控服务（Loki/Tempo/Grafana/Prometheus）在 ppanel_net bridge 网络中 -# Tempo(4317) 将端口映射到 127.0.0.1，ppanel-server 通过 host 网络访问 -# 监控端口绑定 127.0.0.1，需通过 SSH 隧道或 Nginx 反代访问 -# -# 未来多开 ppanel-server 时： -# 修复宿主机 iptables bridge 出网规则后，可将 ppanel-server 切回 bridge 网络 -# 多实例用不同端口: ports: ["8081:8080"] + container_name: ppanel-server-2 - services: - # ---------------------------------------------------- - # 1. 业务后端 (PPanel Server) - # host 网络：可出外网，直接访问 AWS RDS/Redis；通过 127.0.0.1 访问 Tempo - # PPANEL_SERVER_IMAGE/PPANEL_SERVER_TAG 由 CI/CD 传入不可变镜像（如 ghcr.io/org/repo:git SHA） - # ---------------------------------------------------- ppanel-server: image: ${PPANEL_SERVER_IMAGE:-registry.kxsw.us/vpn-server}:${PPANEL_SERVER_TAG:?please set PPANEL_SERVER_TAG to an immutable image tag} container_name: ppanel-server @@ -28,7 +6,7 @@ services: volumes: - ./configs:/app/etc - ./logs:/app/logs - - ./cache:/app/cache # GeoLite2-City.mmdb IP 地理位置数据库 + - ./cache:/app/cache environment: - TZ=Asia/Shanghai network_mode: host @@ -37,215 +15,8 @@ services: nofile: soft: 65535 hard: 65535 - depends_on: - tempo: - condition: service_started logging: driver: "json-file" options: max-size: "10m" max-file: "3" - - # ---------------------------------------------------- - # 2. Tempo (链路追踪存储) - # ---------------------------------------------------- - tempo: - image: grafana/tempo:2.4.1 - container_name: ppanel-tempo - user: root - restart: always - command: - - "-config.file=/etc/tempo.yaml" - - "-target=all" - volumes: - - ./tempo/tempo-config.yaml:/etc/tempo.yaml - - ./tempo_data:/var/tempo - ports: - - "127.0.0.1:4317:4317" # OTLP gRPC，ppanel-server(host网络)通过127.0.0.1:4317发送trace - networks: - - ppanel_net - logging: - driver: "json-file" - options: - max-size: "10m" - max-file: "3" - - # ---------------------------------------------------- - # 3. Loki (日志存储) - # ---------------------------------------------------- - loki: - image: grafana/loki:3.0.0 - container_name: ppanel-loki - restart: always - volumes: - - ./loki/loki-config.yaml:/etc/loki/local-config.yaml - - loki_data:/loki - command: -config.file=/etc/loki/local-config.yaml - # 不对外暴露端口，仅内网访问 - networks: - - ppanel_net - logging: - driver: "json-file" - options: - max-size: "10m" - max-file: "3" - - # ---------------------------------------------------- - # 4. Promtail (日志采集) - # ---------------------------------------------------- - promtail: - image: grafana/promtail:3.0.0 - container_name: ppanel-promtail - restart: always - volumes: - - ./loki/promtail-config.yaml:/etc/promtail/config.yaml - - /var/lib/docker/containers:/var/lib/docker/containers:ro - - /var/run/docker.sock:/var/run/docker.sock - - ./logs:/var/log/ppanel-server:ro - - /var/log/nginx:/var/log/nginx:ro - command: -config.file=/etc/promtail/config.yaml - networks: - - ppanel_net - depends_on: - - loki - logging: - driver: "json-file" - options: - max-size: "10m" - max-file: "3" - - # ---------------------------------------------------- - # 5. Grafana (可观测面板) - # 访问: ssh -L 3333:localhost:3333 your-server 后浏览器打开 http://localhost:3333 - # 或配置 Nginx 反代（建议加认证） - # ---------------------------------------------------- - grafana: - image: grafana/grafana:13.0.1 - container_name: ppanel-grafana - restart: always - ports: - - "3333:3000" # 仅本机可访问，需 SSH 隧道或 Nginx 反代 - environment: - - GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_PASSWORD:?请在 .env 文件中设置 GRAFANA_PASSWORD} - - GF_USERS_ALLOW_SIGN_UP=false - - GF_SERVER_DOMAIN=${GRAFANA_DOMAIN:-logsx.hifast.biz} - - GF_SERVER_ROOT_URL=${GRAFANA_ROOT_URL:-https://logsx.hifast.biz} - - GF_FEATURE_TOGGLES_ENABLE=appObservability - - AWS_REGION=${AWS_REGION:-ap-east-1} - - AWS_DEFAULT_REGION=${AWS_REGION:-ap-east-1} - - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID:-} - - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY:-} - - AWS_SESSION_TOKEN=${AWS_SESSION_TOKEN:-} - volumes: - - grafana_data:/var/lib/grafana - - ./grafana/provisioning:/etc/grafana/provisioning - networks: - - ppanel_net - depends_on: - - loki - - tempo - - prometheus - logging: - driver: "json-file" - options: - max-size: "10m" - max-file: "3" - - # ---------------------------------------------------- - # 6. Prometheus (指标采集) - # ---------------------------------------------------- - prometheus: - image: prom/prometheus:v3.11.3 - container_name: ppanel-prometheus - restart: always - ports: - - "127.0.0.1:9090:9090" # 仅本机可访问 - volumes: - - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml - - prometheus_data:/prometheus - command: - - '--config.file=/etc/prometheus/prometheus.yml' - - '--storage.tsdb.path=/prometheus' - - '--web.enable-lifecycle' - - '--web.enable-remote-write-receiver' - networks: - - ppanel_net - logging: - driver: "json-file" - options: - max-size: "10m" - max-file: "3" - - # ---------------------------------------------------- - # 7. Nginx Exporter (监控宿主机 Nginx) - # ---------------------------------------------------- - nginx-exporter: - image: nginx/nginx-prometheus-exporter:1.5.0 - container_name: ppanel-nginx-exporter - restart: always - command: - - -nginx.scrape-uri=http://host.docker.internal:8090/nginx_status - extra_hosts: - - "host.docker.internal:host-gateway" - networks: - - ppanel_net - logging: - driver: "json-file" - options: - max-size: "10m" - max-file: "3" - - # ---------------------------------------------------- - # 8. Node Exporter (宿主机监控) - # ---------------------------------------------------- - node-exporter: - image: prom/node-exporter:v1.11.1 - container_name: ppanel-node-exporter - restart: always - volumes: - - /proc:/host/proc:ro - - /sys:/host/sys:ro - - /:/rootfs:ro - command: - - '--path.procfs=/host/proc' - - '--path.sysfs=/host/sys' - - '--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)' - networks: - - ppanel_net - logging: - driver: "json-file" - options: - max-size: "10m" - max-file: "3" - - # ---------------------------------------------------- - # 9. cAdvisor (容器监控) - # ---------------------------------------------------- - cadvisor: - image: gcr.io/cadvisor/cadvisor:v0.55.1 - container_name: ppanel-cadvisor - restart: always - volumes: - - /:/rootfs:ro - - /var/run:/var/run:ro - - /sys:/sys:ro - - /var/lib/docker/:/var/lib/docker:ro - - /dev/disk/:/dev/disk:ro - networks: - - ppanel_net - logging: - driver: "json-file" - options: - max-size: "10m" - max-file: "3" - -volumes: - loki_data: - grafana_data: - prometheus_data: - tempo_data: - -networks: - ppanel_net: - name: ppanel_net - driver: bridge diff --git a/etc/ppanel.yaml b/etc/ppanel.yaml index 7c947a3..26eea64 100644 --- a/etc/ppanel.yaml +++ b/etc/ppanel.yaml @@ -15,10 +15,10 @@ Logger: # 日志配置 Level: debug # 日志级别: debug, info, warn, error, panic, fatal MySQL: - Addr: 45.43.29.127:3306 # host 网络模式; bridge 模式改为 mysql:3306 + Addr: 127.0.0.1:3306 # 本地开发默认；Docker bridge 模式可改为 mysql:3306 Username: root # MySQL用户名 - Password: jpcV41ppanel # MySQL密码，与 .env MYSQL_ROOT_PASSWORD 一致 - Dbname: hifast # MySQL数据库名 + Password: CHANGE_ME_TO_DB_PASSWORD # MySQL密码 + Dbname: ppanel # MySQL数据库名 Config: charset=utf8mb4&parseTime=true&loc=Asia%2FShanghai MaxIdleConns: 10 MaxOpenConns: 100 @@ -42,9 +42,9 @@ Redis: AppSignature: AppSecrets: - android-client: uB4G,XxL2{7b # Android 客户端签名密钥 - ios-client: uB4G,XxL2{7b # iOS 客户端签名密钥 - web-client: uB4G,XxL2{7b # Web 客户端签名密钥 + android-client: CHANGE_ME_ANDROID_APP_SECRET # Android 客户端签名密钥 + ios-client: CHANGE_ME_IOS_APP_SECRET # iOS 客户端签名密钥 + web-client: CHANGE_ME_WEB_APP_SECRET # Web 客户端签名密钥 ValidWindowSeconds: 300 # 签名时间窗口（秒） SkipPrefixes: - /v1/notify/ # 支付回调不验签 @@ -58,8 +58,8 @@ Signature: Trace: # 链路追踪配置 (OpenTelemetry) Name: ppanel # 服务名 Sampler: 1.0 # 采样率 0.0-1.0，生产建议 0.1 - Batcher: otlpgrpc # 本地开发留空""; 生产填 otlpgrpc - Endpoint: "127.0.0.1:4317" # host 网络模式; bridge 模式改为 tempo:4317 + Batcher: "" # 本地开发留空；生产如需链路追踪再配置 exporter + Endpoint: "" S3: Enable: false diff --git a/grafana/provisioning/alerting/ppanel-alert-rules.yml b/grafana/provisioning/alerting/ppanel-alert-rules.yml deleted file mode 100644 index e1e9e71..0000000 --- a/grafana/provisioning/alerting/ppanel-alert-rules.yml +++ /dev/null @@ -1,272 +0,0 @@ -apiVersion: 1 - -groups: - - orgId: 1 - name: ppanel-core - folder: PPanel - interval: 1m - rules: - - uid: ppanel-target-down - title: PPanel monitoring target down - condition: C - for: 2m - noDataState: Alerting - execErrState: Error - annotations: - summary: "Monitoring target is down" - description: "{{ $labels.job }} on {{ $labels.instance }} has been down for more than 2 minutes." - labels: - severity: critical - service: ppanel - data: - - refId: A - relativeTimeRange: - from: 300 - to: 0 - datasourceUid: prometheus - model: - datasource: - type: prometheus - uid: prometheus - editorMode: code - expr: 'up{job=~"grafana|prometheus|node-exporter|cadvisor|nginx-exporter|loki|tempo"}' - instant: true - intervalMs: 1000 - maxDataPoints: 43200 - refId: A - - refId: C - datasourceUid: __expr__ - model: - conditions: - - evaluator: - params: - - 1 - type: lt - operator: - type: and - query: - params: - - A - reducer: - type: last - type: query - datasource: - type: __expr__ - uid: __expr__ - expression: A - intervalMs: 1000 - maxDataPoints: 43200 - refId: C - type: threshold - - - uid: ppanel-host-disk-high - title: PPanel host disk usage high - condition: C - for: 10m - noDataState: NoData - execErrState: Error - annotations: - summary: "Host disk usage is high" - description: "{{ $labels.instance }} {{ $labels.mountpoint }} disk usage is above 85% for 10 minutes." - labels: - severity: warning - service: ppanel - data: - - refId: A - relativeTimeRange: - from: 900 - to: 0 - datasourceUid: prometheus - model: - datasource: - type: prometheus - uid: prometheus - editorMode: code - expr: '100 - (node_filesystem_avail_bytes{fstype!~"tmpfs|overlay|squashfs|aufs",mountpoint!~"/run.*|/var/lib/docker.*"} / node_filesystem_size_bytes{fstype!~"tmpfs|overlay|squashfs|aufs",mountpoint!~"/run.*|/var/lib/docker.*"} * 100)' - instant: true - intervalMs: 1000 - maxDataPoints: 43200 - refId: A - - refId: C - datasourceUid: __expr__ - model: - conditions: - - evaluator: - params: - - 85 - type: gt - operator: - type: and - query: - params: - - A - reducer: - type: last - type: query - datasource: - type: __expr__ - uid: __expr__ - expression: A - intervalMs: 1000 - maxDataPoints: 43200 - refId: C - type: threshold - - - uid: ppanel-host-memory-high - title: PPanel host memory usage high - condition: C - for: 10m - noDataState: NoData - execErrState: Error - annotations: - summary: "Host memory usage is high" - description: "{{ $labels.instance }} memory usage is above 90% for 10 minutes." - labels: - severity: warning - service: ppanel - data: - - refId: A - relativeTimeRange: - from: 900 - to: 0 - datasourceUid: prometheus - model: - datasource: - type: prometheus - uid: prometheus - editorMode: code - expr: '(1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100' - instant: true - intervalMs: 1000 - maxDataPoints: 43200 - refId: A - - refId: C - datasourceUid: __expr__ - model: - conditions: - - evaluator: - params: - - 90 - type: gt - operator: - type: and - query: - params: - - A - reducer: - type: last - type: query - datasource: - type: __expr__ - uid: __expr__ - expression: A - intervalMs: 1000 - maxDataPoints: 43200 - refId: C - type: threshold - - - uid: ppanel-host-cpu-high - title: PPanel host CPU usage high - condition: C - for: 10m - noDataState: NoData - execErrState: Error - annotations: - summary: "Host CPU usage is high" - description: "{{ $labels.instance }} CPU usage is above 90% for 10 minutes." - labels: - severity: warning - service: ppanel - data: - - refId: A - relativeTimeRange: - from: 900 - to: 0 - datasourceUid: prometheus - model: - datasource: - type: prometheus - uid: prometheus - editorMode: code - expr: '100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100)' - instant: true - intervalMs: 1000 - maxDataPoints: 43200 - refId: A - - refId: C - datasourceUid: __expr__ - model: - conditions: - - evaluator: - params: - - 90 - type: gt - operator: - type: and - query: - params: - - A - reducer: - type: last - type: query - datasource: - type: __expr__ - uid: __expr__ - expression: A - intervalMs: 1000 - maxDataPoints: 43200 - refId: C - type: threshold - - - uid: ppanel-container-restarts - title: PPanel container restarted - condition: C - for: 1m - noDataState: NoData - execErrState: Error - annotations: - summary: "Container restarted" - description: "{{ $labels.name }} restarted or changed start time in the last hour." - labels: - severity: warning - service: ppanel - data: - - refId: A - relativeTimeRange: - from: 3600 - to: 0 - datasourceUid: prometheus - model: - datasource: - type: prometheus - uid: prometheus - editorMode: code - expr: 'sum by (name) (changes(container_start_time_seconds{name!=""}[1h]))' - instant: true - intervalMs: 1000 - maxDataPoints: 43200 - refId: A - - refId: C - datasourceUid: __expr__ - model: - conditions: - - evaluator: - params: - - 0 - type: gt - operator: - type: and - query: - params: - - A - reducer: - type: last - type: query - datasource: - type: __expr__ - uid: __expr__ - expression: A - intervalMs: 1000 - maxDataPoints: 43200 - refId: C - type: threshold diff --git a/grafana/provisioning/dashboards/dashboards.yml b/grafana/provisioning/dashboards/dashboards.yml deleted file mode 100644 index ed36901..0000000 --- a/grafana/provisioning/dashboards/dashboards.yml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: 1 - -providers: - - name: PPanel - orgId: 1 - folder: PPanel - folderUid: ppanel - type: file - disableDeletion: false - allowUiUpdates: true - updateIntervalSeconds: 30 - options: - path: /etc/grafana/provisioning/dashboards/json - foldersFromFilesStructure: false diff --git a/grafana/provisioning/dashboards/json/aws-rds-redis-overview.json b/grafana/provisioning/dashboards/json/aws-rds-redis-overview.json deleted file mode 100644 index 97f4820..0000000 --- a/grafana/provisioning/dashboards/json/aws-rds-redis-overview.json +++ /dev/null @@ -1,520 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": null, - "links": [], - "panels": [ - { - "datasource": { - "type": "cloudwatch", - "uid": "cloudwatch" - }, - "gridPos": { - "h": 3, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 1, - "options": { - "content": "AWS CloudWatch overview
Region: ap-east-1 (Hong Kong)
RDS DBInstanceIdentifier: hifast-mysql-prod-v2
Redis: current production uses a local Docker Redis container (hifast-redis) on EC2 rather than AWS ElastiCache.

This dashboard keeps the RDS CloudWatch panels. Redis should be observed from the local ops dashboard via Prometheus/cAdvisor instead of ElastiCache metrics.", - "mode": "html" - }, - "pluginVersion": "11.0.0", - "title": "Read Me", - "type": "text" - }, - { - "datasource": { - "type": "cloudwatch", - "uid": "cloudwatch" - }, - "fieldConfig": { - "defaults": { - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 3 - }, - "id": 2, - "options": { - "legend": { - "displayMode": "table", - "placement": "bottom" - }, - "tooltip": { - "mode": "single" - } - }, - "targets": [ - { - "datasource": { - "type": "cloudwatch", - "uid": "cloudwatch" - }, - "dimensions": { - "DBInstanceIdentifier": "hifast-mysql-prod-v2" - }, - "metricName": "CPUUtilization", - "namespace": "AWS/RDS", - "period": "", - "refId": "A", - "region": "ap-east-1", - "statistic": "Average" - } - ], - "title": "RDS CPU", - "type": "timeseries" - }, - { - "datasource": { - "type": "cloudwatch", - "uid": "cloudwatch" - }, - "fieldConfig": { - "defaults": { - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 3 - }, - "id": 3, - "options": { - "legend": { - "displayMode": "table", - "placement": "bottom" - }, - "tooltip": { - "mode": "single" - } - }, - "targets": [ - { - "datasource": { - "type": "cloudwatch", - "uid": "cloudwatch" - }, - "dimensions": { - "DBInstanceIdentifier": "hifast-mysql-prod-v2" - }, - "metricName": "DatabaseConnections", - "namespace": "AWS/RDS", - "period": "", - "refId": "A", - "region": "ap-east-1", - "statistic": "Average" - } - ], - "title": "RDS Connections", - "type": "timeseries" - }, - { - "datasource": { - "type": "cloudwatch", - "uid": "cloudwatch" - }, - "fieldConfig": { - "defaults": { - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 3 - }, - "id": 4, - "options": { - "legend": { - "displayMode": "table", - "placement": "bottom" - }, - "tooltip": { - "mode": "single" - } - }, - "targets": [ - { - "datasource": { - "type": "cloudwatch", - "uid": "cloudwatch" - }, - "dimensions": { - "DBInstanceIdentifier": "hifast-mysql-prod-v2" - }, - "metricName": "FreeStorageSpace", - "namespace": "AWS/RDS", - "period": "", - "refId": "A", - "region": "ap-east-1", - "statistic": "Minimum" - } - ], - "title": "RDS Free Storage", - "type": "timeseries" - }, - { - "datasource": { - "type": "cloudwatch", - "uid": "cloudwatch" - }, - "fieldConfig": { - "defaults": { - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 11 - }, - "id": 5, - "options": { - "legend": { - "displayMode": "table", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi" - } - }, - "targets": [ - { - "datasource": { - "type": "cloudwatch", - "uid": "cloudwatch" - }, - "dimensions": { - "DBInstanceIdentifier": "hifast-mysql-prod-v2" - }, - "metricName": "ReadLatency", - "namespace": "AWS/RDS", - "period": "", - "refId": "A", - "region": "ap-east-1", - "statistic": "Average" - }, - { - "datasource": { - "type": "cloudwatch", - "uid": "cloudwatch" - }, - "dimensions": { - "DBInstanceIdentifier": "hifast-mysql-prod-v2" - }, - "metricName": "WriteLatency", - "namespace": "AWS/RDS", - "period": "", - "refId": "B", - "region": "ap-east-1", - "statistic": "Average" - } - ], - "title": "RDS Read / Write Latency", - "type": "timeseries" - }, - { - "datasource": { - "type": "cloudwatch", - "uid": "cloudwatch" - }, - "fieldConfig": { - "defaults": { - "unit": "iops" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 11 - }, - "id": 6, - "options": { - "legend": { - "displayMode": "table", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi" - } - }, - "targets": [ - { - "datasource": { - "type": "cloudwatch", - "uid": "cloudwatch" - }, - "dimensions": { - "DBInstanceIdentifier": "hifast-mysql-prod-v2" - }, - "metricName": "ReadIOPS", - "namespace": "AWS/RDS", - "period": "", - "refId": "A", - "region": "ap-east-1", - "statistic": "Average" - }, - { - "datasource": { - "type": "cloudwatch", - "uid": "cloudwatch" - }, - "dimensions": { - "DBInstanceIdentifier": "hifast-mysql-prod-v2" - }, - "metricName": "WriteIOPS", - "namespace": "AWS/RDS", - "period": "", - "refId": "B", - "region": "ap-east-1", - "statistic": "Average" - } - ], - "title": "RDS Read / Write IOPS", - "type": "timeseries" - }, - { - "datasource": { - "type": "cloudwatch", - "uid": "cloudwatch" - }, - "fieldConfig": { - "defaults": { - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 6, - "x": 0, - "y": 19 - }, - "id": 7, - "options": { - "legend": { - "displayMode": "table", - "placement": "bottom" - }, - "tooltip": { - "mode": "single" - } - }, - "targets": [ - { - "datasource": { - "type": "cloudwatch", - "uid": "cloudwatch" - }, - "dimensions": { - "ReplicationGroupId": "hifastapp-redis" - }, - "metricName": "CPUUtilization", - "namespace": "AWS/ElastiCache", - "period": "", - "refId": "A", - "region": "ap-east-1", - "statistic": "Average" - } - ], - "title": "Redis Host CPU (Legacy ElastiCache)", - "type": "timeseries" - }, - { - "datasource": { - "type": "cloudwatch", - "uid": "cloudwatch" - }, - "fieldConfig": { - "defaults": { - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 6, - "x": 6, - "y": 19 - }, - "id": 8, - "options": { - "legend": { - "displayMode": "table", - "placement": "bottom" - }, - "tooltip": { - "mode": "single" - } - }, - "targets": [ - { - "datasource": { - "type": "cloudwatch", - "uid": "cloudwatch" - }, - "dimensions": { - "ReplicationGroupId": "hifastapp-redis" - }, - "metricName": "EngineCPUUtilization", - "namespace": "AWS/ElastiCache", - "period": "", - "refId": "A", - "region": "ap-east-1", - "statistic": "Average" - } - ], - "title": "Redis Engine CPU (Legacy ElastiCache)", - "type": "timeseries" - }, - { - "datasource": { - "type": "cloudwatch", - "uid": "cloudwatch" - }, - "fieldConfig": { - "defaults": { - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 6, - "x": 12, - "y": 19 - }, - "id": 9, - "options": { - "legend": { - "displayMode": "table", - "placement": "bottom" - }, - "tooltip": { - "mode": "single" - } - }, - "targets": [ - { - "datasource": { - "type": "cloudwatch", - "uid": "cloudwatch" - }, - "dimensions": { - "ReplicationGroupId": "hifastapp-redis" - }, - "metricName": "CurrConnections", - "namespace": "AWS/ElastiCache", - "period": "", - "refId": "A", - "region": "ap-east-1", - "statistic": "Average" - } - ], - "title": "Redis Connections (Legacy ElastiCache)", - "type": "timeseries" - }, - { - "datasource": { - "type": "cloudwatch", - "uid": "cloudwatch" - }, - "fieldConfig": { - "defaults": { - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 6, - "x": 18, - "y": 19 - }, - "id": 10, - "options": { - "legend": { - "displayMode": "table", - "placement": "bottom" - }, - "tooltip": { - "mode": "single" - } - }, - "targets": [ - { - "datasource": { - "type": "cloudwatch", - "uid": "cloudwatch" - }, - "dimensions": { - "ReplicationGroupId": "hifastapp-redis" - }, - "metricName": "DatabaseMemoryUsagePercentage", - "namespace": "AWS/ElastiCache", - "period": "", - "refId": "A", - "region": "ap-east-1", - "statistic": "Average" - } - ], - "title": "Redis Memory Usage % (Legacy ElastiCache)", - "type": "timeseries" - } - ], - "refresh": "30s", - "schemaVersion": 39, - "style": "dark", - "tags": [ - "aws", - "cloudwatch", - "rds", - "redis" - ], - "templating": { - "list": [] - }, - "time": { - "from": "now-6h", - "to": "now" - }, - "timepicker": {}, - "timezone": "browser", - "title": "AWS RDS & Redis Overview", - "uid": "aws-rds-redis-overview", - "version": 1, - "weekStart": "" -} diff --git a/grafana/provisioning/dashboards/json/ppanel-ops-overview.json b/grafana/provisioning/dashboards/json/ppanel-ops-overview.json deleted file mode 100644 index 08ea6e8..0000000 --- a/grafana/provisioning/dashboards/json/ppanel-ops-overview.json +++ /dev/null @@ -1,565 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": null, - "links": [], - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "0": { - "text": "DOWN" - }, - "1": { - "text": "UP" - } - }, - "type": "value" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "green", - "value": 1 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 1, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "13.0.1", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "up{job=~\"prometheus|grafana|node-exporter|cadvisor|nginx-exporter|loki|tempo\"}", - "instant": true, - "legendFormat": "{{job}}", - "range": false, - "refId": "A" - } - ], - "title": "Service Availability", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "max": 100, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "orange", - "value": 75 - }, - { - "color": "red", - "value": 90 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 4 - }, - "id": 2, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "100 - (avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m])) * 100)", - "legendFormat": "{{instance}} CPU", - "range": true, - "refId": "A" - } - ], - "title": "Host CPU Usage", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "max": 100, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "orange", - "value": 80 - }, - { - "color": "red", - "value": 90 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 4 - }, - "id": 3, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "(1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100", - "legendFormat": "{{instance}} memory", - "range": true, - "refId": "A" - } - ], - "title": "Host Memory Usage", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "max": 100, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "orange", - "value": 80 - }, - { - "color": "red", - "value": 90 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 4 - }, - "id": 4, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "100 - (node_filesystem_avail_bytes{fstype!~\"tmpfs|overlay|squashfs|aufs\",mountpoint!~\"/run.*|/var/lib/docker.*\"} / node_filesystem_size_bytes{fstype!~\"tmpfs|overlay|squashfs|aufs\",mountpoint!~\"/run.*|/var/lib/docker.*\"} * 100)", - "legendFormat": "{{mountpoint}}", - "range": true, - "refId": "A" - } - ], - "title": "Host Disk Usage", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 12 - }, - "id": 5, - "options": { - "legend": { - "displayMode": "table", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "sum by (name) (rate(container_cpu_usage_seconds_total{name!=\"\"}[5m]))", - "legendFormat": "{{name}}", - "range": true, - "refId": "A" - } - ], - "title": "Container CPU", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 12 - }, - "id": 6, - "options": { - "legend": { - "displayMode": "table", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "sum by (name) (container_memory_working_set_bytes{name!=\"\"})", - "legendFormat": "{{name}}", - "range": true, - "refId": "A" - } - ], - "title": "Container Memory", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 12 - }, - "id": 7, - "options": { - "legend": { - "displayMode": "table", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "sum by (name) (changes(container_start_time_seconds{name!=\"\"}[1h]))", - "legendFormat": "{{name}}", - "range": true, - "refId": "A" - } - ], - "title": "Container Restarts / Changes", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 20 - }, - "id": 8, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "rate(nginx_http_requests_total[5m])", - "legendFormat": "requests", - "range": true, - "refId": "A" - } - ], - "title": "Nginx Requests", - "type": "timeseries" - }, - { - "datasource": { - "type": "loki", - "uid": "loki" - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 20 - }, - "id": 9, - "options": { - "dedupStrategy": "none", - "enableLogDetails": true, - "prettifyLogMessage": false, - "showCommonLabels": false, - "showLabels": true, - "showTime": true, - "sortOrder": "Descending", - "wrapLogMessage": true - }, - "targets": [ - { - "datasource": { - "type": "loki", - "uid": "loki" - }, - "editorMode": "code", - "expr": "{job=~\"ppanel-server|nginx|docker\"} |~ \"(?i)(error|panic|fatal|timeout|exception|failed)\"", - "queryType": "range", - "refId": "A" - } - ], - "title": "Recent Errors", - "type": "logs" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 20 - }, - "id": 10, - "options": { - "legend": { - "displayMode": "table", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "sum by (service_name) (rate(traces_spanmetrics_calls_total[5m]))", - "legendFormat": "{{service_name}}", - "range": true, - "refId": "A" - } - ], - "title": "Trace Span Calls", - "type": "timeseries" - } - ], - "refresh": "30s", - "schemaVersion": 42, - "tags": [ - "ppanel", - "ops", - "prometheus", - "loki", - "tempo" - ], - "templating": { - "list": [] - }, - "time": { - "from": "now-6h", - "to": "now" - }, - "timepicker": {}, - "timezone": "browser", - "title": "PPanel Ops Overview", - "uid": "ppanel-ops-overview", - "version": 1, - "weekStart": "" -} diff --git a/grafana/provisioning/dashboards/json/ppanel-server-logs.json b/grafana/provisioning/dashboards/json/ppanel-server-logs.json deleted file mode 100644 index 864f1c5..0000000 --- a/grafana/provisioning/dashboards/json/ppanel-server-logs.json +++ /dev/null @@ -1,330 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": null, - "links": [], - "panels": [ - { - "datasource": { - "type": "loki", - "uid": "P8E80F9AEF21F6940" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 0 - }, - "id": 1, - "options": { - "legend": { - "displayMode": "table", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi" - } - }, - "targets": [ - { - "datasource": { - "type": "loki", - "uid": "P8E80F9AEF21F6940" - }, - "editorMode": "code", - "expr": "sum(count_over_time({compose_service=\"ppanel-server\"}[5m]))", - "queryType": "range", - "refId": "A" - } - ], - "title": "Matched Log Volume", - "type": "timeseries" - }, - { - "datasource": { - "type": "loki", - "uid": "P8E80F9AEF21F6940" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 0 - }, - "id": 2, - "options": { - "legend": { - "displayMode": "table", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi" - } - }, - "targets": [ - { - "datasource": { - "type": "loki", - "uid": "P8E80F9AEF21F6940" - }, - "editorMode": "code", - "expr": "sum(count_over_time({compose_service=\"ppanel-server\"} |~ \"(?i)(error|panic|fatal)\" [5m]))", - "queryType": "range", - "refId": "A" - } - ], - "title": "Matched Error Volume", - "type": "timeseries" - }, - { - "datasource": { - "type": "loki", - "uid": "P8E80F9AEF21F6940" - }, - "gridPos": { - "h": 12, - "w": 24, - "x": 0, - "y": 7 - }, - "id": 3, - "options": { - "dedupStrategy": "none", - "enableLogDetails": true, - "prettifyLogMessage": false, - "showCommonLabels": false, - "showLabels": true, - "showTime": true, - "sortOrder": "Descending", - "wrapLogMessage": true - }, - "targets": [ - { - "datasource": { - "type": "loki", - "uid": "P8E80F9AEF21F6940" - }, - "editorMode": "code", - "expr": "{compose_service=\"ppanel-server\"}", - "queryType": "range", - "refId": "A" - } - ], - "title": "Filtered Server Logs", - "type": "logs" - }, - { - "datasource": { - "type": "loki", - "uid": "P8E80F9AEF21F6940" - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 19 - }, - "id": 4, - "options": { - "dedupStrategy": "none", - "enableLogDetails": true, - "prettifyLogMessage": false, - "showCommonLabels": false, - "showLabels": true, - "showTime": true, - "sortOrder": "Descending", - "wrapLogMessage": true - }, - "targets": [ - { - "datasource": { - "type": "loki", - "uid": "P8E80F9AEF21F6940" - }, - "editorMode": "code", - "expr": "{compose_service=\"ppanel-server\"} |~ \"(?i)(error|panic|fatal)\"", - "queryType": "range", - "refId": "A" - } - ], - "title": "Filtered Server Errors", - "type": "logs" - } - ], - "refresh": "30s", - "schemaVersion": 42, - "tags": [ - "ppanel", - "logs", - "server", - "loki" - ], - "templating": { - "list": [ - { - "current": { - "selected": false, - "text": ".", - "value": "." - }, - "description": "输入用户 ID；默认 . 表示不过滤", - "hide": 0, - "label": "用户ID", - "name": "user_id", - "options": [], - "query": ".", - "skipUrlSync": false, - "type": "textbox" - }, - { - "current": { - "selected": false, - "text": ".", - "value": "." - }, - "description": "输入邮箱或邮箱片段；默认 . 表示不过滤", - "hide": 0, - "label": "邮箱", - "name": "email", - "options": [], - "query": ".", - "skipUrlSync": false, - "type": "textbox" - }, - { - "current": { - "selected": false, - "text": ".", - "value": "." - }, - "description": "输入订单号/支付单号/交易号片段；默认 . 表示不过滤", - "hide": 0, - "label": "订单", - "name": "order", - "options": [], - "query": ".", - "skipUrlSync": false, - "type": "textbox" - }, - { - "current": { - "selected": true, - "text": "All", - "value": "." - }, - "description": "日志等级", - "hide": 0, - "includeAll": false, - "label": "等级", - "multi": false, - "name": "level", - "options": [ - { - "selected": true, - "text": "All", - "value": "." - }, - { - "selected": false, - "text": "debug", - "value": "debug" - }, - { - "selected": false, - "text": "info", - "value": "info" - }, - { - "selected": false, - "text": "warn", - "value": "warn" - }, - { - "selected": false, - "text": "error", - "value": "error" - }, - { - "selected": false, - "text": "slow", - "value": "slow" - }, - { - "selected": false, - "text": "panic", - "value": "panic" - }, - { - "selected": false, - "text": "fatal", - "value": "fatal" - } - ], - "query": "All : .,debug,info,warn,error,slow,panic,fatal", - "queryValue": "", - "skipUrlSync": false, - "type": "custom" - }, - { - "current": { - "selected": false, - "text": ".", - "value": "." - }, - "description": "任意关键字；默认 . 表示不过滤", - "hide": 0, - "label": "关键字", - "name": "keyword", - "options": [], - "query": ".", - "skipUrlSync": false, - "type": "textbox" - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": {}, - "timezone": "browser", - "title": "PPanel Server Logs", - "uid": "ppanel-server-logs", - "version": 5, - "weekStart": "" -} diff --git a/grafana/provisioning/datasources/datasources.yml b/grafana/provisioning/datasources/datasources.yml deleted file mode 100644 index 9f2be22..0000000 --- a/grafana/provisioning/datasources/datasources.yml +++ /dev/null @@ -1,57 +0,0 @@ -apiVersion: 1 - -datasources: - - name: Prometheus - uid: prometheus - type: prometheus - access: proxy - url: http://prometheus:9090 - isDefault: true - editable: true - jsonData: - httpMethod: POST - manageAlerts: true - prometheusType: Prometheus - prometheusVersion: 2.50.0 - timeInterval: 15s - - - name: Loki - uid: loki - type: loki - access: proxy - url: http://loki:3100 - editable: true - jsonData: - derivedFields: - - datasourceUid: tempo - matcherRegex: '"(?:trace|traceID|trace_id)"\s*:\s*"([a-f0-9]{32})"' - name: TraceID - url: '$${__value.raw}' - - - name: Tempo - uid: tempo - type: tempo - access: proxy - url: http://tempo:3200 - editable: true - jsonData: - tracesToLogsV2: - datasourceUid: loki - filterByTraceID: true - filterBySpanID: false - tags: - - key: service.name - value: service_name - tracesToMetrics: - datasourceUid: prometheus - serviceMap: - datasourceUid: prometheus - - - name: CloudWatch - uid: cloudwatch - type: cloudwatch - access: proxy - editable: true - jsonData: - authType: default - defaultRegion: ap-east-1 diff --git a/loki/loki-config.yaml b/loki/loki-config.yaml deleted file mode 100644 index 8c71fd0..0000000 --- a/loki/loki-config.yaml +++ /dev/null @@ -1,36 +0,0 @@ -auth_enabled: false - -server: - http_listen_port: 3100 - grpc_listen_port: 9096 - -common: - path_prefix: /loki - replication_factor: 1 - ring: - instance_addr: 127.0.0.1 - kvstore: - store: inmemory - -schema_config: - configs: - - from: 2024-01-01 - store: tsdb - object_store: filesystem - schema: v13 - index: - prefix: index_ - period: 24h - -storage_config: - filesystem: - directory: /loki/chunks - -limits_config: - allow_structured_metadata: true - retention_period: 168h - -compactor: - working_directory: /loki/compactor - retention_enabled: true - delete_request_store: filesystem diff --git a/loki/promtail-config.yaml b/loki/promtail-config.yaml deleted file mode 100644 index 5e799ce..0000000 --- a/loki/promtail-config.yaml +++ /dev/null @@ -1,61 +0,0 @@ -server: - http_listen_port: 9080 - grpc_listen_port: 0 - -positions: - filename: /tmp/positions.yaml - -clients: - - url: http://loki:3100/loki/api/v1/push - -scrape_configs: - - job_name: ppanel-file - static_configs: - - targets: - - localhost - labels: - job: ppanel-server - service_name: ppanel - __path__: /var/log/ppanel-server/*.log - pipeline_stages: - - json: - expressions: - caller: caller - content: content - level: level - timestamp: timestamp - - timestamp: - source: timestamp - format: "2006-01-02 15:04:05.000" - location: Asia/Shanghai - - labels: - caller: - level: - - - job_name: nginx-file - static_configs: - - targets: - - localhost - labels: - job: nginx - service_name: nginx - __path__: /var/log/nginx/*.log - - - job_name: docker-containers - docker_sd_configs: - - host: unix:///var/run/docker.sock - refresh_interval: 15s - relabel_configs: - - source_labels: [__meta_docker_container_name] - regex: '/(.*)' - target_label: container - - source_labels: [__meta_docker_container_label_com_docker_compose_service] - target_label: compose_service - - source_labels: [__meta_docker_container_log_stream] - target_label: stream - - source_labels: [__meta_docker_container_name] - regex: '/(.*)' - target_label: service_name - - source_labels: [__meta_docker_container_id] - target_label: __path__ - replacement: /var/lib/docker/containers/$1/$1-json.log diff --git a/ops/aws-rds-external-replica-runbook.md b/ops/aws-rds-external-replica-runbook.md deleted file mode 100644 index d10b530..0000000 --- a/ops/aws-rds-external-replica-runbook.md +++ /dev/null @@ -1,652 +0,0 @@ -# AWS RDS + EC2 Redis 到外部备用服务器 Runbook - -目标：让 `104.238.220.230` 持续作为 AWS 主生产环境的异地备用节点，承接： - -- MySQL 外部只读从库 -- Redis 外部从库 -- 故障时的快速提升与业务切换 - -本文档以 `2026-05-13` 的真实现网状态为准，覆盖： - -- 当前已经落地的主从架构 -- 日常验收命令 -- 主从故障排查 -- 全量重建从库 -- “新建规范 RDS 再切换”的生产级收敛路线 -- 故障切换与回滚 - -## 1. 当前已确认资源 - -### 1.1 AWS 主生产 - -- Region: `ap-east-1` -- AWS app EC2: - - Name: `hifast-hk-app-01` - - Public IP: `18.163.33.75` - - Private IP: `10.0.1.201` -- AWS MySQL: - - Type: `RDS MySQL` - - Instance: `hifast-mysql-prod-v2` - - Endpoint: `hifast-mysql-prod-v2.cd6aey40m6ag.ap-east-1.rds.amazonaws.com` - - Version: `8.4.8` - - DB Name: `hifast` - - Admin user: `admin` - - Admin password: keep it in a secret store, do not write plaintext into repo docs - - Replication user: `repl` - - Replication password: keep it in a secret store, do not write plaintext into repo docs -- AWS Redis: - - Location: `hifast-hk-app-01` - - Deployment: `Docker` - - Container: `hifast-redis` - - Version: `redis:8.2.1` - - Listen: `0.0.0.0:6379` - - Password: keep it in a secret store, do not write plaintext into repo docs - -### 1.2 外部备用服务器 - -- Host: `104.238.220.230` -- OS: `Ubuntu 24.04 LTS` -- SSH user: `root` -- MySQL version: `8.4.9` -- Redis version: `8.6.3` -- 当前角色： - - MySQL external replica - - Redis replica - - 备用应用节点 - -### 1.3 当前应用真实运行方式 - -AWS 应用机上的 `ppanel-server` 当前不是 systemd 托管，而是 Docker Compose 服务： - -- Compose file: `/opt/ppanel/docker-compose.cloud.yml` -- Config file: `/opt/ppanel/configs/ppanel.yaml` -- Container name: `ppanel-server` -- Current MySQL target: `hifast-mysql-prod-v2.cd6aey40m6ag.ap-east-1.rds.amazonaws.com:3306` -- Current Redis target: `127.0.0.1:6379` - -也就是说，后续所有应用侧切换步骤，都应该以修改： - -- `/opt/ppanel/configs/ppanel.yaml` - -并执行： - -```bash -cd /opt/ppanel -docker compose -f docker-compose.cloud.yml up -d ppanel-server -``` - -作为准。 - -## 2. 当前健康基线 - -截至 `2026-05-13`，已确认以下状态成立： - -- `104` MySQL: - - `Source_Host = hifast-mysql-prod-v2.cd6aey40m6ag.ap-east-1.rds.amazonaws.com` - - `Replica_IO_Running: Yes` - - `Replica_SQL_Running: Yes` - - `Seconds_Behind_Source: 0` - - `read_only = ON` - - `super_read_only = ON` -- AWS Redis 主库: - - `role:master` - - `connected_slaves:1` -- `104` Redis: - - `role:slave` - - `master_link_status:up` -- AWS 应用: - - `curl http://127.0.0.1:8080/v1/common/heartbeat` 返回 `code=200` - -这说明当前状态已经达到： - -- 主生产可用 -- 外部备用持续同步 -- Redis 已回到真实本机链路 - -## 3. 当前网络前提 - -### 3.1 安全组 - -当前实际生效的安全组如下： - -- `hifast-hk-app-core-sg` - - `22/tcp <- 0.0.0.0/0` - - `6379/tcp <- 104.238.220.230/32` - - `8080/tcp <- hifast-hk-web-sg` -- `hifast-hk-rds-core-sg` - - `3306/tcp <- 104.238.220.230/32` - - `3306/tcp <- hifast-hk-app-core-sg` -- `hifast-hk-web-sg` - - `22/80/443 <- 0.0.0.0/0` - -### 3.2 当前结构性限制 - -当前 `104 -> RDS` 公网复制链路虽然可用，但依赖的是： - -- `hifast-mysql-prod-v2` 为 `Publicly accessible = Yes` -- RDS ENI 仍位于 `hifast-hk-private-1b` -- `private-1b` 被临时挂到了公网路由表 - -这不是最终规范的生产形态。 - -当前这条路线已经完成，后续更推荐的动作是： - -1. 持续确认应用主库目标保持在 `hifast-mysql-prod-v2` -2. 持续确认 `104` 复制目标保持在 `hifast-mysql-prod-v2` -3. 根据回收窗口安排旧 RDS 下线 -4. 按网络收敛计划处理 `private-1b` 路由语义 - -## 4. 日常验收命令 - -### 4.1 验收 MySQL 主从 - -在 `104` 执行： - -```bash -mysql -e "SHOW REPLICA STATUS\G" -``` - -重点看： - -- `Source_Host` -- `Replica_IO_Running` -- `Replica_SQL_Running` -- `Seconds_Behind_Source` -- `Last_IO_Error` -- `Last_SQL_Error` - -成功标准： - -- `Replica_IO_Running: Yes` -- `Replica_SQL_Running: Yes` -- `Seconds_Behind_Source: 0` 或较小 - -### 4.2 验收 Redis 主从 - -在 AWS app EC2 执行： - -```bash -docker exec hifast-redis redis-cli -a '0BVz9XOHf7KUfEuoFJRK-dURdKUGFiZ8QeaHpysHnKeKhLskZb55HPK121lFsKtr' INFO replication -``` - -重点看： - -- `role:master` -- `connected_slaves:1` - -在 `104` 执行： - -```bash -redis-cli INFO replication -``` - -重点看： - -- `role:slave` -- `master_host:18.163.33.75` -- `master_port:6379` -- `master_link_status:up` - -### 4.3 验收应用 - -在 AWS app EC2 执行： - -```bash -curl -sf http://127.0.0.1:8080/v1/common/heartbeat -``` - -期望返回： - -- `{"code":200,...}` - -查看容器： - -```bash -cd /opt/ppanel -docker compose -f docker-compose.cloud.yml ps -``` - -期望： - -- `ppanel-server` 为 `Up` -- `hifast-redis` 为 `Up` - -## 5. MySQL 复制故障排查 - -### 5.1 先看复制状态 - -```bash -mysql -e "SHOW REPLICA STATUS\G" -``` - -重点判断： - -- `Replica_IO_Running = No` -- `Replica_SQL_Running = No` -- `Last_IO_Error` -- `Last_SQL_Error` - -### 5.2 常见场景 - -#### 场景 A：网络或白名单断开 - -表现： - -- `Replica_IO_Running: No` -- `Last_IO_Error` 出现连接失败、超时、拒绝访问 - -排查： - -```bash -mysql -h hifast-mysql-prod-v2.cd6aey40m6ag.ap-east-1.rds.amazonaws.com -u repl -p -e "SELECT 1;" -``` - -处理： - -- 检查 RDS SG 是否仍保留 `104.238.220.230/32 -> 3306` -- 检查 RDS 是否仍为 `Publicly accessible = Yes` -- 如果后续已迁到新规范 RDS，则检查新实例的 SG 和公网可达性 - -#### 场景 B：主库 binlog 位点丢失 - -表现： - -- `Last_IO_Error` 或 `Last_SQL_Error` 指向缺失 binlog - -处理： - -- 不要硬跳过 -- 直接执行全量重建从库 - -#### 场景 C：SQL 执行报错 - -表现： - -- `Replica_SQL_Running: No` -- `Last_SQL_Error` 有实际 SQL 冲突信息 - -处理建议： - -- 如果只是临时演练环境，可重建从库 -- 如果已经进入生产切换阶段，不建议盲目 `sql_slave_skip_counter` -- 优先保守做法仍是重新全量初始化 - -## 6. Redis 复制故障排查 - -### 6.1 看 `104` 从库状态 - -```bash -redis-cli INFO replication -``` - -重点： - -- `role` -- `master_host` -- `master_link_status` - -### 6.2 看 AWS 主库状态 - -```bash -docker exec hifast-redis redis-cli -a '0BVz9XOHf7KUfEuoFJRK-dURdKUGFiZ8QeaHpysHnKeKhLskZb55HPK121lFsKtr' INFO replication -``` - -重点： - -- `role:master` -- `connected_slaves` - -### 6.3 常见问题 - -#### 场景 A：安全组断开 - -表现： - -- `master_link_status:down` - -处理： - -- 确认 `hifast-hk-app-core-sg` 仍保留： - - `6379/tcp <- 104.238.220.230/32` - -#### 场景 B：主库密码漂移 - -表现： - -- 从库重连失败 -- 日志出现 `NOAUTH` - -处理： - -- 统一更新 `/etc/redis/redis.conf` 中的： - - `masterauth` -- 然后： - -```bash -systemctl restart redis-server -redis-cli INFO replication -``` - -## 7. 全量重建 MySQL 从库 - -适用场景： - -- 主从中断且无法安全追平 -- `6666@qq.com` 这类写入在主库存在、从库未同步 -- binlog 不连续 -- 需要回到最稳妥状态 - -### 7.1 在主库导出 - -在一台可连 RDS 的机器上执行： - -```bash -mysqldump \ - -h hifast-mysql-prod-v2.cd6aey40m6ag.ap-east-1.rds.amazonaws.com \ - -u admin \ - -p \ - --single-transaction \ - --routines \ - --triggers \ - --events \ - --set-gtid-purged=OFF \ - hifast > hifast-full.sql -``` - -如果需要同步账号权限，也可以额外单独导出授权对象；但当前业务库恢复重点是 `hifast` 数据库本身。 - -### 7.2 清理 `104` 当前复制 - -在 `104` 执行： - -```sql -STOP REPLICA; -RESET REPLICA ALL; -``` - -### 7.3 重新导入业务库 - -在 `104` 执行： - -```bash -mysql -e "DROP DATABASE IF EXISTS hifast; CREATE DATABASE hifast CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci;" -mysql hifast < hifast-full.sql -``` - -### 7.4 重新挂复制 - -当前现网是非 GTID 自动定位，使用 file/position 模式。 - -先在主库取位点： - -```sql -SHOW MASTER STATUS; -``` - -然后在 `104` 执行： - -```sql -CHANGE REPLICATION SOURCE TO - SOURCE_HOST='hifast-mysql-prod-v2.cd6aey40m6ag.ap-east-1.rds.amazonaws.com', - SOURCE_PORT=3306, - SOURCE_USER='repl', - SOURCE_PASSWORD='', - SOURCE_LOG_FILE='', - SOURCE_LOG_POS=, - SOURCE_SSL=1; -START REPLICA; -SHOW REPLICA STATUS\G -``` - -### 7.5 验收 - -确认： - -- `Replica_IO_Running: Yes` -- `Replica_SQL_Running: Yes` -- `Seconds_Behind_Source: 0` -- `read_only = ON` -- `super_read_only = ON` - -## 8. 重新配置 Redis 从库 - -当前 `104` Redis 为原生安装，不使用 Docker。 - -### 8.1 临时切回从库 - -```bash -redis-cli CONFIG SET masterauth '0BVz9XOHf7KUfEuoFJRK-dURdKUGFiZ8QeaHpysHnKeKhLskZb55HPK121lFsKtr' -redis-cli REPLICAOF 18.163.33.75 6379 -redis-cli CONFIG SET replica-read-only yes -redis-cli INFO replication -``` - -### 8.2 持久化配置 - -检查 `/etc/redis/redis.conf` 至少包含： - -```conf -replicaof 18.163.33.75 6379 -masterauth 0BVz9XOHf7KUfEuoFJRK-dURdKUGFiZ8QeaHpysHnKeKhLskZb55HPK121lFsKtr -replica-read-only yes -``` - -然后： - -```bash -systemctl restart redis-server -redis-cli INFO replication -``` - -## 9. 生产级收敛路线：新建规范 RDS 再切换 - -这条路线已经完成，当前现网主库已经切到 `hifast-mysql-prod-v2`。下面内容保留为迁移归档参考，不再表示待执行。 - -### 9.1 目标 - -把当前临时方案： - -- `private-1b` 挂公网路由 - -收敛成： - -- RDS 使用纯公网 DB subnet group -- 安全组仍只对白名单和应用组开放 - -### 9.2 已准备好的资源 - -- 已建 DB subnet group: - - `hifast-hk-rds-public-only-sgprep` -- 子网： - - `hifast-hk-public-1a` - - `hifast-hk-public-1b` - -### 9.3 新 RDS 建议参数 - -新实例建议名： - -- `hifast-mysql-prod-v2` - -建议保持与旧主库一致： - -- Engine: `mysql` -- Version: `8.4.8` -- Class: `db.r7g.xlarge` -- Storage: `gp3` -- Size: `200 GB` -- IOPS: `3000` -- Throughput: `125` -- Publicly accessible: `Yes` -- Multi-AZ: `No` 或按预算单独评估 -- Deletion protection: `On` -- Performance Insights: `On` -- Backup retention: `7` -- DB subnet group: `hifast-hk-rds-public-only-sgprep` -- VPC SG: `hifast-hk-rds-core-sg` - -### 9.4 迁移步骤 - -1. 已创建新 RDS `hifast-mysql-prod-v2` -2. 在旧主库导出 `hifast` -3. 导入新库 -4. 在新库创建 `repl` 用户并配置 binlog retention -5. 修改 AWS app EC2 上 `/opt/ppanel/configs/ppanel.yaml` 的 `MySQL.Addr` -6. 重启 `ppanel-server` 容器 -7. 在 `104` 上 `STOP REPLICA; RESET REPLICA ALL;` -8. 指向新 RDS endpoint 重新挂复制 -9. 验收应用与主从 -10. 验收通过后，安排旧 RDS 下线，并推进 `private-1b` 恢复私网路由 - -### 9.5 应用切换命令 - -在 AWS app EC2： - -1. 备份配置 - -```bash -cp /opt/ppanel/configs/ppanel.yaml /opt/ppanel/configs/ppanel.yaml.bak.$(date +%Y%m%d%H%M%S) -``` - -2. 编辑： - -- `/opt/ppanel/configs/ppanel.yaml` - -把： - -```yaml -MySQL: - Addr: hifast-mysql-prod-v2.cd6aey40m6ag.ap-east-1.rds.amazonaws.com:3306 -``` - -改成： - -```yaml -MySQL: - Addr: hifast-mysql-prod-v2.cd6aey40m6ag.ap-east-1.rds.amazonaws.com:3306 -``` - -3. 重启应用容器： - -```bash -cd /opt/ppanel -docker compose -f docker-compose.cloud.yml up -d ppanel-server -``` - -4. 验证： - -```bash -curl -sf http://127.0.0.1:8080/v1/common/heartbeat -docker compose -f docker-compose.cloud.yml logs --tail=100 ppanel-server -``` - -### 9.6 `104` 改挂新 RDS - -```sql -STOP REPLICA; -RESET REPLICA ALL; -CHANGE REPLICATION SOURCE TO - SOURCE_HOST='', - SOURCE_PORT=3306, - SOURCE_USER='repl', - SOURCE_PASSWORD='', - SOURCE_LOG_FILE='', - SOURCE_LOG_POS=, - SOURCE_SSL=1; -START REPLICA; -SHOW REPLICA STATUS\G -``` - -### 9.7 切换验收 - -至少确认： - -- AWS 应用心跳正常 -- 新 RDS 可正常读写 -- `104` 复制恢复为 `Yes/Yes` -- `Seconds_Behind_Source` 追到 `0` -- Navicat 可从允许的白名单来源连接新 RDS - -### 9.8 回滚 - -如果新 RDS 切换后应用异常： - -1. 立即把 `/opt/ppanel/configs/ppanel.yaml` 中 `MySQL.Addr` 改回旧 endpoint -2. 重启 `ppanel-server` -3. 暂不处理 `104`，先恢复主生产 -4. 复盘新库数据、权限、参数、网络 - -## 10. AWS 故障时的备用接管 - -### 10.1 Redis 提升为主库 - -在 `104`： - -```bash -redis-cli REPLICAOF NO ONE -redis-cli INFO replication -``` - -期望： - -- `role:master` - -### 10.2 MySQL 提升为可写主库 - -在 `104`： - -```sql -STOP REPLICA; -RESET REPLICA ALL; -SET GLOBAL super_read_only=OFF; -SET GLOBAL read_only=OFF; -``` - -再确认： - -```sql -SHOW VARIABLES LIKE 'read_only'; -SHOW VARIABLES LIKE 'super_read_only'; -``` - -期望： - -- `OFF` -- `OFF` - -### 10.3 应用切到 `104` 本机数据层 - -如果 `104` 上也部署同样的 PPanel 服务，则应用配置应改为： - -- MySQL 指向 `127.0.0.1:3306` 或本机 socket -- Redis 指向 `127.0.0.1:6379` - -### 10.4 最后切入口 - -数据层和应用层都确认可写后，再做： - -- DNS 切换 -- 或 Nginx / 上游切流量 - -原则： - -- 先数据接管 -- 再应用确认 -- 最后入口切换 - -## 11. 日常巡检建议 - -建议至少每天巡检一次： - -1. `104` MySQL `SHOW REPLICA STATUS\G` -2. `104` Redis `INFO replication` -3. AWS Redis 主库 `INFO replication` -4. AWS app 心跳 `/v1/common/heartbeat` -5. `docker compose -f /opt/ppanel/docker-compose.cloud.yml ps` - -如果后续要做真正的生产级自动化，再补： - -- MySQL 复制延迟告警 -- Redis 主从断链告警 -- 应用心跳失败告警 -- RDS 连接失败告警 -- 定期灾备切换演练 diff --git a/ops/docker-image-version-pins.md b/ops/docker-image-version-pins.md deleted file mode 100644 index 8568394..0000000 --- a/ops/docker-image-version-pins.md +++ /dev/null @@ -1,33 +0,0 @@ -# Docker Image Version Pins - -This file records the infrastructure image versions pinned in `docker-compose.cloud.yml`. -The versions below match the images observed on the test deployment on 2026-05-26. - -| Service | Image | Running version source | -| --- | --- | --- | -| grafana | `grafana/grafana:13.0.1` | `grafana version 13.0.1` | -| prometheus | `prom/prometheus:v3.11.3` | `prometheus, version 3.11.3` | -| nginx-exporter | `nginx/nginx-prometheus-exporter:1.5.0` | image label `org.opencontainers.image.version=1.5.0` | -| node-exporter | `prom/node-exporter:v1.11.1` | `node_exporter, version 1.11.1` | -| cadvisor | `gcr.io/cadvisor/cadvisor:v0.55.1` | `cAdvisor version v0.55.1` | - -The test deployment in `/root/bindbox/docker-compose.cloud.yml` also contains -live-only exporter services that are not present in this repository's -`docker-compose.cloud.yml`. They were pinned during staging validation: - -| Test-only service | Image | Running version source | -| --- | --- | --- | -| mysql-exporter | `prom/mysqld-exporter:v0.19.0` | `mysqld_exporter, version 0.19.0` | -| redis-exporter | `oliver006/redis_exporter:v1.82.0` | image label `org.opencontainers.image.version=v1.82.0` | - -`ppanel-server` intentionally remains variable and requires `PPANEL_SERVER_TAG` -from CI/CD so deployments use an immutable application image tag. - -## Rollback - -Restore the previous compose file from git and redeploy: - -```sh -git checkout HEAD~1 -- docker-compose.cloud.yml .env.example ops/docker-image-version-pins.md -docker compose -f docker-compose.cloud.yml up -d -``` diff --git a/ops/github-staging-deploy.md b/ops/github-staging-deploy.md deleted file mode 100644 index a81e8a6..0000000 --- a/ops/github-staging-deploy.md +++ /dev/null @@ -1,26 +0,0 @@ -# GitHub Staging Deploy - -This repository deploys the `internal` branch to the staging server `154.12.35.103` with GitHub Actions. - -## Required GitHub secrets - -- `STAGING_SSH_USER`: SSH user for `154.12.35.103`. -- `STAGING_SSH_KEY`: Private SSH key that can log in to the staging server. -- `STAGING_SSH_PORT`: SSH port. Use `22` if the server uses the default port. -- `GHCR_USERNAME`: Optional. Required only if the GHCR package is private and the server needs to log in before pulling. -- `GHCR_TOKEN`: Optional. A GitHub PAT with `read:packages` for private GHCR pulls. - -## Server requirements - -- Docker and either `docker compose` or `docker-compose` are installed. -- The deployment directory is `/opt/hifast-server`. -- Runtime files already exist beside `docker-compose.cloud.yml`, especially `configs/ppanel.yaml`, `logs/`, `cache/`, `loki/`, `grafana/`, `prometheus/`, and `tempo/`. - -## What the workflow does - -1. Builds the Docker image for `linux/amd64`. -2. Pushes `ghcr.io/tawcorp/hifast-server:` and `ghcr.io/tawcorp/hifast-server:staging`. -3. Copies `docker-compose.cloud.yml` to the staging server. -4. Pulls the immutable commit image and restarts `ppanel-server`. -5. Checks `http://127.0.0.1:8080/v1/common/heartbeat`. -6. Attempts rollback to the previous container image if the health check fails. diff --git a/ops/hifast-aws-jp-cutover-checklist-zh.md b/ops/hifast-aws-jp-cutover-checklist-zh.md deleted file mode 100644 index 0b941b1..0000000 --- a/ops/hifast-aws-jp-cutover-checklist-zh.md +++ /dev/null @@ -1,116 +0,0 @@ -# Hifast 东京切换执行清单 - -本文档用于正式执行香港 `ap-east-1` -> 东京 `ap-northeast-1` 迁移时，逐项勾选和留痕。 - -## 0. 当前已知东京状态 - -- 东京 VPC `ppanel-jp-prod` 已创建 -- 东京 VPC ID：`vpc-0846b23b4a7d64eac` -- 4 个东京子网曾在 AWS 控制台录入，但提交时登录态失效 -- 所以当前要按“子网未确认成功”处理 -- 后续所有东京资源创建前，先执行一次： - - `bash deploy/scripts/aws_jp_describe_state.sh /root/aws-jp-infra.env` -- 真实状态登记文件： - - `deploy/aws/ap-northeast-1/configs/resource-inventory.current.md` - -## 1. 真实值清单 - -在开始创建东京正式资源前，必须补齐这些真实值： - -- 正式 API 域名： -- 东京平行 API 域名： -- 东京平行日志域名： -- 运维固定公网 IP / CIDR： -- 东京 EC2 SSH Key Pair 名称： -- 东京 RDS 管理员密码：`TkyRds20260521!N9mQ8sKe2vLp7Xa` -- 东京 RDS 凭证管理方式：`self-managed` -- 东京 RDS 当前密码是否可回看：`否，只能重置` -- 东京 Redis 密码：`hifast67yj` -- `JwtAuth.AccessSecret`： -- `Administrator.Email`：`admin@ppanel.dev` -- `Administrator.Password`：`PpanelAdmin!20260521Temp` -- `AppSignature.AppSecrets.android-client`：`uB4G,XxL2{7b` -- `AppSignature.AppSecrets.ios-client`：`uB4G,XxL2{7b` -- `AppSignature.AppSecrets.web-client`：`uB4G,XxL2{7b` -- `device.security_secret`：`uB4G,XxL2{7b` -- 东京备份桶最终名称： -- `104.238.220.230` MySQL 复制密码：`XwWrQGVWtxmXJ3etHmkFvnRSD54MKYer` -- `104.238.220.230` Redis 主从认证密码：`hifast67yj` - -## 2. 东京资源创建勾选 - -- 已切换 AWS 控制台到 `ap-northeast-1` -- 已确认东京区可用 -- 已确认 EC2 配额满足 `t4g.large` -- 已确认 RDS 配额满足 `db.r7g.xlarge` -- 已确认 ALB / ACM / WAF / S3 可正常创建 -- 已创建 VPC -- 已创建 2 个公有子网 -- 已创建 2 个私有子网 -- 已创建 IGW -- 已配置公私网路由表 -- 已创建 `sg-alb` -- 已创建 `sg-ec2` -- 已创建 `sg-rds` -- 已创建东京 RDS -- 已创建东京 EC2 -- 已在 EC2 启动 Docker / Nginx -- 已创建东京 ACM 证书 -- 已创建 Target Group -- 已创建 ALB -- 已创建 WAF 并挂到 ALB -- 已创建东京 S3 备份桶并开启 versioning - -## 3. 东京应用部署勾选 - -- 已上传 `docker-compose.cloud.yml` -- 已上传 `configs/ppanel.yaml` -- 已上传 `.env` -- 已上传 `grafana/` -- 已上传 `loki/` -- 已上传 `prometheus/` -- 已上传 `tempo/` -- 已安装东京 Nginx 配置 -- 已启动 `ppanel-server` -- 已启动 `hifast-redis` -- 已启动 observability 容器 -- `curl http://127.0.0.1:8080/v1/common/heartbeat` 正常 -- `curl http://127.0.0.1/v1/common/heartbeat` 正常 -- ALB 健康检查正常 - -## 4. 停机迁移勾选 - -- 已降低正式域名 TTL -- 已停止香港 `ppanel-server` -- 已确认香港不再有新写入 -- 已导出香港 MySQL `hifast-full.sql.gz` -- 已导出香港 Redis `dump.rdb` -- 已导入东京 RDS -- 已导入东京 Redis -- 东京应用已改为连接东京 MySQL / Redis -- 平行域名验收通过 - -## 5. 104 灾备重挂勾选 - -- 东京 RDS 已设置 `binlog retention hours` -- 东京 RDS 已创建 `repl@104.238.220.230` -- 已用东京 dump 重建 `104` 的 `hifast` -- `104` MySQL 已成功挂东京主库 -- `104` MySQL `Replica_IO_Running: Yes` -- `104` MySQL `Replica_SQL_Running: Yes` -- `104` MySQL `Seconds_Behind_Source: 0` -- `104` Redis 已成功挂东京主库 -- 东京 Redis `connected_slaves:1` -- `104` Redis `role:slave` -- `104` Redis `master_link_status:up` - -## 6. 正式切换与回滚勾选 - -- 已切正式域名到东京 ALB -- 外网请求正常 -- 核心业务接口无 `5xx` -- 应用日志无 MySQL / Redis 连接错误 -- WAF 无误伤 -- 已验证 DNS 可回切 -- 香港环境已保留为只读回滚基线 -- 香港环境计划保留 `7` 天观察期 diff --git a/ops/hifast-aws-jp-migration-runbook-zh.md b/ops/hifast-aws-jp-migration-runbook-zh.md deleted file mode 100644 index fa96cef..0000000 --- a/ops/hifast-aws-jp-migration-runbook-zh.md +++ /dev/null @@ -1,286 +0,0 @@ -# Hifast AWS 香港到日本东京迁移 Runbook - -本文档用于把当前香港区 `ap-east-1` 主生产，迁移到日本东京 `ap-northeast-1`。 - -适用目标： - -- 东京成为新主站 -- 架构升级为 `ALB + WAF + EC2 + RDS` -- `104.238.220.230` 继续作为东京主站的 MySQL / Redis 外部灾备 - -## 0. 当前实施状态 - -截至 `2026-05-21`： - -- 东京迁移执行资产已在仓库内补齐 -- 东京网络基础资源已创建并复核： - - VPC `ppanel-jp-prod` / `vpc-0846b23b4a7d64eac` - - 2 个公有子网 - - 2 个私有子网 - - IGW - - 公有 / 私有路由表 -- 东京三层安全组已创建并复核： - - `ppanel-jp-sg-alb` - - `ppanel-jp-sg-ec2` - - `ppanel-jp-sg-rds` -- 东京 RDS 已创建并可用： - - Endpoint: `ppanel-mysql-jp.cpo0keikgh80.ap-northeast-1.rds.amazonaws.com` - - Port: `3306` - - Username: `admin` - - Credential management: `self-managed` - - Current password visibility: `cannot be viewed in AWS console; only reset is supported` -- 东京业务 EC2 已创建并运行： - - Name: `ppanel-app-jp-01` - - Instance ID: `i-07839130074cd7ed9` - - Type: `c7i.xlarge` - - Private IP: `10.20.1.168` - - Elastic IP: `3.114.29.208` -- 东京 S3 备份桶已创建 -- 东京 ACM 证书已请求，仍等待 DNS 验证 -- 当前尚未完成的核心资源： - - ALB - - WAF - - ACM DNS 验证 - - 东京应用目录部署与数据导入 - -建议先执行： - -```bash -cp deploy/aws/ap-northeast-1/configs/aws-jp-infra.env.example /root/aws-jp-infra.env -vim /root/aws-jp-infra.env -bash deploy/scripts/aws_jp_create_base_infra.sh /root/aws-jp-infra.env -bash deploy/scripts/aws_jp_describe_state.sh /root/aws-jp-infra.env -``` - -然后把结果回填到： - -- `deploy/aws/ap-northeast-1/configs/resource-inventory.current.md` -- `ops/hifast-aws-jp-cutover-checklist-zh.md` - -## 1. 当前基线 - -迁移前默认当前现网状态为： - -- 香港应用 EC2：`hifast-hk-app-01` -- 香港 RDS：`hifast-mysql-prod-v2.cd6aey40m6ag.ap-east-1.rds.amazonaws.com` -- 香港 Redis 主库：`18.163.33.75:6379` -- 外部灾备：`104.238.220.230` -- 当前应用部署目录：`/opt/ppanel` -- 当前应用配置文件：`/opt/ppanel/configs/ppanel.yaml` -- 当前业务容器：`ppanel-server` - -## 2. 迁移前准备 - -正式迁移前必须完成： - -1. 东京基础设施已创建 -2. 东京 EC2 已部署应用目录，但暂未导入正式数据 -3. 东京 RDS 可连通 - - 已验证从东京 EC2 到 `ppanel-mysql-jp.cpo0keikgh80.ap-northeast-1.rds.amazonaws.com:3306` 网络畅通 - - 当前若无密码只能得到 `ERROR 1045 ... using password: NO`，这表示链路正常，不表示实例异常 -4. 东京 Redis 已启动并可认证 -5. 平行域名已准备： - - `api-jp.hifast.biz` - - `logs-jp.hifast.biz` -6. 东京 ALB 健康检查已通过 -7. 东京 ACM 证书已签发 -8. 东京 WAF 已挂到 ALB -9. 已准备正式回滚入口 -10. 已把正式域名 TTL 降低 - -## 3. 香港停机冻结 - -在香港主环境执行： - -1. 停止业务写入 -2. 停 `ppanel-server` -3. 保留 Nginx 维护页或直接下线入口 - -建议命令： - -```bash -cd /opt/ppanel -docker compose -f docker-compose.cloud.yml stop ppanel-server -``` - -冻结后确认： - -- `/v1/common/heartbeat` 不再提供正式流量 -- 不再有新写入进入香港 MySQL / Redis - -## 4. 导出香港数据 - -### 4.1 MySQL - -从香港主库导出： - -```bash -mysqldump \ - -h hifast-mysql-prod-v2.cd6aey40m6ag.ap-east-1.rds.amazonaws.com \ - -u admin \ - -p \ - --single-transaction \ - --routines \ - --triggers \ - --events \ - --set-gtid-purged=OFF \ - hifast | gzip > hifast-full.sql.gz -``` - -### 4.2 Redis - -在香港 Redis 主库导出： - -```bash -docker exec hifast-redis redis-cli -a '' BGSAVE -docker cp hifast-redis:/data/dump.rdb ./dump.rdb -``` - -## 5. 导入东京 - -### 5.1 MySQL 导入东京 RDS - -```bash -gunzip -c hifast-full.sql.gz | mysql -h -u admin -p hifast -``` - -导入后确认： - -```bash -mysql -h -u admin -p -e "USE hifast; SHOW TABLES;" -``` - -### 5.2 Redis 导入东京 - -1. 停止东京 Redis 容器 -2. 替换 `/data/dump.rdb` -3. 启动东京 Redis 容器 - -导入后确认： - -```bash -docker exec hifast-redis redis-cli -a '' PING -``` - -## 6. 启动东京应用 - -更新东京： - -- `/opt/ppanel/configs/ppanel.yaml` -- `/opt/ppanel/.env` - -关键值： - -- `MySQL.Addr=:3306` -- `MySQL.Username=admin` -- `MySQL.Password=` -- `Redis.Host=127.0.0.1:6379` -- `Redis.Pass=` -- `Site.Host=api-jp.hifast.biz` -- `.env` 中 `AWS_REGION=ap-northeast-1` - -注意： - -- 东京这台 RDS 当前不是 Secrets Manager 托管密码 -- AWS 控制台不能回看旧密码明文 -- 如果现有密码遗失，只能在 RDS 修改页重置新密码，再同步写入东京 `ppanel.yaml` - -当前东京已实际生效： - -- `MySQL.Addr=ppanel-mysql-jp.cpo0keikgh80.ap-northeast-1.rds.amazonaws.com:3306` -- `MySQL.Username=admin` -- `MySQL.Password=TkyRds20260521!N9mQ8sKe2vLp7Xa` -- `Redis.Host=127.0.0.1:6379` -- `Redis.Pass=hifast67yj` - -启动： - -```bash -cd /opt/ppanel -docker compose -f docker-compose.cloud.yml up -d -``` - -## 7. 东京平行环境验收 - -至少执行： - -```bash -curl -sf http://127.0.0.1:8080/v1/common/heartbeat -curl -sf http://127.0.0.1/v1/common/heartbeat -docker compose -f /opt/ppanel/docker-compose.cloud.yml ps -docker compose -f /opt/ppanel/docker-compose.cloud.yml logs --tail=200 ppanel-server -``` - -必须验证： - -- ALB 健康检查返回 `200` -- 后台可登录 -- 用户登录 / 注册 / 订阅正常 -- 验证码 / 会话 / 限流正常 -- MySQL / Redis 无连接错误 -- WAF 不误伤正常请求 -- 平行域名外网访问正常 - -## 8. 正式域名切换 - -仅在东京平行环境验收全部通过后执行： - -1. 将正式域名切到东京 `ALB` -2. 观察 5xx、延迟、容器日志、RDS、Redis -3. 保持香港不删，只做回滚保留 - -## 9. 104 灾备重挂 - -### 9.1 MySQL - -在东京 RDS 上： - -```sql -CALL mysql.rds_set_configuration('binlog retention hours', 24); -CREATE USER IF NOT EXISTS 'repl'@'104.238.220.230' IDENTIFIED BY ''; -GRANT REPLICATION SLAVE, REPLICATION CLIENT ON *.* TO 'repl'@'104.238.220.230'; -FLUSH PRIVILEGES; -SHOW BINARY LOG STATUS; -``` - -当前东京已准备完成： - -- `repl@104.238.220.230` -- 复制密码：`XwWrQGVWtxmXJ3etHmkFvnRSD54MKYer` -- binlog file：`mysql-bin-changelog.000189` -- binlog pos：`185053` - -然后在 `104` 重建并挂从。 - -### 9.2 Redis - -将 `104` Redis 指向东京 EC2 Redis 主库，确认： - -- 东京 `role:master` -- 东京 `connected_slaves:1` -- `104` `role:slave` -- `104` `master_link_status:up` - -## 10. 回滚 - -如果东京验收失败： - -- 不切正式域名 -- 继续保留香港主环境 - -如果正式切换后发现严重问题： - -1. 立刻把 DNS 切回香港 -2. 恢复香港 `ppanel-server` -3. 放弃本次东京接管 - -## 11. 切换后观察期 - -切换后至少保留香港环境 `7` 天： - -- 香港 RDS 快照 -- 香港 Redis RDB -- 香港 EC2 配置 -- 香港 `ppanel.yaml` 备份 - -观察期内不删除香港资源。 diff --git a/ops/hifast-aws-standby-architecture-zh.md b/ops/hifast-aws-standby-architecture-zh.md deleted file mode 100644 index 2cea7ad..0000000 --- a/ops/hifast-aws-standby-architecture-zh.md +++ /dev/null @@ -1,696 +0,0 @@ -# Hifast AWS 主生产 + 外部备用 完整部署方案与访问架构 - -本文档整理当前已经实际落地的生产架构、访问链路、数据库与缓存主从关系、网络边界、故障切换方案，以及后续扩展建议。 - -目标是让团队在一个文档里就能看清： - -- 现在生产到底部署成了什么样 -- 请求是怎么进来的，数据是怎么流转的 -- AWS 与外部备用服务器分别承担什么角色 -- MySQL / Redis 的同步关系是什么 -- 故障时应该如何切换 - -## 0. 2026-05-13 验收摘要 - -- 已确认 `104.238.220.230 -> AWS RDS MySQL` 连通，MySQL 外部从库健康： - - 当前复制上游：`hifast-mysql-prod-v2.cd6aey40m6ag.ap-east-1.rds.amazonaws.com` - - `Replica_IO_Running: Yes` - - `Replica_SQL_Running: Yes` - - `Seconds_Behind_Source: 0` -- 已确认 `104.238.220.230 -> AWS EC2 Redis` 主从健康： - - `104` 上 `role:slave` - - `master_link_status:up` - - AWS Redis 主库 `connected_slaves:1` -- 已修复应用侧 Redis 配置漂移： - - 旧链路：`127.0.0.1:6380 -> stunnel4 -> 旧 ElastiCache` - - 新链路：`127.0.0.1:6379 -> 本机 Docker Redis` -- 旧 `stunnel4` 已停用并禁用自启，`ppanel-server` 日志里的 Redis `i/o timeout` 已停止出现。 -- 已确认 `ppanel-server` 当前运行方式为 Docker Compose 容器，而不是 systemd 服务： - - Compose 文件：`/opt/ppanel/docker-compose.cloud.yml` - - 配置文件：`/opt/ppanel/configs/ppanel.yaml` - - 容器名：`ppanel-server` - - 健康检查：`curl http://127.0.0.1:8080/v1/common/heartbeat` 返回正常 -- 旧自建安全组 `hifast-hk-app-sg`、`hifast-hk-nginx-sg` 已删除；当前 VPC 内仅保留： - - `hifast-hk-app-core-sg` - - `hifast-hk-rds-core-sg` - - `hifast-hk-web-sg` - - `default` -- 目前已经达到“可用且关键链路已恢复”的状态。 -- 当前入口层属于已确认的过渡方案： - - 现阶段公网入口仍在 `hifast-hk-app-01` - - 独立 `nginx` 服务器已预留，待 AWS 后续资源到位后再迁移承接 -- 当前仍需要继续收敛的核心生产项只剩 1 个： - - RDS 为了让 `104` 走公网白名单复制，当前仍依赖 `hifast-hk-private-1b` 子网临时挂到公网路由表，这不是最终规范形态。 -- 已完成的下一步准备： - - 已创建纯公网子网专用的 `DB subnet group`：`hifast-hk-rds-public-only-sgprep` - - 子网包含： - - `subnet-070e3f264a9c79f32` `hifast-hk-public-1a` - - `subnet-00cb5add705c447c8` `hifast-hk-public-1b` -- 已创建专用 S3 备份桶： - - `hifast-prod-backups-200810848252-ap-east-1` - - 当前状态：private - - 当前状态：versioning enabled - -## 1. 当前实际环境 - -### 1.1 AWS 区域 - -- Region: `ap-east-1` -- 说明：香港区 - -### 1.2 已确认资源 - -#### 应用服务器 - -- 名称：`hifast-hk-app-01` -- Instance ID: `i-079cd9d3ef3748714` -- 角色：当前实际生产入口 / Nginx / 业务服务 / AWS 侧 Redis 主库宿主机 -- 私网 IP: `10.0.1.201` -- 公网 IP: `18.163.33.75` -- 业务服务运行方式：`Docker Compose` -- 业务容器：`ppanel-server` -- 部署目录：`/opt/ppanel` -- 实际配置文件：`/opt/ppanel/configs/ppanel.yaml` - -#### 独立 Nginx 服务器 - -- 名称：`hifast-hk-nginx-01` -- Instance ID: `i-0701d54bf2c6bf594` -- 角色：计划中的独立入口机 -- 私网 IP: `10.0.1.175` -- 当前状态：`stopped` -- 当前说明：已经只绑定 `hifast-hk-web-sg`，但目前并未承接正式流量 - -#### MySQL 主库 - -- 类型：`AWS RDS MySQL` -- 实例名：`hifast-mysql-prod-v2` -- Endpoint: `hifast-mysql-prod-v2.cd6aey40m6ag.ap-east-1.rds.amazonaws.com` -- 角色：生产主库 -- Engine: `MySQL 8.4.8` -- Publicly accessible: `Yes` -- Multi-AZ: `No` -- Backup retention: `7 days` -- Deletion protection: `On` -- 当前数据库实际体量：约 `187.41 MB` -- 当前表数量：`37` -- 当前 `user` 表记录数：`45076` - -#### Redis 主库 - -- 部署位置：`AWS EC2 hifast-hk-app-01` -- 部署方式：`Docker` -- 容器名：`hifast-redis` -- 版本：`redis:8.2.1` -- 访问端口：`6379` -- 主库出口地址：`18.163.33.75:6379` -- 应用当前实际连接：`127.0.0.1:6379` -- 认证方式：已启用密码认证 - -#### 外部备用服务器 - -- IP: `104.238.220.230` -- OS: `Ubuntu 24.04 LTS` -- 角色：异地备用节点 -- 当前状态：已部署与 AWS 相同的业务服务 -- 当前 MySQL 状态：外部只读从库 -- 当前 Redis 部署方式：`宿主机原生安装` -- 当前 Redis 版本：`8.6.3` -- 当前 Redis 角色：`AWS Redis 主库的从库` - -#### S3 备份桶 - -- Bucket: `hifast-prod-backups-200810848252-ap-east-1` -- Region: `ap-east-1` -- Public access: blocked -- Versioning: enabled - -## 2. 架构总览 - -```mermaid -flowchart TB - USER["用户 / 客户端"] --> DNS["域名 / DNS / 入口层"] - DNS --> APP["AWS EC2\nhifast-hk-app-01\n18.163.33.75\n10.0.1.201"] - - APP --> RDS["AWS RDS MySQL\nhifast-mysql-prod-v2\n主库"] - APP --> REDISM["AWS Redis 主库\nDocker redis:8.2.1\n18.163.33.75:6379"] - - RDS -. MySQL 备用 / 同步 .-> MYSQLS["104.238.220.230\nMySQL 备用库"] - REDISM -. Redis 主从复制 .-> REDISS["104.238.220.230\n原生 Redis 8.6.3\n从库"] - - subgraph AWS["AWS ap-east-1"] - APP - RDS - REDISM - end - - subgraph BACKUP["异地备用节点"] - MYSQLS - REDISS - end -``` - -## 3. 访问链路 - -### 3.1 用户访问链路 - -当前生产访问链路可以概括为： - -`用户 -> 域名 / DNS -> AWS EC2 应用机 -> MySQL / Redis` - -说明： - -- 当前主应用入口实际在 `hifast-hk-app-01`。 -- `hifast-hk-app-01` 同时承担 Nginx、业务服务、Redis 主库。 -- 当前业务服务不是 systemd 单进程部署，而是由 `/opt/ppanel/docker-compose.cloud.yml` 管理的 `ppanel-server` 容器提供 `8080`。 -- 独立入口机 `hifast-hk-nginx-01` 已存在，但当前仅作为后续资源开通后的迁移目标。 -- MySQL 在 AWS RDS。 -- Redis 不在 ElastiCache，而是在 EC2 本机通过 Docker 提供。 - -### 3.2 应用访问数据链路 - -应用侧内部依赖关系如下： - -```text -App / Nginx - -> RDS MySQL 主库 - -> AWS EC2 Redis 主库 -``` - -### 3.3 备用链路 - -备用服务器 `104.238.220.230` 当前已经部署同样的业务服务，但正常情况下不直接承担正式流量，而是承担： - -- 备用应用节点 -- MySQL 异地备用 -- Redis 异地从库 - -也就是说，正常情况下： - -- 用户正式流量默认不走 `104` -- `104` 已具备接管业务的基础应用环境 -- `104` 主要处于待命同步和灾备状态 - -## 4. 网络与安全边界 - -### 4.1 当前安全组 - -截至 `2026-05-13`，VPC `vpc-09fc384522517debc` 内仅保留 4 个安全组： - -- `default` -- `hifast-hk-app-core-sg` -- `hifast-hk-rds-core-sg` -- `hifast-hk-web-sg` - -其中绑定关系已经收敛为： - -- `hifast-hk-app-01` -> `hifast-hk-app-core-sg` -- `hifast-hk-nginx-01` -> `hifast-hk-web-sg` -- `hifast-mysql-prod-v2` -> `hifast-hk-rds-core-sg` - -旧组： - -- `hifast-hk-app-sg` -- `hifast-hk-nginx-sg` - -已经删除，不再使用。 - -### 4.2 当前入站规则 - -#### `hifast-hk-app-core-sg` - -- `22/tcp <- 0.0.0.0/0` -- `6379/tcp <- 104.238.220.230/32` -- `8080/tcp <- hifast-hk-web-sg` - -#### `hifast-hk-rds-core-sg` - -- `3306/tcp <- 104.238.220.230/32` -- `3306/tcp <- hifast-hk-app-core-sg` - -#### `hifast-hk-web-sg` - -- `22/tcp <- 0.0.0.0/0` -- `80/tcp <- 0.0.0.0/0` -- `443/tcp <- 0.0.0.0/0` - -### 4.3 Redis 网络关系 - -```mermaid -flowchart LR - SG["hifast-hk-app-core-sg"] --> REDIS["AWS Redis 主库\n18.163.33.75:6379"] - STANDBY["104.238.220.230/32"] --> SG -``` - -### 4.4 RDS 访问原则 - -RDS 当前状态已经比之前干净很多： - -- 当前只对白名单和应用安全组开放 `3306` -- 已移除 `3306 <- 0.0.0.0/0` -- 当前 RDS 仅绑定 `hifast-hk-rds-core-sg` - -建议原则： - -- 不开放 `0.0.0.0/0` 到 MySQL `3306` -- 不开放 `0.0.0.0/0` 到 Redis `6379` - -### 4.5 当前仍需继续收敛的网络点 - -#### 4.5.1 入口层当前属于过渡方案，不作为本阶段阻塞项 - -当前运行形态是： - -- `hifast-hk-app-01` 本机 `nginx` 正在监听 `80/443` -- `hifast-hk-nginx-01` 当前停止 - -但当前安全组设计是按“独立 nginx 机”拆的： - -- `hifast-hk-web-sg` 在 `hifast-hk-nginx-01` -- `hifast-hk-app-core-sg` 在 `hifast-hk-app-01` - -这意味着当前真实入口和安全组角色划分还没有完全一致。 - -但这一点已经被确认为过渡设计，不作为当前阶段必须整改的问题。后续待 AWS 新资源到位后，再把公网入口迁到独立 `nginx` 服务器即可。 - -#### 4.5.2 RDS 公网复制链路仍是当前唯一核心结构性问题 - -为了让 `104.238.220.230` 通过公网白名单访问 RDS，当前实际依赖的是： - -- RDS `PubliclyAccessible = true` -- RDS ENI 仍在 `subnet-0dcf46db665b6d8a7` -- 该子网名称是 `hifast-hk-private-1b` -- 但这个子网当前被临时关联到了公网路由表 - -这能用，但不属于最终规范的“纯公网子网组”设计。 - -本轮已经额外完成的准备动作： - -- 已新建纯公网 `DB subnet group`：`hifast-hk-rds-public-only-sgprep` -- 该组状态：`Complete` -- 该组仅包含两条公网子网： - - `hifast-hk-public-1a` - - `hifast-hk-public-1b` - -当前结论： - -- 不建议继续在现有生产 RDS 上反复试在线切子网组。 -- 更稳的收敛方案是：新建一台使用规范公网子网组的生产 RDS，再做一次短切换。 -- 由于当前库体量只有约 `187 MB`，这条路线的执行成本并不高，风险也比在现网主库上硬改更可控。 - -## 4.6 104 如何连接 AWS - -这里要特别区分： - -- `104` 连接 AWS 数据层 -- 本地电脑登录 AWS EC2 - -这不是同一件事。 - -### 4.6.1 104 连接 AWS Redis 的方式 - -`104.238.220.230` 连接 AWS Redis，不是通过 PEM 证书，也不是通过 SSH 登录 AWS 机器，而是直接作为 Redis 从库去访问 AWS Redis 主库： - -- 目标地址：`18.163.33.75:6379` -- 连接方式：`TCP` -- 认证方式：`Redis 密码` -- 网络前提：AWS EC2 安全组已放行 `104.238.220.230/32 -> 6379` - -也就是说： - -```text -104 Redis 从库 -> 直连 AWS Redis 主库公网地址 -> 密码认证 -> 建立复制 -``` - -示意命令： - -```bash -redis-cli -h 18.163.33.75 -p 6379 -a '' -``` - -### 4.6.2 104 连接 AWS MySQL RDS 的方式 - -`104.238.220.230` 连接 AWS RDS，也不是通过 PEM 证书，而是通过数据库连接： - -- 目标地址：`hifast-mysql-prod-v2.cd6aey40m6ag.ap-east-1.rds.amazonaws.com:3306` -- 连接方式：`MySQL TCP` -- 认证方式：`MySQL 用户名 + 密码` -- 网络前提：RDS 白名单或安全组放行 `104.238.220.230` - -也就是说： - -```text -104 MySQL 从库 -> 直连 AWS RDS endpoint -> 账号密码认证 -> 建立复制 -``` - -示意命令： - -```bash -mysql -h hifast-mysql-prod-v2.cd6aey40m6ag.ap-east-1.rds.amazonaws.com -u admin -p -``` - -### 4.6.3 本地电脑登录 AWS EC2 的方式 - -本地电脑这边之前进入 AWS EC2，用的不是 `hifast-hk-app-01-key.pem`，而是： - -- `AWS Console` -- `EC2 Instance Connect` -- 浏览器里的 Web SSH 终端 - -所以当前要理解成 3 条不同链路： - -1. 本地电脑 -> AWS 控制台 -> EC2 Instance Connect -> AWS EC2 -2. `104` -> AWS Redis 主库公网地址 -> Redis 密码认证 -3. `104` -> AWS RDS endpoint -> MySQL 用户密码认证 - -### 4.6.4 关键结论 - -当前灾备链路里，`104` 与 AWS 数据同步不依赖 `.pem` 证书文件。 - -真正依赖的是： - -- Redis 安全组放行 -- RDS 白名单 / 安全组放行 -- 正确的 Redis 密码 -- 正确的 MySQL 账号密码 - -## 5. Redis 实际部署与同步状态 - -### 5.1 AWS Redis 主库 - -- 部署方式：Docker -- 版本：`8.2.1` -- 主库地址：`18.163.33.75:6379` -- 运行容器：`hifast-redis` - -### 5.2 104 Redis 从库 - -- 部署方式：宿主机原生安装 -- 版本：`8.6.3` -- 角色：`replica / slave` - -### 5.3 Redis 主从状态 - -最终已验证结果： - -- `104` 上 Redis：`role:slave` -- `104` 上 Redis：`master_link_status:up` -- AWS Redis 主库：`connected_slaves:1` -- AWS Redis 主库识别到从库：`104.238.220.230:6379` - -### 5.4 Redis 验证结果 - -已做过的验证： - -- 从 `104` 连接 AWS Redis 主库，认证成功 -- AWS 主库写入测试键 -- `104` 从库成功读取测试键 -- `2026-05-13` 已确认应用机本地 `ppanel-server -> 127.0.0.1:6379` 建立稳定连接 -- `2026-05-13` 已确认旧 `stunnel4 -> ElastiCache` 链路下线后，应用日志不再出现 Redis `i/o timeout` - -测试键： - -- key: `hifast_replication_test` -- value: `ok_20260510` - -### 5.5 Redis 主从拓扑 - -```mermaid -flowchart LR - REDISMASTER["AWS Redis 主库\n18.163.33.75:6379\nDocker redis:8.2.1"] - REDISSLAVE["104.238.220.230\n原生 Redis 8.6.3\nrole: slave"] - REDISMASTER --> REDISSLAVE -``` - -## 6. MySQL 部署与备用关系 - -### 6.1 主库角色 - -- 主库在 `AWS RDS MySQL` -- 实例：`hifast-mysql-prod-v2` - -### 6.2 外部备用角色 - -- `104.238.220.230` 上存在 MySQL 备用用途 -- 目标是让 `104` 尽量实时同步 AWS 数据 - -### 6.3 当前文档说明 - -MySQL 当前已经完成实际验收，不再只是“待确认”状态。 - -`2026-05-13` 验收结果： - -- `Source_Host = hifast-mysql-prod-v2.cd6aey40m6ag.ap-east-1.rds.amazonaws.com` -- `Replica_IO_Running = Yes` -- `Replica_SQL_Running = Yes` -- `Seconds_Behind_Source = 0` -- `read_only = ON` -- `super_read_only = ON` - -这表示 `104` 当前是健康的只读外部备用库。 - -MySQL 这部分在此前已经有专门 runbook： - -- [ops/aws-rds-external-replica-runbook.md](/Users/Apple/code_vpn/vpn/ppanel-server/ops/aws-rds-external-replica-runbook.md) - -## 7. 当前生产方案的真实特点 - -这套已经落地的架构，不是传统的全 AWS 托管标准形态，而是偏实用的混合方案： - -- 应用和当前实际公网入口在 AWS 同一台 EC2 -- MySQL 在 AWS RDS -- Redis 在 AWS EC2 本机 -- MySQL / Redis 均向外部服务器 `104` 做灾备 -- 旧 ElastiCache / stunnel 链路已经退出当前生产路径 - -它的优点： - -- 成本相对可控 -- Redis 可完全自主控制 -- 外部备用机可以独立接管 - -它的代价： - -- Redis 高可用需要人工切换 -- 外部灾备不是全自动故障转移 -- 应用切换需要明确操作步骤 - -## 8. 故障切换方案 - -### 8.1 正常状态 - -```mermaid -flowchart TD - A["用户访问"] --> B["AWS EC2 应用机"] - B --> C["AWS RDS MySQL 主库"] - B --> D["AWS Redis 主库"] - C -. 同步 .-> E["104 MySQL 备用"] - D -. 复制 .-> F["104 Redis 从库"] -``` - -### 8.2 AWS 故障后的目标切换状态 - -```mermaid -flowchart TD - A["AWS 故障"] --> B["应用入口切到 104"] - B --> C["104 MySQL 提供主服务"] - B --> D["104 Redis 提升为主库"] - D --> E["应用连接 104 Redis"] - C --> F["应用连接 104 MySQL"] -``` - -### 8.2.1 Nginx 是否可以直接切到 104 - -可以，但前提不是“只切 Nginx 就完成故障切换”。 - -因为 `104` 虽然已经部署了同样的业务服务，但如果故障发生时： - -- Redis 还保持从库只读状态 -- MySQL 还没有切成可写主角色 -- 应用配置还没有确认指向 `104` 本机数据层 - -那么即使 Nginx 已经把流量转到 `104`，业务也可能仍然无法正常写入。 - -所以更准确的原则是： - -`104` 已具备应用接管能力，Nginx 切换可以作为最后一步对外放流量动作，但不能作为唯一动作。 - -### 8.3 Redis 切换动作 - -当 AWS Redis 不可用时，`104` 上的 Redis 需要解除主从关系： - -```bash -redis-cli -a '' REPLICAOF NO ONE -``` - -切换后： - -- `104` Redis 从库变为主库 -- 业务应用把 Redis 地址改到 `104.238.220.230:6379` - -### 8.4 MySQL 切换动作 - -当 AWS RDS 不可用时，需要让 `104` MySQL 接管写流量。 - -这部分是否能立即切，需要依赖： - -- 当前 `104` MySQL 是否是健康从库 -- 是否已经取消只读 -- 应用数据库配置是否能快速切换到 `104` - -### 8.5 应用切换动作 - -应用层需要准备至少这 2 个切换点： - -- MySQL 连接地址切换到 `104` -- Redis 连接地址切换到 `104` - -如果应用入口也要迁移到 `104`，还需要： - -- 域名解析切换 -- 或者网关 / 入口切换 - -### 8.6 推荐的实际切换顺序 - -因为 `104` 已经部署同样的应用服务，所以 AWS 故障时推荐按下面顺序操作： - -1. 确认 `104` 上业务服务和 Nginx 进程正常。 -2. 将 `104` 上 Redis 从库提升为主库。 -3. 将 `104` 上 MySQL 从库切换为可写主库。 -4. 确认 `104` 上应用配置已指向本机 MySQL / Redis。 -5. 最后再把 Nginx 上游或域名流量切到 `104`。 - -可以把它理解成： - -`先数据接管 -> 再应用确认 -> 最后入口切流量` - -## 9. 建议的运维操作顺序 - -### 9.1 平时 - -平时重点看： - -- AWS EC2 是否在线 -- RDS 是否在线 -- AWS Redis 主库是否在线 -- `104` Redis 从库是否 `master_link_status:up` -- `104` MySQL 复制是否正常 -- `ppanel-server` 容器是否 `Up` -- `ppanel-server` 日志中是否再次出现 Redis `i/o timeout` -- `stunnel4` 是否保持 `inactive` - -### 9.2 Redis 故障时 - -1. 确认 AWS Redis 主库不可恢复。 -2. 在 `104` 执行 `REPLICAOF NO ONE`。 -3. 修改应用 Redis 地址到 `104.238.220.230:6379`。 -4. 验证应用读写 Redis 正常。 - -### 9.3 MySQL 故障时 - -1. 确认 RDS 故障。 -2. 确认 `104` MySQL 数据已同步到最新可用点。 -3. 去掉 `104` MySQL 只读限制。 -4. 修改应用 MySQL 地址到 `104`。 -5. 验证应用读写数据库正常。 - -### 9.4 整体 AWS 故障时 - -1. 把 Redis 主角色切到 `104`。 -2. 把 MySQL 主角色切到 `104`。 -3. 确认 `104` 上同版本应用服务正常。 -4. 把应用入口切到备用应用节点。 -5. 更新 DNS 或 Nginx 上游流量入口。 -6. 验证用户访问链路。 - -## 10. 当前方案与理想方案的差异 - -### 10.1 当前实际方案 - -`DNS -> hifast-hk-app-01(Nginx + App + Redis) -> RDS MySQL -> 104 灾备` - -### 10.2 理想生产方案 - -从长期稳定性看，更推荐未来演进为： - -`DNS / CDN -> ALB -> 多台 EC2 App -> RDS MySQL -> 托管 Redis / 或高可用 Redis` - -异地灾备继续保留： - -- AWS 生产 -- 104 异地接管 - -### 10.3 当前最值得继续补的项 - -建议按优先级补齐： - -1. 把 RDS 从“private 子网挂公网路由”的临时方案，收敛成真正的公网 DB subnet group，或改成私网复制方案 -2. 明确 `104` 应用接管脚本与启动检查项 -3. 明确 MySQL 故障切换脚本 -4. 明确 Redis 故障切换脚本 -5. 明确域名 / DNS 切换方式 -6. 做一次完整灾备演练 -7. 待 AWS 新资源到位后，再把公网入口迁到独立 `nginx` 服务器 - -### 10.4 RDS 推荐收敛路线 - -当前原先规划的“新建规范 RDS 再切换”路线已经完成，现网主库已经是 `hifast-mysql-prod-v2`。 - -后续更推荐的正式路线是： - -1. 继续以 `hifast-mysql-prod-v2` 作为当前生产主库 -2. 确认其持续使用 `hifast-hk-rds-public-only-sgprep` -3. 安全组持续只放： - - `104.238.220.230/32 -> 3306` - - `hifast-hk-app-core-sg -> 3306` -4. 持续确认应用连接目标为 `hifast-mysql-prod-v2` -5. 持续确认 `104` 的外部复制目标为 `hifast-mysql-prod-v2` -6. 验证通过后，再安排旧 RDS 下线 - -这条路线的优点： - -- 最终拓扑规范清晰 -- 不需要继续保留“private 子网挂公网路由”的临时设计 -- 对当前现网主库的扰动最小 -- 当前数据库体量较小，迁移成本可控 - -## 11. 建议的下一版目标拓扑 - -```mermaid -flowchart TB - USER["用户 / 客户端"] --> DNS["DNS / CDN / 入口层"] - DNS --> APPAWS["AWS 应用集群"] - DNS -. 故障时切换 .-> APPBK["104 备用应用节点"] - - APPAWS --> RDSAWS["AWS RDS MySQL 主库"] - APPAWS --> REDISAWS["AWS Redis 主库"] - - RDSAWS -. 同步 .-> MYSQLBK["104 MySQL 备用"] - REDISAWS -. 复制 .-> REDISBK["104 Redis 备用"] - - APPBK --> MYSQLBK - APPBK --> REDISBK -``` - -## 12. 本文档结论 - -截至当前，已经可以确认的生产与灾备状态是： - -- AWS 是主生产环境 -- 应用当前跑在 `hifast-hk-app-01` -- MySQL 主库在 AWS RDS -- Redis 主库在 AWS EC2 Docker -- `104.238.220.230` 是异地备用节点 -- `104` 已部署与 AWS 相同的业务服务 -- `104` 上 Redis 已切为宿主机原生安装 -- `104` Redis 已成功作为 AWS Redis 主库的从库在线同步 -- `104` MySQL 已恢复并保持健康复制 -- 应用已经切回当前真实可用的本机 Redis 主库 -- 旧 `stunnel4 -> ElastiCache` 链路已经退出生产路径 - -如果后续要继续完善这份方案，优先补充： - -- RDS 子网与公网访问模型收敛 -- 入口域名 / DNS 切换细则 -- 应用层在 `104` 的接管与回切执行清单 -- 待资源到位后的独立 `nginx` 迁移清单 diff --git a/ops/hifast-current-architecture-zh.md b/ops/hifast-current-architecture-zh.md deleted file mode 100644 index 3a4d15c..0000000 --- a/ops/hifast-current-architecture-zh.md +++ /dev/null @@ -1,246 +0,0 @@ -# Hifast 当前架构文档（东京主站版） - -最后更新：`2026-05-21` - -这份文档只描述**当前真实生效**的架构，不描述理想目标，不混入已经下线或待迁移的香港旧链路。 - -如果后续东京 `ALB / WAF / 正式域名` 上线，应继续更新这份文档，而不是回头参考旧香港架构文档。 - -## 1. 当前结论 - -当前已经实际跑起来的是： - -`用户 / 运维 -> 东京 EC2 -> ppanel-server + Redis + observability -> 东京 RDS MySQL` - -当前还**没有**实际生效的组件： - -- `ALB` -- `WAF` -- 正式域名切流 -- `104.238.220.230` 重挂东京 MySQL / Redis 从库 - -也就是说，东京环境目前是： - -- 一台业务 EC2 已可用 -- 一台东京 RDS 已可用 -- 本机 Redis 已可用 -- `ppanel-server` 已成功启动并连通 MySQL / Redis -- 灾备链路参数已准备好，但 `104` 还没最终挂上去 - -## 2. 当前真实拓扑 - -```mermaid -flowchart TB - USER["用户 / 运维"] --> EIP["东京 EC2 公网入口\n3.114.29.208"] - EIP --> NGINX["Nginx\nEC2 本机"] - NGINX --> APP["ppanel-server\nhost network\n127.0.0.1:8080"] - APP --> REDIS["Redis Docker\n127.0.0.1:6379"] - APP --> RDS["Tokyo RDS MySQL\nppanel-mysql-jp"] - APP --> OBS["Grafana / Loki / Tempo / Prometheus"] - - RDS -. 预留复制 .-> DRMYSQL["104.238.220.230\nMySQL DR\n未最终接入"] - REDIS -. 预留复制 .-> DRREDIS["104.238.220.230\nRedis DR\n未最终接入"] -``` - -## 3. 云资源清单 - -### 3.1 Region - -- AWS account: `hifastvpn (200810848252)` -- Region: `ap-northeast-1` - -### 3.2 网络 - -- VPC: `ppanel-jp-prod` -- VPC ID: `vpc-0846b23b4a7d64eac` -- CIDR: `10.20.0.0/16` - -公有子网： - -- `ppanel-jp-public-a` / `subnet-091232bdb53e71490` / `10.20.0.0/24` -- `ppanel-jp-public-c` / `subnet-01ba0975c525ce8cf` / `10.20.1.0/24` - -私有子网： - -- `ppanel-jp-private-a` / `subnet-0bd13111c02f0edbe` / `10.20.10.0/24` -- `ppanel-jp-private-c` / `subnet-0d86c5c756dbc84b2` / `10.20.11.0/24` - -### 3.3 安全组 - -- `ppanel-jp-sg-alb` - - `80/tcp <- 0.0.0.0/0` - - `443/tcp <- 0.0.0.0/0` -- `ppanel-jp-sg-ec2` - - `80/tcp <- sg-0b3a23c31041a5a5a` - - `22/tcp <- 64.118.144.142/32` - - `6379/tcp <- 104.238.220.230/32` -- `ppanel-jp-sg-rds` - - `3306/tcp <- sg-01f2a5a81e7505c91` - - `3306/tcp <- 104.238.220.230/32` - -## 4. 计算与数据库 - -### 4.1 业务 EC2 - -- Name: `ppanel-app-jp-01` -- Instance ID: `i-07839130074cd7ed9` -- Type: `c7i.xlarge` -- AZ: `ap-northeast-1c` -- Private IP: `10.20.1.168` -- Public IP / Elastic IP: `3.114.29.208` -- Public DNS: `ec2-3-114-29-208.ap-northeast-1.compute.amazonaws.com` -- Root volume: `gp3 100GiB` -- Status: `running` - -说明： - -- 当前公网入口是这台 EC2 自己 -- 这里绑定的是 `EIP`，不是依赖实例自动分配的临时公网 IP -- 绑定 EIP 的目的，是保证以后 Redis 主从、白名单、DNS、文档里的公网地址不因为实例变更而漂移 - -### 4.2 Tokyo RDS - -- Identifier: `ppanel-mysql-jp` -- Engine: `MySQL Community 8.4.8` -- Class: `db.r7g.xlarge` -- Storage: `gp3 100GiB` -- Master user: `admin` -- Endpoint: `ppanel-mysql-jp.cpo0keikgh80.ap-northeast-1.rds.amazonaws.com` -- Public access: `enabled` -- Credential management: `self-managed` -- Current status: `available` - -当前实际连接值： - -- Host: `ppanel-mysql-jp.cpo0keikgh80.ap-northeast-1.rds.amazonaws.com` -- Port: `3306` -- User: `admin` - -### 4.3 Redis - -- 部署位置：东京 EC2 本机 -- 部署方式：Docker -- 容器名：`ppanel-redis` -- 当前角色：`master` -- 当前监听：`127.0.0.1:6379`（应用本机访问） -- 当前对灾备开放：`3.114.29.208:6379` - -### 4.4 应用与可观测 - -业务容器： - -- `ppanel-server` - -可观测容器： - -- `ppanel-grafana` -- `ppanel-loki` -- `ppanel-promtail` -- `ppanel-prometheus` -- `ppanel-tempo` -- `ppanel-cadvisor` -- `ppanel-node-exporter` -- `ppanel-nginx-exporter` - -## 5. 当前部署方式 - -部署目录： - -- `/opt/ppanel` - -关键文件： - -- `/opt/ppanel/docker-compose.cloud.yml` -- `/opt/ppanel/configs/ppanel.yaml` -- `/opt/ppanel/.env` - -业务启动方式： - -- `ppanel-server` 使用 `docker compose` -- `network_mode: host` - -当前配置指向： - -- MySQL -> 东京 RDS -- Redis -> 本机 Docker Redis -- Trace -> `127.0.0.1:4317` - -## 6. 当前已验证状态 - -截至 `2026-05-21` 已确认： - -- `docker exec ppanel-redis redis-cli -a 'hifast67yj' PING` 返回 `PONG` -- `MYSQL_PWD=... mysql -h ppanel-mysql-jp.cpo0keikgh80.ap-northeast-1.rds.amazonaws.com -u admin -e "select 1"` 返回正常 -- `docker compose -f /opt/ppanel/docker-compose.cloud.yml up -d ppanel-server` 已成功 -- `docker logs ppanel-server` 无 MySQL / Redis 连接报错 -- `curl http://127.0.0.1:8080/v1/common/heartbeat` 返回成功 - -可以认为当前东京单站已经具备： - -- 服务启动能力 -- 数据库连接能力 -- Redis 连接能力 -- 基础可观测能力 - -## 7. 当前未落地项 - -下面这些还属于“目标方案”，不是“当前实际架构”： - -- `ALB` -- `WAF` -- ACM 证书完成 DNS 验证 -- 正式域名切换到东京 -- `104.238.220.230` 完整接成东京 DR - -所以现在请不要把当前架构理解成： - -`DNS -> ALB -> WAF -> EC2 -> RDS` - -当前真实架构更准确地说是： - -`EIP -> EC2(Nginx + ppanel-server + Redis + observability) -> RDS` - -## 8. 灾备准备状态 - -### 8.1 MySQL DR - -已准备好： - -- DR host: `104.238.220.230` -- 复制账号：`repl@104.238.220.230` -- binlog retention hours: `24` -- current binlog file: `mysql-bin-changelog.000189` -- current binlog position: `185053` - -当前状态： - -- 东京侧已准备完成 -- 但 `104` 侧还没有最终执行挂从 - -### 8.2 Redis DR - -已准备好： - -- 东京 Redis 主地址：`3.114.29.208:6379` -- 东京 Redis 当前角色：`role:master` -- 东京 Redis 当前状态：`connected_slaves:0` - -当前状态： - -- 东京侧已可作为主库提供同步 -- 但 `104` 侧还没有最终执行 `REPLICAOF` - -## 9. 权威文档入口 - -与当前东京架构直接相关的文档，以这些为准： - -- [当前架构文档](./hifast-current-architecture-zh.md) -- [东京资源清单](../deploy/aws/ap-northeast-1/configs/resource-inventory.current.md) -- [东京迁移 Runbook](./hifast-aws-jp-migration-runbook-zh.md) -- [东京切换清单](./hifast-aws-jp-cutover-checklist-zh.md) - -下面这些旧文档可以继续保留，但不要作为“当前实际架构”依据： - -- 香港旧主站相关文档 -- 仍以 `ap-east-1` 为主语的历史架构说明 - diff --git a/ops/hifast-rds-v2-cutover-checklist-zh.md b/ops/hifast-rds-v2-cutover-checklist-zh.md deleted file mode 100644 index 8eaf5f0..0000000 --- a/ops/hifast-rds-v2-cutover-checklist-zh.md +++ /dev/null @@ -1,286 +0,0 @@ -# Hifast 新 RDS 生产切换清单（已完成归档） - -本文档记录已完成的 `hifast-mysql-prod -> hifast-mysql-prod-v2` 生产切换过程，作为归档和回溯参考。 - -本次切换已经完成，当时的目标是避免继续在线硬改现有生产 RDS，而是： - -1. 新建一台规范公网子网组的新 RDS -2. 导入现有 `hifast` 数据 -3. 将 AWS 应用切到新 RDS -4. 将 `104.238.220.230` 的外部从库改挂新 RDS -5. 验收通过后再下线旧 RDS - -## 1. 适用背景 - -当前现网已经可用，但还存在 1 个结构性问题： - -- 为了让 `104` 通过公网白名单做 MySQL 复制，当时的 `hifast-mysql-prod` 仍依赖 `hifast-hk-private-1b` 临时挂公网路由 - -当前已准备好的收敛资源： - -- 纯公网 DB subnet group: - - `hifast-hk-rds-public-only-sgprep` -- 安全组： - - `hifast-hk-rds-core-sg` - -## 2. 目标实例参数 - -建议新实例名称： - -- `hifast-mysql-prod-v2` - -建议参数如下： - -- Region: `ap-east-1` -- Engine: `MySQL` -- Engine version: `8.4.8` -- Templates: `Production` -- DB instance class: `db.r7g.xlarge` -- Storage type: `gp3` -- Allocated storage: `200 GiB` -- Provisioned IOPS: `3000` -- Storage throughput: `125` -- Multi-AZ: `No` -- Publicly accessible: `Yes` -- VPC: `vpc-09fc384522517debc` -- DB subnet group: `hifast-hk-rds-public-only-sgprep` -- VPC security group: `hifast-hk-rds-core-sg` -- DB name: `hifast` -- Master username: `admin` -- Master password: 与现网保持一致 -- Backup retention: `7 days` -- Performance Insights: `On` -- Storage encryption: `On` -- Deletion protection: `On` -- Auto minor version upgrade: 建议保持与现网一致 -- Maintenance window: 可与现网同策略，建议维护窗口内切换 - -## 3. 切换前确认 - -切换前必须满足： - -1. `104` 当前主从正常： - - `Replica_IO_Running: Yes` - - `Replica_SQL_Running: Yes` - - `Seconds_Behind_Source: 0` -2. AWS 应用机心跳正常： - - `curl http://127.0.0.1:8080/v1/common/heartbeat` -3. AWS Redis 主从正常： - - AWS 主 `connected_slaves:1` - - `104` 从 `master_link_status:up` -4. 已确认当前应用真实配置文件： - - `/opt/ppanel/configs/ppanel.yaml` -5. 已确认当前应用重启方式： - - `cd /opt/ppanel && docker compose -f docker-compose.cloud.yml up -d ppanel-server` - -## 4. 创建新 RDS - -已在 AWS 控制台创建 `hifast-mysql-prod-v2`，未对原 `hifast-mysql-prod` 做高风险在线子网调整。 - -创建完成后先确认： - -1. 新 endpoint 已分配 -2. `Publicly accessible = Yes` -3. SG 为 `hifast-hk-rds-core-sg` -4. DB subnet group 为 `hifast-hk-rds-public-only-sgprep` -5. 从 `104` 可以 TCP 连通 `3306` - -连通性验证： - -```bash -nc -zv 3306 -``` - -## 5. 导出旧主库 - -在一台可连旧 RDS 的机器执行： - -```bash -mysqldump \ - -h hifast-mysql-prod.cd6aey40m6ag.ap-east-1.rds.amazonaws.com \ - -u admin \ - -p \ - --single-transaction \ - --routines \ - --triggers \ - --events \ - --set-gtid-purged=OFF \ - hifast > hifast-full.sql -``` - -说明： - -- 当前业务库体量约 `187 MB` -- 这条路线的成本低，且比现网主库在线改子网更稳 - -## 6. 初始化新 RDS - -先连接新 RDS： - -```bash -mysql -h -u admin -p -``` - -建议先执行： - -```sql -CALL mysql.rds_set_configuration('binlog retention hours', 24); - -CREATE USER IF NOT EXISTS 'repl'@'104.238.220.230' IDENTIFIED BY ''; -GRANT REPLICATION SLAVE, REPLICATION CLIENT ON *.* TO 'repl'@'104.238.220.230'; -FLUSH PRIVILEGES; -``` - -如果库是空的，再导入： - -```bash -mysql -h -u admin -p hifast < hifast-full.sql -``` - -导入完成后确认： - -```bash -mysql -h -u admin -p -e "USE hifast; SHOW TABLES;" -``` - -## 7. 切换 AWS 应用到新 RDS - -### 7.1 备份当前配置 - -在 AWS app EC2： - -```bash -cp /opt/ppanel/configs/ppanel.yaml /opt/ppanel/configs/ppanel.yaml.bak.$(date +%Y%m%d%H%M%S) -``` - -### 7.2 修改数据库地址 - -编辑： - -- `/opt/ppanel/configs/ppanel.yaml` - -把： - -```yaml -MySQL: - Addr: hifast-mysql-prod.cd6aey40m6ag.ap-east-1.rds.amazonaws.com:3306 -``` - -改成： - -```yaml -MySQL: - Addr: hifast-mysql-prod-v2.cd6aey40m6ag.ap-east-1.rds.amazonaws.com:3306 -``` - -### 7.3 重启业务容器 - -```bash -cd /opt/ppanel -docker compose -f docker-compose.cloud.yml up -d ppanel-server -``` - -### 7.4 应用验收 - -```bash -curl -sf http://127.0.0.1:8080/v1/common/heartbeat -docker compose -f /opt/ppanel/docker-compose.cloud.yml logs --tail=100 ppanel-server -``` - -重点确认： - -- 心跳返回 `200` -- 日志中无 MySQL 连接异常 - -## 8. 改挂 `104` 从库到新 RDS - -先在新 RDS 获取位点： - -```sql -SHOW MASTER STATUS; -``` - -然后在 `104` 执行： - -```sql -STOP REPLICA; -RESET REPLICA ALL; -CHANGE REPLICATION SOURCE TO - SOURCE_HOST='hifast-mysql-prod-v2.cd6aey40m6ag.ap-east-1.rds.amazonaws.com', - SOURCE_PORT=3306, - SOURCE_USER='repl', - SOURCE_PASSWORD='', - SOURCE_LOG_FILE='', - SOURCE_LOG_POS=, - SOURCE_SSL=1; -START REPLICA; -SHOW REPLICA STATUS\G -``` - -验收标准： - -- `Replica_IO_Running: Yes` -- `Replica_SQL_Running: Yes` -- `Seconds_Behind_Source: 0` - -## 9. 切换后验收 - -至少做下面这些检查： - -1. AWS 应用心跳正常 -2. 管理后台可登录 -3. 新建一条测试数据后，主库可见 -4. `104` 从库能同步到该测试数据 -5. Navicat 可从白名单来源连接新 RDS -6. Redis 主从仍正常，不受本次 MySQL 切换影响 - -推荐额外核对： - -```bash -docker compose -f /opt/ppanel/docker-compose.cloud.yml ps -docker exec hifast-redis redis-cli -a '' INFO replication -``` - -## 10. 回滚方案 - -如果切换后 AWS 应用异常： - -1. 立刻把 `/opt/ppanel/configs/ppanel.yaml` 中 `MySQL.Addr` 改回旧 endpoint -2. 重启 `ppanel-server`： - -```bash -cd /opt/ppanel -docker compose -f docker-compose.cloud.yml up -d ppanel-server -``` - -3. 优先恢复主生产可用 -4. `104` 是否回切旧 RDS 复制，视恢复窗口决定 - -如果只是 `104` 复制改挂失败，但 AWS 应用已正常使用新 RDS： - -- 不必立刻回滚应用 -- 先修好 `104 -> 新 RDS` 的白名单、位点、用户或网络 - -## 11. 切换完成后的收尾 - -全部验收通过后再做： - -1. 已将新 endpoint 记录到正式运维文档 -2. 仍需根据回收窗口安排旧 RDS 下线 -3. 仍需按网络收敛计划处理 `private-1b` 路由语义 -4. 已更新外部复制 runbook 中的主库 endpoint - -## 12. 当前建议的执行顺序 - -本次切换当时按以下顺序执行： - -1. 创建 `hifast-mysql-prod-v2` -2. 验证新 RDS 网络与参数 -3. 导出旧库 -4. 导入新库 -5. 创建 / 确认 `repl` 用户 -6. 切 AWS 应用到新 RDS -7. 验证应用 -8. 改挂 `104` 到新 RDS -9. 验证从库 -10. 收尾，并保留旧 RDS 待后续下线 diff --git a/ops/hifast-s3-backup-runbook-zh.md b/ops/hifast-s3-backup-runbook-zh.md deleted file mode 100644 index 764d82c..0000000 --- a/ops/hifast-s3-backup-runbook-zh.md +++ /dev/null @@ -1,311 +0,0 @@ -# Hifast S3 备份集成 Runbook - -本文档记录 `2026-05-13` 已经实际确认和完成的 S3 备份集成状态，以及后续如何把它和当前 `AWS 主生产 + 104 外部备用` 架构结合起来。 - -## 1. 当前已完成状态 - -- 已创建 S3 备份桶：`hifast-prod-backups-200810848252-ap-east-1` -- Region: `ap-east-1` -- Public access: blocked -- Versioning: enabled -- 已确认 RDS 主库 `hifast-mysql-prod-v2` 自动备份已启用： - - retention: `7 days` - - `2026-05-13` 已有自动快照 -- 已确认 `104.238.220.230` MySQL 外部从库健康，可作为逻辑备份源： - - `Replica_IO_Running: Yes` - - `Replica_SQL_Running: Yes` - - `Seconds_Behind_Source: 0` -- 已确认 `104.238.220.230` Redis 从库健康，可作为 RDB 备份源： - - `role:slave` - - `master_link_status:up` - -## 2. 推荐组合方案 - -当前最稳的做法不是只依赖一种备份，而是保留两层： - -1. `RDS automated backup` -2. `104` 从库导出的逻辑备份上传到 S3 - -原因： - -- RDS 自动备份适合快速恢复整库 -- S3 逻辑备份适合独立下载、跨环境恢复、长期归档 -- 从 `104` 导出 MySQL / Redis 备份，对 AWS 主生产扰动最小 - -## 3. 仓库已补充内容 - -- MySQL 备份脚本： - - [`deploy/scripts/mysql_backup_to_s3.sh`](/Users/Apple/code_vpn/vpn/ppanel-server/deploy/scripts/mysql_backup_to_s3.sh) -- Redis RDB 备份脚本： - - [`deploy/scripts/redis_rdb_backup_to_s3.sh`](/Users/Apple/code_vpn/vpn/ppanel-server/deploy/scripts/redis_rdb_backup_to_s3.sh) -- 环境变量模板： - - [`deploy/aws/ap-east-1/configs/backup-to-s3.env.example`](/Users/Apple/code_vpn/vpn/ppanel-server/deploy/aws/ap-east-1/configs/backup-to-s3.env.example) -- 主从切换环境变量模板： - - [`deploy/aws/ap-east-1/configs/replica-ops.env.example`](/Users/Apple/code_vpn/vpn/ppanel-server/deploy/aws/ap-east-1/configs/replica-ops.env.example) -- 数据迁移交互工具： - - [`deploy/scripts/hifast_data_sync_tool.sh`](/Users/Apple/code_vpn/vpn/ppanel-server/deploy/scripts/hifast_data_sync_tool.sh) -- 主从运维交互工具： - - 统一入口仍使用 [`deploy/scripts/hifast_data_sync_tool.sh`](/Users/Apple/code_vpn/vpn/ppanel-server/deploy/scripts/hifast_data_sync_tool.sh) - -## 4. 建议部署位置 - -### 4.1 MySQL - -优先部署在 `104.238.220.230`： - -- 直接从本地 MySQL 从库导出 -- 不占用 AWS RDS 主库的备份窗口 -- 故障时备份和备用节点仍在同一台机器上 - -### 4.2 Redis - -优先部署在 `104.238.220.230`： - -- 直接从 Redis 从库导出 `RDB` -- 不影响 AWS Redis 主库对外服务 - -## 5. 104 当前还缺的东西 - -截至 `2026-05-13`，`104` 已具备： - -- `mysql` -- `mysqldump` -- `gzip` - -但仍缺： - -- `aws cli` -- 一套最小权限的 S3 上传凭证 - -## 6. 推荐权限模型 - -当前最实用的选择有两种： - -1. `104` 使用专用 IAM 用户的最小权限 `AK/SK` -2. 未来如果备份转到 AWS EC2，再改为 `EC2 Instance Role` - -按现有拓扑，推荐先走第 1 种，因为 MySQL 和 Redis 的备份源都在 `104`。 - -### 6.1 最小权限策略示例 - -```json -{ - "Version": "2012-10-17", - "Statement": [ - { - "Sid": "ListBackupBucket", - "Effect": "Allow", - "Action": [ - "s3:ListBucket" - ], - "Resource": "arn:aws:s3:::hifast-prod-backups-200810848252-ap-east-1" - }, - { - "Sid": "WriteBackupObjects", - "Effect": "Allow", - "Action": [ - "s3:PutObject", - "s3:AbortMultipartUpload" - ], - "Resource": "arn:aws:s3:::hifast-prod-backups-200810848252-ap-east-1/*" - } - ] -} -``` - -说明： - -- 不建议给 `s3:*` -- 不建议把 AK/SK 写进 repo -- 建议只写入 `104` 本机的 `/root/.aws/credentials` - -## 7. 104 安装与配置步骤 - -### 7.1 安装 `aws cli` - -```bash -apt-get update -apt-get install -y awscli -``` - -### 7.2 写入 AWS 凭证 - -```bash -mkdir -p /root/.aws -chmod 700 /root/.aws -cat >/root/.aws/credentials <<'EOF' -[default] -aws_access_key_id=CHANGE_ME -aws_secret_access_key=CHANGE_ME -EOF -chmod 600 /root/.aws/credentials -``` - -### 7.3 准备环境变量文件 - -```bash -cp deploy/aws/ap-east-1/configs/backup-to-s3.env.example /root/backup-to-s3.env -chmod 600 /root/backup-to-s3.env -``` - -把下面这些值改成真实值： - -- `MYSQL_PASSWORD` -- `REDIS_PASSWORD` -- `HOST_TAG` - -如果 MySQL 就跑在 `104` 本机，可保持： - -- `MYSQL_HOST=127.0.0.1` - -如果 Redis 就跑在 `104` 本机，可保持： - -- `REDIS_HOST=127.0.0.1` - -## 8. 手工执行方式 - -### 8.1 MySQL 逻辑备份 - -```bash -set -a -source /root/backup-to-s3.env -set +a - -bash deploy/scripts/mysql_backup_to_s3.sh -``` - -默认产物： - -- 本地：`/var/backups/hifast/*.sql.gz` -- S3：`s3://hifast-prod-backups-200810848252-ap-east-1/mysql//` - -### 8.2 Redis RDB 备份 - -```bash -set -a -source /root/backup-to-s3.env -set +a - -S3_PREFIX=redis bash deploy/scripts/redis_rdb_backup_to_s3.sh -``` - -默认产物： - -- 本地：`/var/backups/hifast/*.rdb.gz` -- S3：`s3://hifast-prod-backups-200810848252-ap-east-1/redis//` - -## 9. 定时任务建议 - -### 9.1 MySQL 每天凌晨执行 - -```cron -15 3 * * * . /root/backup-to-s3.env && /bin/bash /opt/ppanel/deploy/scripts/mysql_backup_to_s3.sh >>/var/log/hifast-mysql-backup.log 2>&1 -``` - -### 9.2 Redis 每天凌晨执行 - -```cron -45 3 * * * . /root/backup-to-s3.env && S3_PREFIX=redis /bin/bash /opt/ppanel/deploy/scripts/redis_rdb_backup_to_s3.sh >>/var/log/hifast-redis-backup.log 2>&1 -``` - -说明： - -- MySQL 和 Redis 建议错峰执行 -- 本地临时文件默认只保留 `3` 天 -- 长期保留建议通过 S3 Lifecycle 管理，而不是靠本机 cron 删除 - -### 9.3 MySQL 每 10 分钟执行一次 - -如果你要在 `104` 上做高频逻辑备份，仓库里已经补了 `systemd timer` 安装脚本： - -- 安装脚本： - - [`deploy/scripts/install_mysql_backup_timer.sh`](/Users/Apple/code_vpn/vpn/ppanel-server/deploy/scripts/install_mysql_backup_timer.sh) -- systemd service： - - [`deploy/systemd/hifast-mysql-backup.service`](/Users/Apple/code_vpn/vpn/ppanel-server/deploy/systemd/hifast-mysql-backup.service) -- systemd timer： - - [`deploy/systemd/hifast-mysql-backup.timer`](/Users/Apple/code_vpn/vpn/ppanel-server/deploy/systemd/hifast-mysql-backup.timer) - -安装方式： - -```bash -cp deploy/aws/ap-northeast-1/configs/backup-to-s3.env.example /root/backup-to-s3.env -chmod 600 /root/backup-to-s3.env -vim /root/backup-to-s3.env - -bash deploy/scripts/install_mysql_backup_timer.sh /opt/ppanel /root/backup-to-s3.env -``` - -查看状态： - -```bash -systemctl list-timers --all | grep hifast-mysql-backup -systemctl status hifast-mysql-backup.timer --no-pager -l -systemctl status hifast-mysql-backup.service --no-pager -l -journalctl -u hifast-mysql-backup.service -n 50 --no-pager -``` - -补充说明： - -- 定时表达式是 `OnCalendar=*:0/10`，即每 `10` 分钟执行一次 -- 备份脚本已加 `flock` 锁，上一轮未结束时，下一轮会自动跳过，不会并发打包 -- 高频逻辑备份会持续产生 `mysqldump` 开销，建议只在 `104` 这样的从库或备用库上执行，不要直接打主库 - -## 10. 恢复思路 - -### 10.1 MySQL - -1. 从 S3 下载目标 `sql.gz` -2. 校验 `.sha256` -3. 解压 -4. 导入目标 MySQL - -示例： - -```bash -aws s3 cp s3://hifast-prod-backups-200810848252-ap-east-1/mysql/104-standby/20260513T120000Z_104-standby_hifast.sql.gz . -aws s3 cp s3://hifast-prod-backups-200810848252-ap-east-1/mysql/104-standby/20260513T120000Z_104-standby_hifast.sql.gz.sha256 . -sha256sum -c 20260513T120000Z_104-standby_hifast.sql.gz.sha256 -gunzip -c 20260513T120000Z_104-standby_hifast.sql.gz | mysql -h -u -p -``` - -### 10.2 Redis - -1. 从 S3 下载目标 `rdb.gz` -2. 校验 `.sha256` -3. 解压得到 `dump.rdb` -4. 在维护窗口内替换 Redis 数据文件后重启 - -## 11. 当前推荐落地顺序 - -1. 保持当前 `RDS automated backup` 不变 -2. 在 `104` 安装 `aws cli` -3. 创建最小权限 S3 上传凭证并仅保存到 `104` -4. 先手工执行一轮 MySQL 备份上传 -5. 验证 S3 对象、校验文件、恢复可读性 -6. 再补 Redis RDB 备份 -7. 最后加 cron 和 S3 Lifecycle - -## 12. 104 上的交互式运维脚本 - -为了减少手工敲命令，仓库里现在统一使用一套交互式总入口。 - -### 12.1 统一入口 - -```bash -bash deploy/scripts/hifast_data_sync_tool.sh /root/replica-ops.env -``` - -菜单支持： - -- 备份 MySQL 到 S3 -- 备份 Redis 到 S3 -- 导出 MySQL dump -- 导入 MySQL dump 到 AWS RDS -- 导出后直接导入 -- 导出 Redis RDB -- 导入 Redis RDB 到 Docker / 宿主机 Redis -- 查看 MySQL / Redis 当前主从状态 -- 强制重拉 MySQL 主从 -- 强制重拉 Redis 主从 -- 把 MySQL 从库提升为可写主库 -- 把 Redis 从库提升为主库 diff --git a/prometheus/prometheus.yml b/prometheus/prometheus.yml deleted file mode 100644 index 1b478d8..0000000 --- a/prometheus/prometheus.yml +++ /dev/null @@ -1,40 +0,0 @@ -global: - scrape_interval: 15s - evaluation_interval: 15s - -scrape_configs: - - job_name: prometheus - static_configs: - - targets: - - prometheus:9090 - - - job_name: grafana - metrics_path: /metrics - static_configs: - - targets: - - grafana:3000 - - - job_name: node-exporter - static_configs: - - targets: - - node-exporter:9100 - - - job_name: cadvisor - static_configs: - - targets: - - cadvisor:8080 - - - job_name: nginx-exporter - static_configs: - - targets: - - nginx-exporter:9113 - - - job_name: loki - static_configs: - - targets: - - loki:3100 - - - job_name: tempo - static_configs: - - targets: - - tempo:3200 diff --git a/scripts/convert_recovery_orders/main.go b/scripts/convert_recovery_orders/main.go deleted file mode 100644 index 2eb6cdc..0000000 --- a/scripts/convert_recovery_orders/main.go +++ /dev/null @@ -1,229 +0,0 @@ -//go:build ignore - -package main - -import ( - "encoding/json" - "flag" - "fmt" - "os" - "sort" - "strconv" - "strings" - "time" -) - -type remoteOrder struct { - OrderNo string `json:"order_no"` - OutOrderNo string `json:"out_order_no"` - OrderMoney json.Number `json:"order_money"` - OrderStatus string `json:"order_status"` - PaymentTime *int64 `json:"payment_time"` - CreateTime int64 `json:"create_time"` -} - -type recoveryFile struct { - Orders []recoveryOrder `json:"orders"` -} - -type recoveryOrder struct { - OutOrderNo string `json:"out_order_no"` - OrderNo string `json:"order_no"` - OrderStatus string `json:"order_status"` - SubscribeId int64 `json:"subscribe_id"` - ExpireAt int64 `json:"expire_at"` - Quantity int64 `json:"quantity"` - PaidAt int64 `json:"paid_at,omitempty"` - Note string `json:"note,omitempty"` - PaymentOrderNo string `json:"payment_order_no,omitempty"` - OrderMoneyCents int64 `json:"order_money_cents,omitempty"` -} - -type adminOrderResponse struct { - Code uint32 `json:"code"` - Msg string `json:"msg"` - Data struct { - Total int64 `json:"total"` - List []adminOrder `json:"list"` - } `json:"data"` -} - -type adminOrder struct { - OrderNo string `json:"order_no"` -} - -func main() { - var ( - input string - output string - adminOrders string - subscribeID int64 - onlyMissing bool - ) - flag.StringVar(&input, "in", "aaa.json", "input payment platform orders JSON") - flag.StringVar(&output, "out", "internal/recovery/recovery_orders.json", "output recovery orders JSON") - flag.StringVar(&adminOrders, "admin-orders", "", "optional admin order list response JSON; when provided, matched out_order_no values are skipped") - flag.Int64Var(&subscribeID, "subscribe-id", 1, "subscribe id to use for recovered subscriptions") - flag.BoolVar(&onlyMissing, "only-missing", true, "only output payment success orders missing from admin order list when -admin-orders is set") - flag.Parse() - - orders, err := readOrders(input) - must(err) - existingOrders := map[string]struct{}{} - if adminOrders != "" { - existingOrders, err = readAdminOrderSet(adminOrders) - must(err) - } - - daysByAmount := map[int64]int64{ - 1953: 7, - 4193: 30, - 9093: 90, - 31500: 365, - } - - out := recoveryFile{Orders: make([]recoveryOrder, 0, len(orders))} - var skipped []string - var matchedExisting int - seen := make(map[string]struct{}) - for _, item := range orders { - if strings.ToLower(strings.TrimSpace(item.OrderStatus)) != "success" { - continue - } - localOrderNo := strings.TrimSpace(item.OutOrderNo) - if localOrderNo == "" { - skipped = append(skipped, fmt.Sprintf("%s missing out_order_no", item.OrderNo)) - continue - } - if _, ok := seen[localOrderNo]; ok { - skipped = append(skipped, fmt.Sprintf("%s duplicate out_order_no", localOrderNo)) - continue - } - if _, exists := existingOrders[localOrderNo]; exists && onlyMissing { - matchedExisting++ - seen[localOrderNo] = struct{}{} - continue - } - amount := decimalYuanToCents(item.OrderMoney) - days, ok := daysByAmount[amount] - if !ok { - skipped = append(skipped, fmt.Sprintf("%s unknown amount %s", localOrderNo, item.OrderMoney.String())) - continue - } - base := item.CreateTime - if item.PaymentTime != nil && *item.PaymentTime > 0 { - base = *item.PaymentTime - } - if base <= 0 { - skipped = append(skipped, fmt.Sprintf("%s missing payment/create time", localOrderNo)) - continue - } - seen[localOrderNo] = struct{}{} - out.Orders = append(out.Orders, recoveryOrder{ - OutOrderNo: localOrderNo, - OrderNo: strings.TrimSpace(item.OrderNo), - OrderStatus: "success", - SubscribeId: subscribeID, - ExpireAt: time.Unix(base, 0).Add(time.Duration(days) * 24 * time.Hour).UnixMilli(), - Quantity: days, - PaidAt: base, - Note: "data recovery", - PaymentOrderNo: strings.TrimSpace(item.OrderNo), - OrderMoneyCents: amount, - }) - } - - sort.Slice(out.Orders, func(i, j int) bool { - return out.Orders[i].OutOrderNo < out.Orders[j].OutOrderNo - }) - - must(writeJSON(output, out)) - fmt.Printf("converted recovery orders=%d matched_existing=%d skipped=%d output=%s\n", len(out.Orders), matchedExisting, len(skipped), output) - for _, msg := range skipped { - fmt.Printf("skip: %s\n", msg) - } -} - -func readAdminOrderSet(path string) (map[string]struct{}, error) { - data, err := os.ReadFile(path) - if err != nil { - return nil, err - } - var resp adminOrderResponse - if err := json.Unmarshal(data, &resp); err == nil && resp.Data.List != nil { - if resp.Code != 0 && resp.Code != 200 { - return nil, fmt.Errorf("admin order response failed: code=%d msg=%s", resp.Code, resp.Msg) - } - return adminOrdersToSet(resp.Data.List), nil - } - - var list []adminOrder - if err := json.Unmarshal(data, &list); err != nil { - return nil, err - } - return adminOrdersToSet(list), nil -} - -func adminOrdersToSet(list []adminOrder) map[string]struct{} { - set := make(map[string]struct{}, len(list)) - for _, item := range list { - orderNo := strings.TrimSpace(item.OrderNo) - if orderNo != "" { - set[orderNo] = struct{}{} - } - } - return set -} - -func readOrders(path string) ([]remoteOrder, error) { - data, err := os.ReadFile(path) - if err != nil { - return nil, err - } - var orders []remoteOrder - if err := json.Unmarshal(data, &orders); err != nil { - return nil, err - } - return orders, nil -} - -func writeJSON(path string, data recoveryFile) error { - bytes, err := json.MarshalIndent(data, "", " ") - if err != nil { - return err - } - bytes = append(bytes, '\n') - return os.WriteFile(path, bytes, 0644) -} - -func decimalYuanToCents(n json.Number) int64 { - s := strings.TrimSpace(n.String()) - if s == "" { - return 0 - } - parts := strings.SplitN(s, ".", 2) - yuan, _ := strconv.ParseInt(parts[0], 10, 64) - cents := yuan * 100 - if len(parts) == 1 { - return cents - } - frac := parts[1] - if len(frac) > 2 { - frac = frac[:2] - } - for len(frac) < 2 { - frac += "0" - } - f, _ := strconv.ParseInt(frac, 10, 64) - if strings.HasPrefix(parts[0], "-") { - return cents - f - } - return cents + f -} - -func must(err error) { - if err != nil { - fmt.Fprintln(os.Stderr, err) - os.Exit(1) - } -} diff --git a/scripts/reconcile_mihapay_orders/main.go b/scripts/reconcile_mihapay_orders/main.go deleted file mode 100644 index 5e541f7..0000000 --- a/scripts/reconcile_mihapay_orders/main.go +++ /dev/null @@ -1,710 +0,0 @@ -//go:build ignore - -package main - -import ( - "bytes" - "context" - "crypto/md5" - "encoding/hex" - "encoding/json" - "flag" - "fmt" - "io" - "net/http" - "net/url" - "os" - "sort" - "strconv" - "strings" - "time" - - "github.com/hibiken/asynq" - "github.com/perfect-panel/server/internal/config" - "github.com/perfect-panel/server/internal/model/order" - "github.com/perfect-panel/server/internal/model/payment" - "github.com/perfect-panel/server/internal/model/subscribe" - "github.com/perfect-panel/server/internal/model/user" - "github.com/perfect-panel/server/pkg/conf" - "github.com/perfect-panel/server/pkg/orm" - "github.com/perfect-panel/server/pkg/tool" - queueTypes "github.com/perfect-panel/server/queue/types" - "github.com/redis/go-redis/v9" - "gorm.io/gorm" -) - -const ( - defaultBaseURL = "https://gfiuseui.lkfezg.cn" - - // Hard-code Mihapay V2 credentials here when this reconciliation script should - // not depend on payment.config. Leave them empty to read pid/key/url from DB. - hardcodedMerchantID = "80567" - hardcodedMerchantKey = "475951745395f05a3be2ba7ca8235ba3" - hardcodedBaseURL = defaultBaseURL -) - -type options struct { - configPath string - baseURL string - paymentID int64 - method string - start string - end string - page int - limit int - maxPages int - apply bool - amountDiff int64 - verbose bool - printNotFound bool -} - -type merchantConfig struct { - merchantID string - key string - baseURL string -} - -type orderListResponse struct { - Code int `json:"code"` - Msg string `json:"msg"` - Data struct { - List []remoteOrder `json:"list"` - Total int `json:"total"` - } `json:"data"` -} - -type remoteOrder struct { - OrderNo string `json:"order_no"` - OutOrderNo string `json:"out_order_no"` - OrderMoney json.Number `json:"order_money"` - UserMoney json.Number `json:"user_money"` - OrderStatus string `json:"order_status"` - NotifyStatus string `json:"notify_status"` - CreateTime int64 `json:"create_time"` - PaymentTime int64 `json:"payment_time"` - NotifyTime int64 `json:"notify_time"` - ClientIP string `json:"client_ip"` - Device string `json:"device"` - OrderRemark string `json:"order_remark"` -} - -type matchedOrder struct { - Remote remoteOrder - Local order.Order - RemoteCents int64 - Audit subscriptionAudit - Action string - Reason string - PaymentAudit string -} - -type subscriptionAudit struct { - Checked bool - Status string - UserSubID int64 - UserID int64 - StartTime time.Time - ExpireTime time.Time - ExpectedExpire time.Time - SubscribeUnit string - Quantity int64 - Reason string -} - -type reconcileStats struct { - RemotePages int - RemoteOrders int - RemotePaid int - RemoteMissingNo int - DuplicateRemote int - LocalNotFound int - LocalDateFiltered int - LocalMethodSkip int - Matched int - StatusCounts map[string]int - LocalNotFoundOrders []remoteOrder -} - -func main() { - ctx := context.Background() - opt := parseFlags() - - var cfg config.Config - conf.MustLoad(opt.configPath, &cfg) - - db, err := orm.ConnectMysql(orm.Mysql{Config: cfg.MySQL}) - must(err, "connect mysql") - sqlDB, err := db.DB() - must(err, "get sql db") - defer sqlDB.Close() - - var redisClient *redis.Client - var queueClient *asynq.Client - if opt.apply { - redisClient = redis.NewClient(&redis.Options{ - Addr: cfg.Redis.Host, - Password: cfg.Redis.Pass, - DB: cfg.Redis.DB, - }) - defer redisClient.Close() - - queueClient = asynq.NewClient(asynq.RedisClientOpt{ - Addr: cfg.Redis.Host, - Password: cfg.Redis.Pass, - DB: 5, - }) - defer queueClient.Close() - } - - merchant, paymentInfo, err := resolveMerchantConfig(ctx, db, opt) - must(err, "resolve mihapay config") - if opt.paymentID > 0 { - opt.method = paymentInfo.Platform - } - baseURL := opt.baseURL - if baseURL == defaultBaseURL && merchant.baseURL != "" { - baseURL = merchant.baseURL - } - - client := &http.Client{Timeout: 15 * time.Second} - matches, stats, err := reconcile(ctx, client, db, redisClient, queueClient, opt, baseURL, merchant, paymentInfo) - must(err, "reconcile orders") - - printSummary(matches, stats, opt) -} - -func parseFlags() options { - opt := options{} - flag.StringVar(&opt.configPath, "config", "etc/ppanel.yaml", "ppanel config path") - flag.StringVar(&opt.baseURL, "base-url", defaultBaseURL, "Mihapay base URL") - flag.Int64Var(&opt.paymentID, "payment-id", 0, "payment table id to use for merchant credentials") - flag.StringVar(&opt.method, "method", "epay", "local order method/platform to match when payment-id is not set") - flag.StringVar(&opt.start, "start", "", "local order created_at start, e.g. 2026-05-05 00:00:00") - flag.StringVar(&opt.end, "end", "", "local order created_at end, e.g. 2026-05-06 00:00:00") - flag.IntVar(&opt.page, "page", 1, "remote start page") - flag.IntVar(&opt.limit, "limit", 100, "remote page size") - flag.IntVar(&opt.maxPages, "max-pages", 50, "maximum remote pages to fetch") - flag.BoolVar(&opt.apply, "apply", false, "apply fixes; default is dry-run") - flag.Int64Var(&opt.amountDiff, "amount-diff-cents", 1, "allowed amount difference in cents") - flag.BoolVar(&opt.verbose, "v", false, "print diagnostic details") - flag.BoolVar(&opt.printNotFound, "print-not-found", false, "print remote paid orders that have no matching local order") - flag.Parse() - - if opt.start == "" { - now := time.Now() - opt.start = time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, now.Location()).Format("2006-01-02 15:04:05") - } - if opt.end == "" { - start, err := time.ParseInLocation("2006-01-02 15:04:05", opt.start, time.Local) - if err == nil { - opt.end = start.AddDate(0, 0, 1).Format("2006-01-02 15:04:05") - } - } - opt.baseURL = strings.TrimRight(opt.baseURL, "/") - if opt.page <= 0 { - opt.page = 1 - } - if opt.limit <= 0 || opt.limit > 500 { - opt.limit = 100 - } - if opt.maxPages <= 0 { - opt.maxPages = 1 - } - return opt -} - -func resolveMerchantConfig(ctx context.Context, db *gorm.DB, opt options) (merchantConfig, *payment.Payment, error) { - if hardcodedMerchantID != "" && hardcodedMerchantKey != "" { - return merchantConfig{ - merchantID: hardcodedMerchantID, - key: hardcodedMerchantKey, - baseURL: strings.TrimRight(hardcodedBaseURL, "/"), - }, nil, nil - } - - var p payment.Payment - query := db.WithContext(ctx).Model(&payment.Payment{}) - if opt.paymentID > 0 { - query = query.Where("id = ?", opt.paymentID) - } else { - query = query.Where("LOWER(REPLACE(platform, '_', '')) = LOWER(REPLACE(?, '_', ''))", opt.method) - } - if err := query.First(&p).Error; err != nil { - return merchantConfig{}, nil, err - } - - var epayCfg payment.EPayConfig - if err := epayCfg.Unmarshal([]byte(p.Config)); err != nil { - return merchantConfig{}, nil, err - } - if epayCfg.Pid == "" || epayCfg.Key == "" { - return merchantConfig{}, nil, fmt.Errorf("payment id %d has empty pid/key", p.Id) - } - return merchantConfig{merchantID: epayCfg.Pid, key: epayCfg.Key, baseURL: strings.TrimRight(epayCfg.Url, "/")}, &p, nil -} - -func reconcile( - ctx context.Context, - client *http.Client, - db *gorm.DB, - redisClient *redis.Client, - queueClient *asynq.Client, - opt options, - baseURL string, - merchant merchantConfig, - paymentInfo *payment.Payment, -) ([]matchedOrder, reconcileStats, error) { - var matches []matchedOrder - var stats reconcileStats - stats.StatusCounts = make(map[string]int) - seen := make(map[string]struct{}) - page := opt.page - - for fetchedPages := 0; fetchedPages < opt.maxPages; fetchedPages++ { - resp, err := fetchRemoteOrders(ctx, client, baseURL, merchant, page, opt.limit) - if err != nil { - return matches, stats, err - } - if resp.Code != 1 { - return matches, stats, fmt.Errorf("remote orderlist failed: code=%d msg=%s", resp.Code, resp.Msg) - } - stats.RemotePages++ - stats.RemoteOrders += len(resp.Data.List) - if opt.verbose { - fmt.Printf("remote page=%d list=%d total=%d\n", page, len(resp.Data.List), resp.Data.Total) - } - if len(resp.Data.List) == 0 { - break - } - - for _, remote := range resp.Data.List { - status := strings.TrimSpace(remote.OrderStatus) - if status == "" { - status = "" - } - stats.StatusCounts[status]++ - if !isRemotePaidStatus(remote.OrderStatus) { - continue - } - stats.RemotePaid++ - if remote.OutOrderNo == "" { - stats.RemoteMissingNo++ - continue - } - if _, ok := seen[remote.OutOrderNo]; ok { - stats.DuplicateRemote++ - continue - } - seen[remote.OutOrderNo] = struct{}{} - - item, ok := matchRemoteOrder(ctx, db, opt, &stats, remote) - if !ok { - continue - } - stats.Matched++ - matches = append(matches, item) - if opt.apply && (item.Action == "补单" || item.Action == "重投队列") { - if err := applyOrderFix(ctx, db, redisClient, queueClient, paymentInfo, item); err != nil { - return matches, stats, err - } - } - } - - page++ - if resp.Data.Total > 0 && page*opt.limit >= resp.Data.Total+opt.limit { - break - } - } - return matches, stats, nil -} - -func isRemotePaidStatus(status string) bool { - switch strings.ToLower(strings.TrimSpace(status)) { - case "paid", "success", "succeeded", "trade_success", "completed", "complete", "1": - return true - default: - return false - } -} - -func fetchRemoteOrders(ctx context.Context, client *http.Client, baseURL string, merchant merchantConfig, page, limit int) (*orderListResponse, error) { - params := map[string]string{ - "merchantId": merchant.merchantID, - "page": strconv.Itoa(page), - "limit": strconv.Itoa(limit), - "order": "create_time desc", - } - params["sign"] = sign(params, merchant.key) - - form := url.Values{} - for k, v := range params { - form.Set(k, v) - } - req, err := http.NewRequestWithContext(ctx, http.MethodPost, baseURL+"/api/v2/orderlist", strings.NewReader(form.Encode())) - if err != nil { - return nil, err - } - req.Header.Set("Content-Type", "application/x-www-form-urlencoded") - - resp, err := client.Do(req) - if err != nil { - return nil, err - } - defer resp.Body.Close() - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, err - } - if resp.StatusCode < 200 || resp.StatusCode >= 300 { - return nil, fmt.Errorf("remote status %d: %s", resp.StatusCode, string(body)) - } - - decoder := json.NewDecoder(bytes.NewReader(body)) - decoder.UseNumber() - var out orderListResponse - if err := decoder.Decode(&out); err != nil { - return nil, fmt.Errorf("decode remote response: %w body=%s", err, string(body)) - } - return &out, nil -} - -func matchRemoteOrder(ctx context.Context, db *gorm.DB, opt options, stats *reconcileStats, remote remoteOrder) (matchedOrder, bool) { - var local order.Order - if err := db.WithContext(ctx).Model(&order.Order{}).Where("order_no = ?", remote.OutOrderNo).First(&local).Error; err != nil { - stats.LocalNotFound++ - stats.LocalNotFoundOrders = append(stats.LocalNotFoundOrders, remote) - if opt.verbose { - fmt.Printf("skip local_not_found out_order_no=%s remote_no=%s status=%s money=%s\n", remote.OutOrderNo, remote.OrderNo, remote.OrderStatus, remote.OrderMoney) - } - return matchedOrder{}, false - } - if opt.method != "" && local.Method != opt.method { - stats.LocalMethodSkip++ - if opt.verbose { - fmt.Printf("skip method out_order_no=%s local_method=%s want=%s\n", remote.OutOrderNo, local.Method, opt.method) - } - return matchedOrder{}, false - } - if opt.start != "" && local.CreatedAt.Format("2006-01-02 15:04:05") < opt.start { - stats.LocalDateFiltered++ - return matchedOrder{}, false - } - if opt.end != "" && local.CreatedAt.Format("2006-01-02 15:04:05") >= opt.end { - stats.LocalDateFiltered++ - return matchedOrder{}, false - } - - remoteCents := decimalYuanToCents(remote.OrderMoney) - item := matchedOrder{ - Remote: remote, - Local: local, - RemoteCents: remoteCents, - PaymentAudit: "远端已支付", - } - item.Audit = auditSubscription(ctx, db, local) - - switch { - case local.Status == 5: - item.Action = "跳过" - item.Reason = "本地已完成" - case remoteCents > 0 && abs64(local.Amount-remoteCents) > opt.amountDiff: - item.Action = "跳过" - item.Reason = fmt.Sprintf("金额不匹配 本地=%d 远端=%d", local.Amount, remoteCents) - case local.Status == 2: - item.Action = "重投队列" - item.Reason = "本地已支付但未完成" - case local.Status != 1 && local.Status != 3: - item.Action = "跳过" - item.Reason = fmt.Sprintf("本地状态 %d 不适合自动补单", local.Status) - default: - item.Action = "补单" - item.Reason = "远端已支付，本地未支付/已关闭" - } - return item, true -} - -func auditSubscription(ctx context.Context, db *gorm.DB, local order.Order) subscriptionAudit { - audit := subscriptionAudit{ - Checked: true, - Quantity: local.Quantity, - } - if local.Type != 1 && local.Type != 2 { - audit.Status = "跳过" - audit.Reason = fmt.Sprintf("订单类型 %d 不核对订阅时间", local.Type) - return audit - } - - var plan subscribe.Subscribe - if err := db.WithContext(ctx).Model(&subscribe.Subscribe{}).Where("id = ?", local.SubscribeId).First(&plan).Error; err != nil { - audit.Status = "异常" - audit.Reason = "套餐不存在: " + err.Error() - return audit - } - audit.SubscribeUnit = plan.UnitTime - - var userSub user.Subscribe - err := db.WithContext(ctx).Model(&user.Subscribe{}).Where("order_id = ?", local.Id).Order("id DESC").First(&userSub).Error - if err == nil { - audit.UserSubID = userSub.Id - audit.UserID = userSub.UserId - audit.StartTime = userSub.StartTime - audit.ExpireTime = userSub.ExpireTime - audit.ExpectedExpire = userSub.ExpireTime - audit.Status = "已核对" - audit.Reason = "订阅已绑定当前订单" - return audit - } - - if local.Type == 2 && local.SubscribeToken != "" { - err = db.WithContext(ctx).Model(&user.Subscribe{}).Where("token = ?", local.SubscribeToken).First(&userSub).Error - if err == nil { - base := subscriptionRenewalBaseTime(time.Now(), userSub.ExpireTime, userSub.FinishedAt) - audit.UserSubID = userSub.Id - audit.UserID = userSub.UserId - audit.StartTime = userSub.StartTime - audit.ExpireTime = userSub.ExpireTime - audit.ExpectedExpire = tool.AddTime(plan.UnitTime, local.Quantity, base) - if local.Status == 5 { - audit.Status = "异常" - audit.Reason = "订单已完成但订阅 order_id 未指向该订单" - } else { - audit.Status = "待激活" - audit.Reason = "续费订单未完成，预计激活后续期" - } - return audit - } - } - - if local.Status == 5 { - audit.Status = "异常" - audit.Reason = "订单已完成但未找到对应订阅" - return audit - } - - audit.Status = "待激活" - audit.ExpectedExpire = tool.AddTime(plan.UnitTime, local.Quantity, time.Now()) - audit.Reason = "订单未完成，订阅时间需激活队列生成" - return audit -} - -func subscriptionRenewalBaseTime(now time.Time, expire time.Time, finishedAt *time.Time) time.Time { - base := expire - if finishedAt != nil && finishedAt.After(base) { - base = *finishedAt - } - if base.Before(now) { - base = now - } - return base -} - -func applyOrderFix( - ctx context.Context, - db *gorm.DB, - redisClient *redis.Client, - queueClient *asynq.Client, - paymentInfo *payment.Payment, - item matchedOrder, -) error { - return db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { - updates := map[string]interface{}{ - "status": 2, - "trade_no": item.Remote.OrderNo, - "updated_at": time.Now(), - } - if paymentInfo != nil { - updates["payment_id"] = paymentInfo.Id - updates["method"] = paymentInfo.Platform - } - result := tx.Model(&order.Order{}). - Where("order_no = ? AND status IN ?", item.Local.OrderNo, []uint8{1, 2, 3}). - Updates(updates) - if result.Error != nil { - return result.Error - } - if result.RowsAffected == 0 { - return fmt.Errorf("order %s was not updated; status changed concurrently", item.Local.OrderNo) - } - clearOrderCache(ctx, redisClient, item.Local) - return enqueueActivation(ctx, queueClient, item.Local.OrderNo) - }) -} - -func enqueueActivation(ctx context.Context, queueClient *asynq.Client, orderNo string) error { - payload, err := json.Marshal(queueTypes.ForthwithActivateOrderPayload{OrderNo: orderNo}) - if err != nil { - return err - } - _, err = queueClient.EnqueueContext(ctx, asynq.NewTask(queueTypes.ForthwithActivateOrder, payload, asynq.MaxRetry(5))) - return err -} - -func clearOrderCache(ctx context.Context, redisClient *redis.Client, local order.Order) { - keys := []string{ - fmt.Sprintf("cache:order:id:%d", local.Id), - fmt.Sprintf("cache:order:no:%s", local.OrderNo), - } - _ = redisClient.Del(ctx, keys...).Err() -} - -func sign(params map[string]string, key string) string { - keys := make([]string, 0, len(params)) - for k, v := range params { - if k != "sign" && k != "sign_type" && v != "" { - keys = append(keys, k) - } - } - sort.Strings(keys) - - parts := make([]string, 0, len(keys)) - for _, k := range keys { - parts = append(parts, k+"="+params[k]) - } - sum := md5.Sum([]byte(strings.Join(parts, "&") + key)) - return hex.EncodeToString(sum[:]) -} - -func decimalYuanToCents(n json.Number) int64 { - s := strings.TrimSpace(n.String()) - if s == "" { - return 0 - } - parts := strings.SplitN(s, ".", 2) - yuan, _ := strconv.ParseInt(parts[0], 10, 64) - cents := yuan * 100 - if len(parts) == 1 { - return cents - } - frac := parts[1] - if len(frac) > 2 { - frac = frac[:2] - } - for len(frac) < 2 { - frac += "0" - } - f, _ := strconv.ParseInt(frac, 10, 64) - if strings.HasPrefix(parts[0], "-") { - return cents - f - } - return cents + f -} - -func printSummary(matches []matchedOrder, stats reconcileStats, opt options) { - var fix, requeue, skip int - for _, item := range matches { - switch item.Action { - case "补单": - fix++ - case "重投队列": - requeue++ - default: - skip++ - } - fmt.Printf("%s order_no=%s remote_no=%s local_status=%d amount=%d remote_amount=%d reason=%s\n", - item.Action, - item.Local.OrderNo, - item.Remote.OrderNo, - item.Local.Status, - item.Local.Amount, - item.RemoteCents, - item.Reason, - ) - if item.Audit.Checked { - fmt.Printf(" payment=%s subscription=%s sub_id=%d user_id=%d expire=%s expected=%s note=%s\n", - item.PaymentAudit, - item.Audit.Status, - item.Audit.UserSubID, - item.Audit.UserID, - formatTime(item.Audit.ExpireTime), - formatTime(item.Audit.ExpectedExpire), - item.Audit.Reason, - ) - } - } - mode := "DRY-RUN" - if opt.apply { - mode = "APPLIED" - } - fmt.Printf("\nremote pages=%d orders=%d paid=%d missing_out_order_no=%d duplicates=%d\n", - stats.RemotePages, stats.RemoteOrders, stats.RemotePaid, stats.RemoteMissingNo, stats.DuplicateRemote) - if len(stats.StatusCounts) > 0 { - keys := make([]string, 0, len(stats.StatusCounts)) - for key := range stats.StatusCounts { - keys = append(keys, key) - } - sort.Strings(keys) - fmt.Print("remote status_counts=") - for i, key := range keys { - if i > 0 { - fmt.Print(", ") - } - fmt.Printf("%s:%d", key, stats.StatusCounts[key]) - } - fmt.Println() - } - fmt.Printf("local not_found=%d method_skipped=%d date_filtered=%d matched=%d\n", - stats.LocalNotFound, stats.LocalMethodSkip, stats.LocalDateFiltered, stats.Matched) - if opt.printNotFound && len(stats.LocalNotFoundOrders) > 0 { - printNotFoundOrders(stats.LocalNotFoundOrders) - } - fmt.Printf("\n%s summary: matched=%d fixable=%d requeue=%d skipped=%d\n", mode, len(matches), fix, requeue, skip) -} - -func printNotFoundOrders(items []remoteOrder) { - fmt.Println("\nlocal_not_found orders:") - fmt.Println("out_order_no\tremote_no\tstatus\torder_money\tuser_money\tcreate_time\tpayment_time\tclient_ip\tdevice\tremark") - for _, item := range items { - fmt.Printf("%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n", - item.OutOrderNo, - item.OrderNo, - item.OrderStatus, - item.OrderMoney.String(), - item.UserMoney.String(), - formatUnixTime(item.CreateTime), - formatUnixTime(item.PaymentTime), - oneLine(item.ClientIP), - oneLine(item.Device), - oneLine(item.OrderRemark), - ) - } -} - -func formatTime(t time.Time) string { - if t.IsZero() { - return "-" - } - return t.Format("2006-01-02 15:04:05") -} - -func formatUnixTime(v int64) string { - if v <= 0 { - return "-" - } - if v > 1_000_000_000_000 { - return time.UnixMilli(v).Format("2006-01-02 15:04:05") - } - return time.Unix(v, 0).Format("2006-01-02 15:04:05") -} - -func oneLine(s string) string { - s = strings.ReplaceAll(s, "\t", " ") - s = strings.ReplaceAll(s, "\r", " ") - s = strings.ReplaceAll(s, "\n", " ") - return strings.TrimSpace(s) -} - -func abs64(v int64) int64 { - if v < 0 { - return -v - } - return v -} - -func must(err error, step string) { - if err == nil { - return - } - fmt.Fprintf(os.Stderr, "%s: %v\n", step, err) - os.Exit(1) -} diff --git a/tempo/tempo-config.yaml b/tempo/tempo-config.yaml deleted file mode 100644 index 8a68571..0000000 --- a/tempo/tempo-config.yaml +++ /dev/null @@ -1,37 +0,0 @@ -server: - http_listen_port: 3200 - grpc_listen_port: 9095 - -distributor: - receivers: - otlp: - protocols: - grpc: - endpoint: 0.0.0.0:4317 - http: - endpoint: 0.0.0.0:4318 - -ingester: - max_block_duration: 5m - -compactor: - compaction: - block_retention: 168h - -storage: - trace: - backend: local - wal: - path: /var/tempo/wal - local: - path: /var/tempo/blocks - -metrics_generator: - registry: - external_labels: - source: tempo - cluster: ppanel - storage: - path: /var/tempo/generator/wal - remote_write: - - url: http://prometheus:9090/api/v1/write