65 lines
3.8 KiB
YAML
Executable File
65 lines
3.8 KiB
YAML
Executable File
name: Recover DB (conditional mongo restart)
|
||
|
||
# 磁盘已腾出，但满盘期间 MongoDB 可能卡进坏状态。先用应用自带 driver ping：
|
||
# 卡死/失败 → 重启本机 mongod（共享库，恢复所有连库产品）；健康 → 只重启本产品 API 清连接池。
|
||
# 顺带装 pm2-logrotate，防止失控日志再次撑爆磁盘。
|
||
on:
|
||
workflow_dispatch:
|
||
|
||
jobs:
|
||
recover:
|
||
runs-on: ubuntu-latest
|
||
steps:
|
||
- name: SSH recover DB + prevent log blowup
|
||
uses: appleboy/ssh-action@v1
|
||
with:
|
||
host: ${{ secrets.ECS_HOST }}
|
||
username: ${{ secrets.ECS_USER }}
|
||
password: ${{ secrets.ECS_PASSWORD }}
|
||
command_timeout: 10m
|
||
script: |
|
||
echo "========== 当前磁盘 =========="; df -h / | tail -1
|
||
echo "========== 0) 装 pm2-logrotate 防再次爆盘(幂等) =========="
|
||
pm2 install pm2-logrotate 2>&1 | tail -2 || echo " (logrotate 安装跳过)"
|
||
pm2 set pm2-logrotate:max_size 50M 2>/dev/null
|
||
pm2 set pm2-logrotate:retain 3 2>/dev/null
|
||
pm2 set pm2-logrotate:compress true 2>/dev/null
|
||
echo " pm2-logrotate: max_size=50M retain=3 compress=on"
|
||
echo "========== 1) 用应用 driver ping MongoDB(8s 超时) =========="
|
||
cd /var/www/digitalme-api.invoratec.cn
|
||
PING=$(timeout 14 node -e '
|
||
const { MongoClient } = require("mongodb");
|
||
(async () => {
|
||
const c = new MongoClient("mongodb://localhost:27017", { serverSelectionTimeoutMS: 8000 });
|
||
try { await c.connect(); await c.db("admin").command({ ping: 1 }); console.log("MONGO_OK"); }
|
||
catch (e) { console.log("MONGO_FAIL " + e.message); }
|
||
finally { try { await c.close(); } catch (_) {} }
|
||
})();
|
||
' 2>&1 | grep -aE "MONGO_OK|MONGO_FAIL" | head -1)
|
||
echo " ping 结果: [$PING]"
|
||
if echo "$PING" | grep -q "MONGO_OK"; then
|
||
echo "========== 2) Mongo 健康 → 只重启本产品 API 清连接池 =========="
|
||
pm2 restart digitalme-api --update-env 2>/dev/null | tail -2
|
||
else
|
||
echo "========== 2) Mongo 卡死/无响应 → 重启本机 mongod =========="
|
||
if systemctl list-units --type=service 2>/dev/null | grep -q mongod; then
|
||
systemctl restart mongod 2>&1 | tail -3; sleep 8
|
||
systemctl is-active mongod 2>/dev/null && echo " mongod active" || echo " ⚠ mongod 未 active"
|
||
elif command -v docker >/dev/null && docker ps -a --format '{{.Names}}' 2>/dev/null | grep -qi mongo; then
|
||
MC=$(docker ps -a --format '{{.Names}}' | grep -i mongo | head -1); echo " docker restart $MC"; docker restart "$MC" 2>&1 | tail -2; sleep 8
|
||
else
|
||
echo " ❌ 未识别 mongod 服务(既非 systemd 也非 docker)，需人工处理"
|
||
fi
|
||
echo "========== 3) 重启本产品 API =========="
|
||
pm2 restart digitalme-api --update-env 2>/dev/null | tail -2
|
||
fi
|
||
sleep 8
|
||
echo "========== 4) 验证 =========="
|
||
curl -s -m 8 -o /dev/null -w " /health -> code=%{http_code} time=%{time_total}s\n" http://localhost:3603/health
|
||
curl -s -m 12 -o /dev/null -w " /api/plaza -> code=%{http_code} time=%{time_total}s\n" http://localhost:3603/api/plaza
|
||
curl -s -m 12 -o /dev/null -w " /api/masters-> code=%{http_code} time=%{time_total}s\n" http://localhost:3603/api/masters
|
||
echo "========== pm2 关键进程状态 =========="
|
||
pm2 list 2>/dev/null | grep -iE "digitalme-api|mongo" || true
|
||
echo "========== 最终磁盘 =========="; df -h / | tail -1
|
||
echo "========== 完成 =========="
|