mirror of
https://gitee.com/xmz7891/ykxmz.git
synced 2026-10-18 10:10:43 +00:00
xmz7891
47e08af151
== 为什么重写 ==
certbot 签证书时加了 http -> https 的 301，而监测单元原来判断
"curl 拿到 200 才算健康"、且 curl **默认不跟跳转** -> 拿到 301
-> 判成"网站挂了" -> 又要启动预案 -> 重启中转 -> 再查还是 301。
这是**第二次**踩同一类坑了（上一次是改监听地址忘了改探测地址）。
共同教训：**改了入口的行为，必须回头看所有探测入口的地方。**
== 新设计：分两层探测 ==
PRIMARY https://yk.xxmxxs.top/v1/health 用户实际走的路
FALLBACK http://127.0.0.1:13245/v1/health 中转本身
两个都查才能分清哪一层坏了，进而选对抢救动作：
PRIMARY 挂 + FALLBACK 好 -> edge 层（CF / DNS / nginx）
两个都挂 -> core 层（中转服务本身）
预案据此走不同的阶梯：
edge: 重载 nginx -> 重启 nginx -> 查 443 与证书
core: 重启中转 -> 强杀重拉 -> 中转+nginx 都重启
不区分层次的话只能瞎重启，而 edge 层的问题重启中转一万次也没用。
== 做法上的改进：不再远端打补丁 ==
前几轮一直是"SSH 过去用嵌套 heredoc 现场改 monitor.sh"，
连续踩三次：PowerShell 在本机求值 \、引号被吃、
脚本里的 %{http_code} 和 Python 的 %% 格式化打架。
现在改成：relay/monitor-lightsail.sh 和 relay/rescue-lightsail.sh
是**唯一事实来源**，本地编辑 -> 整份上传 -> 重装，
改了什么一目了然，也能进 git。
== 顺带 ==
因为 Lightsail 防火墙还没放行 443，PRIMARY 一直是 000，
监测会把 fails 一直加上去、白生成抢救报告。
所以暂时把 monitor.timer 停掉，等 443 开了再启用
（脚本 _build/pause_monitor.py，恢复是一条 systemctl start）。
380 lines
14 KiB
Bash
380 lines
14 KiB
Bash
#!/bin/bash
|
||
# ============================================================================
|
||
# 紧急预案（Lightsail 版）—— 网站打不开时按阶梯自动抢救
|
||
# ============================================================================
|
||
#
|
||
# ── 和旧那版的区别 ────────────────────────────────────────────────────
|
||
#
|
||
# 旧机器是围着 Cloudflare 隧道转的（第 1 级重启隧道、第 3 级重施
|
||
# MSS/MTU 调优）。新机器**有公网 IP、没有隧道**，中转跑在 nginx 后面，
|
||
# 所以阶梯完全重写。
|
||
#
|
||
# ── 核心改进：先看是哪一层坏的 ────────────────────────────────────────
|
||
#
|
||
# 监测单元会传进来层次：
|
||
#
|
||
# edge = 中转本身是好的（127.0.0.1:13245 返回 200），
|
||
# 但用户走的路（Cloudflare -> nginx）不通
|
||
# -> 该动 nginx / 查 DNS / 查 Cloudflare
|
||
# -> **重启中转服务毫无用处**
|
||
#
|
||
# core = 中转本身也不通
|
||
# -> 该重启中转
|
||
#
|
||
# 不区分层次的话就只能瞎重启 —— 而 edge 层的问题重启中转一万次也没用，
|
||
# 只会把报告淹掉。
|
||
#
|
||
# ── 保留的核心设计 ────────────────────────────────────────────────────
|
||
#
|
||
# 1. **先记录现场，再动手**（重启会把证据冲掉）
|
||
# 2. **阶梯式，每级都验**：代价从最小到最大，好了就停
|
||
# 3. **全失败要留证据**：生成 txt 报告，可下载
|
||
# 4. **成功的招数记进知识库**：下次同类问题优先试
|
||
#
|
||
# 谁调用：monitor.sh
|
||
# 也可以人工跑：bash /opt/ykxmz-relay/rescue.sh "手工测试"
|
||
# ============================================================================
|
||
|
||
set -u
|
||
|
||
RELAY_DIR="${RELAY_DIR:-/opt/ykxmz-relay}"
|
||
DOMAIN="${DOMAIN:-yk.xxmxxs.top}"
|
||
PORT="${RELAY_PORT:-13245}"
|
||
|
||
PRIMARY_URL="${PRIMARY_URL:-https://$DOMAIN}"
|
||
FALLBACK_URL="${FALLBACK_URL:-http://127.0.0.1:$PORT}"
|
||
|
||
REPORTS_DIR="$RELAY_DIR/rescue_reports"
|
||
KNOWLEDGE="$RELAY_DIR/rescue_knowledge.md"
|
||
LOG="$RELAY_DIR/rescue.log"
|
||
LOCKFILE="/tmp/ykxmz-rescue.lock"
|
||
|
||
SETTLE_FAST=6
|
||
SETTLE_SLOW=12
|
||
|
||
# 是否允许"重启整机"这张底牌。默认关。
|
||
RESCUE_REBOOT="${RESCUE_REBOOT:-0}"
|
||
|
||
mkdir -p "$REPORTS_DIR"
|
||
|
||
if ! mkdir "$LOCKFILE" 2>/dev/null; then
|
||
echo "[rescue] 已经有一个预案在跑，本次退出"
|
||
exit 0
|
||
fi
|
||
trap 'rmdir "$LOCKFILE" 2>/dev/null' EXIT
|
||
|
||
STAMP="$(date +%Y%m%d-%H%M%S)"
|
||
REPORT="$REPORTS_DIR/$STAMP.txt"
|
||
|
||
say() {
|
||
echo "$*"
|
||
echo "$(date '+%F %T') $*" >> "$LOG"
|
||
}
|
||
|
||
# 用户那层通了才算网站是好的
|
||
is_healthy() {
|
||
local code
|
||
code=$(curl -sL -o /dev/null -w '%{http_code}' --max-time 12 \
|
||
"$PRIMARY_URL/v1/health" 2>/dev/null)
|
||
[ "$code" = "200" ]
|
||
}
|
||
|
||
# 中转本身
|
||
core_ok() {
|
||
local code
|
||
code=$(curl -s -o /dev/null -w '%{http_code}' --max-time 6 \
|
||
"$FALLBACK_URL/v1/health" 2>/dev/null)
|
||
[ "$code" = "200" ]
|
||
}
|
||
|
||
# nginx 本机（带 SNI 走 443）
|
||
nginx_ok() {
|
||
local code
|
||
code=$(curl -sk -o /dev/null -w '%{http_code}' --max-time 8 \
|
||
--resolve "$DOMAIN:443:127.0.0.1" \
|
||
"https://$DOMAIN/v1/health" 2>/dev/null)
|
||
[ "$code" = "200" ]
|
||
}
|
||
|
||
# ===========================================================================
|
||
# 现场记录
|
||
# ===========================================================================
|
||
collect() {
|
||
{
|
||
echo "================================================================================"
|
||
echo " ykxmz 紧急预案诊断报告"
|
||
echo " 时间：$(date '+%F %T %Z')"
|
||
echo " 主机：$(hostname) 运行时长：$(uptime -p 2>/dev/null || uptime)"
|
||
echo " 触发原因：$1"
|
||
echo " 层次判断：${LAYER:-未知}"
|
||
echo "================================================================================"
|
||
echo
|
||
echo "---- 一、结论速览 ----"
|
||
echo
|
||
echo " 用户层 $PRIMARY_URL/v1/health -> $(is_healthy && echo '200 正常' || echo 不通)"
|
||
echo " 中转层 $FALLBACK_URL/v1/health -> $(core_ok && echo '200 正常' || echo 不通)"
|
||
echo " nginx https://$DOMAIN (本机带 SNI) -> $(nginx_ok && echo '200 正常' || echo 不通)"
|
||
echo
|
||
case "${LAYER:-}" in
|
||
edge)
|
||
echo " 判断：**中转是好的，问题在边缘层**（Cloudflare / DNS / nginx）。"
|
||
echo " 要动的是 nginx，不是中转。"
|
||
echo " 常见原因："
|
||
echo " * Lightsail 防火墙没开 443（Cloudflare Full 模式回源要用）"
|
||
echo " * Cloudflare 的 SSL/TLS 模式和源站对不上"
|
||
echo " * DNS 记录被改成了灰色云朵，或者指向了别的 IP"
|
||
;;
|
||
core)
|
||
echo " 判断：**中转服务本身不通**，先看第二、三节。"
|
||
;;
|
||
*)
|
||
echo " 判断：两层都通（可能已自愈），或者只是偶发抖动。"
|
||
;;
|
||
esac
|
||
echo
|
||
echo "---- 二、服务状态 ----"
|
||
echo
|
||
for s in ykxmz-relay nginx ykxmz-monitor.timer certbot.timer; do
|
||
printf ' %-24s %s\n' "$s" "$(systemctl is-active $s 2>&1)"
|
||
done
|
||
echo
|
||
echo " --- 中转最近 15 行日志 ---"
|
||
journalctl -u ykxmz-relay --no-pager -n 15 2>/dev/null | sed 's/^/ /'
|
||
echo
|
||
echo " --- nginx 最近 15 行错误日志 ---"
|
||
tail -15 /var/log/nginx/ykxmz-error.log 2>/dev/null | sed 's/^/ /'
|
||
echo
|
||
echo "---- 三、进程与端口 ----"
|
||
echo
|
||
echo " 监听情况："
|
||
ss -tlnp 2>/dev/null | grep -E ':(80|443|'"$PORT"') ' | sed 's/^/ /'
|
||
echo
|
||
echo " 相关进程："
|
||
ps -eo pid,rss,etime,cmd --sort=-rss 2>/dev/null \
|
||
| grep -E 'relay\.py|nginx' | grep -v grep | head -6 | sed 's/^/ /'
|
||
echo
|
||
echo "---- 四、本机各层自测 ----"
|
||
echo
|
||
printf ' %-34s %s\n' "中转 127.0.0.1:$PORT" \
|
||
"$(curl -s -o /dev/null -w '%{http_code}' --max-time 6 http://127.0.0.1:$PORT/v1/health)"
|
||
printf ' %-34s %s\n' "nginx :80（应 301）" \
|
||
"$(curl -s -o /dev/null -w '%{http_code}' --max-time 6 http://127.0.0.1:80/v1/health)"
|
||
printf ' %-34s %s\n' "nginx :443（带 SNI）" \
|
||
"$(curl -sk -o /dev/null -w '%{http_code}' --max-time 8 --resolve "$DOMAIN:443:127.0.0.1" https://$DOMAIN/v1/health)"
|
||
printf ' %-34s %s\n' "用户层 $PRIMARY_URL" \
|
||
"$(curl -sL -o /dev/null -w '%{http_code}' --max-time 12 $PRIMARY_URL/v1/health)"
|
||
echo
|
||
echo "---- 五、证书 ----"
|
||
echo
|
||
certbot certificates 2>/dev/null | grep -E 'Certificate Name|Domains|Expiry' \
|
||
| sed 's/^/ /'
|
||
echo
|
||
echo "---- 六、系统资源 ----"
|
||
echo
|
||
free -m 2>/dev/null | sed 's/^/ /'
|
||
echo
|
||
uptime | sed 's/^/ /'
|
||
echo
|
||
df -h / 2>/dev/null | sed 's/^/ /'
|
||
echo
|
||
echo "---- 七、网络 ----"
|
||
echo
|
||
ip -brief addr show 2>/dev/null | sed 's/^/ /'
|
||
echo
|
||
echo " 默认路由："
|
||
ip route show default 2>/dev/null | sed 's/^/ /'
|
||
echo
|
||
echo " 公网 IP：$(curl -s --max-time 8 https://api.ipify.org 2>/dev/null || echo 取不到)"
|
||
echo
|
||
echo " DNS 解析 $DOMAIN："
|
||
getent hosts "$DOMAIN" 2>/dev/null | sed 's/^/ /' || echo " 解析失败"
|
||
echo
|
||
echo "---- 八、内核异常（OOM / 硬件）----"
|
||
echo
|
||
dmesg -T 2>/dev/null | grep -i -E 'oom|killed process|panic' \
|
||
| tail -10 | sed 's/^/ /'
|
||
echo
|
||
echo "---- 九、最近一次启动 ----"
|
||
echo
|
||
who -b 2>/dev/null | sed 's/^/ /'
|
||
echo " 运行时长：$(cut -d. -f1 /proc/uptime 2>/dev/null) 秒"
|
||
echo
|
||
echo "================================================================================"
|
||
echo " 报告结束。可以在中转站后台的「抢救报告」分区下载。"
|
||
echo "================================================================================"
|
||
} > "$REPORT" 2>&1
|
||
say " 现场已记录 -> $REPORT"
|
||
}
|
||
|
||
# ===========================================================================
|
||
# 阶梯 —— 按层次走
|
||
# ===========================================================================
|
||
|
||
step_edge_1_reload_nginx() {
|
||
say " [edge-1] 重载 nginx 配置（最轻，不断连接）"
|
||
sudo nginx -t >/dev/null 2>&1 && sudo systemctl reload nginx 2>/dev/null
|
||
sleep "$SETTLE_FAST"
|
||
}
|
||
|
||
step_edge_2_restart_nginx() {
|
||
say " [edge-2] 重启 nginx（重载不够时用）"
|
||
sudo systemctl restart nginx 2>/dev/null
|
||
sleep "$SETTLE_FAST"
|
||
}
|
||
|
||
step_edge_3_check_443() {
|
||
say " [edge-3] 查公网 443 通不通（Cloudflare Full 模式回源要用）"
|
||
if timeout 5 bash -c "</dev/tcp/$(curl -s --max-time 6 https://api.ipify.org 2>/dev/null)/443" 2>/dev/null; then
|
||
say " 本机自连 443 通"
|
||
else
|
||
say " ⚠ 本机自连 443 **不通** —— nginx 可能没在 443 上"
|
||
ss -tln 2>/dev/null | grep ':443 ' | sed 's/^/ /'
|
||
fi
|
||
say " ⚠ 提醒：**Lightsail 网页上的防火墙规则**系统内部改不了。"
|
||
say " 如果这里一切正常但公网还是不通，去网页上确认 443 已放行。"
|
||
# 顺手确认证书还在
|
||
if [ -f "/etc/letsencrypt/live/$DOMAIN/fullchain.pem" ]; then
|
||
say " 证书存在，到期日：$(openssl x509 -in "/etc/letsencrypt/live/$DOMAIN/fullchain.pem" -noout -enddate 2>/dev/null | cut -d= -f2)"
|
||
else
|
||
say " ⚠ **找不到证书**！/etc/letsencrypt/live/$DOMAIN/fullchain.pem"
|
||
fi
|
||
sleep "$SETTLE_SLOW"
|
||
}
|
||
|
||
step_core_1_restart_relay() {
|
||
say " [core-1] 重启中转服务"
|
||
sudo systemctl restart ykxmz-relay 2>/dev/null
|
||
sleep "$SETTLE_FAST"
|
||
}
|
||
|
||
step_core_2_kill_and_start() {
|
||
say " [core-2] 强杀残留进程再拉起（进程卡死/端口被占时用）"
|
||
sudo pkill -f "$RELAY_DIR/relay.py" 2>/dev/null
|
||
sleep 3
|
||
sudo systemctl reset-failed ykxmz-relay 2>/dev/null
|
||
sudo systemctl start ykxmz-relay 2>/dev/null
|
||
sleep "$SETTLE_FAST"
|
||
}
|
||
|
||
step_core_3_restart_all() {
|
||
say " [core-3] 中转 + nginx 都重启一遍"
|
||
sudo systemctl restart ykxmz-relay 2>/dev/null
|
||
sleep 2
|
||
sudo systemctl restart nginx 2>/dev/null
|
||
sleep "$SETTLE_SLOW"
|
||
}
|
||
|
||
step_4_reboot() {
|
||
if [ "$RESCUE_REBOOT" != "1" ]; then
|
||
say " [4] 重启整机 —— **已跳过**（默认不开）"
|
||
say " 要允许自动重启，把 RESCUE_REBOOT=1 写进 ykxmz-monitor.service。"
|
||
return 1
|
||
fi
|
||
say " [4] 重启整机（最后的底牌）"
|
||
sync
|
||
sudo systemctl reboot 2>/dev/null
|
||
return 0
|
||
}
|
||
|
||
remember() {
|
||
local step="$1" reason="$2"
|
||
{
|
||
echo
|
||
echo "## $(date '+%F %T') 第 $step 级奏效"
|
||
echo
|
||
echo "- 症状：$reason"
|
||
echo "- 层次：${LAYER:-未知}"
|
||
echo "- 有效步骤：$step"
|
||
echo "- 当时用户层：$(is_healthy && echo 通 || echo 不通)；中转层：$(core_ok && echo 通 || echo 不通)"
|
||
echo "- 详情见 rescue_reports/$(basename "$REPORT")"
|
||
} >> "$KNOWLEDGE"
|
||
say " 已把「第 $step 级奏效」记进 $KNOWLEDGE"
|
||
}
|
||
|
||
# ===========================================================================
|
||
# 主流程
|
||
# ===========================================================================
|
||
|
||
REASON="${1:-monitor 触发}"
|
||
|
||
# 从触发原因里认出层次（monitor.sh 会传"层次 edge"这样的字样）
|
||
LAYER=""
|
||
case "$REASON" in
|
||
*"层次 edge"*) LAYER="edge" ;;
|
||
*"层次 core"*) LAYER="core" ;;
|
||
esac
|
||
# 认不出来就自己判
|
||
if [ -z "$LAYER" ]; then
|
||
if ! core_ok; then LAYER="core"
|
||
elif ! is_healthy; then LAYER="edge"
|
||
fi
|
||
fi
|
||
|
||
say ""
|
||
say "================================================================================"
|
||
say " 紧急预案启动：$REASON"
|
||
say " 层次判断：${LAYER:-两层都通}"
|
||
say "================================================================================"
|
||
|
||
# ⚠ 先复查再记录现场。
|
||
# collect() 会写一份报告文件；如果网站其实已经好了，
|
||
# 那一次也会留下报告，把真正出事时的报告淹掉（后台只显示最近 30 份）。
|
||
if is_healthy; then
|
||
say " 复查：用户层现在是通的，不需要动手 —— 不采现场、不留报告。"
|
||
exit 0
|
||
fi
|
||
|
||
collect "$REASON"
|
||
|
||
if [ "$LAYER" = "edge" ]; then
|
||
STEPS=("重载 nginx" "重启 nginx" "查 443 与证书")
|
||
FUNCS=(step_edge_1_reload_nginx step_edge_2_restart_nginx step_edge_3_check_443)
|
||
else
|
||
STEPS=("重启中转服务" "强杀残留再拉起" "中转+nginx 都重启")
|
||
FUNCS=(step_core_1_restart_relay step_core_2_kill_and_start step_core_3_restart_all)
|
||
fi
|
||
|
||
for i in 0 1 2; do
|
||
${FUNCS[$i]}
|
||
|
||
if is_healthy; then
|
||
say ""
|
||
say " ✅ 第 $((i+1)) 级（${STEPS[$i]}）救回来了，用户层已恢复。"
|
||
remember "$((i+1))" "$REASON"
|
||
exit 0
|
||
fi
|
||
say " 第 $((i+1)) 级没解决，继续下一级"
|
||
done
|
||
|
||
# 兜底：把两边都试一遍（有时候层次判错了）
|
||
say ""
|
||
say " 三轮都没解决，兜底：中转和 nginx 各重启一次"
|
||
sudo systemctl restart ykxmz-relay 2>/dev/null
|
||
sleep 3
|
||
sudo systemctl restart nginx 2>/dev/null
|
||
sleep "$SETTLE_SLOW"
|
||
if is_healthy; then
|
||
say " ✅ 兜底重启救回来了"
|
||
remember "兜底" "$REASON"
|
||
exit 0
|
||
fi
|
||
|
||
say ""
|
||
say " ❌ 全部没解决。再采一次现场（状态已经变了）"
|
||
collect "$REASON（多轮抢救后仍不通）"
|
||
|
||
{
|
||
echo "⚠ 这份报告是「自动抢救全部失败」之后生成的。"
|
||
echo " 机器自己试过了：重载/重启 nginx、重启中转、强杀重拉。"
|
||
echo " （重启整机默认没做。）"
|
||
echo " 需要人工看一眼 —— 特别是 **Lightsail 网页上的防火墙规则**，"
|
||
echo " 那一层系统内部改不了。"
|
||
echo
|
||
cat "$REPORT"
|
||
} > "$REPORT.tmp" && mv "$REPORT.tmp" "$REPORT"
|
||
|
||
ls -1t "$REPORTS_DIR"/*.txt 2>/dev/null | tail -n +31 | xargs -r rm -f
|
||
|
||
say " 报告已生成：$REPORT"
|
||
exit 1
|