#!/bin/bash
# ============================================================================
#  紧急预案（Lightsail 版）—— 网站打不开时按阶梯自动抢救
# ============================================================================
#
#  ── 和旧那版的区别 ────────────────────────────────────────────────────
#
#  旧机器是围着 Cloudflare 隧道转的（第 1 级重启隧道、第 3 级重施
#  MSS/MTU 调优）。新机器**有公网 IP、没有隧道**，中转跑在 nginx 后面，
#  所以阶梯完全重写。
#
#  ── 核心改进：先看是哪一层坏的 ────────────────────────────────────────
#
#  监测单元会传进来层次：
#
#      edge  = 中转本身是好的（127.0.0.1:13245 返回 200），
#              但用户走的路（Cloudflare -> nginx）不通
#              -> 该动 nginx / 查 DNS / 查 Cloudflare
#              -> **重启中转服务毫无用处**
#
#      core  = 中转本身也不通
#              -> 该重启中转
#
#  不区分层次的话就只能瞎重启 —— 而 edge 层的问题重启中转一万次也没用，
#  只会把报告淹掉。
#
#  ── 保留的核心设计 ────────────────────────────────────────────────────
#
#    1. **先记录现场，再动手**（重启会把证据冲掉）
#    2. **阶梯式，每级都验**：代价从最小到最大，好了就停
#    3. **全失败要留证据**：生成 txt 报告，可下载
#    4. **成功的招数记进知识库**：下次同类问题优先试
#
#  谁调用：monitor.sh
#  也可以人工跑：bash /opt/ykxmz-relay/rescue.sh "手工测试"
# ============================================================================

set -u

RELAY_DIR="${RELAY_DIR:-/opt/ykxmz-relay}"
DOMAIN="${DOMAIN:-yk.xxmxxs.top}"
PORT="${RELAY_PORT:-13245}"

PRIMARY_URL="${PRIMARY_URL:-https://$DOMAIN}"
FALLBACK_URL="${FALLBACK_URL:-http://127.0.0.1:$PORT}"

REPORTS_DIR="$RELAY_DIR/rescue_reports"
KNOWLEDGE="$RELAY_DIR/rescue_knowledge.md"
LOG="$RELAY_DIR/rescue.log"
LOCKFILE="/tmp/ykxmz-rescue.lock"

SETTLE_FAST=6
SETTLE_SLOW=12

# 是否允许"重启整机"这张底牌。默认关。
RESCUE_REBOOT="${RESCUE_REBOOT:-0}"

mkdir -p "$REPORTS_DIR"

if ! mkdir "$LOCKFILE" 2>/dev/null; then
    echo "[rescue] 已经有一个预案在跑，本次退出"
    exit 0
fi
trap 'rmdir "$LOCKFILE" 2>/dev/null' EXIT

STAMP="$(date +%Y%m%d-%H%M%S)"
REPORT="$REPORTS_DIR/$STAMP.txt"

say() {
    echo "$*"
    echo "$(date '+%F %T')  $*" >> "$LOG"
}

# 用户那层通了才算网站是好的
is_healthy() {
    local code
    code=$(curl -sL -o /dev/null -w '%{http_code}' --max-time 12 \
           "$PRIMARY_URL/v1/health" 2>/dev/null)
    [ "$code" = "200" ]
}

# 中转本身
core_ok() {
    local code
    code=$(curl -s -o /dev/null -w '%{http_code}' --max-time 6 \
           "$FALLBACK_URL/v1/health" 2>/dev/null)
    [ "$code" = "200" ]
}

# nginx 本机（带 SNI 走 443）
nginx_ok() {
    local code
    code=$(curl -sk -o /dev/null -w '%{http_code}' --max-time 8 \
           --resolve "$DOMAIN:443:127.0.0.1" \
           "https://$DOMAIN/v1/health" 2>/dev/null)
    [ "$code" = "200" ]
}

# ===========================================================================
#  现场记录
# ===========================================================================
collect() {
    {
        echo "================================================================================"
        echo "  ykxmz 紧急预案诊断报告"
        echo "  时间：$(date '+%F %T %Z')"
        echo "  主机：$(hostname)   运行时长：$(uptime -p 2>/dev/null || uptime)"
        echo "  触发原因：$1"
        echo "  层次判断：${LAYER:-未知}"
        echo "================================================================================"
        echo
        echo "---- 一、结论速览 ----"
        echo
        echo "  用户层  $PRIMARY_URL/v1/health  ->  $(is_healthy && echo '200 正常' || echo 不通)"
        echo "  中转层  $FALLBACK_URL/v1/health  ->  $(core_ok && echo '200 正常' || echo 不通)"
        echo "  nginx  https://$DOMAIN (本机带 SNI)  ->  $(nginx_ok && echo '200 正常' || echo 不通)"
        echo
        case "${LAYER:-}" in
            edge)
                echo "  判断：**中转是好的，问题在边缘层**（Cloudflare / DNS / nginx）。"
                echo "        要动的是 nginx，不是中转。"
                echo "        常见原因："
                echo "          * Lightsail 防火墙没开 443（Cloudflare Full 模式回源要用）"
                echo "          * Cloudflare 的 SSL/TLS 模式和源站对不上"
                echo "          * DNS 记录被改成了灰色云朵，或者指向了别的 IP"
                ;;
            core)
                echo "  判断：**中转服务本身不通**，先看第二、三节。"
                ;;
            *)
                echo "  判断：两层都通（可能已自愈），或者只是偶发抖动。"
                ;;
        esac
        echo
        echo "---- 二、服务状态 ----"
        echo
        for s in ykxmz-relay nginx ykxmz-monitor.timer certbot.timer; do
            printf '  %-24s %s\n' "$s" "$(systemctl is-active $s 2>&1)"
        done
        echo
        echo "  --- 中转最近 15 行日志 ---"
        journalctl -u ykxmz-relay --no-pager -n 15 2>/dev/null | sed 's/^/    /'
        echo
        echo "  --- nginx 最近 15 行错误日志 ---"
        tail -15 /var/log/nginx/ykxmz-error.log 2>/dev/null | sed 's/^/    /'
        echo
        echo "---- 三、进程与端口 ----"
        echo
        echo "  监听情况："
        ss -tlnp 2>/dev/null | grep -E ':(80|443|'"$PORT"') ' | sed 's/^/    /'
        echo
        echo "  相关进程："
        ps -eo pid,rss,etime,cmd --sort=-rss 2>/dev/null \
            | grep -E 'relay\.py|nginx' | grep -v grep | head -6 | sed 's/^/    /'
        echo
        echo "---- 四、本机各层自测 ----"
        echo
        printf '    %-34s %s\n' "中转 127.0.0.1:$PORT" \
            "$(curl -s -o /dev/null -w '%{http_code}' --max-time 6 http://127.0.0.1:$PORT/v1/health)"
        printf '    %-34s %s\n' "nginx :80（应 301）" \
            "$(curl -s -o /dev/null -w '%{http_code}' --max-time 6 http://127.0.0.1:80/v1/health)"
        printf '    %-34s %s\n' "nginx :443（带 SNI）" \
            "$(curl -sk -o /dev/null -w '%{http_code}' --max-time 8 --resolve "$DOMAIN:443:127.0.0.1" https://$DOMAIN/v1/health)"
        printf '    %-34s %s\n' "用户层 $PRIMARY_URL" \
            "$(curl -sL -o /dev/null -w '%{http_code}' --max-time 12 $PRIMARY_URL/v1/health)"
        echo
        echo "---- 五、证书 ----"
        echo
        certbot certificates 2>/dev/null | grep -E 'Certificate Name|Domains|Expiry' \
            | sed 's/^/  /'
        echo
        echo "---- 六、系统资源 ----"
        echo
        free -m 2>/dev/null | sed 's/^/  /'
        echo
        uptime | sed 's/^/  /'
        echo
        df -h / 2>/dev/null | sed 's/^/  /'
        echo
        echo "---- 七、网络 ----"
        echo
        ip -brief addr show 2>/dev/null | sed 's/^/  /'
        echo
        echo "  默认路由："
        ip route show default 2>/dev/null | sed 's/^/    /'
        echo
        echo "  公网 IP：$(curl -s --max-time 8 https://api.ipify.org 2>/dev/null || echo 取不到)"
        echo
        echo "  DNS 解析 $DOMAIN："
        getent hosts "$DOMAIN" 2>/dev/null | sed 's/^/    /' || echo "    解析失败"
        echo
        echo "---- 八、内核异常（OOM / 硬件）----"
        echo
        dmesg -T 2>/dev/null | grep -i -E 'oom|killed process|panic' \
            | tail -10 | sed 's/^/  /'
        echo
        echo "---- 九、最近一次启动 ----"
        echo
        who -b 2>/dev/null | sed 's/^/  /'
        echo "  运行时长：$(cut -d. -f1 /proc/uptime 2>/dev/null) 秒"
        echo
        echo "================================================================================"
        echo "  报告结束。可以在中转站后台的「抢救报告」分区下载。"
        echo "================================================================================"
    } > "$REPORT" 2>&1
    say "  现场已记录 -> $REPORT"
}

# ===========================================================================
#  阶梯 —— 按层次走
# ===========================================================================

step_edge_1_reload_nginx() {
    say "  [edge-1] 重载 nginx 配置（最轻，不断连接）"
    sudo nginx -t >/dev/null 2>&1 && sudo systemctl reload nginx 2>/dev/null
    sleep "$SETTLE_FAST"
}

step_edge_2_restart_nginx() {
    say "  [edge-2] 重启 nginx（重载不够时用）"
    sudo systemctl restart nginx 2>/dev/null
    sleep "$SETTLE_FAST"
}

step_edge_3_check_443() {
    say "  [edge-3] 查公网 443 通不通（Cloudflare Full 模式回源要用）"
    if timeout 5 bash -c "</dev/tcp/$(curl -s --max-time 6 https://api.ipify.org 2>/dev/null)/443" 2>/dev/null; then
        say "      本机自连 443 通"
    else
        say "      ⚠ 本机自连 443 **不通** —— nginx 可能没在 443 上"
        ss -tln 2>/dev/null | grep ':443 ' | sed 's/^/        /'
    fi
    say "      ⚠ 提醒：**Lightsail 网页上的防火墙规则**系统内部改不了。"
    say "        如果这里一切正常但公网还是不通，去网页上确认 443 已放行。"
    # 顺手确认证书还在
    if [ -f "/etc/letsencrypt/live/$DOMAIN/fullchain.pem" ]; then
        say "      证书存在，到期日：$(openssl x509 -in "/etc/letsencrypt/live/$DOMAIN/fullchain.pem" -noout -enddate 2>/dev/null | cut -d= -f2)"
    else
        say "      ⚠ **找不到证书**！/etc/letsencrypt/live/$DOMAIN/fullchain.pem"
    fi
    sleep "$SETTLE_SLOW"
}

step_core_1_restart_relay() {
    say "  [core-1] 重启中转服务"
    sudo systemctl restart ykxmz-relay 2>/dev/null
    sleep "$SETTLE_FAST"
}

step_core_2_kill_and_start() {
    say "  [core-2] 强杀残留进程再拉起（进程卡死/端口被占时用）"
    sudo pkill -f "$RELAY_DIR/relay.py" 2>/dev/null
    sleep 3
    sudo systemctl reset-failed ykxmz-relay 2>/dev/null
    sudo systemctl start ykxmz-relay 2>/dev/null
    sleep "$SETTLE_FAST"
}

step_core_3_restart_all() {
    say "  [core-3] 中转 + nginx 都重启一遍"
    sudo systemctl restart ykxmz-relay 2>/dev/null
    sleep 2
    sudo systemctl restart nginx 2>/dev/null
    sleep "$SETTLE_SLOW"
}

step_4_reboot() {
    if [ "$RESCUE_REBOOT" != "1" ]; then
        say "  [4] 重启整机 —— **已跳过**（默认不开）"
        say "      要允许自动重启，把 RESCUE_REBOOT=1 写进 ykxmz-monitor.service。"
        return 1
    fi
    say "  [4] 重启整机（最后的底牌）"
    sync
    sudo systemctl reboot 2>/dev/null
    return 0
}

remember() {
    local step="$1" reason="$2"
    {
        echo
        echo "## $(date '+%F %T')  第 $step 级奏效"
        echo
        echo "- 症状：$reason"
        echo "- 层次：${LAYER:-未知}"
        echo "- 有效步骤：$step"
        echo "- 当时用户层：$(is_healthy && echo 通 || echo 不通)；中转层：$(core_ok && echo 通 || echo 不通)"
        echo "- 详情见 rescue_reports/$(basename "$REPORT")"
    } >> "$KNOWLEDGE"
    say "  已把「第 $step 级奏效」记进 $KNOWLEDGE"
}

# ===========================================================================
#  主流程
# ===========================================================================

REASON="${1:-monitor 触发}"

# 从触发原因里认出层次（monitor.sh 会传"层次 edge"这样的字样）
LAYER=""
case "$REASON" in
    *"层次 edge"*) LAYER="edge" ;;
    *"层次 core"*) LAYER="core" ;;
esac
# 认不出来就自己判
if [ -z "$LAYER" ]; then
    if ! core_ok; then LAYER="core"
    elif ! is_healthy; then LAYER="edge"
    fi
fi

say ""
say "================================================================================"
say "  紧急预案启动：$REASON"
say "  层次判断：${LAYER:-两层都通}"
say "================================================================================"

# ⚠ 先复查再记录现场。
#   collect() 会写一份报告文件；如果网站其实已经好了，
#   那一次也会留下报告，把真正出事时的报告淹掉（后台只显示最近 30 份）。
if is_healthy; then
    say "  复查：用户层现在是通的，不需要动手 —— 不采现场、不留报告。"
    exit 0
fi

collect "$REASON"

if [ "$LAYER" = "edge" ]; then
    STEPS=("重载 nginx" "重启 nginx" "查 443 与证书")
    FUNCS=(step_edge_1_reload_nginx step_edge_2_restart_nginx step_edge_3_check_443)
else
    STEPS=("重启中转服务" "强杀残留再拉起" "中转+nginx 都重启")
    FUNCS=(step_core_1_restart_relay step_core_2_kill_and_start step_core_3_restart_all)
fi

for i in 0 1 2; do
    ${FUNCS[$i]}

    if is_healthy; then
        say ""
        say "  ✅ 第 $((i+1)) 级（${STEPS[$i]}）救回来了，用户层已恢复。"
        remember "$((i+1))" "$REASON"
        exit 0
    fi
    say "      第 $((i+1)) 级没解决，继续下一级"
done

# 兜底：把两边都试一遍（有时候层次判错了）
say ""
say "  三轮都没解决，兜底：中转和 nginx 各重启一次"
sudo systemctl restart ykxmz-relay 2>/dev/null
sleep 3
sudo systemctl restart nginx 2>/dev/null
sleep "$SETTLE_SLOW"
if is_healthy; then
    say "  ✅ 兜底重启救回来了"
    remember "兜底" "$REASON"
    exit 0
fi

say ""
say "  ❌ 全部没解决。再采一次现场（状态已经变了）"
collect "$REASON（多轮抢救后仍不通）"

{
    echo "⚠ 这份报告是「自动抢救全部失败」之后生成的。"
    echo "  机器自己试过了：重载/重启 nginx、重启中转、强杀重拉。"
    echo "  （重启整机默认没做。）"
    echo "  需要人工看一眼 —— 特别是 **Lightsail 网页上的防火墙规则**，"
    echo "  那一层系统内部改不了。"
    echo
    cat "$REPORT"
} > "$REPORT.tmp" && mv "$REPORT.tmp" "$REPORT"

ls -1t "$REPORTS_DIR"/*.txt 2>/dev/null | tail -n +31 | xargs -r rm -f

say "  报告已生成：$REPORT"
exit 1