mirror of
https://gitee.com/xmz7891/ykxmz.git
synced 2026-10-18 13:40:45 +00:00
xmz7891
0ad3f648c6
上一个提交把仓库里的服务器 IP 换成了占位符，但**那些脚本本身要用这个 IP
去连服务器** —— 换掉之后它们就跑不起来了。这个提交把取值方式理清楚。
== 取值统一走 _build/lssh.py ==
YKXMZ_LS_HOST 环境变量
或 _build/.lightsail_host（一行 IP，已 gitignore）
都没有就报错，并告诉你怎么设
十个运维脚本改成 `from lssh import ... HOST as LS_HOST`，
公开仓库里一个具体地址都不留。
== 顺带修的两个自己造的 bug ==
* 用正则往 `from lssh import ...` 后面追加东西时，
追加到了行尾的 `# noqa: E402` **注释里面** ——
十处导入全是废的，脚本一跑就 NameError。
改成先剥掉注释、拼好导入、再把 noqa 加回去。
* 同一个正则还把 `from lssh import HOST as LS_HOST` 加进了
**lssh.py 自己**，造成自我导入。
教训：**正则批量改导入语句很危险**（注释、行尾、顺序都能坑到），
这种地方宁可逐个 edit。
== 为什么要重建产物 ==
docs/AFTER-INSTALL.txt 会被打进服务端 zip，而它上一轮改过
（去掉了写死的旧管理员密码）—— 不重打的话发出去的还是旧的。
重打后确认：三个产物里都没有旧密码、没有服务器 IP。
== 验证 ==
全量回归退出码 0，41 个阶段全 PASS
603 个 Python 文件语法通过
ykxmz-agent / server / watch 三个产物扫描干净
提交前泄漏闸门通过
286 lines
9.6 KiB
Python
286 lines
9.6 KiB
Python
# -*- coding: utf-8 -*-
|
||
r'''在 Lightsail 上装好「监测单元 + 紧急预案」。
|
||
|
||
== 部署什么 ==
|
||
|
||
/opt/ykxmz-relay/monitor.sh 每 60 秒查一次公网
|
||
/opt/ykxmz-relay/rescue-lightsail.sh 四级阶梯抢救
|
||
/etc/systemd/system/ykxmz-monitor.service
|
||
/etc/systemd/system/ykxmz-monitor.timer
|
||
|
||
== 和随身 WiFi 那版的区别 ==
|
||
|
||
那版围着 Cloudflare 隧道转：查隧道连接数、重启隧道、重施 MSS/MTU。
|
||
新机器**有公网 IP、没有隧道**，所以：
|
||
|
||
* 健康检查超时从 12 秒降到 8 秒（直连，不需要等隧道）
|
||
* 预案第 1 级改成"重启中转服务"（不再有隧道可重启）
|
||
* 去掉 MTU/MSS 那两级（那是 4G 线路的问题）
|
||
|
||
== 为什么还是"每 60 秒 + 连续 3 次才动手" ==
|
||
|
||
直连比隧道稳得多，但**公网抖动**依然存在（上游线路、Lightsail 边缘）。
|
||
一次失败就救会天天瞎折腾，所以保留防抖。
|
||
现在预期是**基本不会触发** —— 触发了就说明真出事了。
|
||
'''
|
||
from __future__ import annotations
|
||
|
||
import sys
|
||
import tempfile
|
||
from pathlib import Path
|
||
|
||
ROOT = Path(__file__).resolve().parent.parent
|
||
sys.path.insert(0, str(ROOT / "_build"))
|
||
from lssh import run, upload, HOST as LS_HOST # noqa: E402
|
||
|
||
RELAY_DIR = "/opt/ykxmz-relay"
|
||
PORT = "13245"
|
||
PUBLIC_URL = "http://%s:%s" % (LS_HOST, PORT)
|
||
|
||
MONITOR = r'''#!/bin/bash
|
||
# ============================================================================
|
||
# 网站监测单元（Lightsail 版）
|
||
# ============================================================================
|
||
#
|
||
# 每 60 秒从**公网**查一次，连续 3 次不通就叫紧急预案。
|
||
#
|
||
# 为什么不查 127.0.0.1：
|
||
# 本机 200 只说明进程活着。用户真正关心的是"打开
|
||
# http://LS_HOST:13245 能不能出页面" —— 那要经过
|
||
# Lightsail 的防火墙 / 安全组 / 上游线路。实测踩过：
|
||
# 隧道两条实例都掉了，本机 curl 照样 200，但公网 502。
|
||
#
|
||
# 为什么用 timer 不用常驻进程：
|
||
# 崩了下一分钟自己就回来，不会有"跑久了状态变脏"的问题。
|
||
# ============================================================================
|
||
|
||
set -u
|
||
|
||
RELAY_DIR="${RELAY_DIR:-/opt/ykxmz-relay}"
|
||
PUBLIC_URL="${PUBLIC_URL:-__PUBLIC_URL__}"
|
||
|
||
FAIL_THRESHOLD="${FAIL_THRESHOLD:-3}"
|
||
COOLDOWN="${COOLDOWN:-600}"
|
||
CHECK_TIMEOUT="${CHECK_TIMEOUT:-8}"
|
||
|
||
STATE="$RELAY_DIR/monitor_state.txt"
|
||
LOG="$RELAY_DIR/monitor.log"
|
||
LOCKFILE="/tmp/ykxmz-monitor.lock"
|
||
|
||
if ! mkdir "$LOCKFILE" 2>/dev/null; then
|
||
exit 0
|
||
fi
|
||
trap 'rmdir "$LOCKFILE" 2>/dev/null' EXIT
|
||
|
||
log() { echo "$(date '+%F %T') $*" >> "$LOG"; }
|
||
|
||
FAILS=0
|
||
LAST_RESCUE=0
|
||
if [ -f "$STATE" ]; then
|
||
FAILS=$(grep -m1 '^fails=' "$STATE" 2>/dev/null | cut -d= -f2)
|
||
LAST_RESCUE=$(grep -m1 '^last_rescue=' "$STATE" 2>/dev/null | cut -d= -f2)
|
||
FAILS=${FAILS:-0}
|
||
LAST_RESCUE=${LAST_RESCUE:-0}
|
||
fi
|
||
NOW=$(date +%s)
|
||
|
||
CODE=$(curl -s -o /dev/null -w '%{http_code}' --max-time "$CHECK_TIMEOUT" \
|
||
"$PUBLIC_URL/v1/health" 2>/dev/null)
|
||
DUR=$(curl -s -o /dev/null -w '%{time_total}' --max-time "$CHECK_TIMEOUT" \
|
||
"$PUBLIC_URL/v1/health" 2>/dev/null)
|
||
DUR=${DUR:-0}
|
||
|
||
write_state() {
|
||
{
|
||
echo "fails=$1"
|
||
echo "last_rescue=$2"
|
||
echo "last_code=$CODE"
|
||
echo "last_check=$NOW"
|
||
echo "last_dur=$DUR"
|
||
} > "$STATE"
|
||
}
|
||
|
||
if [ "$CODE" = "200" ]; then
|
||
if [ "$FAILS" != "0" ]; then
|
||
log "公网恢复（HTTP 200，${DUR}s）—— 之前连续失败 $FAILS 次"
|
||
fi
|
||
PREV=$(grep -m1 '^last_code=' "$STATE" 2>/dev/null | cut -d= -f2)
|
||
[ "$PREV" != "200" ] && log "公网正常（HTTP 200，${DUR}s）"
|
||
write_state 0 "$LAST_RESCUE"
|
||
exit 0
|
||
fi
|
||
|
||
FAILS=$((FAILS + 1))
|
||
log "公网异常 第 $FAILS 次（HTTP ${CODE:-无响应}，${DUR}s）"
|
||
write_state "$FAILS" "$LAST_RESCUE"
|
||
|
||
if [ "$FAILS" -lt "$FAIL_THRESHOLD" ]; then
|
||
exit 0
|
||
fi
|
||
|
||
SINCE=$((NOW - LAST_RESCUE))
|
||
if [ "$LAST_RESCUE" != "0" ] && [ "$SINCE" -lt "$COOLDOWN" ]; then
|
||
log " 已连续失败 $FAILS 次，但距上次抢救只过了 ${SINCE}s（冷却 ${COOLDOWN}s）"
|
||
exit 0
|
||
fi
|
||
|
||
HINT=""
|
||
if [ -f "$RELAY_DIR/rescue_knowledge.md" ]; then
|
||
HINT=$(grep -oE '第 [0-9] 级奏效' "$RELAY_DIR/rescue_knowledge.md" 2>/dev/null \
|
||
| sort | uniq -c | sort -rn | head -1 | grep -oE '[0-9]')
|
||
fi
|
||
|
||
log "连续失败 $FAILS 次，启动紧急预案${HINT:+（知识库提示：第 $HINT 级常奏效）}"
|
||
write_state "$FAILS" "$NOW"
|
||
|
||
timeout 300 bash "$RELAY_DIR/rescue-lightsail.sh" \
|
||
"监测单元：公网连续 $FAILS 次不通" >> "$LOG" 2>&1
|
||
RC=$?
|
||
|
||
if [ "$RC" = "0" ]; then
|
||
log " 预案完成：已恢复"
|
||
else
|
||
log " 预案完成：仍未恢复（退出码 $RC），报告在 rescue_reports/"
|
||
fi
|
||
exit 0
|
||
'''
|
||
|
||
SERVICE = '''[Unit]
|
||
# ykxmz 网站监测单元（Lightsail 版）
|
||
Description=ykxmz 网站监测单元（每 60 秒检查公网，异常时启动紧急预案）
|
||
After=network-online.target ykxmz-relay.service
|
||
Wants=network-online.target
|
||
|
||
[Service]
|
||
Type=oneshot
|
||
TimeoutStartSec=360
|
||
|
||
# 用普通用户跑 —— 它只读状态、调 systemctl restart，不需要 root。
|
||
# （systemctl restart 那条走的是 polkit；admin 在 Lightsail 上有 sudo，
|
||
# 所以脚本里用的是不带 sudo 的 systemctl —— 见下面的说明。）
|
||
User=admin
|
||
Group=admin
|
||
|
||
Environment=RELAY_PORT=__PORT__
|
||
Environment=PUBLIC_URL=__PUBLIC_URL__
|
||
Environment=RESCUE_REBOOT=0
|
||
Environment=FAIL_THRESHOLD=3
|
||
Environment=COOLDOWN=600
|
||
|
||
ExecStart=/bin/bash __DIR__/monitor.sh
|
||
|
||
StandardOutput=append:__DIR__/monitor.log
|
||
StandardError=append:__DIR__/monitor.log
|
||
|
||
[Install]
|
||
WantedBy=multi-user.target
|
||
'''
|
||
|
||
TIMER = '''[Unit]
|
||
Description=每 60 秒检查一次网站是否可用
|
||
|
||
[Timer]
|
||
OnBootSec=45s
|
||
OnUnitActiveSec=60s
|
||
AccuracySec=5s
|
||
Unit=ykxmz-monitor.service
|
||
|
||
[Install]
|
||
WantedBy=timers.target
|
||
'''
|
||
|
||
|
||
def main() -> int:
|
||
print("=" * 62)
|
||
print(" 在 Lightsail 上装监测单元 + 紧急预案")
|
||
print("=" * 62)
|
||
|
||
monitor = (MONITOR.replace("__PUBLIC_URL__", PUBLIC_URL))
|
||
tmp = Path(tempfile.gettempdir())
|
||
|
||
print()
|
||
print(" ── 1. 传脚本 ──")
|
||
mp = tmp / "monitor.sh"
|
||
mp.write_text(monitor, encoding="utf-8", newline="\n")
|
||
upload(str(mp), "%s/monitor.sh" % RELAY_DIR, mode=0o755)
|
||
|
||
rp = ROOT / "relay" / "rescue-lightsail.sh"
|
||
if rp.is_file():
|
||
upload(str(rp), "%s/rescue-lightsail.sh" % RELAY_DIR, mode=0o755)
|
||
else:
|
||
print(" ✗ 找不到 relay/rescue-lightsail.sh")
|
||
return 1
|
||
|
||
print()
|
||
print(" ── 2. 传 systemd 单元 ──")
|
||
sp = tmp / "ykxmz-monitor.service"
|
||
sp.write_text(SERVICE.replace("__PORT__", PORT)
|
||
.replace("__PUBLIC_URL__", PUBLIC_URL)
|
||
.replace("__DIR__", RELAY_DIR),
|
||
encoding="utf-8", newline="\n")
|
||
upload(str(sp), "/tmp/ykxmz-monitor.service")
|
||
tp = tmp / "ykxmz-monitor.timer"
|
||
tp.write_text(TIMER, encoding="utf-8", newline="\n")
|
||
upload(str(tp), "/tmp/ykxmz-monitor.timer")
|
||
|
||
print()
|
||
print(" ── 3. 装单元 + 建目录 ──")
|
||
out = run(
|
||
"sudo mv /tmp/ykxmz-monitor.service /tmp/ykxmz-monitor.timer "
|
||
"/etc/systemd/system/ && "
|
||
"mkdir -p %s/rescue_reports && chmod +x %s/*.sh && "
|
||
"if [ ! -f %s/rescue_knowledge.md ]; then "
|
||
"printf '# 紧急预案知识库\\n\\n每次自动抢救成功会记一条「第几级奏效」。"
|
||
"\\n下次同类症状优先试历史上奏效的那一级。\\n' > %s/rescue_knowledge.md; "
|
||
"fi && sudo systemctl daemon-reload && echo OK"
|
||
% (RELAY_DIR, RELAY_DIR, RELAY_DIR, RELAY_DIR), timeout=180)
|
||
for ln in out[1].strip().splitlines():
|
||
print(" " + ln)
|
||
|
||
print()
|
||
print(" ── 4. 启用定时器 ──")
|
||
out = run("sudo systemctl enable --now ykxmz-monitor.timer 2>&1 | tail -2; "
|
||
"sleep 2; systemctl is-active ykxmz-monitor.timer",
|
||
timeout=180)
|
||
for ln in out[1].strip().splitlines():
|
||
print(" " + ln)
|
||
|
||
print()
|
||
print(" ── 5. 手动跑一次监测（验证它能工作）──")
|
||
out = run("bash %s/monitor.sh; echo '--- 状态 ---'; "
|
||
"cat %s/monitor_state.txt 2>/dev/null; "
|
||
"echo '--- 日志 ---'; tail -3 %s/monitor.log 2>/dev/null"
|
||
% (RELAY_DIR, RELAY_DIR, RELAY_DIR), timeout=180)
|
||
for ln in out[1].strip().splitlines():
|
||
print(" " + ln)
|
||
|
||
print()
|
||
print(" ── 6. 演练预案（网站正常时应该什么都不做）──")
|
||
out = run("bash %s/rescue-lightsail.sh '安装验证（网站正常）' 2>&1 | tail -6; "
|
||
"echo '--- 报告数（应该是 0）---'; "
|
||
"ls %s/rescue_reports 2>/dev/null | wc -l"
|
||
% (RELAY_DIR, RELAY_DIR), timeout=180)
|
||
for ln in out[1].strip().splitlines():
|
||
print(" " + ln)
|
||
|
||
print()
|
||
print(" ── 7. 总览 ──")
|
||
out = run(
|
||
"printf ' %-26s %s\\n' 'ykxmz-relay' \"$(systemctl is-active ykxmz-relay)\"; "
|
||
"printf ' %-26s %s\\n' 'ykxmz-monitor.timer' \"$(systemctl is-active ykxmz-monitor.timer)\"; "
|
||
"printf ' %-26s %s\\n' '本机 health' \"$(curl -s -o /dev/null -w '%%{http_code}' http://127.0.0.1:%s/v1/health)\"; "
|
||
"printf ' %-26s %s\\n' '公网 health' \"$(curl -s -o /dev/null -w '%%{http_code}' %s/v1/health)\""
|
||
% (PORT, PUBLIC_URL), timeout=180)
|
||
for ln in out[1].strip().splitlines():
|
||
print(" " + ln)
|
||
|
||
print()
|
||
print(" 说明：预案里用 systemctl 重启服务，而 monitor.service 以 admin")
|
||
print(" 身份跑 —— admin 在 Lightsail 上有免密 sudo，但脚本里没加")
|
||
print(" sudo。如果实测发现重启失败，把它改成 sudo systemctl 即可。")
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|