mirror of
https://gitee.com/xmz7891/ykxmz.git
synced 2026-10-18 12:30:44 +00:00
修监测单元查错地址（改监听后它开始误报网站挂了）
把中转改成只监听 127.0.0.1（公网走 nginx）之后，监测单元还在查 http://13.54.30.196:13245/v1/health —— 那个端口已经不对公网开了， 于是它连续报"公网异常"，fails 涨到 2。 **差一点就是无限重启循环**：再失败一次它会启动紧急预案， 而预案的动作是"重启中转服务"，重启完再查还是 000，再救…… 这是改监听地址时没想到的连带影响。教训写在这里： **改一个服务的暴露方式时，必须把所有"探测它"的地方一起看一遍。** == 改法 == 监测地址改成"用户实际怎么访问"为准，域名优先、IP 兜底： PUBLIC_URL=http://yk.xxmxxs.top 域名（经 Cloudflare + nginx） FALLBACK_URL=http://13.54.30.196:80 绕过 CF 直连 nginx 只要有一个通就算健康。这样域名配好前靠兜底、配好后走真实路径。 == 顺带修的 == * monitor.log 属主还是 root（后来新建的），admin 写不进去 -> 再 chown 一次 * _build/check_lightsail.py 里的"公网"地址也要跟着改， 并且**故意保留**一项查 :13245（预期 000）用来确认 中转确实不再暴露公网 全量状态：三层地址、缓存头、监测、属主全部正确。
This commit is contained in:
xmz7891
@@ -0,0 +1,30 @@
|
||||
修监测单元查错地址（改监听后它开始误报网站挂了）
|
||||
|
||||
把中转改成只监听 127.0.0.1（公网走 nginx）之后，监测单元还在查
|
||||
http://13.54.30.196:13245/v1/health —— 那个端口已经不对公网开了，
|
||||
于是它连续报"公网异常"，fails 涨到 2。
|
||||
|
||||
**差一点就是无限重启循环**：再失败一次它会启动紧急预案，
|
||||
而预案的动作是"重启中转服务"，重启完再查还是 000，再救……
|
||||
|
||||
这是改监听地址时没想到的连带影响。教训写在这里：
|
||||
**改一个服务的暴露方式时，必须把所有"探测它"的地方一起看一遍。**
|
||||
|
||||
== 改法 ==
|
||||
|
||||
监测地址改成"用户实际怎么访问"为准，域名优先、IP 兜底：
|
||||
|
||||
PUBLIC_URL=http://yk.xxmxxs.top 域名（经 Cloudflare + nginx）
|
||||
FALLBACK_URL=http://13.54.30.196:80 绕过 CF 直连 nginx
|
||||
|
||||
只要有一个通就算健康。这样域名配好前靠兜底、配好后走真实路径。
|
||||
|
||||
== 顺带修的 ==
|
||||
|
||||
* monitor.log 属主还是 root（后来新建的），admin 写不进去
|
||||
-> 再 chown 一次
|
||||
* _build/check_lightsail.py 里的"公网"地址也要跟着改，
|
||||
并且**故意保留**一项查 :13245（预期 000）用来确认
|
||||
中转确实不再暴露公网
|
||||
|
||||
全量状态：三层地址、缓存头、监测、属主全部正确。
|
||||
+49
-38
@@ -1,14 +1,14 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
r'''看一眼 Lightsail 上中转服务器的完整状态。
|
||||
|
||||
为什么写成文件：用 `python -c @"..."@` 传带 `$(...)` 和引号的远端命令，
|
||||
PowerShell 会做变量插值、还会在本机执行 `$(systemctl ...)`
|
||||
（报 "无法将 systemctl 项识别为 cmdlet"）。凡是超过两行的远端命令，
|
||||
一律写成文件跑。
|
||||
⚠ 写这个文件的原因：用 `python -c @"..."@` 传带 `$(...)` 的远端命令，
|
||||
PowerShell 会**先在本机求值** `$(systemctl ...)`（报"无法将 systemctl
|
||||
项识别为 cmdlet"），还会吃掉引号导致 Python 语法错。
|
||||
这个毛病我这一轮犯了四五次 —— **凡是超过两行的远端命令，一律写文件。**
|
||||
|
||||
用法：
|
||||
python _build/check_lightsail.py
|
||||
python _build/check_lightsail.py --fix # 顺便修文件属主
|
||||
python _build/check_lightsail.py --fix # 顺便修文件属主
|
||||
'''
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -20,52 +20,63 @@ sys.path.insert(0, str(ROOT / "_build"))
|
||||
from lssh import run # noqa: E402
|
||||
|
||||
PORT = "13245"
|
||||
PUBLIC = "http://13.54.30.196:%s" % PORT
|
||||
DOMAIN = "yk.xxmxxs.top"
|
||||
RELAY_DIR = "/opt/ykxmz-relay"
|
||||
|
||||
# 三层地址各查一遍：
|
||||
# 域名 —— 用户实际走的路（Cloudflare -> nginx -> 中转）
|
||||
# IP:80 —— 绕过 Cloudflare，直连 nginx（域名没配好时靠它）
|
||||
# :13245 —— **应该不通**（中转已只绑 127.0.0.1，不再对公网）
|
||||
DOMAIN_URL = "http://%s" % DOMAIN
|
||||
NGINX_URL = "http://13.54.30.196:80"
|
||||
DIRECT_URL = "http://13.54.30.196:%s" % PORT
|
||||
|
||||
REMOTE = r'''
|
||||
R=__DIR__
|
||||
P=__PORT__
|
||||
|
||||
echo "════ 服务 ════"
|
||||
for s in ykxmz-relay ykxmz-monitor.timer; do
|
||||
printf ' %-24s %s\n' "$s" "$(systemctl is-active $s 2>&1)"
|
||||
for s in ykxmz-relay nginx ykxmz-monitor.timer; do
|
||||
printf ' %-22s %s\n' "$s" "$(systemctl is-active $s 2>&1)"
|
||||
done
|
||||
|
||||
echo
|
||||
echo "════ 健康 ════"
|
||||
printf ' %-24s %s\n' "本机 health" "$(curl -s -o /dev/null -w '%{http_code}' --max-time 8 http://127.0.0.1:$P/v1/health)"
|
||||
printf ' %-24s %s\n' "公网 health" "$(curl -s -o /dev/null -w '%{http_code}' --max-time 15 __PUBLIC__/v1/health)"
|
||||
printf ' %-24s %s\n' "公网首页" "$(curl -s -o /dev/null -w '%{http_code}' --max-time 15 __PUBLIC__/)"
|
||||
printf ' %-24s %s\n' "本机延迟" "$(curl -s -o /dev/null -w '%{time_total}s' --max-time 8 http://127.0.0.1:$P/v1/health)"
|
||||
printf ' %-24s %s\n' "公网延迟" "$(curl -s -o /dev/null -w '%{time_total}s' --max-time 15 __PUBLIC__/v1/health)"
|
||||
echo "════ 三层地址（用户路径 / 绕 CF / 直连中转）════"
|
||||
printf ' %-26s %s\n' "域名 __DOMAIN__" "$(curl -s -o /dev/null -w '%{http_code}' --max-time 12 __DOMAIN_URL__/v1/health)"
|
||||
printf ' %-26s %s\n' "IP:80（nginx）" "$(curl -s -o /dev/null -w '%{http_code}' --max-time 12 __NGINX_URL__/v1/health)"
|
||||
printf ' %-26s %s\n' ":__PORT__（应 000）" "$(curl -s -o /dev/null -w '%{http_code}' --max-time 8 __DIRECT_URL__/v1/health)"
|
||||
printf ' %-26s %s\n' "本机 :__PORT__" "$(curl -s -o /dev/null -w '%{http_code}' --max-time 8 http://127.0.0.1:$P/v1/health)"
|
||||
printf ' %-26s %s\n' "IP:80 延迟" "$(curl -s -o /dev/null -w '%{time_total}s' --max-time 12 __NGINX_URL__/v1/health)"
|
||||
|
||||
echo
|
||||
echo "════ 首页角标 ════"
|
||||
echo " 最新版: $(curl -s --max-time 10 http://127.0.0.1:$P/ | grep -o '最新版 [0-9.]*' | head -1)"
|
||||
echo " IP:80 -> $(curl -s --max-time 10 __NGINX_URL__/ | grep -o '最新版 [0-9.]*' | head -1)"
|
||||
echo " 域名 -> $(curl -s --max-time 12 __DOMAIN_URL__/ | grep -o '最新版 [0-9.]*' | head -1)"
|
||||
|
||||
echo
|
||||
echo "════ 缓存头（决定 Cloudflare 缓不缓存）════"
|
||||
printf ' %-20s %s 条\n' "下载" "$(curl -sI --max-time 10 __NGINX_URL__/download/ykxmz-setup-2.1.3.exe | grep -ci cache-control)"
|
||||
curl -sI --max-time 10 __NGINX_URL__/download/ykxmz-setup-2.1.3.exe \
|
||||
| grep -i '^cache-control' | sed 's/^/ /'
|
||||
printf ' %-20s %s\n' "长轮询" "$(curl -sI --max-time 10 __NGINX_URL__/v1/health | grep -i '^cache-control' | tr -d '\r')"
|
||||
|
||||
echo
|
||||
echo "════ 监听 / 进程 ════"
|
||||
ss -tlnp 2>/dev/null | grep ":$P " | sed 's/^/ /'
|
||||
ps -eo pid,rss,etime,cmd --sort=-rss 2>/dev/null | grep 'relay\.py' | grep -v grep | head -2 | sed 's/^/ /'
|
||||
ss -tlnp 2>/dev/null | grep -E ":(80|$P) " | sed 's/^/ /'
|
||||
ps -eo pid,rss,etime,cmd --sort=-rss 2>/dev/null | grep 'relay\.py' | grep -v grep | head -1 | sed 's/^/ /'
|
||||
|
||||
echo
|
||||
echo "════ 监测单元 ════"
|
||||
cat $R/monitor_state.txt 2>/dev/null | sed 's/^/ /'
|
||||
echo " --- monitor.log 尾部 ---"
|
||||
tail -4 $R/monitor.log 2>/dev/null | sed 's/^/ /'
|
||||
echo " --- 探测的是什么地址 ---"
|
||||
grep -E '^(PUBLIC_URL|FALLBACK_URL)=' $R/monitor.sh | sed 's/^/ /'
|
||||
echo " --- monitor.log 尾部（只挑非 Permission 的）---"
|
||||
grep -v 'Permission denied' $R/monitor.log 2>/dev/null | tail -4 | sed 's/^/ /'
|
||||
|
||||
echo
|
||||
echo "════ 抢救报告 ════"
|
||||
echo " 共 $(ls $R/rescue_reports 2>/dev/null | wc -l) 份"
|
||||
ls -la $R/rescue_reports/ 2>/dev/null | tail -n +4 | awk '{print " " $6" "$7" "$8" "$9}'
|
||||
|
||||
echo
|
||||
echo "════ 知识库 ════"
|
||||
if [ -f $R/rescue_knowledge.md ]; then
|
||||
echo " $(wc -l < $R/rescue_knowledge.md) 行"
|
||||
else
|
||||
echo " （还没有）"
|
||||
fi
|
||||
echo "════ 抢救报告 / 知识库 ════"
|
||||
echo " 报告 $(ls $R/rescue_reports 2>/dev/null | wc -l) 份"
|
||||
echo " 知识库 $(wc -l < $R/rescue_knowledge.md 2>/dev/null || echo 0) 行"
|
||||
|
||||
echo
|
||||
echo "════ 资源 ════"
|
||||
@@ -74,11 +85,7 @@ df -h / | tail -1 | awk '{printf " 磁盘 %s 用 %s\n", $2, $5}'
|
||||
uptime | sed 's/^/ /'
|
||||
|
||||
echo
|
||||
echo "════ 定时器下次触发 ════"
|
||||
systemctl list-timers ykxmz-monitor.timer --no-pager 2>/dev/null | head -3 | sed 's/^/ /'
|
||||
|
||||
echo
|
||||
echo "════ 文件属主（admin 要能写日志）════"
|
||||
echo "════ 文件属主（admin 必须能写日志）════"
|
||||
ls -la $R/*.log $R/monitor_state.txt 2>/dev/null | awk '{print " " $3":"$4" "$9}'
|
||||
'''
|
||||
|
||||
@@ -87,12 +94,16 @@ def main() -> int:
|
||||
if "--fix" in sys.argv:
|
||||
print(" ── 修属主 ──")
|
||||
out = run("sudo chown -R admin:admin %s && sudo chmod 755 %s && echo OK"
|
||||
% (RELAY_DIR, RELAY_DIR), timeout=120)
|
||||
% (RELAY_DIR, RELAY_DIR), timeout=120, quiet=True)
|
||||
print(" " + out[1].strip())
|
||||
|
||||
cmd = (REMOTE.replace("__DIR__", RELAY_DIR)
|
||||
cmd = (REMOTE
|
||||
.replace("__DIR__", RELAY_DIR)
|
||||
.replace("__PORT__", PORT)
|
||||
.replace("__PUBLIC__", PUBLIC))
|
||||
.replace("__DOMAIN__", DOMAIN)
|
||||
.replace("__DOMAIN_URL__", DOMAIN_URL)
|
||||
.replace("__NGINX_URL__", NGINX_URL)
|
||||
.replace("__DIRECT_URL__", DIRECT_URL))
|
||||
out = run(cmd, timeout=300, quiet=True)
|
||||
print(out[1] if isinstance(out, tuple) else out)
|
||||
return 0
|
||||
|
||||
@@ -0,0 +1,165 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
r'''修监测单元：它现在查错了地址，在**误报网站挂了**。
|
||||
|
||||
== 怎么发现的 ==
|
||||
|
||||
把中转改成只监听 127.0.0.1（公网只能走 nginx）之后，跑状态检查看到：
|
||||
|
||||
公网 health 000 <- 因为 :13245 已经不对公网开了
|
||||
fails=2 <- 监测单元连续两次"失败"
|
||||
|
||||
也就是说监测单元**误以为网站挂了**，再失败一次就会启动紧急预案 ——
|
||||
而它抢救的方式是"重启中转服务"，重启完它再查还是 000，再救……
|
||||
**会陷入无限重启循环**。
|
||||
|
||||
这是我自己改监听地址时没想到的连带影响。教训：改一个服务的
|
||||
暴露方式时，必须把所有"探测它"的地方一起看一遍。
|
||||
|
||||
== 改法 ==
|
||||
|
||||
监测的地址应该以"**用户实际怎么访问**"为准，现在应该是：
|
||||
|
||||
http://yk.xxmxxs.top/v1/health （经过 Cloudflare + nginx）
|
||||
|
||||
但域名还没配好（DNS 还没指过来），所以加一个**回退**：
|
||||
域名不通就用 IP:80 试，只要有一个通就算健康。
|
||||
|
||||
这样：
|
||||
* 域名配好之前 -> 走 IP:80，正常监控
|
||||
* 域名配好之后 -> 走域名，监控的是用户真实路径
|
||||
* 两个都不通 -> 才是真挂了
|
||||
|
||||
== 顺带 ==
|
||||
|
||||
monitor.log 的属主问题：monitor.service 以 admin 跑，
|
||||
而 relay.log 之类的之前是 root 建的。已经 chown 过一次，
|
||||
但 monitor.log 是后来新生成的（仍是 root）—— 这次一并修掉，
|
||||
并且在 unit 里加一行让 systemd 自己保证目录属主。
|
||||
'''
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
sys.path.insert(0, str(ROOT / "_build"))
|
||||
from lssh import run # noqa: E402
|
||||
|
||||
RELAY_DIR = "/opt/ykxmz-relay"
|
||||
DOMAIN = "yk.xxmxxs.top"
|
||||
FALLBACK = "http://13.54.30.196:80"
|
||||
|
||||
|
||||
def main() -> int:
|
||||
print("=" * 62)
|
||||
print(" 修监测单元：改探测地址 + 修日志属主")
|
||||
print("=" * 62)
|
||||
|
||||
print()
|
||||
print(" ── 1. 改 monitor.sh 的健康检查（域名优先，IP 兜底）──")
|
||||
# 把 PUBLIC_URL 的默认值换掉，并把 is_healthy 改成"两个都试"
|
||||
old_url = 'PUBLIC_URL="${PUBLIC_URL:-http://13.54.30.196:13245}"'
|
||||
new_url = (
|
||||
'# 探测地址：域名优先（用户实际走的路），IP:80 兜底\n'
|
||||
'# （域名 DNS 还没指过来时用它，否则监测会一直误报）\n'
|
||||
'PUBLIC_URL="${PUBLIC_URL:-http://%s}"\n'
|
||||
'FALLBACK_URL="${FALLBACK_URL:-%s}"' % (DOMAIN, FALLBACK))
|
||||
|
||||
out = run(
|
||||
"grep -n 'PUBLIC_URL=' %s/monitor.sh | head -3" % RELAY_DIR,
|
||||
timeout=60, quiet=True)
|
||||
print(" 改前:")
|
||||
for ln in out[1].strip().splitlines():
|
||||
print(" " + ln)
|
||||
|
||||
script = (
|
||||
"python3 - <<'PYEOF'\n"
|
||||
"from pathlib import Path\n"
|
||||
"p = Path('%s/monitor.sh')\n"
|
||||
"t = p.read_text(encoding='utf-8')\n"
|
||||
"old = [l for l in t.splitlines() if l.startswith('PUBLIC_URL=')]\n"
|
||||
"if not old:\n"
|
||||
" print(' x 找不到 PUBLIC_URL 行')\n"
|
||||
"else:\n"
|
||||
" t = t.replace(old[0], %r, 1)\n"
|
||||
" # 健康检查改成两个都试\n"
|
||||
" a = 'CODE=$(curl -s -o /dev/null -w \\'%%{http_code}\\' --max-time "
|
||||
"\"$CHECK_TIMEOUT\" \\\\\\n \"$PUBLIC_URL/v1/health\" 2>/dev/null)'\n"
|
||||
" b = ('PROBE=\"$PUBLIC_URL\"\\n'\n"
|
||||
" 'CODE=$(curl -s -o /dev/null -w \\'%%{http_code}\\' "
|
||||
"--max-time \"$CHECK_TIMEOUT\" \"$PROBE/v1/health\" 2>/dev/null)\\n'\n"
|
||||
" 'if [ \"$CODE\" != \"200\" ] && [ -n \"$FALLBACK_URL\" ]; then\\n'\n"
|
||||
" ' PROBE=\"$FALLBACK_URL\"\\n'\n"
|
||||
" ' CODE=$(curl -s -o /dev/null -w \\'%%{http_code}\\' "
|
||||
"--max-time \"$CHECK_TIMEOUT\" \"$PROBE/v1/health\" 2>/dev/null)\\n'\n"
|
||||
" 'fi')\n"
|
||||
" if a in t:\n"
|
||||
" t = t.replace(a, b, 1)\n"
|
||||
" print(' ok 健康检查改成域名优先 + IP 兜底')\n"
|
||||
" else:\n"
|
||||
" print(' ! 健康检查那段没匹配上，手工看')\n"
|
||||
" p.write_text(t, encoding='utf-8', newline='\\n')\n"
|
||||
"PYEOF" % (RELAY_DIR, new_url))
|
||||
|
||||
out = run(script, timeout=120, quiet=True)
|
||||
for ln in out[1].strip().splitlines():
|
||||
print(" " + ln)
|
||||
|
||||
out = run("grep -n 'PUBLIC_URL=\\|FALLBACK_URL=\\|PROBE=' %s/monitor.sh "
|
||||
"| head -8" % RELAY_DIR, timeout=60, quiet=True)
|
||||
print(" 改后:")
|
||||
for ln in out[1].strip().splitlines():
|
||||
print(" " + ln)
|
||||
|
||||
print()
|
||||
print(" ── 2. 修日志属主 + 让 systemd 保证它 ──")
|
||||
out = run(
|
||||
"sudo chown -R admin:admin %s && "
|
||||
"sudo sed -i '/^User=admin$/i "
|
||||
"# 日志文件和状态文件都要 admin 能写（服务以 admin 跑）\\n"
|
||||
"# 之前 relay.log / monitor.log 是 root 建的，导致 "
|
||||
"monitor.sh 报 Permission denied\\n"
|
||||
"PermissionsStartOnly=true' "
|
||||
"/etc/systemd/system/ykxmz-monitor.service 2>/dev/null; "
|
||||
"sudo touch %s/monitor.log %s/monitor_state.txt && "
|
||||
"sudo chown admin:admin %s/*.log %s/*.txt %s/*.md 2>/dev/null; "
|
||||
"ls -la %s/*.log | awk '{print \" \" $3\":\"$4\" \" $9}'"
|
||||
% (RELAY_DIR, RELAY_DIR, RELAY_DIR, RELAY_DIR, RELAY_DIR,
|
||||
RELAY_DIR, RELAY_DIR), timeout=180, quiet=True)
|
||||
for ln in out[1].strip().splitlines():
|
||||
print(ln)
|
||||
|
||||
print()
|
||||
print(" ── 3. 重置监测状态 + 跑一次 ──")
|
||||
out = run(
|
||||
"sudo systemctl daemon-reload; "
|
||||
"printf 'fails=0\\nlast_rescue=0\\nlast_code=200\\n' > "
|
||||
"%s/monitor_state.txt; "
|
||||
"bash %s/monitor.sh; echo '--- 状态 ---'; "
|
||||
"cat %s/monitor_state.txt; echo '--- 日志尾 ---'; "
|
||||
"tail -3 %s/monitor.log" % (RELAY_DIR, RELAY_DIR, RELAY_DIR,
|
||||
RELAY_DIR),
|
||||
timeout=180, quiet=True)
|
||||
for ln in out[1].strip().splitlines():
|
||||
print(" " + ln)
|
||||
|
||||
print()
|
||||
print(" ── 4. 验证探测链路 ──")
|
||||
out = run(
|
||||
'printf " %-28s %s\\n" "域名(还没配)" '
|
||||
'"$(curl -s -o /dev/null -w \'%{http_code}\' --max-time 8 '
|
||||
'http://%s/v1/health)"; '
|
||||
'printf " %-28s %s\\n" "IP:80 (兜底)" '
|
||||
'"$(curl -s -o /dev/null -w \'%{http_code}\' --max-time 8 '
|
||||
'http://13.54.30.196:80/v1/health)"; '
|
||||
'printf " %-28s %s\\n" "IP:13245 (已关闭)" '
|
||||
'"$(curl -s -o /dev/null -w \'%{http_code}\' --max-time 8 '
|
||||
'http://13.54.30.196:13245/v1/health)"' % DOMAIN,
|
||||
timeout=120, quiet=True)
|
||||
for ln in out[1].strip().splitlines():
|
||||
print(ln)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user