diff --git a/_build/_c10.txt b/_build/_c10.txt new file mode 100644 index 0000000..4db4c3b --- /dev/null +++ b/_build/_c10.txt @@ -0,0 +1,23 @@ +看门狗:连接数掉到 1 就立刻重启,不等攒够 3 次 + +排查公网间歇 530 时发现的: + watchdog 日志显示「公网不通(HTTP 530,ha_connections=1)」 + —— 出问题那段时间隧道**只剩 1 条连接**。 + +原来 watchdog 只把 CONNS == 0 算"零连接", +CONNS == 1 不当回事。但实测**只剩一条连接时跨机房回源就会失败** +(请求落 SEA,隧道只剩 LAX 一条)。 + +改法: + * CONNS <= 1 都计入降级计数 + * 公网不通 **且** 连接数已在危险区 = 确认的故障(不是短抖动) + -> 立刻重启,不等 3 次 + +每次检查间隔 20 秒,等 3 次就是白断 40 秒。现在能省下来。 + +实测:改完公网 15/15 = 100%,连接数 4 条。 + +顺带说明 watchdog 是怎么跑的: + systemd timer(lanremote-watchdog.timer),每 20 秒调一次 + watchdog.service。之前查"服务是 inactive"一度以为它没在跑, + 其实是 timer 触发型的 —— oneshot 服务跑完就是 inactive,正常。 diff --git a/_build/check_tunnel_state.sh b/_build/check_tunnel_state.sh new file mode 100644 index 0000000..505f1b4 --- /dev/null +++ b/_build/check_tunnel_state.sh @@ -0,0 +1,66 @@ +#!/bin/bash +# 公网间歇 530 —— 查隧道当前真实状态 +set -u + +echo " relay=$(systemctl is-active lanremote-relay) tunnel=$(systemctl is-active lanremote-tunnel)" +echo " 隧道连接数: $(curl -s --max-time 5 http://127.0.0.1:20241/metrics 2>/dev/null | grep '^cloudflared_tunnel_ha_connections' | awk '{print $2}')" + +echo +echo "════════ watchdog 最近 15 行 ════════" +tail -15 /root/lanremote-relay/watchdog.log 2>/dev/null | sed 's/^/ /' + +echo +echo "════════ watchdog 怎么被调起来的 ════════" +echo " 服务: $(systemctl is-active lanremote-watchdog 2>&1)" +crontab -l 2>/dev/null | grep -i watch | sed 's/^/ cron: /' +systemctl list-timers --all 2>/dev/null | grep -i watch | sed 's/^/ timer: /' + +echo +echo "════════ 隧道最近的重连频率(1500 行内)════════" +for kw in 'Registered tunnel connection' 'Lost connection' 'Retrying connection' 'i/o timeout' 'context deadline'; do + n=$(tail -1500 /root/lanremote-relay/tunnel.log 2>/dev/null | grep -c "$kw") + printf " %-34s %s 次\n" "$kw" "$n" +done + +echo +echo "════════ 出口 IP 可达性 ════════" +python3 - <<'PY' +import socket, time +ips = ["198.41.192.107", "198.41.200.43", "198.41.192.77", + "198.41.200.113", "198.41.192.227", "198.41.200.193", + "198.41.192.37", "198.41.200.63"] +ok = 0 +for ip in ips: + s = socket.socket(); s.settimeout(3) + t = time.perf_counter() + try: + s.connect((ip, 7844)) + print(" %-16s %4.0f ms" % (ip, (time.perf_counter()-t)*1000)); ok += 1 + except OSError: + print(" %-16s 不通" % ip) + finally: + s.close() +print(" => %d/%d 可达" % (ok, len(ips))) +PY + +echo +echo "════════ 公网连打 12 次 ════════" +ok=0 +for i in $(seq 1 12); do + c=$(curl -s -o /dev/null -w '%{http_code}' --max-time 15 \ + -H 'User-Agent: LanRemote/1.4.0' https://relay.xxmxxs.top/v1/health) + [ "$c" = "200" ] && ok=$((ok+1)) + printf " %2d: %s\n" "$i" "$c" + sleep 1 +done +echo " 可用 $ok/12" + +echo +echo "════════ 本机 relay(排除源站问题)════════" +for i in 1 2 3; do + printf " %d: HTTP %s\n" "$i" "$(curl -s -o /dev/null -w '%{http_code}' --max-time 8 http://127.0.0.1:8080/v1/health)" +done + +echo +echo "════════ 网络质量 ════════" +ping -c 10 -W 2 1.1.1.1 2>/dev/null | tail -2 | sed 's/^/ /' diff --git a/_build/final_page_check.sh b/_build/final_page_check.sh new file mode 100644 index 0000000..b2c661b --- /dev/null +++ b/_build/final_page_check.sh @@ -0,0 +1,37 @@ +#!/bin/bash +# 最后确认:页面在浏览器里真的能看了 +set -u + +BASE=http://127.0.0.1:8080 +PASS=$(cat /root/lanremote-relay/data/admin_password.txt 2>/dev/null) + +curl -s -c /tmp/ck.txt -o /dev/null --max-time 15 \ + -d "username=xmz7891&password=$PASS" "$BASE/admin/login" + +ct() { curl -sI "$@" --max-time 15 | grep -i '^content-type' | tr -d '\r'; } + +echo " 管理页 Content-Type: $(ct -b /tmp/ck.txt $BASE/zhongzhuanzhan)" +echo " 登录页 Content-Type: $(ct $BASE/zhongzhuanzhan)" +echo " 官网 Content-Type: $(ct $BASE/)" +echo " 截图 Content-Type: $(ct $BASE/shot/pc-main.png)" +echo " favicon Content-Type: $(ct $BASE/favicon.ico)" + +echo +echo " 管理页开头(应该是 ,不是被当文本):" +curl -s -b /tmp/ck.txt --max-time 15 "$BASE/zhongzhuanzhan" | head -c 60 +echo +echo + +echo " HEAD 支持:" +for p in "/" "/v1/health" "/favicon.ico" "/zhongzhuanzhan"; do + printf " %-20s HEAD HTTP %s\n" "$p" \ + "$(curl -sI -o /dev/null -w '%{http_code}' --max-time 10 "$BASE$p")" +done + +echo +echo " 公网:" +for i in 1 2 3; do + printf " %d: HTTP %s\n" "$i" \ + "$(curl -s -o /dev/null -w '%{http_code}' --max-time 20 \ + -H 'User-Agent: LanRemote/1.4.0' https://relay.xxmxxs.top/zhongzhuanzhan)" +done diff --git a/_build/verify_watchdog.sh b/_build/verify_watchdog.sh new file mode 100644 index 0000000..63c78fc --- /dev/null +++ b/_build/verify_watchdog.sh @@ -0,0 +1,38 @@ +#!/bin/bash +# 验证改进后的 watchdog +set -u + +echo " 一、语法检查" +if bash -n /root/lanremote-relay/watchdog.sh 2>&1; then + echo " ✓ watchdog.sh 语法 OK" +else + echo " ✗ 语法有问题" +fi + +echo +echo " 二、手动跑一次(看它怎么判断)" +bash /root/lanremote-relay/watchdog.sh 2>&1 | tail -4 | sed 's/^/ /' + +echo +echo " 三、公网连打 15 次" +ok=0 +for i in $(seq 1 15); do + c=$(curl -s -o /dev/null -w '%{http_code}' --max-time 15 \ + -H 'User-Agent: LanRemote/1.4.0' https://relay.xxmxxs.top/v1/health) + [ "$c" = "200" ] && ok=$((ok + 1)) + printf " %2d: %s\n" "$i" "$c" + sleep 1 +done +echo " 可用 $ok/15" + +echo +echo " 四、当前连接数" +echo " $(curl -s --max-time 5 http://127.0.0.1:20241/metrics 2>/dev/null | grep '^cloudflared_tunnel_ha_connections' | awk '{print $2}') 条" + +echo +echo " 五、watchdog 定时器" +systemctl list-timers lanremote-watchdog.timer --no-pager 2>/dev/null | head -3 | sed 's/^/ /' + +echo +echo " 六、watchdog 日志最后 6 行" +tail -6 /root/lanremote-relay/watchdog.log 2>/dev/null | sed 's/^/ /' diff --git a/_tests/shots/1-屏幕中间.png b/_tests/shots/1-屏幕中间.png index 4d89526..549668f 100644 Binary files a/_tests/shots/1-屏幕中间.png and b/_tests/shots/1-屏幕中间.png differ diff --git a/_tests/shots/3-自定义640x400.png b/_tests/shots/3-自定义640x400.png index d0daadc..4e38939 100644 Binary files a/_tests/shots/3-自定义640x400.png and b/_tests/shots/3-自定义640x400.png differ diff --git a/relay/watchdog.sh b/relay/watchdog.sh index 0c61e9c..6a06b6a 100644 --- a/relay/watchdog.sh +++ b/relay/watchdog.sh @@ -127,14 +127,28 @@ if [ "$LOG_STALE" = "1" ] && [ "$FAIL" -ge 1 ]; then fi FAIL=$((FAIL + 1)) -if [ -n "$CONNS" ] && [ "$CONNS" = "0" ]; then +# 连接数 <= 1 也算"降级"。 +# +# 原来是 == 0 才算,但实测**只剩 1 条连接时跨机房回源就会失败** +# (用户撞上的 530 就是这种:请求落 SEA,隧道只剩 LAX 一条)。 +# 所以 1 和 0 一样危险,都计入降级计数。 +DEGRADED=0 +if [ -n "$CONNS" ] && [ "$CONNS" -le 1 ] 2>/dev/null; then ZERO=$((ZERO + 1)) + DEGRADED=1 fi -log "公网不通($DETAIL),连续第 $FAIL 次(零连接 $ZERO 次)" +log "公网不通($DETAIL),连续第 $FAIL 次(降级 $ZERO 次)" + +# 公网不通 + 连接数已经降到危险区 = **确认的故障**,不是短抖动。 +# 这种情况立刻重启,不等攒够 3 次 —— 每次检查间隔 20 秒, +# 等 3 次就是白断 40 秒。 +if [ "$DEGRADED" = "1" ]; then + restart_tunnel "公网不通且只剩 ${CONNS} 条隧道连接(确认故障,立即重启)" +fi if [ "$FAIL" -ge "$THRESHOLD" ] || [ "$ZERO" -ge "$THRESHOLD" ]; then - restart_tunnel "连续 $FAIL 次公网不通(含 $ZERO 次零连接)" + restart_tunnel "连续 $FAIL 次公网不通(含 $ZERO 次降级)" fi write_state "$ZERO" "$FAIL"