mirror of
https://gitee.com/xmz7891/ykxmz.git
synced 2026-10-18 11:20:43 +00:00
看门狗：连接数掉到 1 就立刻重启，不等攒够 3 次
排查公网间歇 530 时发现的：
watchdog 日志显示「公网不通（HTTP 530，ha_connections=1）」
—— 出问题那段时间隧道**只剩 1 条连接**。
原来 watchdog 只把 CONNS == 0 算"零连接"，
CONNS == 1 不当回事。但实测**只剩一条连接时跨机房回源就会失败**
（请求落 SEA，隧道只剩 LAX 一条）。
改法：
* CONNS <= 1 都计入降级计数
* 公网不通 **且** 连接数已在危险区 = 确认的故障（不是短抖动）
-> 立刻重启，不等 3 次
每次检查间隔 20 秒，等 3 次就是白断 40 秒。现在能省下来。
实测：改完公网 15/15 = 100%，连接数 4 条。
顺带说明 watchdog 是怎么跑的：
systemd timer（lanremote-watchdog.timer），每 20 秒调一次
watchdog.service。之前查"服务是 inactive"一度以为它没在跑，
其实是 timer 触发型的 —— oneshot 服务跑完就是 inactive，正常。
This commit is contained in:
xmz7891
@@ -0,0 +1,23 @@
|
||||
看门狗：连接数掉到 1 就立刻重启，不等攒够 3 次
|
||||
|
||||
排查公网间歇 530 时发现的：
|
||||
watchdog 日志显示「公网不通（HTTP 530，ha_connections=1）」
|
||||
—— 出问题那段时间隧道**只剩 1 条连接**。
|
||||
|
||||
原来 watchdog 只把 CONNS == 0 算"零连接"，
|
||||
CONNS == 1 不当回事。但实测**只剩一条连接时跨机房回源就会失败**
|
||||
（请求落 SEA，隧道只剩 LAX 一条）。
|
||||
|
||||
改法：
|
||||
* CONNS <= 1 都计入降级计数
|
||||
* 公网不通 **且** 连接数已在危险区 = 确认的故障（不是短抖动）
|
||||
-> 立刻重启，不等 3 次
|
||||
|
||||
每次检查间隔 20 秒，等 3 次就是白断 40 秒。现在能省下来。
|
||||
|
||||
实测：改完公网 15/15 = 100%，连接数 4 条。
|
||||
|
||||
顺带说明 watchdog 是怎么跑的：
|
||||
systemd timer（lanremote-watchdog.timer），每 20 秒调一次
|
||||
watchdog.service。之前查"服务是 inactive"一度以为它没在跑，
|
||||
其实是 timer 触发型的 —— oneshot 服务跑完就是 inactive，正常。
|
||||
@@ -0,0 +1,66 @@
|
||||
#!/bin/bash
|
||||
# 公网间歇 530 —— 查隧道当前真实状态
|
||||
set -u
|
||||
|
||||
echo " relay=$(systemctl is-active lanremote-relay) tunnel=$(systemctl is-active lanremote-tunnel)"
|
||||
echo " 隧道连接数: $(curl -s --max-time 5 http://127.0.0.1:20241/metrics 2>/dev/null | grep '^cloudflared_tunnel_ha_connections' | awk '{print $2}')"
|
||||
|
||||
echo
|
||||
echo "════════ watchdog 最近 15 行 ════════"
|
||||
tail -15 /root/lanremote-relay/watchdog.log 2>/dev/null | sed 's/^/ /'
|
||||
|
||||
echo
|
||||
echo "════════ watchdog 怎么被调起来的 ════════"
|
||||
echo " 服务: $(systemctl is-active lanremote-watchdog 2>&1)"
|
||||
crontab -l 2>/dev/null | grep -i watch | sed 's/^/ cron: /'
|
||||
systemctl list-timers --all 2>/dev/null | grep -i watch | sed 's/^/ timer: /'
|
||||
|
||||
echo
|
||||
echo "════════ 隧道最近的重连频率（1500 行内）════════"
|
||||
for kw in 'Registered tunnel connection' 'Lost connection' 'Retrying connection' 'i/o timeout' 'context deadline'; do
|
||||
n=$(tail -1500 /root/lanremote-relay/tunnel.log 2>/dev/null | grep -c "$kw")
|
||||
printf " %-34s %s 次\n" "$kw" "$n"
|
||||
done
|
||||
|
||||
echo
|
||||
echo "════════ 出口 IP 可达性 ════════"
|
||||
python3 - <<'PY'
|
||||
import socket, time
|
||||
ips = ["198.41.192.107", "198.41.200.43", "198.41.192.77",
|
||||
"198.41.200.113", "198.41.192.227", "198.41.200.193",
|
||||
"198.41.192.37", "198.41.200.63"]
|
||||
ok = 0
|
||||
for ip in ips:
|
||||
s = socket.socket(); s.settimeout(3)
|
||||
t = time.perf_counter()
|
||||
try:
|
||||
s.connect((ip, 7844))
|
||||
print(" %-16s %4.0f ms" % (ip, (time.perf_counter()-t)*1000)); ok += 1
|
||||
except OSError:
|
||||
print(" %-16s 不通" % ip)
|
||||
finally:
|
||||
s.close()
|
||||
print(" => %d/%d 可达" % (ok, len(ips)))
|
||||
PY
|
||||
|
||||
echo
|
||||
echo "════════ 公网连打 12 次 ════════"
|
||||
ok=0
|
||||
for i in $(seq 1 12); do
|
||||
c=$(curl -s -o /dev/null -w '%{http_code}' --max-time 15 \
|
||||
-H 'User-Agent: LanRemote/1.4.0' https://relay.xxmxxs.top/v1/health)
|
||||
[ "$c" = "200" ] && ok=$((ok+1))
|
||||
printf " %2d: %s\n" "$i" "$c"
|
||||
sleep 1
|
||||
done
|
||||
echo " 可用 $ok/12"
|
||||
|
||||
echo
|
||||
echo "════════ 本机 relay（排除源站问题）════════"
|
||||
for i in 1 2 3; do
|
||||
printf " %d: HTTP %s\n" "$i" "$(curl -s -o /dev/null -w '%{http_code}' --max-time 8 http://127.0.0.1:8080/v1/health)"
|
||||
done
|
||||
|
||||
echo
|
||||
echo "════════ 网络质量 ════════"
|
||||
ping -c 10 -W 2 1.1.1.1 2>/dev/null | tail -2 | sed 's/^/ /'
|
||||
@@ -0,0 +1,37 @@
|
||||
#!/bin/bash
|
||||
# 最后确认：页面在浏览器里真的能看了
|
||||
set -u
|
||||
|
||||
BASE=http://127.0.0.1:8080
|
||||
PASS=$(cat /root/lanremote-relay/data/admin_password.txt 2>/dev/null)
|
||||
|
||||
curl -s -c /tmp/ck.txt -o /dev/null --max-time 15 \
|
||||
-d "username=xmz7891&password=$PASS" "$BASE/admin/login"
|
||||
|
||||
ct() { curl -sI "$@" --max-time 15 | grep -i '^content-type' | tr -d '\r'; }
|
||||
|
||||
echo " 管理页 Content-Type: $(ct -b /tmp/ck.txt $BASE/zhongzhuanzhan)"
|
||||
echo " 登录页 Content-Type: $(ct $BASE/zhongzhuanzhan)"
|
||||
echo " 官网 Content-Type: $(ct $BASE/)"
|
||||
echo " 截图 Content-Type: $(ct $BASE/shot/pc-main.png)"
|
||||
echo " favicon Content-Type: $(ct $BASE/favicon.ico)"
|
||||
|
||||
echo
|
||||
echo " 管理页开头（应该是 <!doctype html>，不是被当文本）："
|
||||
curl -s -b /tmp/ck.txt --max-time 15 "$BASE/zhongzhuanzhan" | head -c 60
|
||||
echo
|
||||
echo
|
||||
|
||||
echo " HEAD 支持:"
|
||||
for p in "/" "/v1/health" "/favicon.ico" "/zhongzhuanzhan"; do
|
||||
printf " %-20s HEAD HTTP %s\n" "$p" \
|
||||
"$(curl -sI -o /dev/null -w '%{http_code}' --max-time 10 "$BASE$p")"
|
||||
done
|
||||
|
||||
echo
|
||||
echo " 公网:"
|
||||
for i in 1 2 3; do
|
||||
printf " %d: HTTP %s\n" "$i" \
|
||||
"$(curl -s -o /dev/null -w '%{http_code}' --max-time 20 \
|
||||
-H 'User-Agent: LanRemote/1.4.0' https://relay.xxmxxs.top/zhongzhuanzhan)"
|
||||
done
|
||||
@@ -0,0 +1,38 @@
|
||||
#!/bin/bash
|
||||
# 验证改进后的 watchdog
|
||||
set -u
|
||||
|
||||
echo " 一、语法检查"
|
||||
if bash -n /root/lanremote-relay/watchdog.sh 2>&1; then
|
||||
echo " ✓ watchdog.sh 语法 OK"
|
||||
else
|
||||
echo " ✗ 语法有问题"
|
||||
fi
|
||||
|
||||
echo
|
||||
echo " 二、手动跑一次（看它怎么判断）"
|
||||
bash /root/lanremote-relay/watchdog.sh 2>&1 | tail -4 | sed 's/^/ /'
|
||||
|
||||
echo
|
||||
echo " 三、公网连打 15 次"
|
||||
ok=0
|
||||
for i in $(seq 1 15); do
|
||||
c=$(curl -s -o /dev/null -w '%{http_code}' --max-time 15 \
|
||||
-H 'User-Agent: LanRemote/1.4.0' https://relay.xxmxxs.top/v1/health)
|
||||
[ "$c" = "200" ] && ok=$((ok + 1))
|
||||
printf " %2d: %s\n" "$i" "$c"
|
||||
sleep 1
|
||||
done
|
||||
echo " 可用 $ok/15"
|
||||
|
||||
echo
|
||||
echo " 四、当前连接数"
|
||||
echo " $(curl -s --max-time 5 http://127.0.0.1:20241/metrics 2>/dev/null | grep '^cloudflared_tunnel_ha_connections' | awk '{print $2}') 条"
|
||||
|
||||
echo
|
||||
echo " 五、watchdog 定时器"
|
||||
systemctl list-timers lanremote-watchdog.timer --no-pager 2>/dev/null | head -3 | sed 's/^/ /'
|
||||
|
||||
echo
|
||||
echo " 六、watchdog 日志最后 6 行"
|
||||
tail -6 /root/lanremote-relay/watchdog.log 2>/dev/null | sed 's/^/ /'
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 1.7 MiB After Width: | Height: | Size: 12 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 608 KiB After Width: | Height: | Size: 3.1 KiB |
+17
-3
@@ -127,14 +127,28 @@ if [ "$LOG_STALE" = "1" ] && [ "$FAIL" -ge 1 ]; then
|
||||
fi
|
||||
|
||||
FAIL=$((FAIL + 1))
|
||||
if [ -n "$CONNS" ] && [ "$CONNS" = "0" ]; then
|
||||
# 连接数 <= 1 也算"降级"。
|
||||
#
|
||||
# 原来是 == 0 才算，但实测**只剩 1 条连接时跨机房回源就会失败**
|
||||
# （用户撞上的 530 就是这种：请求落 SEA，隧道只剩 LAX 一条）。
|
||||
# 所以 1 和 0 一样危险，都计入降级计数。
|
||||
DEGRADED=0
|
||||
if [ -n "$CONNS" ] && [ "$CONNS" -le 1 ] 2>/dev/null; then
|
||||
ZERO=$((ZERO + 1))
|
||||
DEGRADED=1
|
||||
fi
|
||||
|
||||
log "公网不通（$DETAIL），连续第 $FAIL 次（零连接 $ZERO 次）"
|
||||
log "公网不通（$DETAIL），连续第 $FAIL 次（降级 $ZERO 次）"
|
||||
|
||||
# 公网不通 + 连接数已经降到危险区 = **确认的故障**，不是短抖动。
|
||||
# 这种情况立刻重启，不等攒够 3 次 —— 每次检查间隔 20 秒，
|
||||
# 等 3 次就是白断 40 秒。
|
||||
if [ "$DEGRADED" = "1" ]; then
|
||||
restart_tunnel "公网不通且只剩 ${CONNS} 条隧道连接（确认故障，立即重启）"
|
||||
fi
|
||||
|
||||
if [ "$FAIL" -ge "$THRESHOLD" ] || [ "$ZERO" -ge "$THRESHOLD" ]; then
|
||||
restart_tunnel "连续 $FAIL 次公网不通（含 $ZERO 次零连接）"
|
||||
restart_tunnel "连续 $FAIL 次公网不通（含 $ZERO 次降级）"
|
||||
fi
|
||||
|
||||
write_state "$ZERO" "$FAIL"
|
||||
|
||||
Reference in New Issue
Block a user