mirror of
https://gitee.com/xmz7891/ykxmz.git
synced 2026-10-18 13:40:45 +00:00
修真凶:4G 链路静默断连导致隧道"假活"(用户撞上 Error 1033)
现象:用户早上打开 relay.xxmxxs.top 看到 Cloudflare Error 1033。
watchdog 日志把过程记下来了:
公网不通(HTTP 000,ha_connections=2) <- 隧道显示连着,公网不通
公网不通(HTTP 502,ha_connections=2)
公网恢复(HTTP 200) <- 自己又好了
也就是说:**间歇性假活**,短样本测不出来。
我之前只测 15~25 次就说"可用率 100%",太草率了。
根因:4G 链路会**静默断开** TCP 连接 —— 不发 FIN/RST,双方都以为还连着。
Cloudflare 把请求发到这条僵尸连接上,就回 502/530/1033。
Linux 默认 tcp_keepalive_time = **7200 秒(2 小时)**才发现连接死了,
所以中间这段时间服务一直"看着在线、实际打不开"。
tunnel.log 最近 3000 行的佐证:
i/o timeout 212 次
Retrying connection 127 次
Lost connection 22 次
修法:
1. TCP keepalive 调到 30 秒 + 每 10 秒探测 3 次
=> 最多约 60 秒发现僵尸连接,cloudflared 立刻重连
(/etc/sysctl.d/97-ykxmz-keepalive.conf)
2. 隧道连接数 2 -> 4,多几条冗余
3. 修 relay 从不清理设备记录的问题:
self.devices 只增不减,被控端每换端口/重装就多一条,
实测一天攒了 75 条僵尸记录,/v1/health 从几 KB 涨到 20KB。
加 DEVICE_TTL(6 小时)在 snapshot() 里清掉。
实测(修完立刻验证):
设备侧 公网 15/15 HTTP 200
Windows 公网 25/25,中位 520 ms,最快 463 ms
health 20931 字节 / 75 台 -> 152 字节 / 0 台
This commit is contained in:
+44
-50
@@ -1,59 +1,53 @@
|
||||
#!/bin/bash
|
||||
# 深挖:"被控端一提交配对码,隧道就掉(1033)"
|
||||
#
|
||||
# 1033 = cloudflared 没连上 Cloudflare(不是 502 那种源站问题)
|
||||
# 所以要看:cloudflared 是不是被杀 / OOM / 崩了
|
||||
# 隧道挂了(Cloudflare Error 1033)—— 全套体检
|
||||
set -u
|
||||
|
||||
RELAY_DIR=/root/lanremote-relay
|
||||
URL=https://relay.xxmxxs.top
|
||||
|
||||
echo "===== 1. 内存(460MB 的小机器,OOM 是头号嫌疑)====="
|
||||
free -m | sed 's/^/ /'
|
||||
echo -n " swap: "
|
||||
swapon --show 2>/dev/null | tail -1 || echo "(没有 swap)"
|
||||
|
||||
echo
|
||||
echo "===== 2. 内核有没有杀进程(OOM killer)====="
|
||||
dmesg 2>/dev/null | grep -iE 'killed process|out of memory|oom' | tail -8 | sed 's/^/ /' || echo " 读不到 dmesg"
|
||||
echo " journal 里的 OOM:"
|
||||
journalctl -k --no-pager 2>/dev/null | grep -iE 'killed process|oom' | tail -6 | sed 's/^/ /' || echo " 没有"
|
||||
|
||||
echo
|
||||
echo "===== 3. 服务重启过几次 ====="
|
||||
for s in lanremote-relay lanremote-tunnel; do
|
||||
printf " %-22s NRestarts=%s 状态=%s\n" "$s" \
|
||||
"$(systemctl show $s -p NRestarts --value 2>/dev/null)" \
|
||||
"$(systemctl is-active $s 2>/dev/null)"
|
||||
echo " 最近 3 条日志:"
|
||||
journalctl -u "$s" --no-pager -n 3 2>/dev/null | sed 's/^/ /'
|
||||
echo "════════ 1. 服务状态 ════════"
|
||||
for svc in lanremote-tunnel lanremote-relay lanremote-nettune; do
|
||||
printf " %-22s %s\n" "$svc" "$(systemctl is-active $svc 2>&1)"
|
||||
done
|
||||
|
||||
echo
|
||||
echo "===== 4. 现在各进程吃多少内存 ====="
|
||||
ps -eo rss,comm --sort=-rss 2>/dev/null | head -10 | sed 's/^/ /'
|
||||
echo "════════ 2. cloudflared 进程 ════════"
|
||||
ps aux 2>/dev/null | grep '[c]loudflared' | head -3 | sed 's/^/ /'
|
||||
echo " 进程数: $(ps aux 2>/dev/null | grep -c '[c]loudflared')"
|
||||
|
||||
echo
|
||||
echo "===== 5. 隧道连接数 + 公网 ====="
|
||||
echo -n " ha_connections = "
|
||||
echo "════════ 3. 隧道 unit 的当前配置 ════════"
|
||||
grep -E 'ExecStart|Restart' /etc/systemd/system/lanremote-tunnel.service 2>/dev/null | sed 's/^/ /'
|
||||
|
||||
echo
|
||||
echo "════════ 4. 最近的隧道日志(关键行)════════"
|
||||
tail -60 /root/lanremote-relay/tunnel.log 2>/dev/null \
|
||||
| grep -E 'ERR|WRN|Registered|Initial protocol|error|failed|timeout' \
|
||||
| tail -14 | sed 's/^/ /'
|
||||
|
||||
echo
|
||||
echo "════════ 5. 日志最后 8 行(不分类型)════════"
|
||||
tail -8 /root/lanremote-relay/tunnel.log 2>/dev/null | sed 's/^/ /'
|
||||
|
||||
echo
|
||||
echo "════════ 6. /etc/hosts 有没有被我改坏 ════════"
|
||||
echo " argotunnel 条目数: $(grep -c argotunnel /etc/hosts 2>/dev/null || echo 0)"
|
||||
echo " hosts 总行数: $(wc -l < /etc/hosts)"
|
||||
echo " hosts 内容:"
|
||||
cat /etc/hosts | sed 's/^/ /'
|
||||
|
||||
echo
|
||||
echo "════════ 7. 隧道凭证文件还在吗 ════════"
|
||||
ls -la /root/.cloudflared/ 2>/dev/null | sed 's/^/ /'
|
||||
|
||||
echo
|
||||
echo "════════ 8. 本机 relay 还活着吗 ════════"
|
||||
curl -s -o /dev/null -w ' relay 本机 HTTP %{http_code}\n' --max-time 8 http://127.0.0.1:8080/v1/health
|
||||
|
||||
echo
|
||||
echo "════════ 9. 拨测公网 ════════"
|
||||
curl -s -o /dev/null -w ' 公网 HTTP %{http_code}\n' --max-time 25 \
|
||||
-H 'User-Agent: LanRemote/1.4.0' https://relay.xxmxxs.top/v1/health
|
||||
|
||||
echo
|
||||
echo "════════ 10. cloudflared 版本和 metrics ════════"
|
||||
cloudflared --version 2>&1 | sed 's/^/ /'
|
||||
curl -s --max-time 5 http://127.0.0.1:20241/metrics 2>/dev/null \
|
||||
| grep '^cloudflared_tunnel_ha_connections' | awk '{print $2}'
|
||||
echo " 公网: HTTP $(timeout 15 curl -4 -s -o /dev/null -w '%{http_code}' \
|
||||
-H 'User-Agent: LanRemote/1.4.0' $URL/v1/health 2>/dev/null)"
|
||||
|
||||
echo
|
||||
echo "===== 6. 看门狗最近动作 ====="
|
||||
tail -10 "$RELAY_DIR/watchdog.log" 2>/dev/null | sed 's/^/ /'
|
||||
|
||||
echo
|
||||
echo "===== 7. 中转自身的内存上限 ====="
|
||||
echo " relay 的 MemoryMax: $(systemctl show lanremote-relay -p MemoryMax --value 2>/dev/null)"
|
||||
echo " tunnel 的 MemoryMax: $(systemctl show lanremote-tunnel -p MemoryMax --value 2>/dev/null)"
|
||||
echo " relay 的 MemoryCurrent: $(systemctl show lanremote-relay -p MemoryCurrent --value 2>/dev/null)"
|
||||
echo " tunnel 的 MemoryCurrent: $(systemctl show lanremote-tunnel -p MemoryCurrent --value 2>/dev/null)"
|
||||
|
||||
echo
|
||||
echo "===== 8. 系统整体 OOM 配额 ====="
|
||||
cat /proc/sys/vm/panic_on_oom 2>/dev/null | sed 's/^/ panic_on_oom=/'
|
||||
cat /proc/sys/vm/oom_kill_allocating_task 2>/dev/null | sed 's/^/ oom_kill_allocating_task=/'
|
||||
echo " 设备内存: $(grep MemTotal /proc/meminfo | awk '{print int($2/1024)}') MB"
|
||||
| grep -E '^cloudflared_tunnel_(ha_connections|request_errors|total_requests)' | sed 's/^/ /'
|
||||
|
||||
Reference in New Issue
Block a user