From 8c4ee6df08325385b52a7bc9545cfc56e645d47d Mon Sep 17 00:00:00 2001 From: xmz7891 Date: Wed, 7 Oct 2026 09:15:27 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BF=AE=E7=9C=9F=E5=87=B6=EF=BC=9A4G=20?= =?UTF-8?q?=E9=93=BE=E8=B7=AF=E9=9D=99=E9=BB=98=E6=96=AD=E8=BF=9E=E5=AF=BC?= =?UTF-8?q?=E8=87=B4=E9=9A=A7=E9=81=93"=E5=81=87=E6=B4=BB"=EF=BC=88?= =?UTF-8?q?=E7=94=A8=E6=88=B7=E6=92=9E=E4=B8=8A=20Error=201033=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 现象：用户早上打开 relay.xxmxxs.top 看到 Cloudflare Error 1033。 watchdog 日志把过程记下来了： 公网不通（HTTP 000，ha_connections=2） <- 隧道显示连着，公网不通 公网不通（HTTP 502，ha_connections=2） 公网恢复（HTTP 200） <- 自己又好了 也就是说：**间歇性假活**，短样本测不出来。 我之前只测 15~25 次就说"可用率 100%"，太草率了。 根因：4G 链路会**静默断开** TCP 连接 —— 不发 FIN/RST，双方都以为还连着。 Cloudflare 把请求发到这条僵尸连接上，就回 502/530/1033。 Linux 默认 tcp_keepalive_time = **7200 秒（2 小时）**才发现连接死了， 所以中间这段时间服务一直"看着在线、实际打不开"。 tunnel.log 最近 3000 行的佐证： i/o timeout 212 次 Retrying connection 127 次 Lost connection 22 次 修法： 1. TCP keepalive 调到 30 秒 + 每 10 秒探测 3 次 => 最多约 60 秒发现僵尸连接，cloudflared 立刻重连 （/etc/sysctl.d/97-ykxmz-keepalive.conf） 2. 隧道连接数 2 -> 4，多几条冗余 3. 修 relay 从不清理设备记录的问题： self.devices 只增不减，被控端每换端口/重装就多一条， 实测一天攒了 75 条僵尸记录，/v1/health 从几 KB 涨到 20KB。 加 DEVICE_TTL（6 小时）在 snapshot() 里清掉。 实测（修完立刻验证）： 设备侧 公网 15/15 HTTP 200 Windows 公网 25/25，中位 520 ms，最快 463 ms health 20931 字节 / 75 台 -> 152 字节 / 0 台 --- _build/commit_keepalive_fix.py | 93 ++++++++++++++++++++++ _build/diag_1033.sh | 94 +++++++++++----------- _build/diag_502.sh | 47 +++++++++++ _build/diag_memory.sh | 61 +++++++++++++++ _build/fix_keepalive.sh | 137 +++++++++++++++++++++++++++++++++ _build/probe_502.sh | 45 +++++++++++ _build/verify_after_fix.sh | 37 +++++++++ _build/watch_availability.py | 113 +++++++++++++++++++++++++++ relay/relay.py | 26 +++++++ 9 files changed, 603 insertions(+), 50 deletions(-) create mode 100644 _build/commit_keepalive_fix.py create mode 100644 _build/diag_502.sh create mode 100644 _build/diag_memory.sh create mode 100644 _build/fix_keepalive.sh create mode 100644 _build/probe_502.sh create mode 100644 _build/verify_after_fix.sh create mode 100644 _build/watch_availability.py diff --git a/_build/commit_keepalive_fix.py b/_build/commit_keepalive_fix.py new file mode 100644 index 0000000..9909988 --- /dev/null +++ b/_build/commit_keepalive_fix.py @@ -0,0 +1,93 @@ +# -*- coding: utf-8 -*- +r'''提交这轮修复，顺便看看有没有现成的手表截图能放到官网上。''' +from __future__ import annotations + +import os +import subprocess +import sys +from pathlib import Path + +ROOT = Path(r"C:\Users\shu\Desktop\deepseekagent\LanRemote") +GIT = r"C:\Program Files\Git\bin\git.exe" + +MSG = """修真凶：4G 链路静默断连导致隧道"假活"（用户撞上 Error 1033） + +现象：用户早上打开 relay.xxmxxs.top 看到 Cloudflare Error 1033。 + watchdog 日志把过程记下来了： + 公网不通（HTTP 000，ha_connections=2） <- 隧道显示连着，公网不通 + 公网不通（HTTP 502，ha_connections=2） + 公网恢复（HTTP 200） <- 自己又好了 + 也就是说：**间歇性假活**，短样本测不出来。 + 我之前只测 15~25 次就说"可用率 100%"，太草率了。 + +根因：4G 链路会**静默断开** TCP 连接 —— 不发 FIN/RST，双方都以为还连着。 + Cloudflare 把请求发到这条僵尸连接上，就回 502/530/1033。 + Linux 默认 tcp_keepalive_time = **7200 秒（2 小时）**才发现连接死了， + 所以中间这段时间服务一直"看着在线、实际打不开"。 + + tunnel.log 最近 3000 行的佐证： + i/o timeout 212 次 + Retrying connection 127 次 + Lost connection 22 次 + +修法： + 1. TCP keepalive 调到 30 秒 + 每 10 秒探测 3 次 + => 最多约 60 秒发现僵尸连接，cloudflared 立刻重连 + （/etc/sysctl.d/97-ykxmz-keepalive.conf） + 2. 隧道连接数 2 -> 4，多几条冗余 + 3. 修 relay 从不清理设备记录的问题： + self.devices 只增不减，被控端每换端口/重装就多一条， + 实测一天攒了 75 条僵尸记录，/v1/health 从几 KB 涨到 20KB。 + 加 DEVICE_TTL（6 小时）在 snapshot() 里清掉。 + +实测（修完立刻验证）： + 设备侧 公网 15/15 HTTP 200 + Windows 公网 25/25，中位 520 ms，最快 463 ms + health 20931 字节 / 75 台 -> 152 字节 / 0 台 +""" + + +def commit() -> None: + subprocess.run([GIT, "add", "-A"], cwd=str(ROOT), check=False) + msg_file = ROOT / "_build" / "_cmsg6.txt" + msg_file.write_bytes(MSG.encode("utf-8")) + r = subprocess.run([GIT, "commit", "-q", "-F", str(msg_file)], + cwd=str(ROOT), capture_output=True, text=True, + encoding="utf-8", errors="replace") + msg_file.unlink(missing_ok=True) + print(" commit exit=%s" % r.returncode) + show = subprocess.run([GIT, "log", "--oneline", "-1"], cwd=str(ROOT), + capture_output=True, text=True, encoding="utf-8", + errors="replace") + print(" " + (show.stdout or "").strip()) + env = dict(os.environ) + env["GIT_SSH_COMMAND"] = r'"C:\Program Files\Git\usr\bin\ssh.exe"' + push = subprocess.run([GIT, "push", "origin", "main"], cwd=str(ROOT), + capture_output=True, text=True, encoding="utf-8", + errors="replace", env=env) + for line in (push.stderr or push.stdout or "").strip().splitlines()[-2:]: + print(" " + line) + + +def main() -> int: + commit() + + print() + print(" 官网现在有什么图可用：") + for folder in ("_tests/shots", "android", "windows", "docs", "_build"): + base = ROOT / folder + if not base.is_dir(): + continue + pics = [p for p in base.rglob("*") + if p.suffix.lower() in (".png", ".jpg", ".jpeg", ".webp") + and p.is_file() and p.stat().st_size > 3000] + if pics: + print(" %s/ （%d 张）" % (folder, len(pics))) + for p in sorted(pics)[:8]: + print(" %-46s %6.1f KB" + % (p.name, p.stat().st_size / 1024)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/_build/diag_1033.sh b/_build/diag_1033.sh index c59f99d..1aee180 100644 --- a/_build/diag_1033.sh +++ b/_build/diag_1033.sh @@ -1,59 +1,53 @@ #!/bin/bash -# 深挖："被控端一提交配对码，隧道就掉（1033）" -# -# 1033 = cloudflared 没连上 Cloudflare（不是 502 那种源站问题） -# 所以要看：cloudflared 是不是被杀 / OOM / 崩了 +# 隧道挂了（Cloudflare Error 1033）—— 全套体检 set -u -RELAY_DIR=/root/lanremote-relay -URL=https://relay.xxmxxs.top - -echo "===== 1. 内存（460MB 的小机器，OOM 是头号嫌疑）=====" -free -m | sed 's/^/ /' -echo -n " swap: " -swapon --show 2>/dev/null | tail -1 || echo "（没有 swap）" - -echo -echo "===== 2. 内核有没有杀进程（OOM killer）=====" -dmesg 2>/dev/null | grep -iE 'killed process|out of memory|oom' | tail -8 | sed 's/^/ /' || echo " 读不到 dmesg" -echo " journal 里的 OOM：" -journalctl -k --no-pager 2>/dev/null | grep -iE 'killed process|oom' | tail -6 | sed 's/^/ /' || echo " 没有" - -echo -echo "===== 3. 服务重启过几次 =====" -for s in lanremote-relay lanremote-tunnel; do - printf " %-22s NRestarts=%s 状态=%s\n" "$s" \ - "$(systemctl show $s -p NRestarts --value 2>/dev/null)" \ - "$(systemctl is-active $s 2>/dev/null)" - echo " 最近 3 条日志：" - journalctl -u "$s" --no-pager -n 3 2>/dev/null | sed 's/^/ /' +echo "════════ 1. 服务状态 ════════" +for svc in lanremote-tunnel lanremote-relay lanremote-nettune; do + printf " %-22s %s\n" "$svc" "$(systemctl is-active $svc 2>&1)" done echo -echo "===== 4. 现在各进程吃多少内存 =====" -ps -eo rss,comm --sort=-rss 2>/dev/null | head -10 | sed 's/^/ /' +echo "════════ 2. cloudflared 进程 ════════" +ps aux 2>/dev/null | grep '[c]loudflared' | head -3 | sed 's/^/ /' +echo " 进程数: $(ps aux 2>/dev/null | grep -c '[c]loudflared')" echo -echo "===== 5. 隧道连接数 + 公网 =====" -echo -n " ha_connections = " +echo "════════ 3. 隧道 unit 的当前配置 ════════" +grep -E 'ExecStart|Restart' /etc/systemd/system/lanremote-tunnel.service 2>/dev/null | sed 's/^/ /' + +echo +echo "════════ 4. 最近的隧道日志（关键行）════════" +tail -60 /root/lanremote-relay/tunnel.log 2>/dev/null \ + | grep -E 'ERR|WRN|Registered|Initial protocol|error|failed|timeout' \ + | tail -14 | sed 's/^/ /' + +echo +echo "════════ 5. 日志最后 8 行（不分类型）════════" +tail -8 /root/lanremote-relay/tunnel.log 2>/dev/null | sed 's/^/ /' + +echo +echo "════════ 6. /etc/hosts 有没有被我改坏 ════════" +echo " argotunnel 条目数: $(grep -c argotunnel /etc/hosts 2>/dev/null || echo 0)" +echo " hosts 总行数: $(wc -l < /etc/hosts)" +echo " hosts 内容:" +cat /etc/hosts | sed 's/^/ /' + +echo +echo "════════ 7. 隧道凭证文件还在吗 ════════" +ls -la /root/.cloudflared/ 2>/dev/null | sed 's/^/ /' + +echo +echo "════════ 8. 本机 relay 还活着吗 ════════" +curl -s -o /dev/null -w ' relay 本机 HTTP %{http_code}\n' --max-time 8 http://127.0.0.1:8080/v1/health + +echo +echo "════════ 9. 拨测公网 ════════" +curl -s -o /dev/null -w ' 公网 HTTP %{http_code}\n' --max-time 25 \ + -H 'User-Agent: LanRemote/1.4.0' https://relay.xxmxxs.top/v1/health + +echo +echo "════════ 10. cloudflared 版本和 metrics ════════" +cloudflared --version 2>&1 | sed 's/^/ /' curl -s --max-time 5 http://127.0.0.1:20241/metrics 2>/dev/null \ - | grep '^cloudflared_tunnel_ha_connections' | awk '{print $2}' -echo " 公网: HTTP $(timeout 15 curl -4 -s -o /dev/null -w '%{http_code}' \ - -H 'User-Agent: LanRemote/1.4.0' $URL/v1/health 2>/dev/null)" - -echo -echo "===== 6. 看门狗最近动作 =====" -tail -10 "$RELAY_DIR/watchdog.log" 2>/dev/null | sed 's/^/ /' - -echo -echo "===== 7. 中转自身的内存上限 =====" -echo " relay 的 MemoryMax: $(systemctl show lanremote-relay -p MemoryMax --value 2>/dev/null)" -echo " tunnel 的 MemoryMax: $(systemctl show lanremote-tunnel -p MemoryMax --value 2>/dev/null)" -echo " relay 的 MemoryCurrent: $(systemctl show lanremote-relay -p MemoryCurrent --value 2>/dev/null)" -echo " tunnel 的 MemoryCurrent: $(systemctl show lanremote-tunnel -p MemoryCurrent --value 2>/dev/null)" - -echo -echo "===== 8. 系统整体 OOM 配额 =====" -cat /proc/sys/vm/panic_on_oom 2>/dev/null | sed 's/^/ panic_on_oom=/' -cat /proc/sys/vm/oom_kill_allocating_task 2>/dev/null | sed 's/^/ oom_kill_allocating_task=/' -echo " 设备内存: $(grep MemTotal /proc/meminfo | awk '{print int($2/1024)}') MB" + | grep -E '^cloudflared_tunnel_(ha_connections|request_errors|total_requests)' | sed 's/^/ /' diff --git a/_build/diag_502.sh b/_build/diag_502.sh new file mode 100644 index 0000000..be656df --- /dev/null +++ b/_build/diag_502.sh @@ -0,0 +1,47 @@ +#!/bin/bash +# 502 但不是 1033：说明 Cloudflare 连上了隧道，是**回源**失败。 +# 本机 curl 200，所以问题在 cloudflared -> relay 这一段。 +set -u + +echo "════════ 1. 隧道 ingress 配置 ════════" +cat /root/.cloudflared/config.yml 2>/dev/null | sed 's/^/ /' + +echo +echo "════════ 2. relay 到底在听哪个地址 ════════" +ss -lntp 2>/dev/null | grep -E '8080|LISTEN' | head -8 | sed 's/^/ /' +echo " ---" +echo " 127.0.0.1:8080 $(curl -s -o /dev/null -w '%{http_code}' --max-time 5 http://127.0.0.1:8080/v1/health)" +echo " localhost:8080 $(curl -s -o /dev/null -w '%{http_code}' --max-time 5 http://localhost:8080/v1/health)" +LANIP=$(hostname -I | awk '{print $1}') +echo " $LANIP:8080 $(curl -s -o /dev/null -w '%{http_code}' --max-time 5 http://$LANIP:8080/v1/health)" + +echo +echo "════════ 3. relay 服务日志（最近 20 行）════════" +journalctl -u lanremote-relay -n 20 --no-pager 2>/dev/null | sed 's/^/ /' + +echo +echo "════════ 4. relay 有没有报错 ════════" +journalctl -u lanremote-relay -n 100 --no-pager 2>/dev/null \ + | grep -iE 'error|exception|traceback|refused|trace' | tail -8 | sed 's/^/ /' +echo " （没输出=没有明显报错）" + +echo +echo "════════ 5. 本机打一次完整请求，看 relay 怎么回的 ════════" +curl -sv --max-time 8 http://127.0.0.1:8080/v1/health 2>&1 \ + | grep -E '^< HTTP|^< [A-Z]|^> GET|Connection' | head -10 | sed 's/^/ /' + +echo +echo "════════ 6. 隧道 metrics 全量 ════════" +curl -s --max-time 5 http://127.0.0.1:20241/metrics 2>/dev/null \ + | grep -E '^cloudflared_tunnel' | sed 's/^/ /' + +echo +echo "════════ 7. cloudflared 进程和它的命令行 ════════" +pid=$(pgrep -f 'cloudflared tunnel' | head -1) +echo " PID=$pid" +[ -n "$pid" ] && tr '\0' ' ' < /proc/$pid/cmdline 2>/dev/null | sed 's/^/ /' +echo + +echo "════════ 8. cloudflared 自己测一次回源 ════════" +# cloudflared 有 diagnose 子命令，能报出配置和连通性 +cloudflared tunnel --config /root/.cloudflared/config.yml ingress validate 2>&1 | tail -12 | sed 's/^/ /' diff --git a/_build/diag_memory.sh b/_build/diag_memory.sh new file mode 100644 index 0000000..14b087b --- /dev/null +++ b/_build/diag_memory.sh @@ -0,0 +1,61 @@ +#!/bin/bash +# 查隧道反复重连的根因：内存限制？OOM？DNS 差异？ +set -u + +echo "════════ 1. 内存总览 ════════" +free -m | sed 's/^/ /' + +echo +echo "════════ 2. 各进程内存占用 ════════" +ps aux --sort=-rss 2>/dev/null | head -8 | awk '{printf " %-8s %6s MB %s\n", $1, int($6/1024), $11}' + +echo +echo "════════ 3. 隧道 unit 里的内存限制 ════════" +grep -E 'MemoryMax|MemoryHigh|Restart' /etc/systemd/system/lanremote-tunnel.service | sed 's/^/ /' +echo " 当前实际用量:" +systemctl show lanremote-tunnel -p MemoryCurrent -p MemoryMax --value 2>/dev/null | sed 's/^/ /' + +echo +echo "════════ 4. 有没有被 OOM 杀过 ════════" +journalctl -u lanremote-tunnel --since '6 hours ago' --no-pager 2>/dev/null \ + | grep -iE 'oom|killed|memory|out of memory' | tail -8 | sed 's/^/ /' +echo " dmesg 里的 OOM:" +dmesg 2>/dev/null | grep -iE 'oom|killed process' | tail -6 | sed 's/^/ /' +echo " （没输出=没被 OOM 杀）" + +echo +echo "════════ 5. 隧道服务重启过几次 ════════" +journalctl -u lanremote-tunnel --since '6 hours ago' --no-pager 2>/dev/null \ + | grep -cE 'Started|Stopped|Deactivated' | sed 's/^/ 系统层面事件数: /' +echo " 最近的启动/停止:" +journalctl -u lanremote-tunnel --since '6 hours ago' --no-pager 2>/dev/null \ + | grep -E 'Started|Stopped|Deactivated|killed' | tail -8 | sed 's/^/ /' + +echo +echo "════════ 6. 隧道日志里重连的频率 ════════" +echo " 最近 6 小时的日志行数: $(wc -l < /root/lanremote-relay/tunnel.log)" +for kw in 'Registered tunnel connection' 'Lost connection' 'Retrying connection' 'i/o timeout'; do + n=$(tail -3000 /root/lanremote-relay/tunnel.log 2>/dev/null | grep -c "$kw") + printf " %-32s %s 次\n" "$kw" "$n" +done + +echo +echo "════════ 7. DNS：设备这边解析到哪 ════════" +getent hosts relay.xxmxxs.top | sed 's/^/ /' +echo " 直接问 1.1.1.1:" +nslookup relay.xxmxxs.top 1.1.1.1 2>/dev/null | grep -A2 'Name:' | sed 's/^/ /' + +echo +echo "════════ 8. relay.log 里的错误（最后 15 条）════════" +tail -400 /root/lanremote-relay/relay.log 2>/dev/null \ + | grep -iE 'error|exception|traceback|broken|reset' | tail -15 | sed 's/^/ /' +echo " （没输出=没有明显错误）" + +echo +echo "════════ 9. relay.log 最后 12 行（看它在干什么）════════" +tail -12 /root/lanremote-relay/relay.log 2>/dev/null | sed 's/^/ /' + +echo +echo "════════ 10. watchdog 在干什么 ════════" +tail -12 /root/lanremote-relay/watchdog.log 2>/dev/null | sed 's/^/ /' +echo " watchdog 服务: $(systemctl is-active lanremote-watchdog 2>&1)" diff --git a/_build/fix_keepalive.sh b/_build/fix_keepalive.sh new file mode 100644 index 0000000..6e99b92 --- /dev/null +++ b/_build/fix_keepalive.sh @@ -0,0 +1,137 @@ +#!/bin/bash +# 真修：让死掉的 TCP 连接被**快速发现** +# +# 根因（watchdog 日志 + 隧道日志看出来的）： +# 隧道显示 ha_connections=2（本地认为连着），但公网 HTTP 000/502/530。 +# 隧道日志最近 3000 行里 i/o timeout 212 次、Retrying 127 次。 +# +# => 4G 链路会把 TCP 连接**静默断开**：连接实际死了， +# 但双方都没有收到 FIN/RST，于是都以为还连着。 +# Cloudflare 把请求发到这条"僵尸连接"上，就回 502/530/1033。 +# +# Linux 默认参数： +# tcp_keepalive_time = 7200 秒（2 小时！）才发现连接死了 +# tcp_keepalive_intvl = 75 秒 +# tcp_keepalive_probes = 9 次 +# => 最坏要 2 小时才知道连接没了。中间这段时间服务就是"假活"。 +# +# 修法： +# 1. 把 keepalive 调到 30 秒 + 10 秒探测 3 次 +# => 最多 ~60 秒发现僵尸连接，cloudflared 立刻重连 +# 2. ha-connections 从 2 提到 4，多几条冗余，一条死了还有别的 +# 3. 顺手清掉测试留下的 ~90 台僵尸设备（health 接口都被撑到 20KB 了） +set -u + +RELAY_DIR=/root/lanremote-relay + +echo "════════ 1. 改 TCP keepalive（核心修复）════════" +cat > /etc/sysctl.d/97-ykxmz-keepalive.conf <<'EOF' +# ykxmz：快速发现"静默断开"的 TCP 连接 +# +# 背景：这台设备走 4G，运营商链路会静默丢连接（不发 FIN/RST）。 +# Linux 默认 7200 秒才发现，期间 cloudflared 以为自己连着、 +# Cloudflare 也以为隧道活着，但请求发过来就是 502/1033。 +# 把 keepalive 调到 30 秒，最多一分钟就能发现并重连。 +net.ipv4.tcp_keepalive_time = 30 +net.ipv4.tcp_keepalive_intvl = 10 +net.ipv4.tcp_keepalive_probes = 3 +EOF +sysctl -p /etc/sysctl.d/97-ykxmz-keepalive.conf 2>&1 | sed 's/^/ /' + +echo +echo " 生效确认:" +for k in tcp_keepalive_time tcp_keepalive_intvl tcp_keepalive_probes; do + printf " %-26s = %s\n" "$k" "$(sysctl -n net.ipv4.$k)" +done + +echo +echo "════════ 2. 隧道连接数提到 4 ════════" +CURRENT=$(grep -o 'ha-connections [0-9]*' /etc/systemd/system/lanremote-tunnel.service) +echo " 当前: $CURRENT" +sed -i 's/--ha-connections [0-9]*/--ha-connections 4/' /etc/systemd/system/lanremote-tunnel.service +NEW=$(grep -o 'ha-connections [0-9]*' /etc/systemd/system/lanremote-tunnel.service) +echo " 改为: $NEW" + +echo +echo "════════ 3. 清掉测试留下的僵尸设备 ════════" +python3 - <<'PY' +import json +import os +import time +from pathlib import Path + +# 中转把设备登记在 data 目录里，找找在哪个文件 +base = Path("/root/lanremote-relay/data") +if not base.is_dir(): + print(" 没有 data 目录") + raise SystemExit(0) + +for path in sorted(base.glob("*.json")): + try: + data = json.loads(path.read_text(encoding="utf-8")) + except Exception as exc: + print(" %s 读不了: %s" % (path.name, exc)) + continue + if not isinstance(data, dict): + print(" %s 不是字典，跳过" % path.name) + continue + # 找设备表 + for key in ("devices", "deviceMap", "device_map", "clients"): + if key in data and isinstance(data[key], (dict, list)): + items = data[key] + before = len(items) + now = time.time() + if isinstance(items, dict): + keep = {k: v for k, v in items.items() + if isinstance(v, dict) + and now - float(v.get("lastSeen", 0) or 0) < 3600} + removed = before - len(keep) + if removed: + data[key] = keep + else: + keep = [v for v in items + if isinstance(v, dict) + and now - float(v.get("lastSeen", 0) or 0) < 3600] + removed = before - len(keep) + if removed: + data[key] = keep + if removed: + path.write_text(json.dumps(data, ensure_ascii=False), + encoding="utf-8") + print(" %s: %s 从 %d 清到 %d（删了 %d）" + % (path.name, key, before, len(keep), removed)) + else: + print(" %s: %s 共 %d 条，都是新的" % (path.name, key, before)) +PY + +echo +echo " 清理后 health 大小:" +curl -s --max-time 10 http://127.0.0.1:8080/v1/health -o /tmp/h.json +echo " $(wc -c < /tmp/h.json) 字节" +python3 -c " +import json +d = json.load(open('/tmp/h.json')) +print(' 设备数:', len(d.get('devices', []))) +" 2>/dev/null | sed 's/^/ /' + +echo +echo "════════ 4. 重启隧道让新参数生效 ════════" +systemctl daemon-reload +systemctl restart lanremote-tunnel +sleep 30 +echo " 连接数: $(curl -s --max-time 5 http://127.0.0.1:20241/metrics 2>/dev/null | grep '^cloudflared_tunnel_ha_connections' | awk '{print $2}')" +echo " 落点:" +grep -E 'Registered tunnel connection' "$RELAY_DIR/tunnel.log" 2>/dev/null | tail -4 \ + | grep -oE 'location=[a-z0-9]+ protocol=[a-z0-9]+' | sed 's/^/ /' + +echo +echo "════════ 5. 立刻测公网 ════════" +ok=0 +for i in $(seq 1 10); do + c=$(curl -s -o /dev/null -w '%{http_code}' --max-time 15 \ + -H 'User-Agent: LanRemote/1.4.0' https://relay.xxmxxs.top/v1/health) + [ "$c" = "200" ] && ok=$((ok+1)) + printf " %2d: %s\n" "$i" "$c" + sleep 1 +done +echo " 可用 $ok/10" diff --git a/_build/probe_502.sh b/_build/probe_502.sh new file mode 100644 index 0000000..56e1fc7 --- /dev/null +++ b/_build/probe_502.sh @@ -0,0 +1,45 @@ +#!/bin/bash +# 发一个公网请求，紧跟着看隧道日志处理了什么 —— 定位 502 到底出在哪一段 +set -u + +RELAY_DIR=/root/lanremote-relay + +echo "════════ 1. 打标记，好区分新旧日志 ════════" +echo "---- PROBE $(date -u +%H:%M:%S) ----" >> "$RELAY_DIR/tunnel.log" +before=$(wc -l < "$RELAY_DIR/tunnel.log") +echo " 日志当前 $before 行" + +echo +echo "════════ 2. 从设备自己请求公网（发夹回路，但能看出链路上通不通）════════" +curl -s -o /tmp/pub.out -w ' HTTP %{http_code} 用时 %{time_total}s 大小 %{size_download}\n' \ + --max-time 30 -H 'User-Agent: LanRemote/1.4.0' https://relay.xxmxxs.top/v1/health +echo " 响应头 5 行:" +head -5 /tmp/pub.out | sed 's/^/ /' + +echo +echo "════════ 3. 请求短路径（首页，只有 15KB）════════" +curl -s -o /tmp/pub2.out -w ' HTTP %{http_code} 用时 %{time_total}s 大小 %{size_download}\n' \ + --max-time 30 -H 'User-Agent: LanRemote/1.4.0' https://relay.xxmxxs.top/ +head -3 /tmp/pub2.out | sed 's/^/ /' + +echo +echo "════════ 4. 请求一个极小的路径 ════════" +curl -s -o /dev/null -w ' /favicon.ico HTTP %{http_code} 用时 %{time_total}s 大小 %{size_download}\n' \ + --max-time 30 -H 'User-Agent: LanRemote/1.4.0' https://relay.xxmxxs.top/favicon.ico + +echo +echo "════════ 5. 这几条请求之后，隧道日志新增了什么 ════════" +after=$(wc -l < "$RELAY_DIR/tunnel.log") +echo " 新增 $((after - before)) 行:" +tail -n +$((before + 1)) "$RELAY_DIR/tunnel.log" | tail -30 | sed 's/^/ /' + +echo +echo "════════ 6. 隧道 metrics 变化 ════════" +curl -s --max-time 5 http://127.0.0.1:20241/metrics 2>/dev/null \ + | grep -E '^cloudflared_tunnel_(total_requests|request_errors|response_by_code)' | sed 's/^/ /' + +echo +echo "════════ 7. relay 自己的日志（不是 journal，是它可能写的文件）════════" +ls -la "$RELAY_DIR"/*.log 2>/dev/null | sed 's/^/ /' +echo " relay 最近通过隧道服务的请求（如果有访问日志）:" +journalctl -u lanremote-relay --since '2 minutes ago' --no-pager 2>/dev/null | tail -10 | sed 's/^/ /' diff --git a/_build/verify_after_fix.sh b/_build/verify_after_fix.sh new file mode 100644 index 0000000..57b5eb9 --- /dev/null +++ b/_build/verify_after_fix.sh @@ -0,0 +1,37 @@ +#!/bin/bash +# 验证：僵尸设备清掉了 + 公网稳定 +set -u + +BASE=http://127.0.0.1:8080 +PUB=https://relay.xxmxxs.top + +echo " 服务状态: relay=$(systemctl is-active lanremote-relay) tunnel=$(systemctl is-active lanremote-tunnel)" + +curl -s --max-time 15 "$BASE/v1/health" -o /tmp/h.json +size=$(wc -c < /tmp/h.json) +count=$(python3 -c "import json; print(len(json.load(open('/tmp/h.json')).get('devices',[])))" 2>/dev/null || echo "?") +echo " health: ${size} 字节，设备 $count 台" + +echo +echo " TCP keepalive:" +for k in tcp_keepalive_time tcp_keepalive_intvl tcp_keepalive_probes; do + printf " %-26s = %s\n" "$k" "$(sysctl -n net.ipv4.$k)" +done +echo " 隧道参数: $(grep -o 'ha-connections [0-9]*' /etc/systemd/system/lanremote-tunnel.service)" +echo " 隧道连接数: $(curl -s --max-time 5 http://127.0.0.1:20241/metrics 2>/dev/null | grep '^cloudflared_tunnel_ha_connections' | awk '{print $2}')" + +echo +echo " 公网 15 次:" +ok=0 +for i in $(seq 1 15); do + c=$(curl -s -o /dev/null -w '%{http_code}' --max-time 20 -H 'User-Agent: LanRemote/1.4.0' "$PUB/v1/health") + [ "$c" = "200" ] && ok=$((ok+1)) + printf " %2d: %s\n" "$i" "$c" + sleep 1 +done +echo " 可用 $ok/15" + +echo +echo " 官网公网:" +curl -s -o /dev/null -w ' HTTP %{http_code} %{size_download} 字节 %{time_total}s\n' \ + --max-time 25 -H 'User-Agent: LanRemote/1.4.0' "$PUB/" diff --git a/_build/watch_availability.py b/_build/watch_availability.py new file mode 100644 index 0000000..ea9bb85 --- /dev/null +++ b/_build/watch_availability.py @@ -0,0 +1,113 @@ +# -*- coding: utf-8 -*- +r'''长时间观察公网可用率 —— 这才是判断"稳不稳"的正确方法。 + +我之前只测了 15~25 次就连着说"可用率 100%"，太草率了。 +用户早上打开就撞上 Error 1033 —— 说明**间断性掉线**， +短样本根本看不出来。 + +这个脚本跑 10 分钟，每 3 秒一次，统计： + * 总可用率 + * 连续失败的最长时长（这个最要命：用户点进去就是白页） + * 失败的时间分布（是零星抖动还是成片断） +''' +from __future__ import annotations + +import statistics +import sys +import time +import urllib.error +import urllib.request + +URL = "https://relay.xxmxxs.top/v1/health" +HEADERS = {"User-Agent": "LanRemote/1.4.0"} +DURATION = 600 # 观察 10 分钟 +INTERVAL = 3 # 每 3 秒一次 + + +def probe(timeout: float = 12.0) -> tuple: + """返回 (是否成功, 耗时秒, 状态码或错误名)。""" + start = time.perf_counter() + try: + with urllib.request.urlopen( + urllib.request.Request(URL, headers=HEADERS), + timeout=timeout) as resp: + resp.read() + ok = resp.status == 200 + return ok, time.perf_counter() - start, str(resp.status) + except urllib.error.HTTPError as exc: + return False, time.perf_counter() - start, "HTTP%d" % exc.code + except Exception as exc: # noqa: BLE001 + return False, time.perf_counter() - start, exc.__class__.__name__ + + +def main() -> int: + print(" 观察 %d 分钟，每 %d 秒一次" % (DURATION // 60, INTERVAL)) + print() + + total = ok_count = 0 + times = [] + failures = [] + streak = 0 + worst_streak = 0 + worst_start = None + current_start = None + + deadline = time.time() + DURATION + while time.time() < deadline: + ok, elapsed, note = probe() + total += 1 + if ok: + ok_count += 1 + times.append(elapsed) + streak = 0 + current_start = None + else: + failures.append((time.strftime("%H:%M:%S"), note)) + if streak == 0: + current_start = time.time() + streak += 1 + if streak > worst_streak: + worst_streak = streak + worst_start = current_start + # 每分钟报一次进度 + if total % 20 == 0: + print(" 已测 %3d 次，成功 %3d，失败 %2d" + % (total, ok_count, total - ok_count)) + time.sleep(INTERVAL) + + print() + print(" ================ 结果 ================") + rate = ok_count / total * 100 if total else 0 + print(" 样本 %d 次（约 %d 分钟）" % (total, DURATION // 60)) + print(" 可用率 %.1f%% (%d/%d)" % (rate, ok_count, total)) + + if times: + print(" 成功时延迟 中位 %.0f ms 最快 %.0f ms 最慢 %.0f ms" + % (statistics.median(times) * 1000, + min(times) * 1000, max(times) * 1000)) + + if failures: + span = worst_streak * INTERVAL + print(" **最长连续失败 %d 次（约 %d 秒）**" % (worst_streak, span)) + if worst_start: + print(" 开始于 %s" % time.strftime("%H:%M:%S", + time.localtime(worst_start))) + print(" 失败明细（最多列 20 条）：") + for stamp, note in failures[:20]: + print(" %s %s" % (stamp, note)) + else: + print(" 全程没有失败") + + print() + print(" 判读：") + if rate >= 99.5: + print(" ≥99.5% —— 可以接受") + elif rate >= 97: + print(" 97~99.5% —— 有抖动，用户偶尔会遇到打不开") + else: + print(" <97% —— **不稳定**，必须查根因（用户会经常撞上 Error 1033）") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/relay/relay.py b/relay/relay.py index 65a8d67..934eaff 100644 --- a/relay/relay.py +++ b/relay/relay.py @@ -81,6 +81,15 @@ CODE_RE = re.compile(r"^\d{6}$") # 设备多久没心跳算掉线 DEVICE_TIMEOUT = 90.0 +# 设备多久没动静就**从列表里彻底删掉**。 +# +# 和 DEVICE_TIMEOUT（90 秒）不一样：那个只是标记"离线"， +# 记录还留着，方便手表看到"我以前连过这台"。 +# 但记录只增不减会出问题 —— 被控端每次换端口/重装都用新 deviceId +# 注册一次，老的永远留着。实测一天攒了 75 条僵尸记录， +# /v1/health 从几 KB 涨到 20KB，手表和网页每次白下载一大坨。 +# 6 小时足够覆盖"上课中途断一下"，真回来会重新注册。 +DEVICE_TTL = float(os.environ.get("RELAY_DEVICE_TTL", str(6 * 3600))) # 一个设备最多同时挂几个手表会话 MAX_SESSIONS_PER_DEVICE = 3 @@ -645,6 +654,23 @@ class Relay: def snapshot(self) -> dict: with self.lock: now = time.time() + + # 先把长时间没动静的设备清掉。 + # + # 为什么必须清：这个 dict 只增不减。被控端每次启动/换端口/ + # 重装都会用新的 deviceId 注册一次，老的永远留着 —— + # 实测跑一天就攒了 75 条僵尸记录，/v1/health 从几 KB + # 涨到 20KB，手表和网页每次都要白下载一大坨没用的东西。 + # 设备超过 6 小时没上报就当它下线了（真回来会重新注册）。 + stale = [did for did, dev in self.devices.items() + if now - getattr(dev, "last_seen", now) > DEVICE_TTL] + for did in stale: + self.devices.pop(did, None) + # 它的会话也一起收掉，免得挂着一堆死会话 + for sid in [s.session_id for s in self.sessions.values() + if s.device_id == did]: + self.sessions.pop(sid, None) + devices = [] for device in self.devices.values(): sessions = [s for s in self.sessions.values()