ykxmz/_tests/audit_encoding.py

# -*- coding: utf-8 -*-
"""全项目编码审计（收窄版）。

上一版太吵：把 Java 字节码、测试用的 .bin、还有审计脚本自己
（源码里就写着乱码样本）全当成文本查了。

这一版：
    * 用**内容**判断是不是二进制（不是只看扩展名）
    * 跳过编译产物和测试数据目录
    * 跳过审计脚本自己

只查真正要交付和要跑的东西。
"""
from __future__ import annotations

import sys
import zipfile
from pathlib import Path

ROOT = Path(__file__).resolve().parent.parent
SELF = Path(__file__).resolve()

# 这些目录是产物/数据，不是源码
SKIP_DIRS = {
    "_build", "__pycache__", ".git", "dist_exe", "dist", "pyinstaller_work",
    "node_modules", "out_java", "received", "gen", "javacheck", "javacheck2",
    "javacheck3", "javacheck4", "javacheck5", "build_tmp", "toolchain",
}
SKIP_SUFFIX = {
    ".pyc", ".png", ".jpg", ".jpeg", ".gif", ".ico", ".apk", ".exe", ".dll",
    ".pyd", ".zip", ".idsig", ".so", ".class", ".bin", ".jar", ".keystore",
    ".jks", ".der", ".pem", ".7z",
}

MOJIBAKE = [
    "锟斤拷", "�", "â€", "ä", "é", "è", "а", "Ñ€",
    "\ufffd",
    "ä¸", "å¤", "æµ", "è¯", "é€",
]

# 只查这些扩展名（其余一律不问）
TEXT_SUFFIX = {".py", ".java", ".ps1", ".sh", ".bat", ".cmd", ".txt",
               ".md", ".json", ".yml", ".yaml", ".xml", ".gradle", ".cfg",
               ".ini", ".properties", ".pro", ".service", ".timer"}

problems = []
stats = {}
scanned = 0


def report(level: str, path, message: str) -> None:
    label = str(path.relative_to(ROOT)) if isinstance(path, Path) \
        and str(path).startswith(str(ROOT)) else str(path)
    problems.append((level, label, message))


def looks_binary(body: bytes) -> bool:
    if b"\x00" in body[:4096]:
        return True
    # 非文本字节比例过高就当二进制
    sample = body[:4096]
    if not sample:
        return False
    weird = sum(1 for byte in sample
                if byte < 8 or (14 <= byte < 32 and byte != 27))
    return weird / len(sample) > 0.10


def check_bytes(path, name: str, suffix: str, raw: bytes) -> None:
    """对一份内容做编码检查。path 只用于报错定位。"""
    global scanned
    scanned += 1
    stats[suffix] = stats.get(suffix, 0) + 1

    has_bom = raw.startswith(b"\xef\xbb\xbf")
    body = raw[3:] if has_bom else raw

    # .bat 走特殊逻辑：cmd.exe 在中文 Windows 上默认代码页 936，
    # 所以 .bat 里要写中文就必须是 **GBK**（UTF-8 的中文会变乱码）。
    # 纯 ASCII 的 .bat 两种代码页下都对，最保险。
    if suffix in (".bat", ".cmd"):
        if has_bom:
            report("警告", path, f"{name} 有 BOM（cmd 会把 BOM 当命令）")
        try:
            text = body.decode("gbk")
            encoding_name = "GBK"
        except UnicodeDecodeError:
            try:
                text = body.decode("utf-8")
                if any("\u4e00" <= ch <= "\u9fff" for ch in text):
                    report("严重", path,
                           f"{name} 是 UTF-8 且含中文 —— cmd.exe 下会乱码，应为 GBK")
                encoding_name = "UTF-8(纯ASCII)"
            except UnicodeDecodeError as exc:
                report("严重", path, f"{name} GBK/UTF-8 都解不开：{exc}")
                return
        crlf = body.count(b"\r\n")
        lf_only = body.count(b"\n") - crlf
        if lf_only:
            report("警告", path, f"{name} 有 {lf_only} 处纯 LF（.bat 应为 CRLF）")
        stats[suffix] = stats.get(suffix, 0) + 1
        scanned += 1
        return

    try:
        text = body.decode("utf-8")
    except UnicodeDecodeError as exc:
        report("严重", path, f"{name} 不是合法 UTF-8：{exc}")
        return

    if suffix == ".ps1" and not has_bom:
        report("严重", path, f"{name} 缺 UTF-8 BOM（PowerShell 5.1 会按 GBK 读）")
    if suffix in (".py", ".java", ".sh") and has_bom:
        report("警告", path, f"{name} 不应有 BOM")
    if suffix in (".bat", ".cmd") and has_bom:
        report("警告", path, f"{name} 不应有 BOM")

    for token in MOJIBAKE:
        if token in text:
            line = next((i + 1 for i, l in enumerate(text.splitlines())
                         if token in l), 0)
            report("严重", path, f"{name} 疑似乱码 {token!r}（第 {line} 行）")
            break

    crlf = body.count(b"\r\n")
    lf_only = body.count(b"\n") - crlf
    cr_only = body.count(b"\r") - crlf

    if suffix == ".sh" and (crlf or cr_only):
        report("严重", path, f"{name} 有 CR（CRLF {crlf} 处）—— Linux 上跑不起来")
    if suffix in (".bat", ".cmd") and lf_only:
        report("警告", path, f"{name} 有 {lf_only} 处纯 LF")
    if cr_only:
        report("警告", path, f"{name} 有 {cr_only} 处孤立 CR")
    if crlf and lf_only and suffix not in (".bat", ".cmd"):
        report("提示", path, f"{name} 行尾混用（CRLF {crlf} / LF {lf_only}）")


def check_zip(zip_path: Path) -> None:
    if not zip_path.is_file():
        return
    print(f"  交付包：{zip_path.name}")
    with zipfile.ZipFile(zip_path) as archive:
        for name in archive.namelist():
            if name.endswith("/"):
                continue
            suffix = Path(name).suffix.lower()
            if suffix not in TEXT_SUFFIX:
                continue
            raw = archive.read(name)
            if not raw or looks_binary(raw):
                continue
            check_bytes(zip_path, f"包内 {name}", suffix, raw)


def main() -> int:
    print("=" * 74)
    print("  编码审计（只查要交付 / 要跑的文本文件）")
    print("=" * 74)

    targets = []
    for path in ROOT.rglob("*"):
        if not path.is_file() or path.resolve() == SELF:
            continue
        rel = path.relative_to(ROOT)
        if set(rel.parts) & SKIP_DIRS:
            continue
        if path.suffix.lower() in SKIP_SUFFIX:
            continue
        if path.suffix.lower() not in TEXT_SUFFIX:
            continue
        targets.append(path)

    print(f"  源码目录扫描 {len(targets)} 个文本文件…")
    for path in sorted(targets):
        raw = path.read_bytes()
        if not raw or looks_binary(raw):
            continue
        check_bytes(path, path.name, path.suffix.lower(), raw)

    print()
    check_zip(ROOT / "dist" / "ykxmz-1.4.0.zip")

    print()
    print("  按类型统计：")
    for suffix, count in sorted(stats.items(), key=lambda x: -x[1]):
        print(f"    {suffix:<12} {count:>5} 个")

    print()
    print("=" * 74)
    if not problems:
        print("  ✓ 全部通过，没有编码问题")
        return 0

    by_level: dict = {}
    for level, path, message in problems:
        by_level.setdefault(level, []).append((path, message))

    for level in ("严重", "警告", "提示"):
        items = by_level.get(level, [])
        if not items:
            continue
        print(f"  【{level}】{len(items)} 条")
        for path, message in items[:20]:
            print(f"    {path}")
            print(f"      {message}")
        if len(items) > 20:
            print(f"    …还有 {len(items) - 20} 条")
        print()
    print(f"  合计 {len(problems)} 条")
    return 1 if by_level.get("严重") else 0


if __name__ == "__main__":
    sys.exit(main())