#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# SPDX-License-Identifier: MulanPSL-2.0
"""
vram_excep - CUDA 显存/地址异常诊断工具（单一入口）

将原先的 cuda_gdb_inject.sh + cuda_gdb_parse.py 两段串联为一个 Python 主控：
  1) attach 容器内（或裸宿主机）训练进程
  2) 用 cuda-gdb hook-stop 在 CUDA 异常触发时 dump 现场
  3) 解析 dump，输出统一的 JSON 报告（对齐 gpudiag 顶层格式：分析详情 + 结果）

退出码约定（与 gpudiag 一致）:
  0  全部检查 OK / 未触发异常
  1  存在 WARN（hook-stop 报错、解析告警等）
  2  存在 ERROR（依赖缺失、容器不可访问、捕获到 CUDA 异常等）

用法:
  python3 vram_excep.py <PID> [-t SECS] [-f OUT.json] [--no-parse] [--keep-logs]
"""

import argparse
import datetime
import json
import os
import re
import secrets
import shutil
import signal
import subprocess
import sys
import tempfile

# 解析模块
try:
    from cuda_gdb_parse import parse_dump, print_report
except ImportError:
    # 当 vram_excep.py 被独立部署时，尝试加载同目录下的 cuda_gdb_parse.py
    _here = os.path.dirname(os.path.abspath(__file__))
    sys.path.insert(0, _here)
    from cuda_gdb_parse import parse_dump, print_report  # noqa: E402

# SLS 上传模块
try:
    from sls_uploader import upload_report, upload_error
except ImportError:
    _here = os.path.dirname(os.path.abspath(__file__))
    if _here not in sys.path:
        sys.path.insert(0, _here)
    from sls_uploader import upload_report, upload_error  # noqa: E402


# ---------------------------------------------------------------------------
# 常量
# ---------------------------------------------------------------------------
CUDAGDB_PATH_DEFAULT = "/usr/local/cuda/bin/cuda-gdb"
DEFAULT_TIMEOUT_SEC  = 0     # 0 表示永久等待（不指定 -t 时默认永久）
HOST_DEPS = ["nsenter", "awk", "grep", "tee", "timeout"]

# 中心端联动默认路径
DEFAULT_CHECK_DIR = "/var/log/sysak/vram_excep"
# 复用 livetrace 的 prof_kmsg ilog 文件，避免新增 ilogtail 采集配置
# subsystem=vram_excep_detail 与 livetrace/vram_excep 的 vram_except 区分
DEFAULT_ILOG_FILE = "/var/sysom/ilog/prof_kmsg"


# ---------------------------------------------------------------------------
# 结果累积（对齐 gpudiag 顶层 schema）
# ---------------------------------------------------------------------------
_results = []   # [{check, severity, message[, suggestion]}, ...]
_logs    = []   # ["...", "..."]

# preflight 实时落盘文件句柄（preflight_single 设置，_report/_log 同时写入）
_check_log_fp = None


def _log(msg):
    _logs.append(msg)
    if _check_log_fp is not None:
        try:
            _check_log_fp.write(msg + "\n")
            _check_log_fp.flush()
        except OSError:
            pass


def _report(check, severity, message, suggestion=None):
    entry = {"check": check, "severity": severity, "message": message}
    if suggestion is not None:
        entry["suggestion"] = suggestion
    _results.append(entry)
    line = "[{:<5}] {}: {}".format(severity, check, message)
    _logs.append(line)
    if _check_log_fp is not None:
        try:
            _check_log_fp.write(line + "\n")
            _check_log_fp.flush()
        except OSError:
            pass


def gen_traceid():
    """生成 16 位 hex traceid，用于关联阶段1（stdout）和阶段2（ilog）。"""
    return secrets.token_hex(8)


# ---------------------------------------------------------------------------
# 工具函数
# ---------------------------------------------------------------------------
def _run(cmd, timeout=10, check=False, shell=False):
    """运行命令并返回 (stdout, stderr, rc)。超时返回 rc=-2。

    cmd 可以是 list（推荐，避免 shell 注入）或 str（shell=True 时）。
    """
    try:
        proc = subprocess.run(
            cmd, shell=shell,
            stdout=subprocess.PIPE, stderr=subprocess.PIPE,
            timeout=timeout, universal_newlines=True,
        )
        return proc.stdout, proc.stderr, proc.returncode
    except subprocess.TimeoutExpired:
        return None, "timed out after {}s".format(timeout), -2
    except OSError as exc:
        return None, str(exc), -3


def _is_alive(pid):
    try:
        os.kill(pid, 0)
        return True
    except (ProcessLookupError, PermissionError):
        return False
    except OSError:
        return False


def _read_status_field(pid, field):
    """读取 /proc/<pid>/status 中以 'field:' 起始的行，返回拆分后的值列表。"""
    path = "/proc/{}/status".format(pid)
    try:
        with open(path, "r", errors="replace") as fh:
            for line in fh:
                if line.startswith(field + ":"):
                    return line.split()[1:]
    except OSError:
        return None
    return None


# ---------------------------------------------------------------------------
# Pre-flight 检查（依赖、PID、命名空间、容器内 cuda-gdb）
# ---------------------------------------------------------------------------
def preflight(args):
    """单 PID 兼容入口：等价于 preflight_single(args.pid, args)。

    返回 (ok: bool, ctx: dict)
    ctx 包含: train_pid, container_pid, sudo, host_container_tmp 等
    """
    return preflight_single(args.pid, args)


def preflight_single(pid, args, log_path=None):
    """对单个 PID 做 preflight，实时落盘到 log_path。

    返回 (ok: bool, ctx: dict)。
    """
    global _check_log_fp

    fp = None
    if log_path:
        try:
            os.makedirs(os.path.dirname(log_path), exist_ok=True)
            fp = open(log_path, "a", encoding="utf-8")
            fp.write("\n==== preflight start pid={} ts={} ====\n".format(
                pid, datetime.datetime.now().isoformat()))
            fp.flush()
        except OSError:
            fp = None
    _check_log_fp = fp

    try:
        ctx = {"train_pid": pid, "sudo": ""}

        # ── 依赖检查 ──
        missing = [b for b in HOST_DEPS if not shutil.which(b)]
        if missing:
            _report("host_deps", "ERROR",
                    "宿主机缺失必备命令: {}".format(", ".join(missing)),
                    suggestion="请安装 util-linux / coreutils 等基础包")
            return False, ctx
        _report("host_deps", "OK", "宿主机基础命令齐全")

        # ── 是否需要 sudo ──
        if os.geteuid() != 0:
            if shutil.which("sudo"):
                ctx["sudo"] = "sudo"
                _report("privilege", "OK", "非 root，将通过 sudo 执行 nsenter")
            else:
                _report("privilege", "ERROR",
                        "非 root 且系统无 sudo，nsenter 无法执行",
                        suggestion="以 root 运行，或安装 sudo")
                return False, ctx
        else:
            _report("privilege", "OK", "以 root 身份运行")

        # ── PID 存活 ──
        if not _is_alive(pid):
            _report("target_pid", "ERROR",
                    "进程 {} 不存在或无权限访问".format(pid))
            return False, ctx
        _report("target_pid", "OK", "目标进程 {} 存活".format(pid))

        # ── TracerPid 冲突 ──
        tracer = _read_status_field(pid, "TracerPid")
        if tracer and tracer[0] != "0":
            _report("tracer_conflict", "ERROR",
                    "进程 {} 已被 PID={} attach（gdb/strace 等）".format(
                        pid, tracer[0]),
                    suggestion="先 detach 现有 tracer 再重试")
            return False, ctx
        _report("tracer_conflict", "OK", "未检测到其他 tracer attach")

        # ── NSpid（容器内 PID）──
        nspid = _read_status_field(pid, "NSpid")
        if nspid and len(nspid) > 1:
            ctx["container_pid"] = nspid[-1]
            _report("ns_pid", "OK",
                    "容器内 PID = {} (host PID = {})".format(
                        ctx["container_pid"], pid))
        else:
            ctx["container_pid"] = str(pid)
            _report("ns_pid", "OK",
                    "未检测到 NSpid，使用宿主 PID（裸宿主机进程或低版本内核）")

        # ── libcuda 检查（非阻塞）──
        maps_path = "/proc/{}/maps".format(pid)
        if os.access(maps_path, os.R_OK):
            try:
                with open(maps_path, "r", errors="replace") as fh:
                    if "libcuda" in fh.read():
                        _report("cuda_loaded", "OK", "进程已加载 libcuda")
                    else:
                        _report("cuda_loaded", "WARN",
                                "进程未加载 libcuda，仍继续尝试 attach")
            except OSError:
                _report("cuda_loaded", "WARN", "无法读取 /proc/{}/maps".format(pid))
        else:
            _report("cuda_loaded", "WARN", "无权读取 /proc/{}/maps".format(pid))

        # ── 容器内 cuda-gdb ──
        cmd = ["nsenter", "-t", str(pid), "--mount", "--", "test", "-x", args.cuda_gdb]
        if ctx.get("sudo"):
            cmd = [ctx["sudo"]] + cmd
        _, _, rc = _run(cmd, timeout=10, shell=False)
        if rc != 0:
            _report("cuda_gdb_exists", "ERROR",
                    "容器内未找到 cuda-gdb: {}".format(args.cuda_gdb),
                    suggestion="确认训练镜像包含 CUDA toolkit，或用 --cuda-gdb 指定路径")
            return False, ctx
        _report("cuda_gdb_exists", "OK", "cuda-gdb 可执行: {}".format(args.cuda_gdb))

        # ── 容器内 cuda-gdb 版本 ──
        # 13.3 改进了 attach/detach 可靠性；12.6 之前存在多个 crash 风险
        cmd_ver = ["nsenter", "-t", str(pid), "--mount", "--", args.cuda_gdb, "--version"]
        if ctx.get("sudo"):
            cmd_ver = [ctx["sudo"]] + cmd_ver
        ver_out, _, ver_rc = _run(cmd_ver, timeout=10, shell=False)
        if ver_rc == 0 and ver_out:
            # cuda-gdb --version 输出中通常同时包含 CUDA 版本（如 NVIDIA (R) cuda-gdb 12.8）
            # 和 GNU gdb 基线版本（如 Based on GNU gdb 13.2）。
            # 必须优先匹配 NVIDIA cuda-gdb/CUDA Debugger 行，避免把 GNU gdb 基线版本误判为 CUDA 版本。
            ver_match = re.search(r'NVIDIA \(R\) (?:cuda-gdb|CUDA Debugger)\s+(\d+)\.(\d+)', ver_out, re.IGNORECASE)
            if not ver_match:
                # 兼容 "NVIDIA (R) CUDA Debugger\n12.3 release" 这种两行格式
                ver_match = re.search(r'NVIDIA \(R\) (?:cuda-gdb|CUDA Debugger)[^\n]*\n[^\d]*(\d+)\.(\d+)(?:\s+release)?', ver_out, re.IGNORECASE)
            if not ver_match:
                # 兼容只有 "X.Y release" 的老格式
                ver_match = re.search(r'(\d+)\.(\d+)\s+release', ver_out, re.IGNORECASE)
            if not ver_match:
                # 最后兜底：匹配第一个版本号（兼容非常规输出）
                ver_match = re.search(r'(\d+)\.(\d+)', ver_out)
            if ver_match:
                version = (int(ver_match.group(1)), int(ver_match.group(2)))
                ver_str = "{}.{}".format(version[0], version[1])
                if version < (12, 6):
                    _report("cuda_gdb_version", "ERROR",
                            "cuda-gdb 版本 {} 偏低，attach 模式存在 crash 风险，拒绝执行".format(ver_str),
                            suggestion="升级到 12.6+ 以使用 attach 模式，建议 13.3+ 获得最佳稳定性")
                    return False, ctx
                elif version < (13, 3):
                    _report("cuda_gdb_version", "OK",
                            "cuda-gdb 版本 {} 可用".format(ver_str),
                            suggestion="13.3 改进了 attach 可靠性，建议升级")
                else:
                    _report("cuda_gdb_version", "OK",
                            "cuda-gdb 版本 {}".format(ver_str))
            else:
                _report("cuda_gdb_version", "WARN",
                        "无法解析 cuda-gdb 版本号: {}".format(ver_out.splitlines()[0] if ver_out else ""))
        else:
            _report("cuda_gdb_version", "WARN", "无法获取 cuda-gdb 版本（--version 执行失败）")

        # ── 容器内 /tmp 可访问 ──
        host_container_tmp = "/proc/{}/root/tmp".format(pid)
        if not os.path.isdir(host_container_tmp):
            _report("container_tmp", "ERROR",
                    "容器内 /tmp 不可访问: {}".format(host_container_tmp))
            return False, ctx
        ctx["host_container_tmp"] = host_container_tmp
        _report("container_tmp", "OK", "容器内 /tmp 可访问")

        _report("hook_stop", "OK", "preflight 通过，准备 attach")
        return True, ctx
    finally:
        if fp:
            try:
                fp.write("==== preflight end pid={} ====\n".format(pid))
                fp.close()
            except OSError:
                pass
        _check_log_fp = None


def preflight_multi(pids, args, traceid, check_dir):
    """串行对多个 PID 做 preflight，返回 (results, warnings)。

    results:   {pid: ctx or None, ...}
    warnings: [{pid, message}, ...]
    """
    results = {}
    warnings = []
    for pid in pids:
        log_path = os.path.join(check_dir, traceid, "{}_check.log".format(pid))
        ok, ctx = preflight_single(pid, args, log_path=log_path)
        if ok:
            results[pid] = ctx
        else:
            results[pid] = None
            warnings.append({
                "pid": pid,
                "message": "preflight 失败，详见 {}/{}_check.log".format(
                    os.path.join(check_dir, traceid), pid),
            })
    return results, warnings


# ---------------------------------------------------------------------------
# 写 cmds.txt
# ---------------------------------------------------------------------------
GDB_CMDS_TEMPLATE = r"""# 捕获常见 CPU 信号
handle SIGABRT           stop nopass
handle SIGSEGV           stop nopass
handle SIGFPE            stop nopass

# 捕获 CUDA 异常
# handle 的异常编号与根因映射见 cuda_gdb_parse.py 的 EXCEPTION_CODE_CAUSES
handle CUDA_EXCEPTION_3  stop print
handle CUDA_EXCEPTION_9  stop print
handle CUDA_EXCEPTION_10 stop print
handle CUDA_EXCEPTION_11 stop print
handle CUDA_EXCEPTION_13 stop print
handle CUDA_EXCEPTION_14 stop print

define hook-stop
  shell mkdir -p /tmp/cuda_dump
  shell echo dump_start >> /tmp/cuda_dump/alert.log

  echo \n==== DUMP START ====\n

  echo \n---- [KERNELS] ----\n
  info cuda kernels

  echo \n---- [THREADS] ----\n
  info cuda threads

  echo \n---- [DEVICES] ----\n
  info cuda devices

  cuda thread 0

  echo \n---- [GPU_BACKTRACE] ----\n
  bt

  echo \n---- [GPU_ARGS] ----\n
  info args

  echo \n---- [GPU_LOCALS] ----\n
  info locals

  echo \n---- [SOURCE] ----\n
  list

  echo \n---- [DISASM] ----\n
  disassemble /m

  echo \n---- [REGISTERS] ----\n
  info registers

  echo \n---- [CPU_THREADS] ----\n
  info threads

  echo \n---- [CPU_BACKTRACE] ----\n
  thread 1
  bt full

  echo \n---- [PYTHON_BACKTRACE] ----\n
  py-bt

  echo \n==== DUMP END ====\n
  shell echo dump_done >> /tmp/cuda_dump/alert.log
end

continue
quit
"""


def write_cmds_file(host_container_tmp, train_pid):
    """在容器视角的 /tmp 下写入独立 cmds.txt，返回 (host_path, container_path)。"""
    prefix = "cuda_gdb_cmds_{}_{}_".format(train_pid, os.getpid())
    fd, host_path = tempfile.mkstemp(
        dir=host_container_tmp,
        prefix=prefix,
        suffix=".txt",
    )
    try:
        with os.fdopen(fd, "w", encoding="utf-8", closefd=True) as fh:
            fh.write(GDB_CMDS_TEMPLATE)
        os.chmod(host_path, 0o644)
    except Exception:
        try:
            os.unlink(host_path)
        except OSError:
            pass
        raise
    name = os.path.basename(host_path)
    container_path = "/tmp/" + name
    return host_path, container_path


# ---------------------------------------------------------------------------
# 运行 cuda-gdb
# ---------------------------------------------------------------------------
def run_cuda_gdb(ctx, args, container_cmds_path, log_dir):
    """
    阻塞执行 cuda-gdb，输出实时落到 dump.log。
    返回: (gdb_exit_code, dump_log_path)
    """
    dump_log = os.path.join(log_dir, "dump.log")

    # 使用 list 传参，避免 shell 注入
    nsenter_cmd = [
        "nsenter",
        "-t", str(ctx["train_pid"]),
        "--mount", "--pid", "--net", "--uts", "--",
        args.cuda_gdb,
        "--batch",
        "-p", str(ctx["container_pid"]),
        "-x", container_cmds_path,
    ]
    # timeout=0 或不指定时，不包 timeout 命令，永久等待
    if args.timeout and args.timeout > 0:
        cmd = ["timeout", str(args.timeout)] + nsenter_cmd
    else:
        cmd = nsenter_cmd
    if ctx.get("sudo"):
        cmd = [ctx["sudo"]] + cmd

    timeout_desc = "{}s".format(args.timeout) if args.timeout and args.timeout > 0 else "永久"
    _log("[INFO] 启动 cuda-gdb (timeout={})".format(timeout_desc))
    _log("[INFO] dump.log -> {}".format(dump_log))

    # Popen 模式：stdout 同时落盘和打印
    rc = 0
    proc = None
    with open(dump_log, "w", encoding="utf-8", errors="replace") as fh:
        try:
            proc = subprocess.Popen(
                cmd,
                stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
                universal_newlines=True,
            )
            # 流式读取，避免 buffer 满死锁
            assert proc.stdout is not None
            for line in proc.stdout:
                sys.stdout.write(line)
                sys.stdout.flush()
                fh.write(line)
            rc = proc.wait()
        except KeyboardInterrupt:
            if proc is not None:
                try:
                    proc.send_signal(signal.SIGINT)
                    proc.wait(timeout=5)
                except Exception:
                    proc.kill()
            rc = 130
            _report("cuda_gdb", "WARN", "用户中断 (Ctrl+C)")

    return rc, dump_log


# ---------------------------------------------------------------------------
# JSON 输出（对齐 gpudiag 顶层 schema）
# ---------------------------------------------------------------------------
def build_json_output(args, dump_payload):
    """
    顶层 schema:
    {
      "分析详情": ["...", ...],
      "结果": {
        "timestamp":   "<ISO8601>",
        "hostname":    "<str>",
        "tool":        "vram_excep",
        "target_pid":  <int>,
        "details":     [{check, severity, message, suggestion?}, ...],
        "dump":        <parse_dump 的完整结构 或 null>
      }
    }
    """
    return {
        "分析详情": list(_logs),
        "结果": {
            "timestamp":  datetime.datetime.now().astimezone().isoformat(),
            "hostname":   os.uname().nodename,
            "tool":       "vram_excep",
            "target_pid": args.pid,
            "details":    list(_results),
            "dump":       dump_payload,
        },
    }


def output_json(report, output_file=None):
    text = json.dumps(report, ensure_ascii=False, indent=2)
    if output_file:
        try:
            with open(output_file, "w", encoding="utf-8") as fh:
                fh.write(text)
                fh.write("\n")
        except OSError as exc:
            print("[ERROR] 无法写入 JSON 报告 '{}': {}".format(output_file, exc),
                  file=sys.stderr)
            sys.exit(2)
    else:
        print(text)


# ---------------------------------------------------------------------------
# 退出码计算
# ---------------------------------------------------------------------------
def compute_exit_code():
    if any(r["severity"] == "ERROR" for r in _results):
        return 2
    if any(r["severity"] == "WARN" for r in _results):
        return 1
    return 0


# ---------------------------------------------------------------------------
# 参数解析
# ---------------------------------------------------------------------------
def parse_args():
    p = argparse.ArgumentParser(
        description="vram_excep - CUDA 显存/地址异常诊断工具",
    )
    p.add_argument("pid", type=int, nargs="+", help="宿主机训练进程 PID（支持多个）")
    p.add_argument("-t", "--timeout", type=int, default=DEFAULT_TIMEOUT_SEC,
                   metavar="SECS",
                   help="cuda-gdb 等待异常的最长时间，0 表示永久等待（默认 {}）".format(
                       "永久" if DEFAULT_TIMEOUT_SEC == 0 else "{}s".format(DEFAULT_TIMEOUT_SEC)))
    p.add_argument("-f", "--output-file", metavar="FILE",
                   help="阶段1 JSON 写入路径（默认输出到 stdout）")
    p.add_argument("--cuda-gdb", default=CUDAGDB_PATH_DEFAULT,
                   metavar="PATH",
                   help="容器内 cuda-gdb 路径（默认 {}）".format(CUDAGDB_PATH_DEFAULT))
    p.add_argument("--no-parse", action="store_true",
                   help="只采集 dump.log，不进入解析阶段")
    p.add_argument("--keep-logs", action="store_true", default=True,
                   help="保留宿主端日志目录（默认保留；保留参数以兼容平台调用）")
    p.add_argument("--text-report", action="store_true",
                   help="额外打印人类可读的终端报告（仅 --foreground 模式生效）")
    # 中心端联动参数
    p.add_argument("--check-dir", default=DEFAULT_CHECK_DIR, metavar="DIR",
                   help="阶段1 检查日志目录（默认 {}）".format(DEFAULT_CHECK_DIR))
    p.add_argument("--ilog-file", default=DEFAULT_ILOG_FILE, metavar="FILE",
                   help="阶段2 ilog 输出路径（默认 {}）".format(DEFAULT_ILOG_FILE))
    p.add_argument("--foreground", action="store_true",
                   help="调试用：不 fork 后台，主进程阻塞等异常（兼容旧行为）")
    p.add_argument("--instance", default=None, metavar="INSTANCE",
                   help="节点 instance ID，默认不传（由中心端按 hostname 聚合）")
    return p.parse_args()


# ---------------------------------------------------------------------------
# 阶段1 汇总输出
# ---------------------------------------------------------------------------
def build_phase1_output(args, traceid, pids, results, warnings):
    """阶段1 汇总 JSON：中心端拿到后可知道哪些 PID 已挂上 hook-stop。"""
    result = {
        "分析详情": list(_logs),
        "结果": {
            "traceid": traceid,
            "phase": 1,
            "timestamp": datetime.datetime.now().astimezone().isoformat(),
            "hostname": os.uname().nodename,
            "tool": "vram_excep",
            "target_pids": list(pids),
            "attached_pids": [p for p, c in results.items() if c],
            "failed_pids": [p for p, c in results.items() if not c],
            "severity": "OK" if any(results.values()) else "ERROR",
            "warnings": warnings,
            "check_dir": args.check_dir,
            "ilog_file": args.ilog_file,
        },
    }
    # instance 仅在显式传入时才输出，默认不传
    if args.instance:
        result["结果"]["instance"] = args.instance
    return result


# ---------------------------------------------------------------------------
# 阶段2：fork 后台守护
# ---------------------------------------------------------------------------
def daemonize_and_wait(args, traceid, results, check_dir):
    """fork 出后台子进程，每 PID 一个，主进程立即返回。"""
    for pid, ctx in results.items():
        if not ctx:
            continue
        try:
            pid_child = os.fork()
        except OSError as exc:
            _report("fork", "WARN", "fork 子进程失败 (pid={}): {}".format(pid, exc))
            continue
        if pid_child == 0:
            # 子进程：脱离终端
            try:
                os.setsid()
            except OSError:
                pass
            # 重定向 stdin/stdout/stderr 到 /dev/null
            # 使用 os.dup2 操作底层 fd，比 sys.stdin = open(...) 更彻底
            # 避免 stdout/stderr 仍指向父进程终端，终端关闭后写操作触发 SIGPIPE
            try:
                devnull_fd = os.open("/dev/null", os.O_RDWR)
                os.dup2(devnull_fd, 0)  # stdin
                os.dup2(devnull_fd, 1)  # stdout
                os.dup2(devnull_fd, 2)  # stderr
                if devnull_fd > 2:
                    os.close(devnull_fd)
            except OSError:
                pass
            # 子进程跑阶段2，不返回
            _phase2_child(pid, ctx, args, traceid, return_dump=False)
            os._exit(0)
        # 父进程继续 fork 下一个
    # 父进程不等子进程，直接返回


def _phase2_child(pid, ctx, args, traceid, return_dump=False):
    """阶段2 主体：跑 cuda-gdb 阻塞等异常，解析后上传 ilog。

    Args:
        pid         : 宿主机 PID
        ctx         : preflight 返回的 ctx
        args        : argparse.Namespace
        traceid     : 16 位 hex
        return_dump : True 时返回 dump_payload（foreground 模式用）；
                      False 时 upload_report 到 ilog 并返回 None。
    """
    hostname = os.uname().nodename
    base_fields = {"hostname": hostname}
    # instance 仅在显式传入时才携带，默认不传
    if args.instance:
        base_fields["instance"] = args.instance

    # 写 cmds.txt
    try:
        host_cmds, container_cmds = write_cmds_file(ctx["host_container_tmp"], pid)
    except OSError as exc:
        if return_dump:
            _report("cmds_file", "ERROR", "无法写入 cmds.txt: {}".format(exc))
            return None
        upload_error(traceid, pid, args.ilog_file,
                     "write_cmds_file 失败: {}".format(exc),
                     base_fields=base_fields)
        return None

    ts = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
    log_dir = "/tmp/cuda_debug_{}_{}".format(pid, ts)
    try:
        os.makedirs(log_dir, exist_ok=True)
    except OSError:
        log_dir = "/tmp"

    if return_dump:
        _log("[INFO] log_dir = {}".format(log_dir))
        _log("[INFO] cmds (容器视角) = {}".format(container_cmds))

    host_cmds_to_clean = host_cmds
    try:
        # ── 运行 cuda-gdb ──
        gdb_rc, dump_log = run_cuda_gdb(ctx, args, container_cmds, log_dir)

        if return_dump:
            if gdb_rc == 124:
                _report("cuda_gdb", "OK",
                        "{}s 内未触发 CUDA 异常（超时退出，正常情况）".format(args.timeout))
            elif gdb_rc == 0:
                _report("cuda_gdb", "OK", "cuda-gdb 正常退出")
            elif gdb_rc == 130:
                pass
            else:
                _report("cuda_gdb", "WARN",
                        "cuda-gdb 退出码={}，可能已捕获到异常".format(gdb_rc))

        # ── 读取 dump ──
        try:
            with open(dump_log, "r", errors="replace") as fh:
                content = fh.read()
        except OSError as exc:
            if return_dump:
                _report("dump_read", "ERROR", "无法读取 dump.log: {}".format(exc))
                return None
            upload_error(traceid, pid, args.ilog_file,
                         "无法读取 dump.log: {}".format(exc),
                         base_fields=base_fields)
            return None

        if "DUMP START" not in content:
            # 未触发异常
            if return_dump:
                _report("cuda_exception", "OK",
                        "未捕获到 CUDA 异常现场（进程正常 / 超时未触发）")
                return None
            upload_report(None, traceid, pid, args.ilog_file,
                          base_fields=dict(base_fields, gdb_rc=gdb_rc))
            return None

        # 捕获到异常
        if return_dump:
            _report("cuda_exception", "ERROR",
                    "捕获到 CUDA 异常现场，详见 dump",
                    suggestion="查阅 details.dump 中 exception/location/cpu_backtrace 定位问题")

        if args.no_parse:
            report = {
                "meta": {"dump_file": dump_log, "parse_time":
                         datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
                         "caught": True},
                "warnings": ["--no-parse 模式，未解析 dump"],
            }
        else:
            try:
                report = parse_dump(content, dump_file=dump_log)
            except Exception as exc:
                if return_dump:
                    _report("parse", "WARN", "解析 dump.log 失败: {}".format(exc))
                    return None
                upload_error(traceid, pid, args.ilog_file,
                             "parse_dump 失败: {}".format(exc),
                             base_fields=base_fields)
                return None

        if return_dump:
            if report.get("warnings"):
                for w in report["warnings"]:
                    _report("parse_warn", "WARN", w)
            if args.text_report:
                print_report(report)
            return report

        # daemon 模式：upload_report 到 ilog
        upload_report(report, traceid, pid, args.ilog_file,
                      base_fields=dict(base_fields, gdb_rc=gdb_rc))
        return None
    except Exception as exc:
        # 子进程任何异常都兜底
        if return_dump:
            _report("phase2", "ERROR", "阶段2 异常: {}".format(exc))
            return None
        try:
            upload_error(traceid, pid, args.ilog_file,
                         "子进程异常: {}".format(exc),
                         base_fields=base_fields)
        except Exception:
            pass
        return None
    finally:
        # 清理 cmds.txt
        try:
            if host_cmds_to_clean and os.path.exists(host_cmds_to_clean):
                os.unlink(host_cmds_to_clean)
        except OSError:
            pass
        # 按需清理日志目录
        if not args.keep_logs:
            try:
                if os.path.isdir(log_dir):
                    shutil.rmtree(log_dir)
            except OSError:
                pass


# ---------------------------------------------------------------------------
# Main
# ---------------------------------------------------------------------------
def main():
    args = parse_args()

    if args.timeout < 0:
        print("ERROR: --timeout 不能为负数", file=sys.stderr)
        sys.exit(1)

    traceid = gen_traceid()
    try:
        os.makedirs(os.path.join(args.check_dir, traceid), exist_ok=True)
    except OSError as exc:
        print("[ERROR] 无法创建 check-dir: {}".format(exc), file=sys.stderr)
        sys.exit(2)

    timeout_desc = "{}s".format(args.timeout) if args.timeout and args.timeout > 0 else "永久"
    _log("[INFO] vram_excep 启动 (pids={}, timeout={}, traceid={})".format(
        args.pid, timeout_desc, traceid))

    # ── 阶段1：多 PID 校验 ──
    results, warnings = preflight_multi(args.pid, args, traceid, args.check_dir)
    phase1 = build_phase1_output(args, traceid, args.pid, results, warnings)
    output_json(phase1, args.output_file)

    # 全部失败则不进入阶段2
    if not any(results.values()):
        sys.exit(2 if phase1["结果"]["severity"] == "ERROR" else 1)

    # ── 阶段2 ──
    if args.foreground:
        # 调试模式：不 fork，主进程阻塞等异常
        all_reports = []
        for pid, ctx in results.items():
            if not ctx:
                continue
            dump_payload = _phase2_child(pid, ctx, args, traceid, return_dump=True)
            all_reports.append((pid, dump_payload))

        if len(all_reports) == 1:
            # 单 PID：兼容旧行为，输出 build_json_output
            pid, dump_payload = all_reports[0]
            args.pid = pid
            report = build_json_output(args, dump_payload)
            output_json(report, args.output_file)
        else:
            # 多 PID：输出汇总报告
            report = {
                "分析详情": list(_logs),
                "结果": {
                    "traceid": traceid,
                    "phase": 2,
                    "timestamp": datetime.datetime.now().astimezone().isoformat(),
                    "hostname": os.uname().nodename,
                    "tool": "vram_excep",
                    "reports": {str(p): d for p, d in all_reports},
                },
            }
            output_json(report, args.output_file)
    else:
        # 默认模式：fork 后台守护
        daemonize_and_wait(args, traceid, results, args.check_dir)

    sys.exit(compute_exit_code())


if __name__ == "__main__":
    main()
