#!/usr/bin/env python3
"""汇总 qwen-eye 实验结果: 混淆矩阵 + 逐维一致率 + 稳定性"""
import json
from pathlib import Path
from collections import defaultdict

OUT = Path("/home/kai/shared/qwen-eye-exp/results")
recs = [json.loads(l) for l in open(OUT / 'judge_raw.jsonl')]

# tag: pos(正配对,期望pass) / neg(负对照,期望fail) / rep(重复,看一致性)
def key(r): return (r['tag'], r['video'], r['spec_of'])

seen = {}
for r in recs:
    seen[key(r)] = r  # 后写覆盖前写 (rep 用 tag 区分不冲突)

tp = fn = fp = tn = 0
dim_stats = defaultdict(lambda: [0, 0])  # dim -> [ok, total]
fails_pos, passes_neg = [], []
parse_fails = []

for r in recs:
    p = r['parsed']
    if not p:
        parse_fails.append(r)
        continue
    v = p.get('verdict')
    if r['tag'] == 'pos':
        if v == 'pass': tp += 1
        else:
            fn += 1
            fails_pos.append((r['video'], r['raw'][:200]))
    elif r['tag'] == 'neg':
        if v == 'fail': tn += 1
        else:
            fp += 1
            passes_neg.append((r['video'], r['spec_of'], r['raw'][:200]))
    if r['tag'] in ('pos', 'neg'):
        for dim in ['framing_ok', 'camera_ok', 'composition_ok', 'axis_ok']:
            if p.get(dim) is True: dim_stats[dim][0] += 1
            dim_stats[dim][1] += 1

# 稳定性: rep 中同 video 两次 verdict 是否一致
rep_group = defaultdict(list)
for r in recs:
    if r['tag'] == 'rep': rep_group[r['video']].append(r['parsed'].get('verdict') if r['parsed'] else 'PF')
stab = {k: v for k, v in rep_group.items() if len(v) >= 2}

lat = [r['elapsed_s'] for r in recs]
pos_lat = [r['elapsed_s'] for r in recs if r['tag'] == 'pos']

report = {
    "n_total": len(recs), "n_parse_fail": len(parse_fails),
    "confusion": {"tp": tp, "fn": fn, "fp": fp, "tn": tn},
    "sensitivity_pos_pass_rate": round(tp / (tp + fn), 3) if tp + fn else None,
    "specificity_neg_fail_rate": round(tn / (tn + fp), 3) if tn + fp else None,
    "accuracy": round((tp + tn) / max(1, tp + fn + fp + tn), 3),
    "dim_ok_rate_when_parsed": {k: round(v[0] / v[1], 3) for k, v in dim_stats.items() if v[1]},
    "pos_latency_s": {"mean": round(sum(pos_lat) / max(1, len(pos_lat)), 1), "max": max(pos_lat) if pos_lat else None},
    "stability_repeat": stab,
    "pos_fails": fails_pos, "neg_leaked_pass": passes_neg,
}
json.dump(report, open(OUT / 'report.json', 'w'), ensure_ascii=False, indent=2)
print(json.dumps(report, ensure_ascii=False, indent=2))