"""Anchoring criterion: a candidate must not overturn an ordering that MiniMax's public data separates.
docs/Role-Play-Bench-可信评测改进方案-2026-10-26.md §8. Read-only: SQLite plus the run indexes; no network, no API calls.

Reference: MiniMax's released per-session scores (11 models × 95 scenarios × 3), overall at the current weights, averaged per
scenario. A pair is "separated" when the scenario-paired difference survives Holm over all 55 pairs at .05.
For every separated pair A > B that a candidate covers, on the scenarios the candidate shares:
  reversal     the candidate's mean A − B is negative and significant (Holm over the candidate's covered pairs)
  conflict     the candidate's mean A − B is negative and its gap differs significantly from MiniMax's on the same scenarios
  compressed   order kept, but the gap is significantly smaller than MiniMax's on the same scenarios (reported, not a failure)
A candidate fails on any reversal or conflict. It passes when, for every separated pair among its models, the one-sided 95% lower
bound of its A − B exceeds −MARGIN (non-inferiority; all pairs must pass, an intersection-union test, so no multiplicity
adjustment); otherwise the result is undecided. With 5 or fewer shared scenarios the sign-flip test cannot reach .05 at all.
Also reported: the gap retention, sum of the candidate's gaps over the sum of MiniMax's on the same scenarios."""
import argparse, itertools, json, math, random, sqlite3, statistics as st
from collections import defaultdict
from pathlib import Path

# MARGIN is half the smallest separated gap on the public data (+2.02), fixed before any candidate was scored.
ALPHA, MARGIN, MC = 0.05, 1.0, 20000
OV = lambda d: 0.5 * (d['basics'] + d['logic'] + d['knowledge']) / 3 + 0.25 * (4 * d['diversity'] + 2 * d['content_logic']) / 6 + 0.25 * d['interaction']
# Platform v6 runs (tools/experiments/v7/runs_rubric.json labels) and the MiniMax model each one reproduces.
V6_MODEL = {'GPT-5.1': 'mm-gpt-5.1', 'M2-her': 'mm-minimax-m2-her', 'V3.1': 'mm-deepseek-v3.1', 'V3.2': 'mm-deepseek-v3.2'}


def sign_flip_p(d, rng):
    """Two-sided p of the mean under random sign flips: exact up to 16 values, Monte Carlo above."""
    obs, n = abs(st.mean(d)), len(d)
    if n <= 16:
        sims = [abs(sum(x * s for x, s in zip(d, signs)) / n) for signs in itertools.product((1, -1), repeat=n)]
    else:
        sims = [abs(sum(x if rng.random() < .5 else -x for x in d) / n) for _ in range(MC)]
    return sum(s >= obs - 1e-12 for s in sims) / len(sims)


def holm(ps):
    order = sorted(range(len(ps)), key=lambda i: ps[i]); out, run = [0.0] * len(ps), 0.0
    for j, i in enumerate(order):
        run = max(run, min(1.0, (len(ps) - j) * ps[i])); out[i] = run
    return out


def boot(d, rng, n=4000):
    return sorted(st.mean(rng.choice(d) for _ in d) for _ in range(n))


def boot_ci(d, rng, n=4000):
    b = boot(d, rng, n); return b[int(.025 * n)], b[int(.975 * n) - 1]


def session_scores(db, run_id):
    per = defaultdict(list)
    for seed, res in db.execute("select seed_id, result_json from sessions where run_id=? and status='done'", (run_id,)):
        per[seed].append(OV(json.loads(res)['combined']))
    return per


def reference(db, rng):
    runs = [r for (r,) in db.execute("select id from runs where id like 'mm-%' order by id")]
    scores = {r: {s: st.mean(v) for s, v in session_scores(db, r).items()} for r in runs}
    pairs = []
    for a, b in itertools.combinations(runs, 2):
        seeds = sorted(set(scores[a]) & set(scores[b])); d = [scores[a][s] - scores[b][s] for s in seeds]
        if st.mean(d) < 0: a, b, d = b, a, [-x for x in d]
        pairs.append({'a': a, 'b': b, 'n': len(d), 'diff': st.mean(d), 'p': sign_flip_p(d, rng)})
    for p, h in zip(pairs, holm([p['p'] for p in pairs])): p['holm'] = h
    return scores, [p for p in pairs if p['holm'] < ALPHA], pairs


def judge_candidate(db, run_id):
    """Our judges on MiniMax's own dialogues: per model and scenario, ours and MiniMax's score of the same dialogues."""
    cfg = json.loads(db.execute('select config_json from runs where id=?', (run_id,)).fetchone()[0])['calibration']
    ours, theirs = defaultdict(lambda: defaultdict(list)), defaultdict(lambda: defaultdict(list))
    for sid, orig in cfg.items():
        a = db.execute("select result_json, seed_id from sessions where id=? and status='done'", (sid,)).fetchone()
        if not a: continue
        res, model = db.execute('select result_json, run_id from sessions where id=?', (orig,)).fetchone()
        ours[model][a[1]].append(OV(json.loads(a[0])['combined'])); theirs[model][a[1]].append(OV(json.loads(res)['combined']))
    m = lambda x: {k: {s: st.mean(v) for s, v in d.items()} for k, d in x.items()}
    return m(ours), m(theirs)


def protocol_candidate(db, root, ref_scores):
    """Platform v6 dialogues on the same scenarios; MiniMax's side is its released per-scenario mean."""
    runs = json.load(open(root / 'tools/experiments/v7/runs_rubric.json'))
    ours = {V6_MODEL[l]: {s: st.mean(v) for s, v in session_scores(db, rid).items()} for l, rid in runs.items() if l in V6_MODEL}
    return ours, {m: ref_scores[m] for m in ours}


def check(name, layer, ours, theirs, separated, rng):
    rows = []
    for p in separated:
        a, b = p['a'], p['b']
        if a not in ours or b not in ours: continue
        seeds = sorted(set(ours[a]) & set(ours[b]) & set(theirs[a]) & set(theirs[b]))
        if not seeds: continue
        d = [ours[a][s] - ours[b][s] for s in seeds]; e = [theirs[a][s] - theirs[b][s] for s in seeds]
        g = [x - y for x, y in zip(d, e)]
        rows.append({'a': a, 'b': b, 'n': len(seeds), 'ref_diff_95': p['diff'], 'ref_diff_same': st.mean(e), 'cand_diff': st.mean(d),
                     'cand_ci': boot_ci(d, rng), 'cand_lower95_one_sided': boot(d, rng)[200], 'gap_change': st.mean(g), 'gap_change_ci': boot_ci(g, rng),
                     'p_cand': sign_flip_p(d, rng), 'p_gap': sign_flip_p(g, rng)})
    for k in ('p_cand', 'p_gap'):
        for r, h in zip(rows, holm([r[k] for r in rows])): r[k + '_holm'] = h
    for r in rows:
        neg = r['cand_diff'] < 0
        r['status'] = ('reversal' if neg and r['p_cand_holm'] < ALPHA else
                       'conflict' if neg and r['gap_change'] < 0 and r['p_gap_holm'] < ALPHA else
                       'compressed' if not neg and r['gap_change'] < 0 and r['p_gap_holm'] < ALPHA else
                       'direction kept' if not neg else 'reversed, not significant')
    failed = any(r['status'] in ('reversal', 'conflict') for r in rows)
    ni = sum(r['cand_lower95_one_sided'] > -MARGIN for r in rows)
    verdict = 'fail' if failed else 'pass' if rows and ni == len(rows) else 'undecided'
    retention = sum(r['cand_diff'] for r in rows) / sum(r['ref_diff_same'] for r in rows) if rows else None
    print(f'\n== {name} [{layer}]: {verdict}; {len(rows)} separated pairs covered, point direction kept '
          f"{sum(r['cand_diff'] >= 0 for r in rows)}/{len(rows)}, non-inferior (lower bound > -{MARGIN}) {ni}/{len(rows)}, "
          f'gap retention {retention:.2f}')
    for r in sorted(rows, key=lambda r: r['status']):
        if r['status'] != 'direction kept':
            print(f"  {r['status']:26} {r['a'][3:]} > {r['b'][3:]}: MiniMax {r['ref_diff_same']:+.2f} (95 scenarios {r['ref_diff_95']:+.2f}), "
                  f"candidate {r['cand_diff']:+.2f} [{r['cand_ci'][0]:+.2f}, {r['cand_ci'][1]:+.2f}], gap change {r['gap_change']:+.2f} "
                  f"[{r['gap_change_ci'][0]:+.2f}, {r['gap_change_ci'][1]:+.2f}] (p {r['p_gap']:.3f}, Holm {r['p_gap_holm']:.3f}), n={r['n']}")
    return {'name': name, 'layer': layer, 'verdict': verdict, 'gap_retention': retention, 'pairs': rows}


if __name__ == '__main__':
    ap = argparse.ArgumentParser()
    ap.add_argument('--root', type=Path, default=Path.cwd())
    ap.add_argument('--output', type=Path)
    args = ap.parse_args(); root = args.root.resolve()
    db = sqlite3.connect(f'file:{root}/data/rpbench.sqlite?mode=ro', uri=True); rng = random.Random(20261026)
    ref_scores, separated, all_pairs = reference(db, rng)
    print(f'MiniMax public data: {len(separated)} of {len(all_pairs)} pairs separated (Holm, alpha {ALPHA})')
    for p in sorted(separated, key=lambda p: p['diff']): print(f"  {p['a'][3:]:30} > {p['b'][3:]:30} {p['diff']:+6.2f}  Holm p {p['holm']:.4f}")
    ledger = json.load(open(root / 'tools/experiments/ledger.json'))['p2']
    results = [check('screening panel on MiniMax dialogues', 'judge', *judge_candidate(db, ledger['screening-110']), separated, rng),
               check('official panel on MiniMax dialogues', 'judge', *judge_candidate(db, ledger['official-44']), separated, rng),
               check('platform v6 protocol (simulated user), screening panel', 'protocol', *protocol_candidate(db, root, ref_scores), separated, rng)]
    if args.output:
        args.output.write_text(json.dumps({'alpha': ALPHA, 'margin': MARGIN, 'separated': separated, 'all_pairs': all_pairs,
                                           'candidates': results}, ensure_ascii=False, indent=1))