"""Deliverable 3: judge reliability of the official panel.
Part A: official-panel full-protocol platform runs (group a). Part B: calibration runs (group b) vs MiniMax's released scores.
Inputs: raw/runs_*.json, raw/export/sessions.jsonl, raw/export/judgments.jsonl, raw/export/simmodels.jsonl.
Outputs: out/reliability_a.md, out/reliability_a_*.csv, out/reliability_b.md, out/reliability_b_*.csv, out/minimax_separated_pairs.csv.
Random seeds fixed (20261028); bootstrap B = 2000; split-half repetitions = 2000."""
import sys, itertools; sys.path.insert(0, '${RPB_AUDIT_DIR}/scripts')
from common import *
B = 2000; REPS = 2000
rng = random.Random(20261028)
runs = load_runs()
sess = defaultdict(list)
for s in load_jsonl('sessions'):
    s['result'] = json.loads(s['result_json']) if s['result_json'] else None; sess[s['run_id']].append(s)
sid = {s['id']: s for v in sess.values() for s in v}
judg = defaultdict(dict)
for j in load_jsonl('judgments'):
    if j['status'] == 'done' and j['dims']: judg[j['session_id']][(j['chunk_idx'], j['judge_key'])] = json.loads(j['dims'])
simmain = {}
for m in load_jsonl('simmodels'):
    if m['source'] == 'user_sim' and m['model']:
        c = simmain.setdefault(m['session_id'], Counter()); c[m['model'].split('/')[-1]] += m['n']
BIN = {'basics', 'logic'}
ZERO = {k: 0 for k in DIMS}

def judge_dims(s, k, apply_rule=True):
    r = s['result']; d = r['judges'].get(k, {}).get('dims')
    if d is None: return None
    return dict(ZERO) if (apply_rule and r.get('penalty')) else d

GA_COMPLETE = ['1547acce317f4fc3', '44e958def4a748ad']; GA_PARTIAL = ['cd542f9e50224bfe']
NAME = {rid: runs[rid]['display_name'] for rid in runs}
out = ['# Judge reliability — official panel (Claude Opus 5.5 high + GPT-6.1 Sol medium)', '',
       'Source: production D1 export (scripts/export.py). Session scores: `result_json` (`judges.judge_a/judge_b.dims` = each judge\'s session mean over 5 segments; `combined` = 0.5/0.5 average, set to 0 by the wrong-language rule). Segment scores: `judgments.scores_json.dims`. Overall = published weights (Worlds 0.5, Stories 0.25 with Diversity 4 : Content logic 2, Preferences 0.25).', '']

# ---------------- Part A ----------------
def scored(rid):
    return [s for s in sess[rid] if s['status'] == 'done' and s['result'] and not s['result'].get('excluded')]
A_all = {rid: scored(rid) for rid in GA_COMPLETE + GA_PARTIAL}
out += ['## A. Official-panel full-protocol platform runs (group a)', '',
        md_table(['run', 'model', 'status', 'scored sessions', 'seeds', 'sim assignment', 'penalised', 'substituted'],
                 [[rid[:10], NAME[rid], runs[rid]['status'], len(v), len({s['seed_id'] for s in v}), runs[rid]['usersim'].get('assignment') or 'hash',
                   sum(bool(s['result'].get('penalty')) for s in v), sum(bool(s['result'].get('substituted')) for s in v)] for rid, v in A_all.items()]), '',
        f'Only {len(GA_COMPLETE)} official-panel full-protocol runs are complete (two models); a third (qwen-plus-character) is still running with {len(A_all[GA_PARTIAL[0]])} of 285 sessions scored. Model-level statistics below therefore compare 2 models (3 with the partial run, marked †). With 2–3 models, rank correlations can only take the values ±1 (2 models) or ±1, ±1/3 (3 models) and carry almost no information about a 10+-model leaderboard.', '']

# A1 session-level agreement
def agreement_rows(sessions, label):
    rows = []
    pairs = [(judge_dims(s, 'judge_a', False), judge_dims(s, 'judge_b', False)) for s in sessions if not s['result'].get('substituted')]
    pairs = [(a, b) for a, b in pairs if a and b]
    for k in DIMS + ['overall']:
        x = [ov(a) if k == 'overall' else a[k] for a, b in pairs]; y = [ov(b) if k == 'overall' else b[k] for a, b in pairs]
        rows.append({'set': label, 'level': 'session', 'dimension': k, 'n': len(x), 'mean_a': round(mean(x), 2), 'mean_b': round(mean(y), 2),
                     'pearson': round(pearson(x, y), 3), 'spearman': round(spearman(x, y), 3), 'mad': round(mean([abs(a - b) for a, b in zip(x, y)]), 2),
                     'icc21': round(icc21(x, y), 3), 'sd_a': round(st.pstdev(x), 2), 'sd_b': round(st.pstdev(y), 2)})
    return rows

def segment_rows(sessions, label):
    rows = []
    segs = []
    for s in sessions:
        if s['result'].get('substituted'): continue
        J = judg.get(s['id'], {})
        for c in range(5):
            if (c, 'judge_a') in J and (c, 'judge_b') in J: segs.append((J[(c, 'judge_a')], J[(c, 'judge_b')]))
    for k in DIMS + ['overall']:
        x = [ov(a) if k == 'overall' else a[k] for a, b in segs]; y = [ov(b) if k == 'overall' else b[k] for a, b in segs]
        row = {'set': label, 'level': 'segment', 'dimension': k, 'n': len(x), 'mean_a': round(mean(x), 2), 'mean_b': round(mean(y), 2), 'pearson': round(pearson(x, y), 3),
               'spearman': round(spearman(x, y), 3), 'mad': round(mean([abs(a - b) for a, b in zip(x, y)]), 2), 'icc21': round(icc21(x, y), 3), 'sd_a': round(st.pstdev(x), 2), 'sd_b': round(st.pstdev(y), 2)}
        if k in BIN:
            kap, po = kappa([a >= 50 for a in x], [b >= 50 for b in y])
            row['kappa'] = round(kap, 3); row['pct_agree'] = round(po, 3)
            row['fail_rate_a'] = round(sum(a < 50 for a in x) / len(x), 3); row['fail_rate_b'] = round(sum(b < 50 for b in y) / len(y), 3)
        rows.append(row)
    return rows

agr = []
pool = [s for rid in GA_COMPLETE for s in A_all[rid]]
pool_p = pool + A_all[GA_PARTIAL[0]]
agr += agreement_rows(pool, 'complete runs (2)') + agreement_rows(pool_p, 'complete + partial (3)†')
for rid in GA_COMPLETE + GA_PARTIAL: agr += agreement_rows(A_all[rid], NAME[rid])
agr += segment_rows(pool, 'complete runs (2)') + segment_rows(pool_p, 'complete + partial (3)†')
keys = ['set', 'level', 'dimension', 'n', 'mean_a', 'mean_b', 'sd_a', 'sd_b', 'pearson', 'spearman', 'mad', 'icc21', 'kappa', 'pct_agree', 'fail_rate_a', 'fail_rate_b']
with open(f'{OUT}/reliability_a_agreement.csv', 'w', newline='') as f:
    w = csv.DictWriter(f, fieldnames=keys); w.writeheader(); w.writerows(agr)
show = [r for r in agr if r['set'] in ('complete runs (2)', 'complete + partial (3)†')]
out += ['### A1. Inter-judge agreement (judge_a = Claude Opus 5.5, judge_b = GPT-6.1 Sol; sessions with substituted segments excluded)', '',
        md_table(['set', 'level', 'dimension', 'n', 'mean A', 'mean B', 'SD A', 'SD B', 'Pearson', 'Spearman', 'MAD', 'ICC(2,1)', 'κ (pass/fail)', '% agree'],
                 [[r['set'], r['level'], r['dimension'], r['n'], r['mean_a'], r['mean_b'], r['sd_a'], r['sd_b'], r['pearson'], r['spearman'], r['mad'], r['icc21'], r.get('kappa', ''), r.get('pct_agree', '')] for r in show]),
        '', 'Per-run rows: out/reliability_a_agreement.csv. Basics and Logic are pass/fail per segment (100 or 0); κ treats them as binary. Knowledge, Content logic and Interaction have little variance (most segments 100), so correlations on them rest on few non-ceiling cases.', '']

# A1b model-level inter-judge
ml = []
for rid in GA_COMPLETE + GA_PARTIAL:
    ss = A_all[rid]
    a = agg([judge_dims(s, 'judge_a') for s in ss if judge_dims(s, 'judge_a')])[1]; b = agg([judge_dims(s, 'judge_b') for s in ss if judge_dims(s, 'judge_b')])[1]
    c = agg([s['result']['combined'] for s in ss])[1]
    ml.append([NAME[rid] + ('†' if rid in GA_PARTIAL else ''), len(ss), round(a, 2), round(b, 2), round(a - b, 2), round(c, 2)])
out += ['### A1b. Model level: overall by each judge alone (wrong-language rule applied to both)', '', md_table(['model', 'sessions', 'judge_a only', 'judge_b only', 'A − B', 'official (combined)'], ml), '']
ta = kendall_tau_b([r[2] for r in ml[:2]], [r[3] for r in ml[:2]]); tp = kendall_tau_b([r[2] for r in ml], [r[3] for r in ml])
out += [f'Kendall τ between the judge_a-only and judge_b-only rankings: {ta:+.2f} (2 complete models), {tp:+.2f} (3 models incl. partial†).', '']

# A1c disputes
dis = []
for rid in GA_COMPLETE + GA_PARTIAL:
    ss = A_all[rid]; cfg = runs[rid]['dispute_cfg']
    dd = Counter(d['dimension'] for s in ss for d in s['result'].get('disputes', []))
    cd = Counter(d['dimension'] for s in ss for d in s['result'].get('chunkDisputes', []))
    nseg = sum(min(5, len([1 for c in range(5) if (c, 'judge_a') in judg.get(s['id'], {}) and (c, 'judge_b') in judg.get(s['id'], {})])) for s in ss)
    dis.append([NAME[rid], len(ss), f"{sum(s['result'].get('disputed', False) for s in ss)} ({sum(s['result'].get('disputed', False) for s in ss) / len(ss):.1%})",
                ', '.join(f'{k} {v}' for k, v in dd.most_common()), nseg, f"{sum(cd.values())} ({sum(cd.values()) / nseg:.2f}/segment)", ', '.join(f'{k} {v}' for k, v in cd.most_common()), f"session ≥{cfg['sessionThreshold']}, segment ≥{cfg['chunkThreshold']}"])
out += ['### A1c. Dispute rates as recorded (result_json.disputed / disputes / chunkDisputes)', '',
        md_table(['model', 'sessions', 'disputed sessions', 'session disputes by dimension', 'segments with both judges', 'segment disputes', 'by dimension', 'thresholds (snapshot)'], dis),
        '', 'A pass/fail dimension (Basics, Logic) is a session dispute only when the judges split on ≥ 2 segments (engine/results.ts); every split pass/fail segment is a segment dispute (gap 100 ≥ 30).', '']

# A2 discrimination
def seed_means(ss, f=lambda s: ov(s['result']['combined'])):
    d = defaultdict(list)
    for s in ss: d[s['seed_id']].append(f(s))
    return {k: mean(v) for k, v in d.items()}

def boot_sessions(ss):
    rows = [s['result']['combined'] for s in ss]
    return [agg([rows[rng.randrange(len(rows))] for _ in rows])[1] for _ in range(B)]

def boot_seeds(ss):
    by = defaultdict(list)
    for s in ss: by[s['seed_id']].append(s['result']['combined'])
    keys = list(by)
    res = []
    for _ in range(B):
        rows = []
        for _ in keys: rows += by[keys[rng.randrange(len(keys))]]
        res.append(agg(rows)[1])
    return res

disc = {}
drows = []
for rid in GA_COMPLETE + GA_PARTIAL:
    ss = A_all[rid]; point = agg([s['result']['combined'] for s in ss])[1]
    bs, bc = boot_sessions(ss), boot_seeds(ss)
    disc[rid] = (point, pct(bs), pct(bc))
    drows.append([NAME[rid] + ('†' if rid in GA_PARTIAL else ''), len(ss), len({s['seed_id'] for s in ss}), f'{point:.2f}', f'[{pct(bs)[0]:.2f}, {pct(bs)[1]:.2f}]', f'[{pct(bc)[0]:.2f}, {pct(bc)[1]:.2f}]',
                  f"{(pct(bc)[1] - pct(bc)[0]) / (pct(bs)[1] - pct(bs)[0]):.2f}"])
out += ['### A2. Discrimination: overall with 95% bootstrap CIs (B = 2000)', '', md_table(['model', 'sessions', 'seeds', 'overall', 'CI over sessions', 'CI over scenarios (cluster by seed)', 'width ratio cluster/session'], drows), '']
prow = []
models = GA_COMPLETE + GA_PARTIAL
for a, b in itertools.combinations(models, 2):
    ma, mb = seed_means(A_all[a]), seed_means(A_all[b]); shared = sorted(set(ma) & set(mb))
    d = [ma[s] - mb[s] for s in shared]
    bd = sorted(mean([d[rng.randrange(len(d))] for _ in d]) for _ in range(B))
    ci = pct(bd); p = sign_flip_p(d, rng)
    ov_ = not (disc[a][2][0] > disc[b][2][1] or disc[b][2][0] > disc[a][2][1])
    prow.append([f"{NAME[a]} − {NAME[b]}" + (' †' if a in GA_PARTIAL or b in GA_PARTIAL else ''), len(shared), f'{mean(d):+.2f}', f'[{ci[0]:+.2f}, {ci[1]:+.2f}]', f'{p:.4f}', 'overlap' if ov_ else 'separated', 'yes' if ci[0] > 0 or ci[1] < 0 else 'no'])
out += ['Pairs (paired bootstrap over shared scenarios, seed-level means; sign-flip p, Monte Carlo 20000):', '', md_table(['pair', 'shared seeds', 'mean diff', 'paired 95% CI', 'sign-flip p', 'cluster CIs', 'separated (paired CI excludes 0)'], prow), '']

# A3 ranking stability
def model_seed_scores(rid, scorer):
    return seed_means(A_all[rid], scorer)
sm = {rid: seed_means(A_all[rid]) for rid in models}
shared_all = sorted(set.intersection(*[set(sm[r]) for r in models])); shared_c = sorted(set.intersection(*[set(sm[r]) for r in GA_COMPLETE]))
def rank_dist(mods, shared):
    cnt = {m: Counter() for m in mods}
    for _ in range(B):
        samp = [shared[rng.randrange(len(shared))] for _ in shared]
        sc = {m: mean([sm[m][s] for s in samp]) for m in mods}
        order = sorted(mods, key=lambda m: -sc[m])
        for i, m in enumerate(order): cnt[m][i + 1] += 1
    return cnt
rd2 = rank_dist(GA_COMPLETE, shared_c); rd3 = rank_dist(models, shared_all)
out += ['### A3. Ranking stability', '', f'Rank distribution under scenario bootstrap (B = 2000; resampling the shared seeds, seed-level means): 2 complete models on {len(shared_c)} seeds; 3 models† on {len(shared_all)} seeds.', '',
        md_table(['model', 'set', 'P(rank 1)', 'P(rank 2)', 'P(rank 3)'], [[NAME[m], '2 models', f"{rd2[m][1] / B:.3f}", f"{rd2[m][2] / B:.3f}", ''] for m in GA_COMPLETE] +
                 [[NAME[m], '3 models†', f"{rd3[m][1] / B:.3f}", f"{rd3[m][2] / B:.3f}", f"{rd3[m][3] / B:.3f}"] for m in models]), '']
# language & judges & split halves
def overall_view(rid, view=None, judge=None):
    ss = [s for s in A_all[rid] if view is None or s['lang'] == view]
    rows = [judge_dims(s, judge) for s in ss] if judge else [s['result']['combined'] for s in ss]
    return agg([r for r in rows if r])[1]
lang_rows = []
for m in models: lang_rows.append([NAME[m] + ('†' if m in GA_PARTIAL else ''), f"{overall_view(m, 'zh'):.2f}", f"{overall_view(m, 'en'):.2f}", f"{overall_view(m, None, 'judge_a'):.2f}", f"{overall_view(m, None, 'judge_b'):.2f}"])
t_lang2 = kendall_tau_b([overall_view(m, 'zh') for m in GA_COMPLETE], [overall_view(m, 'en') for m in GA_COMPLETE]); t_lang3 = kendall_tau_b([overall_view(m, 'zh') for m in models], [overall_view(m, 'en') for m in models])
out += [md_table(['model', 'zh only', 'en only', 'judge_a only', 'judge_b only'], lang_rows), '',
        f'Kendall τ zh-only vs en-only: {t_lang2:+.2f} (2 models), {t_lang3:+.2f} (3†). judge_a-only vs judge_b-only: {ta:+.2f} (2), {tp:+.2f} (3†).', '']
def split_half(mods, shared):
    taus = []
    for _ in range(REPS):
        sh = shared[:]; rng.shuffle(sh); h1, h2 = sh[:len(sh) // 2], sh[len(sh) // 2:]
        taus.append(kendall_tau_b([mean([sm[m][s] for s in h1]) for m in mods], [mean([sm[m][s] for s in h2]) for m in mods]))
    return Counter(round(t, 3) for t in taus)
sh2, sh3 = split_half(GA_COMPLETE, shared_c), split_half(models, shared_all)
out += [f'Split-half (random halves of the shared scenarios, {REPS} repetitions): τ distribution, 2 models: ' + ', '.join(f'τ={k:+.2f}: {v / REPS:.1%}' for k, v in sorted(sh2.items())) +
        '; 3 models†: ' + ', '.join(f'τ={k:+.2f}: {v / REPS:.1%}' for k, v in sorted(sh3.items())) + '.', '']

# A4 generalisation across scenarios
mat2 = [[sm[m][s] for m in GA_COMPLETE] for s in shared_c]; mat3 = [[sm[m][s] for m in models] for s in shared_all]
W2, W3 = kendall_w(mat2), kendall_w(mat3)
overall_order = sorted(GA_COMPLETE, key=lambda m: -disc[m][0])
agree2 = sum((r[GA_COMPLETE.index(overall_order[0])] > r[GA_COMPLETE.index(overall_order[1])]) for r in mat2); ties2 = sum(r[0] == r[1] for r in mat2)
order3 = sorted(models, key=lambda m: -disc[m][0])
same3 = sum(sorted(models, key=lambda m: -r[models.index(m)]) == order3 for r in mat3)
out += ['### A4. Generalisation across scenarios', '',
        f'Kendall\'s W over scenarios (each shared seed ranks the models by its 3-sample mean; tie-corrected): W = {W2:.3f} for the 2 complete models over {len(shared_c)} seeds; W = {W3:.3f} for 3 models† over {len(shared_all)} seeds.',
        f'2 models: the overall leader ({NAME[overall_order[0]]}) is ahead on {agree2} of {len(shared_c)} seeds, behind on {len(shared_c) - agree2 - ties2}, tied on {ties2}. 3 models†: {same3} of {len(shared_all)} seeds reproduce the overall order.', '']
strat = []
for view in ['zh', 'en']:
    strat.append([f'language {view}'] + [f"{overall_view(m, view):.2f}" for m in models] + [' > '.join(NAME[m][:14] for m in sorted(models, key=lambda m: -overall_view(m, view)))])
for simname in ['claude-sonnet-5.5', 'gpt-6-luna']:
    vals = []
    for m in models:
        ss = [s for s in A_all[m] if simmain.get(s['id']) and simmain[s['id']].most_common(1)[0][0] == simname]
        vals.append(agg([s['result']['combined'] for s in ss])[1] if ss else float('nan'))
    strat.append([f'simulator {simname}'] + [f'{v:.2f}' for v in vals] + [' > '.join(NAME[m][:14] for m, _ in sorted(zip(models, vals), key=lambda x: -x[1]))])
strat.append(['all'] + [f'{disc[m][0]:.2f}' for m in models] + [' > '.join(NAME[m][:14] for m in order3)])
out += ['Strata (the seed data carry no scenario category field — worker/src/benchmarks/minimax-rp/data/seeds.*.jsonl have id, names, settings, prologue, first user input only — so language and simulator model are the only available strata):', '',
        md_table(['stratum'] + [NAME[m] + ('†' if m in GA_PARTIAL else '') for m in models] + ['order'], strat), '',
        'Simulator strata: 1547acce31 used hash assignment, so its simulator strata are not the same scenario/sample slots as the slot runs.', '']
open(f'{OUT}/reliability_a.md', 'w').write('\n'.join(out))
print('\n'.join(out))

# ---------------- Part B ----------------
o = ['# Calibration runs (group b) against MiniMax\'s released scores', '']
mm = {k: scored(v) for k, v in MM_RUN.items()}
mm_by_id = {s['id']: s for v in mm.values() for s in v}
def mm_ov(s, rule): return ov(s['result']['combined'] if rule else (s['result'].get('original') or s['result']['combined']))
recomp = {k: agg([s['result']['combined'] for s in v])[1] for k, v in mm.items()}
recomp_pre = {k: agg([s['result'].get('original') or s['result']['combined'] for s in v])[1] for k, v in mm.items()}
ks = list(PUBLISHED)
o += [f"Released-score recomputation (all 285 sessions per model, published weights) vs published leaderboard: Spearman ρ = {spearman([recomp_pre[k] for k in ks], [PUBLISHED[k] for k in ks]):.3f} before the wrong-language rule, {spearman([recomp[k] for k in ks], [PUBLISHED[k] for k in ks]):.3f} after it.", '']
# 41 pairs
def separated(rule):
    sc = {k: defaultdict(list) for k in ks}
    for k in ks:
        for s in mm[k]: sc[k][s['seed_id']].append(mm_ov(s, rule))
    sc = {k: {s: mean(v) for s, v in d.items()} for k, d in sc.items()}
    pairs = []
    r2 = random.Random(20261026)
    for a, b in itertools.combinations(ks, 2):
        seeds = sorted(set(sc[a]) & set(sc[b])); d = [sc[a][s] - sc[b][s] for s in seeds]
        if mean(d) < 0: a, b, d = b, a, [-x for x in d]
        pairs.append({'a': a, 'b': b, 'n': len(d), 'diff': mean(d), 'p': sign_flip_p(d, r2)})
    for p, h in zip(pairs, holm([p['p'] for p in pairs])): p['holm'] = h
    return pairs, sc
pairs_pre, sc_pre = separated(False); pairs_post, sc_post = separated(True)
sep_pre = [p for p in pairs_pre if p['holm'] < 0.05]; sep_post = [p for p in pairs_post if p['holm'] < 0.05]
with open(f'{OUT}/minimax_separated_pairs.csv', 'w', newline='') as f:
    w = csv.writer(f); w.writerow(['rule', 'a', 'b', 'n_seeds', 'diff', 'p_signflip', 'p_holm', 'separated'])
    for tag, P in (('without_rule', pairs_pre), ('with_rule', pairs_post)):
        for p in P: w.writerow([tag, p['a'], p['b'], p['n'], round(p['diff'], 3), round(p['p'], 5), round(p['holm'], 5), p['holm'] < 0.05])
o += [f'MiniMax pairs separated by the reference rule (scenario-paired sign-flip test, Holm over 55 pairs, α = .05; tools/experiments/anchor_check.py method, MC 20000, seed 20261026): {len(sep_pre)} of 55 on the released scores before the wrong-language rule, {len(sep_post)} of 55 after it (out/minimax_separated_pairs.csv).', '']
crows, cpairs, cagree = [], [], []
for rid, r in sorted(runs.items(), key=lambda kv: kv[1]['created_at']):
    if r['origin'] != 'calibration': continue
    cal = r['config'].get('calibration', {})
    ours = {}
    for s in scored(rid):
        orig = cal.get(s['id'])
        if not orig or orig not in mm_by_id: continue
        ours[s['id']] = (s, mm_by_id[orig])
    if not ours:
        crows.append([rid[:10], r['pv'], panel(r), judges_label(r), 0, '', '', '', '', '', '', '', '', '']); continue
    model_of = {k: v for k, v in MM_RUN.items()}; inv = {v: k for k, v in MM_RUN.items()}
    per_model = defaultdict(list); per_model_mm = defaultdict(list); seeds_of = defaultdict(set)
    for s, m in ours.values():
        k = inv[m['run_id']]; per_model[k].append(ov(s['result']['combined'])); per_model_mm[k].append(mm_ov(m, False)); seeds_of[k].add(s['seed_id'])
    mods = [k for k in ks if k in per_model]
    our_m = [mean(per_model[k]) for k in mods]
    xs = [ov(s['result']['combined']) for s, m in ours.values()]; ys = [mm_ov(m, False) for s, m in ours.values()]
    # session-level judge agreement in this run
    pa = [(judge_dims(s, 'judge_a', False), judge_dims(s, 'judge_b', False)) for s, m in ours.values()]
    pa = [(a, b) for a, b in pa if a and b]
    ja = pearson([ov(a) for a, b in pa], [ov(b) for a, b in pa]) if pa else float('nan'); jm = mean([abs(ov(a) - ov(b)) for a, b in pa]) if pa else float('nan')
    # separated pairs: direction on model means (unpaired) and on shared seeds
    kept_m = sum(1 for p in sep_pre if p['a'] in per_model and p['b'] in per_model and mean(per_model[p['a']]) > mean(per_model[p['b']]))
    cov_m = sum(1 for p in sep_pre if p['a'] in per_model and p['b'] in per_model)
    cov_s = kept_s = 0
    for p in sep_pre:
        if p['a'] not in per_model or p['b'] not in per_model: continue
        sh = seeds_of[p['a']] & seeds_of[p['b']]
        if not sh: continue
        cov_s += 1
        da = mean([ov(s['result']['combined']) for s, m in ours.values() if inv[m['run_id']] == p['a'] and s['seed_id'] in sh])
        db = mean([ov(s['result']['combined']) for s, m in ours.values() if inv[m['run_id']] == p['b'] and s['seed_id'] in sh])
        kept_s += da > db
    crows.append([rid[:10], r['pv'], panel(r), judges_label(r), len(ours), len(mods), f'{pearson(xs, ys):.3f}', f'{mean([x - y for x, y in zip(xs, ys)]):+.2f}', f'{mean([abs(x - y) for x, y in zip(xs, ys)]):.2f}',
                  f"{spearman(our_m, [mean(per_model_mm[k]) for k in mods]):.3f}", f"{spearman(our_m, [recomp_pre[k] for k in mods]):.3f}", f"{spearman(our_m, [PUBLISHED[k] for k in mods]):.3f}",
                  f'{kept_m}/{cov_m}', f'{kept_s}/{cov_s}', f'{ja:.3f} / {jm:.2f} (n={len(pa)})'])
o += ['Per calibration run (sessions = released MiniMax sessions re-scored by the run\'s judges; MiniMax side = released session scores before the wrong-language rule; model mean over that run\'s 1–2 sessions per model):', '',
      md_table(['run', 'protocol', 'panel', 'judges', 'sessions', 'models', 'session r vs MiniMax', 'mean bias (ours − MM)', 'MAE', 'ρ model vs MM same sessions', 'ρ vs MM recomputation (285)', 'ρ vs published', f'separated pairs ({len(sep_pre)}) direction kept, model means', '…on shared seeds', 'judge A vs B: session r / MAD'], crows), '']
o += ['None of these calibration runs uses the judge prompt in force for v6.0: the official judge_system prompt of runs 1547acce31/44e958def4/cd542f9e50 (sha256 prefix 9f6fd89243, raw/runs_prompts.json) first appears in calibration run cbfe9afde2 (v4.0, Sol-low + Jev panel). The official-panel calibration runs used the v1.2–v2.4 prompts (four different judge_system texts) or, for v5.0 (026e02768d, paused, 0 of 2 sessions completed; judge_a batch never returned), the current prompt. Their model-level numbers rest on 1 session per model (11-session runs) or 2 (22-session runs); none of the separated pairs can be tested for significance on them.', '']
open(f'{OUT}/reliability_b.md', 'w').write('\n'.join(o))
print('\n'.join(o))

# ---------------- Part B extension: per-dimension agreement, disputes, rankings within calibration runs ----------------
o2 = ['', '## B2. Inter-judge agreement, disputes and rankings inside the official-panel calibration runs', '']
off_cal = [rid for rid, r in sorted(runs.items(), key=lambda kv: kv[1]['created_at']) if r['origin'] == 'calibration' and panel(r) == 'official' and scored(rid)]
agr_b = []
pool_b = [s for rid in off_cal for s in scored(rid)]
agr_b += agreement_rows(pool_b, f'official calibration runs pooled ({len(off_cal)} runs, v1.2–v2.4 prompts)') + segment_rows(pool_b, f'official calibration runs pooled ({len(off_cal)} runs)')
for rid in off_cal: agr_b += agreement_rows(scored(rid), f"{rid[:10]} {runs[rid]['pv']}")
with open(f'{OUT}/reliability_b_agreement.csv', 'w', newline='') as f:
    w = csv.DictWriter(f, fieldnames=keys); w.writeheader(); w.writerows(agr_b)
o2 += ['Pooled over the six completed official-panel calibration runs (the same released dialogues re-scored under six prompt versions, so sessions are not independent across runs):', '',
       md_table(['set', 'level', 'dimension', 'n', 'mean A', 'mean B', 'Pearson', 'Spearman', 'MAD', 'ICC(2,1)', 'κ', '% agree'],
                [[r['set'], r['level'], r['dimension'], r['n'], r['mean_a'], r['mean_b'], r['pearson'], r['spearman'], r['mad'], r['icc21'], r.get('kappa', ''), r.get('pct_agree', '')] for r in agr_b if 'pooled' in r['set']]), '',
       'Per-run rows: out/reliability_b_agreement.csv.', '']
rows_b = []
inv = {v: k for k, v in MM_RUN.items()}
for rid in off_cal:
    r = runs[rid]; cal = r['config']['calibration']; ss = scored(rid)
    mod = lambda s: inv[mm_by_id[cal[s['id']]]['run_id']]
    per = defaultdict(list)
    for s in ss: per[mod(s)].append(s)
    mods = [k for k in ks if k in per]
    ma = [agg([judge_dims(s, 'judge_a', False) for s in per[k]])[1] for k in mods]; mb = [agg([judge_dims(s, 'judge_b', False) for s in per[k]])[1] for k in mods]
    dis_s = sum(s['result'].get('disputed', False) for s in ss); dis_c = sum(len(s['result'].get('chunkDisputes', [])) for s in ss)
    tz = ''
    if all(len({s['lang'] for s in per[k]}) == 2 for k in mods):
        zh = [ov(next(s for s in per[k] if s['lang'] == 'zh')['result']['combined']) for k in mods]; en = [ov(next(s for s in per[k] if s['lang'] == 'en')['result']['combined']) for k in mods]
        tz = f"{kendall_tau_b(zh, en):+.2f}; W(2 scenarios) = {kendall_w([zh, en]):.2f}"
    rows_b.append([rid[:10], r['pv'], len(ss), len(mods), f'{kendall_tau_b(ma, mb):+.2f}', f'{spearman(ma, mb):+.2f}', f'{max(ma + mb) - min(ma + mb):.1f}',
                   f'{dis_s} ({dis_s / len(ss):.0%})', f'{dis_c} ({dis_c / (5 * len(ss)):.2f}/segment)', tz or 'n/a (1 session per model)'])
o2 += [md_table(['run', 'protocol', 'sessions', 'models', 'Kendall τ judge_a vs judge_b (model means)', 'Spearman', 'range of model means', 'disputed sessions', 'segment disputes', 'τ zh vs en (models); Kendall W'], rows_b), '',
       'Discrimination and ranking stability cannot be estimated inside the calibration runs: each model has 1 session (11-session runs) or 2 (one zh, one en; 22-session runs), so there is nothing to resample within a model.', '']
open(f'{OUT}/reliability_b.md', 'a').write('\n'.join(o2))
print('\n'.join(o2))