"""Writes web/src/pages/characterbenchAudit.ts, the data behind the reproduction report's verification section, from the outputs of the audit scripts — no number in that section is typed by hand. Platform runs enter only as totals: the report is about the evaluation pipeline, not about any model (except the paper anchor, which checks the pipeline against the paper). python3 -I tools/experiments/characterbench_audit_module.py audit.json judge_audit.json spec_check.txt anchor.json \ DEPLOYMENT_ID OUT.ts [leaderboard.json] [characterjudge_server_check.json] anchor.json, leaderboard.json and the server check may be "-" (not available yet). """ import json import re import sys def main(audit_p, judge_p, spec_p, anchor_p, deployment, out, leaderboard_p="-", weights_p="-"): a = json.load(open(audit_p, encoding="utf-8")) j = json.load(open(judge_p, encoding="utf-8")) spec = open(spec_p, encoding="utf-8").read() anchor = json.load(open(anchor_p, encoding="utf-8")) if anchor_p != "-" else None lb = json.load(open(leaderboard_p, encoding="utf-8")) if leaderboard_p != "-" else None w = json.load(open(weights_p, encoding="utf-8")) if weights_p != "-" else None runs = list(a["runs"].values()) m = re.search(r"(\d+)/(\d+) checks passed; protocol (.+)", spec) finish = {} for r in runs: for k, v in r["reply_finish_reasons"].items(): finish[k] = finish.get(k, 0) + v data = { "protocol": m.group(3).strip(), "deployment": deployment, "spec": {"passed": int(m.group(1)), "total": int(m.group(2)), "ids": sorted(set(re.findall(r"^PASS (C\d+\w?)", spec, re.M)))}, "data": { "files": a["A_data"]["repository_files"], "fileHashMismatches": len(a["A_data"]["repo_vs_production_sync_hash_mismatch"]), "raw": a["A_data"]["raw_samples"], "keptPerLang": a["A_data"]["official_kept_per_lang"], "production": a["A_data"]["production_items"], "onlyOfficial": len(a["A_data"]["only_official"]), "onlyProduction": len(a["A_data"]["only_production"]), "fieldMismatches": sum(len(v) for v in a["A_data"]["field_diffs"].values()), "rebuildMismatches": len(a["A_data"]["production_vs_rebuild_item_diffs"]), "unusedFields": a["A_data"].get("production_unused_fields", {}), "dropped": [{"file": d["file"], "id": d["id"], "reason": d["reason"]} for d in a["A_data"]["official_dropped"]], }, "prompts": { "roleplayCompared": a["B_prompts"]["roleplay_vs_paper_table11"]["compared"], "roleplayMismatch": a["B_prompts"]["roleplay_vs_paper_table11"]["mismatch"], "roleplayEqualToCodePrompt": a["B_prompts"]["roleplay_vs_code_roleplay_prompt_py"]["equal"], "judgeCompared": a["B_prompts"]["judge_input_placeholder_reply"]["compared"], "judgeMismatch": a["B_prompts"]["judge_input_placeholder_reply"]["mismatch"], "realRuns": len(runs), "realCompared": sum(r["judge_input_real_replies"]["compared"] for r in runs), "realMismatch": sum(r["judge_input_real_replies"]["mismatch"] for r in runs), "snapshotTemplateDiffs": sum(len(r["snapshot_prompts_vs_current_templates"]) for r in runs), }, "generation": { "replies": sum(finish.values()), "finish": finish, "excludedSessions": sum(len(r["excluded_pairs"]) for r in runs), }, "scoring": { "runs": len(runs), "judged": sum(r["judged_samples"] for r in runs), "outputs": 10 * sum(r["judged_samples"] for r in runs), "sampleDiffs": sum(r["per_sample_score_diffs"] for r in runs), "stops": sum(r["outputs_where_all_eval_stops_count"] for r in runs), "unreadable": sum(r["unreadable_outputs_platform_count"] for r in runs), "outOfRange": sum(sum(r["out_of_range_outputs"].values()) for r in runs), "cells": sum(r["cells_compared"] for r in runs), "cellsDiffering": sum(len(r["cells_differing_at_two_decimals"]) for r in runs), "cellsDifferingUnder20": sum(len(r["cells_differing_at_two_decimals_under_2_0_aggregation"]) for r in runs), "maxUnroundedDiff": max(v for r in runs for v in r["max_abs_column_diff_unrounded"].values()), "rankingSame": all(v["same"] for v in a["E_ranking"].values()), }, "judge": { lang: { "n": v["n"], "avgPearson": v["base"]["avg"], "paperAvg": v["base"]["paper_avg"], "within005": v["base"]["columns_within_0.05_of_paper"], "repeat": v.get("repeat_same_seed"), "seeds": v.get("seed_variation"), "releasedMad": v["vs_released_sc"]["mean_abs_diff_raw_points"], "releasedPearson": v["vs_released_sc"]["pearson_released_sc_avg"], "biasVsHuman": v["judge_minus_human_5pt"], "biasVsReleased": v["vs_released_sc"]["mean_diff_5pt_per_column"], "outOfRange": sum(v["sampled_out_of_range"].values()), "stops": v["sampled_where_all_eval_stops"], } for lang, v in j.items() }, "leaderboard": lb and {k: lb[k] for k in ("runs", "eligible", "otherProtocol", "otherProtocolEligible")}, "weights": w and { "checkedAt": w["checked_at"], "matched": sum(1 for f in w["files"].values() if f["status"] == "MATCH"), "total": len(w["files"]), "engine": f"vLLM {w.get('vllm_version', '?')}" + next((f", {a[i + 1]}" for a in [w.get("serving_process", {}).get("argv", [])] for i, x in enumerate(a) if x == "--dtype" and i + 1 < len(a)), ""), "modelDir": w.get("model_dir"), }, "anchor": anchor and { "row": anchor["paper_row"], "model": anchor["runs"][0]["model"], "runs": len(anchor["runs"]), "compare": anchor["compare"], "judgeBiasCorrelation": anchor.get("judge_bias_correlation"), }, } with open(out, "w", encoding="utf-8") as f: f.write("// Generated by tools/experiments/characterbench_audit_module.py from the audit scripts' outputs (see the report's\n") f.write("// verification section); do not edit by hand.\n\n") f.write(f"export const CB_AUDIT = {json.dumps(data, ensure_ascii=False, indent=2)} as const;\n") print("written", out) if __name__ == "__main__": main(*sys.argv[1:9])