"""CharacterBench audit, judge layer: statistics of the deployed CharacterJudge on the 3,250 annotated test responses per language, from the raw outputs of characterbench_judge_validation.py (`run`). python3 -I tools/experiments/characterbench_judge_audit.py OUT.json BASE_zh.jsonl BASE_en.jsonl \ [--repeat REP_zh.jsonl REP_en.jsonl] [--seeds S1_zh.jsonl S1_en.jsonl S2_zh.jsonl S2_en.jsonl ...] BASE: the validation run (greedy + 10 seeded samples). --repeat: the same requests sent again with the same seeds (CB_SC_ONLY=1): how exactly the deployment repeats itself. --seeds: the same requests with other seeds (CB_SEED_OFFSET=k): how much self-consistency sampling moves correlations and scores. Every output is read as all_eval.py's check() reads it; a sample's score is the mean of its 10 readings (protocol 2.0.1). """ import collections import json import statistics import sys FILES = [("memory_consistency", 4), ("fact_accuracy", 4), ("boundary_consistency", 3), ("attribute_consistency_bot", 4), ("attribute_consistency_human", 4), ("behavior_consistency_bot", 4), ("behavior_consistency_human", 3), ("emotion_self_regulation", 4), ("empathetic_responsiveness", 4), ("morality_stability", 2), ("morality_robustness", 2), ("human_likeness", 5), ("engagement", 5)] MAX = dict(FILES) PAPER_SC = {"zh": [80, 92, 71, 80, 63, 62, 65, 67, 65, 66, 61, 52, 58], "en": [81, 88, 65, 76, 57, 57, 58, 65, 62, 64, 55, 53, 53]} PAPER_AVG = {"zh": 68, "en": 64} def check(generated): # all_eval.py check(), followed by float() as there; None where the script would stop first_line = "\n" in generated if first_line: generated = generated.split("\n")[0] try: return float(generated) except Exception: return None if first_line else 3.0 def read(text): v = check(text) return 3.0 if v is None else v def load(path): return {r["key"]: r for r in (json.loads(l) for l in open(path, encoding="utf-8") if l.strip())} def pearson(x, y): mx, my = statistics.fmean(x), statistics.fmean(y) sx = sum((a - mx) ** 2 for a in x) ** 0.5 sy = sum((b - my) ** 2 for b in y) ** 0.5 return sum((a - mx) * (b - my) for a, b in zip(x, y)) / (sx * sy) if sx and sy else None def sc_score(rec): return statistics.fmean(read(t) for t in rec["sc"]) def per_column(recs, value): out = {} for f, _ in FILES: rs = [r for r in recs.values() if r["file"] == f] out[f] = pearson([value(r) for r in rs], [float(r["human"]) for r in rs]) return out def summary(recs, lang): cols = per_column(recs, sc_score) return {"columns": {f: round(v, 4) for f, v in cols.items()}, "avg": round(statistics.fmean(cols.values()), 4), "paper_avg": PAPER_AVG[lang] / 100, "columns_within_0.05_of_paper": sum(1 for (f, _), p in zip(FILES, PAPER_SC[lang]) if abs(cols[f] - p / 100) <= 0.05)} def main(out, args): base = {"zh": load(args[0]), "en": load(args[1])} rest = args[2:] repeat, seeds = None, [] i = 0 while i < len(rest): if rest[i] == "--repeat": repeat = {"zh": load(rest[i + 1]), "en": load(rest[i + 2])} i += 3 elif rest[i] == "--seeds": i += 1 while i + 1 < len(rest) and not rest[i].startswith("--"): seeds.append({"zh": load(rest[i]), "en": load(rest[i + 1])}) i += 2 else: raise SystemExit(f"unknown argument {rest[i]}") report = {} for lang in ("zh", "en"): b = base[lang] r = {"n": len(b), "base": summary(b, lang)} # Inference against the authors' own released predictions (same inputs, their deployment and decoding). both = [x for x in b.values() if x.get("released_sc") is not None] diffs = collections.defaultdict(list) for x in both: diffs[x["file"]].append(sc_score(x) - float(x["released_sc"])) r["vs_released_sc"] = { "n": len(both), "mean_abs_diff_raw_points": round(statistics.fmean(abs(d) for v in diffs.values() for d in v), 4), "mean_diff_5pt_per_column": {f: round(statistics.fmean(diffs[f]) / m * 5, 4) for f, m in FILES}, "pearson_released_sc_avg": round(statistics.fmean(per_column({k: v for k, v in b.items() if v.get("released_sc") is not None}, lambda x: float(x["released_sc"])).values()), 4), } # Systematic bias against the human scores, on the 5-point scale used for results. r["judge_minus_human_5pt"] = {f: round((statistics.fmean(sc_score(x) for x in b.values() if x["file"] == f) - statistics.fmean(float(x["human"]) for x in b.values() if x["file"] == f)) / m * 5, 4) for f, m in FILES} oor = collections.Counter() stops = 0 for x in b.values(): for t in x["sc"]: v = check(t) if v is None: stops += 1 elif not 1 <= v <= MAX[x["file"]]: oor[x["file"]] += 1 r["sampled_out_of_range"] = dict(oor) r["sampled_where_all_eval_stops"] = stops if repeat: rp = repeat[lang] keys = [k for k in b if k in rp] same = sum(1 for k in keys if b[k]["sc"] == rp[k]["sc"]) d = [abs(sc_score(b[k]) - sc_score(rp[k])) for k in keys] r["repeat_same_seed"] = {"n": len(keys), "identical_10_outputs": same, "share": round(same / len(keys), 4), "mean_abs_score_diff": round(statistics.fmean(d), 4), "max_abs_score_diff": max(d), "avg_pearson": summary({k: rp[k] for k in keys}, lang)["avg"] if len(keys) == len(b) else None} if seeds: avgs = [r["base"]["avg"]] + [summary(s[lang], lang)["avg"] for s in seeds if len(s[lang]) == len(b)] col_scores = [] # column means on the 5-point scale for each seed set: the judge's contribution to score noise for s in [b] + [s[lang] for s in seeds if len(s[lang]) == len(b)]: col_scores.append({f: statistics.fmean(sc_score(x) for x in s.values() if x["file"] == f) / m * 5 for f, m in FILES}) r["seed_variation"] = { "seed_sets": len(avgs), "avg_pearson_per_seed_set": avgs, "avg_pearson_range": round(max(avgs) - min(avgs), 4), "column_score_range_5pt": {f: round(max(c[f] for c in col_scores) - min(c[f] for c in col_scores), 4) for f, _ in FILES}, "avg_score_per_seed_set": [round(statistics.fmean(c.values()), 4) for c in col_scores], } report[lang] = r json.dump(report, open(out, "w"), indent=1, ensure_ascii=False) for lang, r in report.items(): print(lang, "SC avg Pearson", r["base"]["avg"], "paper", r["base"]["paper_avg"], "within 0.05:", r["base"]["columns_within_0.05_of_paper"], "/13") if "repeat_same_seed" in r: print(" repeat (same seed):", r["repeat_same_seed"]) if "seed_variation" in r: print(" seeds:", r["seed_variation"]["avg_pearson_per_seed_set"], "avg score", r["seed_variation"]["avg_score_per_seed_set"]) print(" vs released SC:", {k: v for k, v in r["vs_released_sc"].items() if k != "mean_diff_5pt_per_column"}, "stops:", r["sampled_where_all_eval_stops"]) if __name__ == "__main__": main(sys.argv[1], sys.argv[2:])