"""CharacterBench audit bundle: runs every step of the reproduction audit with a log per step, and assembles what a third
party needs to check and re-run it, for publication with the report (web/public/audit/characterbench/<date>/).

Steps (each logged with its command, start and end time, exit status and full output):
  01 spec_check        every protocol setting against the paper and the authors' code
  02 prompt_check      the authors' functions build every prompt (hashes) / 03 the platform's / 04 compare
  05 audit_platform    the platform's functions on what production holds and ran (exported from its database)
  06 audit             the authors' functions on the same material: data, prompts, judge input, scoring
  07 judge_audit       judge statistics: human correlation, repeatability, seed variation, released predictions
  08 anchor            the paper anchor against the paper's row
  09 audit_module      the report's data module

Published (no test-set text, no model reply, no dialogue; the test set stays with its authors):
  code/                the audit scripts and the platform source they execute, as run
  results/             each step's output (audit.json, judge_audit.json, anchor.json, spec_check.txt, prompt hashes)
  frozen/              the inputs the scoring can be recomputed from: every judge output of every audited run with the
                       platform's reading of it, every sampled validation output, and per run its protocol snapshot,
                       configuration and result; plus the leaderboard state (leaderboard.json)
  logs/                one log per step
  SHA256SUMS           every file of the bundle

    python3 -I tools/experiments/characterbench_audit_bundle.py EXPORT REPO EPRINT VALDIR OUT [WEIGHTS.json]

EXPORT: the D1 exports (characterbench_audit_platform.ts); REPO: thu-coai/CharacterBench at 66c113c; EPRINT: the
arXiv e-print; VALDIR: the judge validation outputs (val_{zh,en}.jsonl and judge_rep{0..3}_{zh,en}.jsonl); OUT: the
bundle directory; WEIGHTS: the operator's characterjudge_server_check.py output, when available.
"""
import datetime
import hashlib
import json
import os
import platform
import shutil
import subprocess
import sys

ROOT = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", ".."))
CODE = [
    "tools/experiments/characterbench_spec_check.py",
    "tools/experiments/characterbench_prompt_check.py",
    "tools/experiments/characterbench_prompt_hashes.ts",
    "tools/experiments/characterbench_audit_platform.ts",
    "tools/experiments/characterbench_audit.py",
    "tools/experiments/characterbench_judge_validation.py",
    "tools/experiments/characterbench_judge_audit.py",
    "tools/experiments/characterbench_anchor.py",
    "tools/experiments/characterbench_audit_module.py",
    "tools/experiments/characterbench_audit_bundle.py",
    "tools/experiments/characterjudge_server_check.py",
    "tools/experiments/characterbench_recompute.py",
    "server/src/benchmarks/characterbench/build.ts",
    "server/src/benchmarks/characterbench/data.ts",
    "server/src/benchmarks/characterbench/index.ts",
    "server/src/benchmarks/characterbench/meta.ts",
    "worker/src/benchmarks/characterbench/build.ts",
    "worker/src/benchmarks/characterbench/index.ts",
    "worker/src/engine/results.ts",
    "worker/src/engine/runner.ts",
    "worker/src/scoring.ts",
]


def sha256(path):
    h = hashlib.sha256()
    with open(path, "rb") as f:
        for chunk in iter(lambda: f.read(1 << 20), b""):
            h.update(chunk)
    return h.hexdigest()


# Local paths in logs are replaced by these names (set in main), so the logs name inputs, not a machine.
PLACES = {}


def public(text):
    for path, label in sorted(PLACES.items(), key=lambda x: -len(x[0])):
        text = text.replace(path, label)
    return text


def run(logs, n, name, cmd):
    """Runs one step from the repository root and writes its log; stops the bundle when the step fails."""
    start = datetime.datetime.now(datetime.timezone.utc)
    r = subprocess.run(cmd, cwd=ROOT, capture_output=True, text=True)
    end = datetime.datetime.now(datetime.timezone.utc)
    with open(os.path.join(logs, f"{n:02d}-{name}.log"), "w", encoding="utf-8") as f:
        f.write(public(f"$ {' '.join(['python3 -I' if c == sys.executable else c for c in cmd if c != '-I'])}\n# started {start.isoformat(timespec='seconds')}  ended {end.isoformat(timespec='seconds')}  exit {r.returncode}\n\n"))
        f.write(public(r.stdout))
        if r.stderr.strip():
            f.write(public("\n# stderr\n" + "\n".join(l for l in r.stderr.splitlines() if "ExperimentalWarning" not in l and "MODULE_TYPELESS" not in l)))
    print(f"{n:02d} {name}: exit {r.returncode}", flush=True)
    if r.returncode != 0:
        raise SystemExit(f"step {name} failed; see its log")
    return r.stdout


def main(export, repo, eprint, valdir, out, weights=None):
    export, repo, eprint, valdir, out = (os.path.abspath(p) for p in (export, repo, eprint, valdir, out))
    weights = os.path.abspath(weights) if weights else None
    tmp = os.path.join(os.path.dirname(out), f".work-{os.getpid()}")
    PLACES.update({os.path.abspath(tmp): "<work>", os.path.abspath(out): "<bundle>", os.path.abspath(export): "<export>", os.path.abspath(repo): "<repo>",
                   os.path.abspath(eprint): "<eprint>", os.path.abspath(valdir): "<validation>", ROOT: "<platform>", os.path.expanduser("~"): "<home>"})
    if weights:
        PLACES[os.path.dirname(weights)] = "<operator-check>"
    for d in ("code", "results", "frozen", "logs"):
        os.makedirs(os.path.join(out, d), exist_ok=True)
    logs, res, frozen = (os.path.join(out, d) for d in ("logs", "results", "frozen"))
    os.makedirs(tmp, exist_ok=True)
    node = ["node", "--disable-warning=ExperimentalWarning"]
    py = [sys.executable, "-I"]
    with open(os.path.join(logs, "00-environment.log"), "w") as f:
        f.write(f"generated {datetime.datetime.now(datetime.timezone.utc).isoformat(timespec='seconds')}\n")
        f.write(f"python {platform.python_version()}  node {subprocess.run(['node', '--version'], capture_output=True, text=True).stdout.strip()}  {platform.platform()}\n")
        f.write(f"repository commit {subprocess.run(['git', '-C', repo, 'rev-parse', 'HEAD'], capture_output=True, text=True).stdout.strip()}\n")
        f.write(f"e-print files: {', '.join(sorted(os.listdir(eprint)))}\n")
        f.write(f"exported runs: {', '.join(sorted(f[4:-5] for f in os.listdir(export) if f.startswith('run_') and f.endswith('.json')))}\n")

    val = lambda name: os.path.join(valdir, name)
    run(logs, 1, "spec_check", py + ["tools/experiments/characterbench_spec_check.py", eprint, repo, val("val_zh.jsonl"), val("val_en.jsonl")])
    table11 = os.path.join(tmp, "table11.json")
    with open(table11, "w") as f:
        f.write(run(logs, 2, "table11", py + ["tools/experiments/characterbench_spec_check.py", "table11", eprint]))
    run(logs, 3, "prompt_check_authors", py + ["tools/experiments/characterbench_prompt_check.py", repo, repo, table11, os.path.join(res, "prompt_hashes_authors.json")])
    run(logs, 4, "prompt_check_platform", node + ["tools/experiments/characterbench_prompt_hashes.ts", os.path.join(res, "prompt_hashes_platform.json")])
    run(logs, 5, "prompt_check_compare", py + ["tools/experiments/characterbench_prompt_check.py", "--compare", os.path.join(res, "prompt_hashes_authors.json"), os.path.join(res, "prompt_hashes_platform.json")])
    platform_json = os.path.join(tmp, "platform.json")
    run(logs, 6, "audit_platform", node + ["tools/experiments/characterbench_audit_platform.ts", export, repo, platform_json])
    run(logs, 7, "audit", py + ["tools/experiments/characterbench_audit.py", repo, table11, export, platform_json, os.path.join(res, "audit.json")])
    reps = [val(f"judge_rep{k}_{l}.jsonl") for k in (1, 2, 3) for l in ("zh", "en")]
    run(logs, 8, "judge_audit", py + ["tools/experiments/characterbench_judge_audit.py", os.path.join(res, "judge_audit.json"), val("val_zh.jsonl"), val("val_en.jsonl"),
                                       "--repeat", val("judge_rep0_zh.jsonl"), val("judge_rep0_en.jsonl"), "--seeds", *reps])
    anchors = [os.path.join(export, f) for f in sorted(os.listdir(export)) if f.startswith("run_") and json.load(open(os.path.join(export, f))).get("config_json", "").find('"experiment"') >= 0
               and json.loads(json.load(open(os.path.join(export, f)))["config_json"]).get("mode") == "full"]
    run(logs, 9, "anchor", py + ["tools/experiments/characterbench_anchor.py", eprint, os.path.join(res, "anchor.json"), "--judge", os.path.join(res, "judge_audit.json"), *anchors])
    spec_txt = os.path.join(res, "spec_check.txt")
    shutil.copy(os.path.join(logs, "01-spec_check.log"), spec_txt)

    # Frozen inputs of the scoring: judge outputs as stored, keyed like the audit (language | test file | index).
    items = {}
    for line in open(os.path.join(export, "cb_items.jsonl"), encoding="utf-8"):
        r = json.loads(line)
        it = json.loads(r["item_json"])
        items[r["seed_id"]] = f"{it['lang']}|{it['file']}|{it['index']}"
    runs_meta = []
    for f in sorted(os.listdir(export)):
        if not (f.startswith("run_") and f.endswith(".json")):
            continue
        run_row = json.load(open(os.path.join(export, f), encoding="utf-8"))
        rid = run_row["id"]
        rows = 0
        with open(os.path.join(frozen, f"judge_outputs_{rid}.jsonl"), "w", encoding="utf-8") as o:
            for line in open(os.path.join(export, f"sessions_{rid}.jsonl"), encoding="utf-8"):
                s = json.loads(line)
                for j in json.loads(s["judg"]):
                    if j["status"] != "done":
                        continue
                    p = json.loads(j["parsed"])
                    o.write(json.dumps({"seed": s["seed_id"], "key": items.get(s["seed_id"]), "outputs": j["raw"].split(" | "), "platform_samples": p["samples"], "platform_score": p["score"], "dimension": p["dimension"], "max": p["max"]}, ensure_ascii=False) + "\n")
                    rows += 1
                result = json.loads(s["result_json"]) if s["result_json"] else None
                if result and result.get("excluded"):
                    o.write(json.dumps({"seed": s["seed_id"], "key": items.get(s["seed_id"]), "excluded": result["excluded"]}, ensure_ascii=False) + "\n")
        snap = json.loads(run_row["snapshot_json"])
        runs_meta.append({
            "id": rid, "model": run_row["target_model"], "status": run_row["status"], "valid": run_row["valid"],
            "protocol": snap.get("protocolVersion"), "datasetRevision": snap.get("datasetRevision"), "judges": snap.get("judges"), "engine": snap.get("engine"),
            "config": {k: v for k, v in json.loads(run_row["config_json"]).items() if k in ("langs", "mode", "targetParams", "experiment", "rescored", "expectedSessions")},
            "result": {k: v for k, v in (json.loads(run_row["result_json"]) if run_row.get("result_json") else {}).items() if k in ("perLang", "excludedSessions", "sessions", "eligibility")},
            "judged_samples": rows,
        })
    json.dump(runs_meta, open(os.path.join(frozen, "runs.json"), "w"), ensure_ascii=False, indent=1)
    for name in sorted(os.listdir(valdir)):
        if name.startswith(("val_", "judge_rep")) and name.endswith(".jsonl"):
            shutil.copy(os.path.join(valdir, name), os.path.join(frozen, name))
    lb = json.load(open(os.path.join(export, "leaderboard_runs.json"), encoding="utf-8"))
    current = runs_meta[0]["protocol"] if runs_meta else None
    summary = {
        "runs": len(lb),
        "eligible": sum(1 for r in lb if r["valid"]),
        "otherProtocol": sum(1 for r in lb if r["protocol"] and not r["protocol"].startswith("characterbench-2.0")),
        "otherProtocolEligible": sum(1 for r in lb if r["valid"] and r["protocol"] and not r["protocol"].startswith("characterbench-2.0")),
        "detail": lb,
    }
    json.dump(summary, open(os.path.join(frozen, "leaderboard.json"), "w"), ensure_ascii=False, indent=1)
    if weights:
        shutil.copy(weights, os.path.join(res, "characterjudge_server_check.json"))
        # The script exactly as the operator ran it, next to its output (named in the output's script_as_run).
        as_run = os.path.join(os.path.dirname(weights), "characterjudge_server_check.as_run.py")
        if os.path.exists(as_run):
            shutil.copy(as_run, os.path.join(res, "characterjudge_server_check.as_run.py"))

    # The report's data module, from these results.
    module_args = [os.path.join(res, "audit.json"), os.path.join(res, "judge_audit.json"), spec_txt, os.path.join(res, "anchor.json"), "bundle", os.path.join(ROOT, "web/src/pages/characterbenchAudit.ts"),
                   os.path.join(frozen, "leaderboard.json"), weights or "-"]
    run(logs, 10, "audit_module", py + ["tools/experiments/characterbench_audit_module.py", *module_args])

    # The check a third party can run with only this bundle and the authors' repository.
    run(logs, 11, "recompute", py + ["tools/experiments/characterbench_recompute.py", repo, frozen])
    for p in CODE:
        dst = os.path.join(out, "code", p)
        os.makedirs(os.path.dirname(dst), exist_ok=True)
        shutil.copy(os.path.join(ROOT, p), dst)
    shutil.rmtree(tmp, ignore_errors=True)
    sums = []
    for base, _, files in os.walk(out):
        for name in sorted(files):
            path = os.path.join(base, name)
            rel = os.path.relpath(path, out)
            if rel != "SHA256SUMS":
                sums.append(f"{sha256(path)}  {rel}")
    with open(os.path.join(out, "SHA256SUMS"), "w") as f:
        f.write("\n".join(sorted(sums, key=lambda l: l.split("  ", 1)[1])) + "\n")
    write_index(out, sums, runs_meta)
    print(f"bundle: {len(sums)} files in {out}")


ABOUT = {
    "code/": "The audit scripts and the platform source they execute, exactly as run.",
    "results/audit.json": "Data, prompts, judge input and scoring: the authors' functions against the platform, per run.",
    "results/judge_audit.json": "Judge against human scores, the authors' predictions, a same-seed repeat and three other seeds.",
    "results/anchor.json": "The paper anchor (GPT-4o) against the paper's appendix row.",
    "results/spec_check.txt": "Every protocol setting against the paper and the authors' code.",
    "results/prompt_hashes_authors.json": "SHA-1 of every role-play and judge prompt built by the authors' functions.",
    "results/prompt_hashes_platform.json": "The same, built by the platform's functions.",
    "results/characterjudge_server_check.json": "The operator's check of the served judge's files against Hugging Face (host name and unrelated servers left out).",
    "results/characterjudge_server_check.as_run.py": "The check script exactly as the operator ran it.",
    "frozen/judge_outputs_": "Every stored judge output of one run (10 per sample) with the platform's reading; excluded samples marked.",
    "frozen/runs.json": "Each audited run's protocol snapshot (judges, engine), configuration and result.",
    "frozen/leaderboard.json": "Every CharacterBench run's protocol and eligibility at the time of the audit.",
    "frozen/val_": "The deployed judge on the 3,250 annotated test responses per language (greedy and 10 seeded samples).",
    "frozen/judge_rep0_": "The same requests a day later with the same seeds.",
    "frozen/judge_rep": "The same requests with other seeds (offset 1, 2, 3).",
    "index.html": "This page.",
    "logs/": "One log per step: command, start and end time, exit status and full output.",
    "SHA256SUMS": "SHA-256 of every file here.",
}


def write_index(out, sums, runs_meta):
    """A plain page (no script) listing the bundle, so it reads in any browser and any crawler."""
    import html

    def about(rel):
        for k in sorted(ABOUT, key=len, reverse=True):
            if rel.startswith(k):
                return ABOUT[k]
        return ""

    rows = []
    for line in sorted(sums, key=lambda l: l.split("  ", 1)[1]):
        digest, rel = line.split("  ", 1)
        size = os.path.getsize(os.path.join(out, rel))
        rows.append(f'<tr><td><a href="{html.escape(rel)}">{html.escape(rel)}</a></td><td>{size:,}</td><td><code>{digest[:16]}…</code></td><td>{html.escape(about(rel))}</td></tr>')
    runs = "".join(f"<li><code>{html.escape(r['id'])}</code>: {html.escape(str(r['protocol']))}, {r['judged_samples']:,} judged samples"
                   f"{' (paper anchor, not on the leaderboard)' if (r['config'].get('experiment')) else ''}</li>" for r in runs_meta)
    page = f"""<!doctype html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
<title>CharacterBench reproduction audit bundle · SociaBench</title>
<meta name="description" content="Re-runnable audit of the SociaBench CharacterBench reproduction: scripts, frozen judge outputs, results and execution logs.">
<style>body{{font:15px/1.55 system-ui,sans-serif;max-width:1100px;margin:32px auto;padding:0 16px;color:#111;background:#fff}}
table{{border-collapse:collapse;width:100%;font-size:13px}}td,th{{border-bottom:1px solid #ddd;padding:5px 8px;text-align:left;vertical-align:top}}
code,pre{{font:12.5px ui-monospace,monospace;background:#f4f4f4;padding:1px 4px}}pre{{padding:10px;overflow-x:auto}}
@media (prefers-color-scheme:dark){{body{{color:#eee;background:#111}}code,pre{{background:#222}}td,th{{border-color:#333}}a{{color:#8ab4ff}}}}</style></head><body>
<h1>CharacterBench reproduction audit bundle</h1>
<p>Everything behind Section 6 of the <a href="/b/characterbench/methodology">CharacterBench reproduction report</a>: the scripts as run, their results
and logs, and the frozen inputs from which the scoring can be recomputed without access to the platform. Nothing here contains test-set text,
dialogue or a model's reply: the test set stays with its authors (github.com/thu-coai/CharacterBench), and replies are referred to only by sample.</p>
<h2>Check without running anything</h2>
<pre>shasum -a 256 -c SHA256SUMS      # in a copy of this directory</pre>
<p>Every file's SHA-256 is listed below and in <a href="SHA256SUMS">SHA256SUMS</a>.</p>
<h2>Recompute the scores from the frozen judge outputs</h2>
<p>Each line of <code>frozen/judge_outputs_&lt;run&gt;.jsonl</code> holds a sample's ten judge outputs as returned. Read each with
<code>check()</code> from the authors' <code>all_eval.py</code>, average the ten, and per language and test file take
<code>mean / max(human scores of the kept samples) × 5</code>; AVG is the mean of the 13 columns. The result must equal the run's
<code>result.perLang</code> in <code>frozen/runs.json</code> at two decimals. <code>code/tools/experiments/characterbench_recompute.py</code>
does exactly this with only this bundle and the authors' repository (log: <a href="logs/11-recompute.log">logs/11-recompute.log</a>):</p>
<pre>python3 -I code/tools/experiments/characterbench_recompute.py repo frozen</pre>
<h2>Re-run the whole audit</h2>
<pre>git clone https://github.com/thu-coai/CharacterBench repo &amp;&amp; git -C repo checkout 66c113c
curl -L -o eprint.tar.gz https://arxiv.org/e-print/2412.11912v1 &amp;&amp; mkdir eprint &amp;&amp; tar xzf eprint.tar.gz -C eprint
python3 -I code/tools/experiments/characterbench_spec_check.py eprint repo frozen/val_zh.jsonl frozen/val_en.jsonl
python3 -I code/tools/experiments/characterbench_judge_audit.py out.json frozen/val_zh.jsonl frozen/val_en.jsonl \
    --repeat frozen/judge_rep0_zh.jsonl frozen/judge_rep0_en.jsonl --seeds frozen/judge_rep{{1,2,3}}_{{zh,en}}.jsonl</pre>
<p>The prompt and pipeline checks (<code>characterbench_prompt_check.py</code>, <code>characterbench_audit_platform.ts</code>,
<code>characterbench_audit.py</code>) also need the platform's samples and the runs' replies; their outputs and logs are in
<code>results/</code> and <code>logs/</code>, and the logs give each command as run. The judge requests themselves can be repeated against
any server running the checkpoint with <code>characterbench_judge_validation.py</code> (sampled outputs repeat only on the same deployment).</p>
<h2>Runs audited</h2><ul>{runs}</ul>
<h2>Files</h2><table><tr><th>File</th><th>Bytes</th><th>SHA-256</th><th>Content</th></tr>{''.join(rows)}</table>
</body></html>
"""
    with open(os.path.join(out, "index.html"), "w", encoding="utf-8") as f:
        f.write(page)


if __name__ == "__main__":
    main(*sys.argv[1:7])