"""CoSER audit bundle: runs every scripted check behind the reproduction report with a log per step, and assembles what a third
party needs to check and re-run them, for publication with the report (web/public/audit/coser/<date>/).

Steps (each logged with its command, start and end time, exit status and full output):
  01 spec_check     the platform's protocol module against the official code (matrix V01-V05, V07, V08, V10-V12, generated files)
  02 prompt_check   the paper's printed prompts against the strings the platform sends, word by word
  03 paper          the main table read again from the LaTeX source; published.ts must be unchanged by it
  04 table_checks   the paper's own numbers and the released files
  05 rouge_check    ROUGE-L: every release of the rouge package on the three released values
The outputs of the local end-to-end runs (a local Worker with coser_e2e_mock.py as every model) are copied from E2E: they need
a Worker and a browserless API session, so they are not re-run here.

Published (no test-set text: the data stay with their authors):
  code/      the scripts and the platform source they execute, as run
  results/   spec_check.txt, prompt_check.json, table_checks.json, rouge_check.json, e2e/ (outputs of the local runs), the anchor and the
             judge recheck (numbers only; they called live models and are copied, not re-run)
  reviews/   the three independent reviews (Chinese) and the search behind Section 9, as written, and the reviewers' own scripts and
             outputs (without their copies of the authors' files), except that every line quoting test-set text is replaced by a marker
  logs/      one log per step
  SHA256SUMS every file of the bundle

    <python with nltk==3.9.1 and regex> -I tools/experiments/coser_audit_bundle.py REPO NLTK_DATA EPRINT REVIEWS E2E OUT

REPO: Neph0s/CoSER with commits 93c1474 and 624efc5; NLTK_DATA: holds tokenizers/punkt_tab/english; EPRINT: holds v1 v2 v3
of the arXiv e-print; REVIEWS: holds coser-pass{1,2,3}-*.md and coser_third_party_search.md; E2E: holds out_*.txt; OUT: the bundle.
"""
import datetime
import hashlib
import html
import json
import os
import platform
import shutil
import subprocess
import sys

ROOT = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", ".."))
CODE = [
    "tools/experiments/coser_reference.py",
    "tools/experiments/coser_spec_check.py",
    "tools/experiments/coser_gca_dump.ts",
    "tools/experiments/coser_prompt_check.py",
    "tools/experiments/coser_table_checks.py",
    "tools/experiments/coser_paper.py",
    "tools/experiments/coser_rouge_check.py",
    "tools/experiments/coser_e2e_mock.py",
    "tools/experiments/coser_e2e_check.py",
    "tools/experiments/coser_anchor.py",
    "tools/experiments/coser_judge_recheck.py",
    "tools/experiments/coser_method_validity.py",
    "tools/experiments/coser_audit_bundle.py",
    "docs/复现规范/CoSER-核查矩阵.md",
    "worker/src/benchmarks/coser/index.ts",
    "worker/src/benchmarks/coser/meta.ts",
    "worker/src/benchmarks/coser/data.ts",
    "worker/src/benchmarks/coser/published.ts",
    "worker/src/benchmarks/coser/gca/index.ts",
    "worker/src/benchmarks/coser/gca/setup.ts",
    "worker/src/benchmarks/coser/gca/family.ts",
    "worker/src/benchmarks/coser/gca/loop.ts",
    "worker/src/benchmarks/coser/gca/judge.ts",
    "worker/src/benchmarks/coser/gca/text.ts",
    "worker/src/benchmarks/coser/gca/py.ts",
    "worker/src/benchmarks/coser/gca/rng.ts",
    "worker/src/benchmarks/coser/gca/bleu.ts",
    "worker/src/benchmarks/coser/gca/crmath.ts",
    "worker/src/benchmarks/coser/gca/aggregate.ts",
    "worker/src/benchmarks/coser/gca/strings.gen.ts",
    "worker/src/benchmarks/coser/gca/punkt.gen.ts",
    "worker/src/scoring.ts",
    "worker/src/engine/results.ts",
]
REVIEWS = {
    "coser-pass1-paper-first.md": "First independent review: from the paper to the platform.",
    "coser-pass2-code-first.md": "Second independent review: from the official code to the platform.",
    "coser-pass3-numbers.md": "Third independent review: every number of the report recomputed from its sources.",
    "coser_third_party_search.md": "The search for later work reporting CoSER scores (Section 9): queries, sources and what was verified.",
}
E2E_FILES = {
    "out_regular.txt": "18 sessions of a local run: each replayed through the reference; requests, temperatures, limits, records, scores and aggregates.",
    "out_faults.txt": "Failure injection: empty replies, refusals, a transient 500; temperature sequences and stored lines (V06).",
    "out_gateway403.txt": "A gateway 403 whose text matches the moderation pattern: the session fails, no scored line (V06).",
    "out_extra.txt": "Temperature, limit, top-p and seed placed in the settings: what the model under test received (V14).",
    "out_full.txt": "The full configuration: 600 sessions with stand-in models; eligibility under the anchor flag.",
}
PLACES = {}


def public(text):
    for place, name in sorted(PLACES.items(), key=lambda kv: -len(kv[0])):
        text = text.replace(place, name)
    return text


def sha256(path):
    h = hashlib.sha256()
    with open(path, "rb") as f:
        for chunk in iter(lambda: f.read(1 << 20), b""):
            h.update(chunk)
    return h.hexdigest()


def run(logs, n, name, cmd, env=None):
    started = datetime.datetime.now(datetime.timezone.utc).isoformat(timespec="seconds")
    p = subprocess.run(cmd, cwd=ROOT, capture_output=True, text=True, env=env)
    ended = datetime.datetime.now(datetime.timezone.utc).isoformat(timespec="seconds")
    body = p.stdout + (("\n[stderr]\n" + p.stderr) if p.stderr.strip() else "")
    with open(os.path.join(logs, f"{n:02d}-{name}.log"), "w", encoding="utf-8") as f:
        f.write(public(f"command: {' '.join(cmd)}\nstarted: {started}\nended:   {ended}\nexit status: {p.returncode}\n\n{body}"))
    if p.returncode != 0:
        raise SystemExit(f"step {name} failed with status {p.returncode}\n{body[-2000:]}")
    return body


def dataset_strings(repo):
    """Texts of the test set long enough to identify a quotation: the reviews must not carry them."""
    raw = subprocess.run(["git", "-C", repo, "show", "624efc5:data/test/test_set.json"], capture_output=True, check=True).stdout
    out = set()
    for s in json.loads(raw):
        for d in s["dialogues"]:
            if len(d["message"]) >= 40:
                out.add(d["message"][:40])
        if len(s["scenario"]) >= 40:
            out.add(s["scenario"][:40])
        for p in s["character_profiles"].values():
            if len(p) >= 40:
                out.add(p[:40])
    return out


def redact(text, strings):
    lines, removed = [], 0
    for line in text.split("\n"):
        if any(x in line for x in strings):
            lines.append("[line removed: it quoted the authors' test set]")
            removed += 1
        else:
            lines.append(line)
    return "\n".join(lines), removed


def main():
    repo, nltk_data, eprint, reviews, e2e, out = [os.path.abspath(x) for x in sys.argv[1:7]]
    PLACES.update({repo: "<repo>", nltk_data: "<nltk_data>", eprint: "<eprint>", reviews: "<reviews>", e2e: "<e2e>", out: "<bundle>", ROOT: "<platform>",
                   os.path.dirname(e2e): "<work>", os.path.dirname(reviews): "<work>", os.path.dirname(sys.executable): "<python>", os.path.expanduser("~"): "<home>"})
    tmp = os.path.join(os.path.dirname(out), f".work-{os.getpid()}")
    PLACES[tmp] = "<work>"
    shutil.rmtree(out, ignore_errors=True)
    for d in ("code", "results/e2e", "reviews", "logs"):
        os.makedirs(os.path.join(out, d), exist_ok=True)
    os.makedirs(tmp, exist_ok=True)
    logs, res = os.path.join(out, "logs"), os.path.join(out, "results")
    py = [sys.executable, "-I"]
    with open(os.path.join(logs, "00-environment.log"), "w", encoding="utf-8") as f:
        f.write(f"generated {datetime.datetime.now(datetime.timezone.utc).isoformat(timespec='seconds')}\n")
        f.write(f"python {platform.python_version()}  node {subprocess.run(['node', '--version'], capture_output=True, text=True).stdout.strip()}  {platform.system()} {platform.machine()}\n")
        for label, c in (("624efc5", "624efc5"), ("93c1474", "93c1474")):
            f.write(f"repository commit {label}: {subprocess.run(['git', '-C', repo, 'rev-parse', c], capture_output=True, text=True).stdout.strip()}\n")
        for v in ("v1", "v2", "v3"):
            tgz = os.path.join(eprint, f"{v}.tgz")
            if os.path.exists(tgz):
                f.write(f"arXiv e-print {v} sha256 {sha256(tgz)}\n")
        f.write(f"platform commit {subprocess.run(['git', '-C', ROOT, 'rev-parse', 'HEAD'], capture_output=True, text=True).stdout.strip()} (the source as run is under code/; it includes uncommitted changes of protocol 1.1 until committed)\n")

    spec = run(logs, 1, "spec_check", py + ["tools/experiments/coser_spec_check.py", repo, nltk_data])
    open(os.path.join(res, "spec_check.txt"), "w", encoding="utf-8").write(public(spec))
    run(logs, 2, "prompt_check", py + ["tools/experiments/coser_prompt_check.py", os.path.join(eprint, "v3"), "--json", os.path.join(res, "prompt_check.json")])
    committed = os.path.join(ROOT, "tools/experiments/coser_prompt_check.result.json")
    if sha256(committed) != sha256(os.path.join(res, "prompt_check.json")):
        raise SystemExit("coser_prompt_check.result.json is out of date: the report's Appendix A would not match")
    published = [os.path.join(ROOT, s, "src/benchmarks/coser/published.ts") for s in ("worker", "server")]
    before = [sha256(p) for p in published]
    run(logs, 3, "paper", py + ["tools/experiments/coser_paper.py", os.path.join(eprint, "v3")])
    after = [sha256(p) for p in published]
    with open(os.path.join(logs, "03-paper.log"), "a", encoding="utf-8") as f:
        f.write(f"\npublished.ts (worker, server) SHA-256 before {before}, after {after}: {'unchanged' if before == after else 'CHANGED'}\n")
    if before != after:
        raise SystemExit("published.ts changed: the paper extraction no longer reproduces the leaderboard data")
    run(logs, 4, "table_checks", py + ["tools/experiments/coser_table_checks.py", eprint, repo, "--json", os.path.join(res, "table_checks.json")])
    run(logs, 5, "rouge_check", py + ["tools/experiments/coser_rouge_check.py", repo, os.path.join(tmp, "rouge"), "--json", os.path.join(res, "rouge_check.json")])
    for name in E2E_FILES:
        with open(os.path.join(e2e, name), encoding="utf-8") as f:
            open(os.path.join(res, "e2e", name), "w", encoding="utf-8").write(public(f.read()))
    for name in ("coser_anchor.result.json", "coser_anchor.rows.json", "coser_judge_recheck.result.json", "coser_judge_recheck_variants.result.json", "coser_method_validity.result.json", "coser_original_as_simulation.result.json"):
        shutil.copy(os.path.join(ROOT, "tools/experiments", name), os.path.join(res, name))
    for rel in CODE:
        dst = os.path.join(out, "code", rel)
        os.makedirs(os.path.dirname(dst), exist_ok=True)
        shutil.copy(os.path.join(ROOT, rel), dst)
    strings = dataset_strings(repo)
    removed = {}
    for name in REVIEWS:
        text, removed[name] = redact(public(open(os.path.join(reviews, name), encoding="utf-8").read()), strings)
        open(os.path.join(out, "reviews", name), "w", encoding="utf-8").write(text)
    for name in sorted(os.listdir(reviews)):
        src = os.path.join(reviews, name)
        if not (name.endswith("-scripts") and os.path.isdir(src)):
            continue
        for d, dirs, fs in os.walk(src):
            dirs[:] = [x for x in dirs if x != "data"]  # the reviewers' copies of the authors' files
            for f in fs:
                if f.startswith("utils_") or f == "apply_edits.py":  # copies of the official code; a reviewer's scratch edit
                    continue
                rel = os.path.relpath(os.path.join(d, f), reviews)
                text, n = redact(public(open(os.path.join(d, f), encoding="utf-8", errors="replace").read()), strings)
                removed[rel] = n
                dst = os.path.join(out, "reviews", rel)
                os.makedirs(os.path.dirname(dst), exist_ok=True)
                open(dst, "w", encoding="utf-8").write(text)
    with open(os.path.join(logs, "06-redaction.log"), "w", encoding="utf-8") as f:
        f.write(f"lines removed from the reviews because they quote the authors' test set ({len(strings):,} strings checked): {removed}\n")
    shutil.rmtree(tmp)

    files = sorted(os.path.relpath(os.path.join(d, f), out) for d, _, fs in os.walk(out) for f in fs if f not in ("SHA256SUMS", "index.html"))
    with open(os.path.join(out, "SHA256SUMS"), "w") as f:
        for rel in files:
            f.write(f"{sha256(os.path.join(out, rel))}  {rel}\n")
    describe = {"code": "The check scripts, the matrix and the platform source they execute, as run.", "logs": "One log per step: command, start and end time (UTC), exit status, full output.",
                "results/spec_check.txt": "The protocol module against the official code.", "results/prompt_check.json": "The paper's prompts against the platform's strings, with every difference and its meaning.",
                "results/table_checks.json": "The paper's own numbers and the released files.", "results/rouge_check.json": "ROUGE-L: every release of the rouge package on the released values.",
                "results/coser_anchor.result.json": "The anchor (V09, not reproduced): verdict, scores, splits, run statistics, lengths, line counts, cost.",
                "results/coser_anchor.rows.json": "The anchor by session: scene id, run, lines, scores, severity sums, flaw counts; numbers only.",
                "results/coser_judge_recheck.result.json": "The authors' three released simulations judged today against their stored critiques (V09a).",
                "results/coser_judge_recheck_variants.result.json": "Character fidelity on the same simulations under variants of the judge (V09a).",
                "results/coser_method_validity.result.json": "What the score does on the anchor's sessions: repeatability, the length term, the floor, against the paper's differences between models (Section 7.4).",
                "results/coser_original_as_simulation.result.json": "The book's own conversation judged as a simulation, against GPT-4o on the same scenes (V09b; numbers only)."}
    describe.update({f"reviews/{k}": v for k, v in REVIEWS.items()})
    describe.update({f"reviews/pass{i}-scripts": f"The scripts and outputs of review {i}, without its copies of the authors' files." for i in (1, 2, 3)})
    describe.update({f"results/e2e/{k}": v for k, v in E2E_FILES.items()})
    desc = lambda rel: next((describe[k] for k in ("/".join(rel.split("/")[:i]) for i in range(rel.count("/") + 1, 0, -1)) if k in describe), "")
    rows = "".join(f'<tr><td><a href="{html.escape(rel)}">{html.escape(rel)}</a></td><td>{os.path.getsize(os.path.join(out, rel)):,}</td><td><code>{sha256(os.path.join(out, rel))[:12]}…</code></td><td>{html.escape(desc(rel))}</td></tr>' for rel in files)
    page = f"""<!doctype html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
<title>CoSER reproduction audit bundle · SociaBench</title>
<meta name="description" content="Re-runnable checks of the SociaBench CoSER reproduction: scripts, results, independent reviews and execution logs.">
<style>body{{font:15px/1.55 system-ui,sans-serif;max-width:1100px;margin:32px auto;padding:0 16px;color:#111;background:#fff}}
table{{border-collapse:collapse;width:100%;font-size:13px}}td,th{{border-bottom:1px solid #ddd;padding:5px 8px;text-align:left;vertical-align:top}}
code,pre{{font:12.5px ui-monospace,monospace;background:#f4f4f4;padding:1px 4px}}pre{{padding:10px;overflow-x:auto}}
@media (prefers-color-scheme:dark){{body{{color:#eee;background:#111}}code,pre{{background:#222}}td,th{{border-color:#333}}a{{color:#8ab4ff}}}}</style></head><body>
<h1>CoSER reproduction audit bundle</h1>
<p>Everything behind Sections 2, 3, 6, 7 and 9 of the <a href="/b/coser/methodology">CoSER reproduction report</a>: the check scripts and the platform source as run, their results and logs, the outputs of the local
end-to-end runs, the anchor and the judge recheck, and the three independent reviews. The checks call no model; the anchor and the recheck, which did, are copied as they were written. Nothing here contains test-set text: the data stay with their authors (github.com/Neph0s/CoSER); lines of the reviews that quoted them are replaced by a marker
(logs/06-redaction.log). The reviews are by AI agents the platform's operator ran; no outside party has reviewed this. The platform's CoSER leaderboard was withdrawn on 2026-11-01; the report says why.</p>
<h2>Check without running anything</h2>
<pre>sha256sum -c SHA256SUMS</pre>
<h2>Files</h2>
<table><thead><tr><th>File</th><th>Bytes</th><th>SHA-256</th><th>What it is</th></tr></thead><tbody>{rows}</tbody></table>
</body></html>
"""
    open(os.path.join(out, "index.html"), "w", encoding="utf-8").write(page)
    print(f"{len(files)} files in {public(out)}")


if __name__ == "__main__":
    main()