"""CoSER anchor (matrix V09): judges a finished full run of GPT-4o (2024-08-06) against the paper's GPT-4o row, by the criteria
written on 2026-11-01 before any measurement. Reads the platform's API; calls no model. Writes one JSON file with the verdict and
everything the matrix says is reported with it; it holds numbers and ids only, no text of the test set or of a reply.

    python3 -I tools/experiments/coser_anchor.py <api base> <curl config holding the session cookie> <run id> <out.json>

Criteria (delta = platform - main table):
  reproduced    |delta Average| <= 1.0 and every dimension |delta| <= 2.5
  same range    |delta Average| <= 3.0 and every dimension |delta| <= 5
  not reproduced otherwise
"""
import collections
import concurrent.futures
import json
import os
import re
import statistics
import subprocess
import sys

ROOT = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", ".."))
COLS = ["sc", "an", "cf", "sq"]
TARGET_MODEL = "openai/gpt-4o-2024-08-06"


def paper_row(name):
    text = open(os.path.join(ROOT, "worker/src/benchmarks/coser/published.ts"), encoding="utf-8").read()
    rows = json.loads(text[text.index("PAPER_ROWS: PaperRow[] = ") + len("PAPER_ROWS: PaperRow[] = "):].rstrip().rstrip(";"))
    return next(r for r in rows if r["name"] == name)


def split_means():
    """The paper's table of in-domain and out-of-domain scores for GPT-4o, from the output of coser_table_checks.py."""
    data = json.load(open(os.path.join(ROOT, "web/public/audit/coser/2026-11-01/results/table_checks.json"), encoding="utf-8"))
    cells = [c for c in data["IDOOD"]["cells"] if c["model"] == "GPT-4o"]
    return {s: {c["col"]: c[s] for c in cells} for s in ("id", "ood")}


def by_length(rows):
    """Mean scores by the number of character lines: does a dimension follow the length of the simulation?"""
    out = []
    for lo, hi in ((3, 6), (7, 9), (10, 12), (13, 15), (16, 20)):
        own = [r for r in rows if lo <= r["R"] <= hi]
        dims = {k: sum(r["dims"][k] for r in own) / len(own) for k in COLS}
        out.append({"R": [lo, hi], "sessions": len(own), **dims, "avg": sum(dims.values()) / 4})
    return out


def main():
    base, cfg, run_id, out = sys.argv[1:5]

    def get(path):
        for attempt in range(5):
            p = subprocess.run(["curl", "-s", "-m", "60", "-K", cfg, base + path], capture_output=True)
            try:
                return json.loads(p.stdout)
            except ValueError:
                continue
        raise SystemExit(f"cannot read {path}")

    run = get(f"/runs/{run_id}")
    result = run["result"]
    assert run["status"] == "completed", run["status"]
    assert run.get("targetModel") == TARGET_MODEL, run.get("targetModel")
    view = result["perLang"]["en"]
    sessions = get(f"/runs/{run_id}/sessions")
    assert len(sessions) == 600 and all(s["status"] == "done" for s in sessions), collections.Counter(s["status"] for s in sessions)

    def read(s):
        d = get(f"/sessions/{s['id']}")
        msgs = [m for m in d["messages"] if m["round"] >= 2]
        row = {"scene": d["seed_id"], "sample": d["sample_idx"], "R": sum(m["role"] == "ai" for m in msgs), "env": sum(m["role"] == "env" for m in msgs), "dims": {}, "S": {}, "n": {}, "upper": {}}
        for j in d["judgments"]:
            sc = j["scores"]
            k = next(a for a in sc["dims"] if a != "bleu")
            row["dims"][k] = sc["dims"][k]
            row["S"][k] = int(re.search(r"严重度之和 (-?\d+)", sc["notes"][k]).group(1))
            row["n"][k] = len(sc["issues"])
            # The variant with the ceiling of 100 (official code since 0e11992): the protocol's score, capped.
            row["upper"][k] = min(sc["dims"][k], 100)
        row["bleu"] = d["result"]["combined"].get("bleu")
        return row

    with concurrent.futures.ThreadPoolExecutor(8) as pool:
        rows = list(pool.map(read, sessions))

    mean = lambda xs: sum(xs) / len(xs)
    paper = paper_row("GPT-4o")
    platform = {k: mean([r["dims"][k] for r in rows]) for k in COLS}
    platform["avg"] = mean([platform[k] for k in COLS])
    delta = {k: platform[k] - paper["scores"][k] for k in COLS + ["avg"]}
    reproduced = abs(delta["avg"]) <= 1.0 and all(abs(delta[k]) <= 2.5 for k in COLS)
    same_range = abs(delta["avg"]) <= 3.0 and all(abs(delta[k]) <= 5 for k in COLS)
    verdict = "reproduced" if reproduced else "same range" if same_range else "not reproduced"

    capped = {k: mean([r["upper"][k] for r in rows]) for k in COLS}
    capped["avg"] = mean([capped[k] for k in COLS])
    scene = collections.defaultdict(list)
    for r in rows:
        scene[r["scene"]].append(mean([r["dims"][k] for k in COLS]))
    rs = [r["R"] for r in rows]
    lines = result["gca"]["lines"]
    report = {
        "run": run_id,
        "model": TARGET_MODEL,
        "protocol": run["snapshot"].get("protocolVersion"),
        "criteria": {"reproduced": {"avg": 1.0, "dim": 2.5}, "same_range": {"avg": 3.0, "dim": 5}},
        "paper_main_table": {k: {"mean": paper["scores"][k], "pm": paper["spread"][k]} for k in COLS + ["avg", "bleu"]},
        "platform": platform,
        "delta": delta,
        "verdict": verdict,
        "platform_summary_rounded": {"overall": view["overall"], "ci": view["ci"], "dims": view["dims"]},
        "bleu": {"platform": mean([r["bleu"] for r in rows]), "paper": paper["scores"]["bleu"]},
        "paper_splits": split_means(),
        "platform_splits": result["gca"]["splits"],
        "runs": result["gca"]["runs"],
        "across_runs": result["gca"]["across"],
        "with_ceiling_100": {"means": capped, "sessions_dims_above_100": sum(r["dims"][k] > 100 for r in rows for k in COLS)},
        "floor": {"session_dims_at_0": {k: sum(r["dims"][k] == 0 for r in rows) for k in COLS}},
        "R": {"mean": mean(rs), "median": statistics.median(rs), "min": min(rs), "max": max(rs), "histogram": dict(sorted(collections.Counter(rs).items()))},
        "lines": lines,
        "nsp": {"fallback_rate": lines["nsp"]["fallback"] / lines["nsp"]["lines"], "ended_rate_per_session": lines["nsp"]["ended"] / 600},
        "flaws_per_session": {k: mean([r["n"][k] for r in rows]) for k in COLS},
        "flaws_ge_10_share": {k: sum(r["n"][k] >= 10 for r in rows) / len(rows) for k in COLS},
        "by_length": by_length(rows),
        "severity_sum_per_session": {k: mean([r["S"][k] for r in rows]) for k in COLS},
        "hosts": run.get("hosts"),
        "usage": result["usage"]["total"],
        "eligibility": result["eligibility"],
    }
    with open(out, "w", encoding="utf-8") as f:
        json.dump(report, f, ensure_ascii=False, indent=1)
    with open(out.replace(".result.json", ".rows.json"), "w", encoding="utf-8") as f:
        json.dump(rows, f)
    print(f"verdict: {verdict}")
    for k in COLS + ["avg"]:
        print(f"{k}: platform {platform[k]:.2f}, paper {paper['scores'][k]:.2f}, delta {delta[k]:+.2f}")
    print(f"cost ${result['usage']['total']['cost']:.2f}; mean R {mean(rs):.2f}; NSP fallback {report['nsp']['fallback_rate']:.3f}; dims above 100: {report['with_ceiling_100']['sessions_dims_above_100']}")
    return 0


if __name__ == "__main__":
    sys.exit(main())