"""CoSER: does today's judge score the authors' released simulations as the authors' stored critiques do? (matrix V09a)

The official repository holds three simulated scenes (exp/simulation/test_set_gpt-4o.json) with the critiques the authors stored
for them (exp/evaluation/test_set_gpt-4o.json). This script sends the same simulations through the official judge prompts, built by
the verbatim official functions in coser_reference.py, to openai/gpt-4o-2024-08-06 on OpenAI through OpenRouter, at temperature 0,
twice each, and compares. No simulation is generated, so the next-speaker model and the model under test play no part: a difference
can only come from the judge. It writes numbers only.

Criteria, written on 2026-11-01 before the first request. Over the three scenes, per dimension, with S the sum of severities:
  consistent  every dimension's S within 30% of the stored S, and no critique lists 10 or more flaws
  harsher     Character Fidelity's or Anthropomorphism's S more than 30% above the stored S, or a critique lists 10 or more flaws
  other       anything else
Three scenes are few: in the anchor 25% of sessions list 10 or more Character Fidelity flaws, so three scenes without one would occur
by chance about 42% of the time. "Consistent" therefore rules out a large shift in severity, not every change of behaviour.

    <python with nltk and regex> -I tools/experiments/coser_judge_recheck.py <CoSER repo> <file holding an OpenRouter key> <out.json>

With --variants (Character Fidelity only, the dimension that moved): the same three simulations with the placeholder
{major_characters} filled in, as the paper's table reads, and with the literal placeholder on other hosts and GPT-4o snapshots. The
question is which of these brings the stored critiques back; no criterion was fixed for it.
"""
import importlib.util
import json
import os
import re
import sys
import urllib.error
import urllib.request

HERE = os.path.dirname(os.path.abspath(__file__))
MODEL = "openai/gpt-4o-2024-08-06"
PASSES = 2
COST_LIMIT = 0.60


def load_ref():
    spec = importlib.util.spec_from_file_location("coser_reference", os.path.join(HERE, "coser_reference.py"))
    mod = importlib.util.module_from_spec(spec)
    spec.loader.exec_module(mod)
    return mod


def call(keys, messages, temperature=0, model=MODEL, host="openai"):
    body = json.dumps({"model": model, "messages": messages, "temperature": temperature, "max_tokens": 8196, "provider": {"only": [host], "allow_fallbacks": False}, "usage": {"include": True}}).encode()
    last = None
    for key in keys:
        req = urllib.request.Request("https://openrouter.ai/api/v1/chat/completions", data=body, headers={"Authorization": f"Bearer {key}", "Content-Type": "application/json"})
        for _ in range(4):
            try:
                with urllib.request.urlopen(req, timeout=180) as r:
                    j = json.loads(r.read())
                if j.get("choices"):
                    return j["choices"][0]["message"].get("content") or "", j.get("usage", {}).get("cost") or 0.0, j.get("provider")
                last = str(j.get("error"))[:200]
            except urllib.error.HTTPError as e:
                last = f"HTTP {e.code}"
                if e.code in (401, 402, 403):
                    break
            except OSError as e:
                last = type(e).__name__
    raise SystemExit(f"no reply: {last}")


def main():
    repo, keyfile, out = sys.argv[1:4]
    ref = load_ref()
    ref.load_critic_prompts(repo)
    keys = list(dict.fromkeys(re.findall(r"sk-or-v1-[A-Za-z0-9]+", open(keyfile, encoding="utf-8").read())))
    if not keys:
        raise SystemExit("no OpenRouter key in that file")
    sims = json.loads(ref.git_show(repo, ref.PINNED, "exp/simulation/test_set_gpt-4o.json"))
    stored = json.loads(ref.git_show(repo, ref.PINNED, "exp/evaluation/test_set_gpt-4o.json"))["cases"]
    dims = ref.dimensions
    rows, spent, hosts = [], 0.0, set()
    for r in sims:
        key = f"{r['book_title']}-{r['i_p']}-{r['i_c']}"
        case = stored[key]
        circ = {**r["circumstance"], "book": r["book_title"]}
        Rn = len([m for m in r["simulation"] if m["role"] not in ("NSP", "Environment")])
        for dim in dims:
            old = case["critique"][dim]
            old_s = sum(f["severity"] for f in old["flaws"] if isinstance(f["severity"], int))
            row = {"scene": hashlib_short(key), "dim": dim, "R": Rn, "stored": {"flaws": len(old["flaws"]), "S": old_s, "score": old["score"], "instance_chars": round(sum(len(str(f.get("instance", ""))) for f in old["flaws"]) / max(1, len(old["flaws"])))}, "now": []}
            messages = ref.judge_messages(circ, r["simulation"], r["involved_character_profiles"], dim)
            for p in range(PASSES):
                if spent > COST_LIMIT:
                    raise SystemExit(f"cost limit reached: ${spent:.3f}")
                temperature = 0
                value = None
                for attempt in range(3):
                    text, cost, host = call(keys, messages, temperature)
                    spent += cost
                    hosts.add(host)
                    m = re.search(r"\{.*\}", text, re.S)
                    try:
                        value = json.loads(m.group(0))[dim]
                        break
                    except (AttributeError, KeyError, ValueError, TypeError):
                        temperature = 1
                if value is None:
                    row["now"].append({"unreadable": True})
                    continue
                flaws = value.get("flaws", [])
                s = sum(f.get("severity", 1) for f in flaws if isinstance(f.get("severity", 1), int))
                row["now"].append({"flaws": len(flaws), "S": s, "score": max(0, 100 - (s - 0.3 * Rn) * 5), "instance_chars": round(sum(len(str(f.get("instance", ""))) for f in flaws) / max(1, len(flaws)))})
            rows.append(row)
            print(row["scene"], dim[:12].ljust(12), "R", Rn, "stored", row["stored"]["flaws"], row["stored"]["S"], row["stored"]["score"], "now", [(x.get("flaws"), x.get("S"), x.get("score")) for x in row["now"]], flush=True)
    summary = {}
    for dim in dims:
        own = [x for x in rows if x["dim"] == dim]
        old = sum(x["stored"]["S"] for x in own)
        now = sum(sum(y["S"] for y in x["now"] if "S" in y) / max(1, sum("S" in y for y in x["now"])) for x in own)
        summary[dim] = {"stored_S": old, "now_S": now, "ratio": now / old if old else None, "max_flaws": max(y.get("flaws", 0) for x in own for y in x["now"]), "stored_score": sum(x["stored"]["score"] for x in own) / len(own),
                        "now_score": sum(sum(y["score"] for y in x["now"] if "score" in y) / max(1, sum("score" in y for y in x["now"])) for x in own) / len(own)}
    enumerated = any(summary[d]["max_flaws"] >= 10 for d in dims)
    if all(abs(summary[d]["ratio"] - 1) <= 0.30 for d in dims) and not enumerated:
        verdict = "consistent"
    elif enumerated or any(summary[d]["ratio"] > 1.30 for d in ("Character Fidelity", "Anthropomorphism")):
        verdict = "harsher"
    else:
        verdict = "other"
    report = {"model": MODEL, "passes": PASSES, "hosts": sorted(str(h) for h in hosts), "cost": spent, "verdict": verdict, "summary": summary, "rows": rows}
    with open(out, "w", encoding="utf-8") as f:
        json.dump(report, f, ensure_ascii=False, indent=1)
    print("verdict:", verdict, "cost:", round(spent, 4), "hosts:", report["hosts"])
    for d in dims:
        print(d, {k: (round(v, 2) if isinstance(v, float) else v) for k, v in summary[d].items()})
    return 0


def variants(repo, keyfile, out):
    ref = load_ref()
    ref.load_critic_prompts(repo)
    keys = list(dict.fromkeys(re.findall(r"sk-or-v1-[A-Za-z0-9]+", open(keyfile, encoding="utf-8").read())))
    sims = json.loads(ref.git_show(repo, ref.PINNED, "exp/simulation/test_set_gpt-4o.json"))
    stored = json.loads(ref.git_show(repo, ref.PINNED, "exp/evaluation/test_set_gpt-4o.json"))["cases"]
    dim = "Character Fidelity"
    plan = [("placeholder filled, 2024-08-06, OpenAI", True, MODEL, "openai", 2), ("literal, 2024-08-06, Azure", False, MODEL, "azure", 1), ("literal, 2024-05-13, OpenAI", False, "openai/gpt-4o-2024-05-13", "openai", 1),
            ("literal, 2024-11-20, OpenAI", False, "openai/gpt-4o-2024-11-20", "openai", 1), ("literal, gpt-4o alias, OpenAI", False, "openai/gpt-4o", "openai", 1)]
    results, spent = [], 0.0
    for name, filled, model, host, passes in plan:
        row = {"variant": name, "scenes": []}
        for r in sims:
            key = f"{r['book_title']}-{r['i_p']}-{r['i_c']}"
            circ = {**r["circumstance"], "book": r["book_title"]}
            Rn = len([m for m in r["simulation"] if m["role"] not in ("NSP", "Environment")])
            messages = ref.judge_messages(circ, r["simulation"], r["involved_character_profiles"], dim)
            assert "{major_characters}" in messages[0]["content"]
            if filled:
                messages = [{**messages[0], "content": messages[0]["content"].replace("{major_characters}", ", ".join(circ["major_characters"]))}, messages[1]]
            old = stored[key]["critique"][dim]
            scene = {"scene": hashlib_short(key), "R": Rn, "stored": {"flaws": len(old["flaws"]), "S": sum(f["severity"] for f in old["flaws"] if isinstance(f["severity"], int))}, "now": []}
            for _ in range(passes):
                if spent > COST_LIMIT:
                    raise SystemExit(f"cost limit reached: ${spent:.3f}")
                try:
                    text, cost, h = call(keys, messages, 0, model, host)
                except SystemExit as e:
                    scene["now"].append({"error": str(e)[:80]})
                    continue
                spent += cost
                try:
                    flaws = json.loads(re.search(r"\{.*\}", text, re.S).group(0))[dim].get("flaws", [])
                except (AttributeError, KeyError, ValueError, TypeError):
                    scene["now"].append({"unreadable": True})
                    continue
                s = sum(f.get("severity", 1) for f in flaws if isinstance(f.get("severity", 1), int))
                scene["now"].append({"flaws": len(flaws), "S": s, "host": h, "instance_chars": round(sum(len(str(f.get("instance", ""))) for f in flaws) / max(1, len(flaws)))})
            row["scenes"].append(scene)
        vals = [y["S"] for x in row["scenes"] for y in x["now"] if "S" in y]
        row["S_per_scene"] = sum(vals) / len(vals) if vals else None
        results.append(row)
        print(name.ljust(40), "S per scene", row["S_per_scene"], [[(y.get("flaws"), y.get("S"), y.get("error", "")[:30]) for y in x["now"]] for x in row["scenes"]], flush=True)
    stored_mean = sum(x["stored"]["S"] for x in results[0]["scenes"]) / 3
    print("stored S per scene", stored_mean, "cost", round(spent, 4))
    with open(out, "w", encoding="utf-8") as f:
        json.dump({"dimension": dim, "stored_S_per_scene": stored_mean, "cost": spent, "variants": results}, f, ensure_ascii=False, indent=1)
    return 0


def hashlib_short(key):
    import hashlib

    return hashlib.sha256(key.encode()).hexdigest()[:8]


if __name__ == "__main__":
    if "--variants" in sys.argv:
        sys.exit(variants(*[a for a in sys.argv[1:] if a != "--variants"]))
    sys.exit(main())