"""CoSER: can ROUGE-L be recomputed? Runs every released version of the `rouge` package, as the official
utils.calculate_bleu_rouge calls it (Rouge().get_scores(simulation_str, reference_str)[0]['rouge-l']['f'], no lower-casing),
on the three examples the authors released (exp/simulation/test_set_gpt-4o.json with exp/evaluation/test_set_gpt-4o.json),
and compares with the stored values. The strings are built as main.py builds them. Network: `pip` downloads the packages
from PyPI into separate directories under <work>. Writes the result as JSON; prints no book text. Matrix C29, V12.

    python3 -I tools/experiments/coser_rouge_check.py <CoSER repo> <work dir> [--json out.json]
"""
import argparse
import importlib.util
import json
import os
import subprocess
import sys

HERE = os.path.dirname(os.path.abspath(__file__))
VERSIONS = ["0.2", "0.2.1", "0.3.0", "0.3.1", "0.3.2", "1.0.0rc0", "1.0.0rc1", "1.0.0rc2", "1.0.0", "1.0.1"]
RUNNER = r"""
import json, sys
sys.path.insert(0, sys.argv[1])
from rouge import Rouge
pairs = json.load(open(sys.argv[2], encoding="utf-8"))
out = []
for sim, ref in pairs:
    try:
        out.append(Rouge().get_scores(sim, ref)[0]["rouge-l"]["f"])
    except Exception as e:
        out.append("error: " + type(e).__name__)
print(json.dumps(out))
"""


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("repo")
    ap.add_argument("work")
    ap.add_argument("--json")
    args = ap.parse_args()
    spec = importlib.util.spec_from_file_location("coser_reference", os.path.join(HERE, "coser_reference.py"))
    ref = importlib.util.module_from_spec(spec)
    spec.loader.exec_module(ref)
    sims = json.loads(ref.git_show(args.repo, ref.PINNED, "exp/simulation/test_set_gpt-4o.json"))
    ev = json.loads(ref.git_show(args.repo, ref.PINNED, "exp/evaluation/test_set_gpt-4o.json"))
    pairs, stored = [], []
    for r in sims:
        simulation = [m for m in r["simulation"] if m["role"] != "NSP"]
        simulation = [m if m["role"] == "Environment" else {**m, "content": ref.remove_inner_thoughts(m["content"])} for m in simulation]
        reference = [m if m["character"] == "Environment" else {**m, "message": ref.remove_inner_thoughts(m["message"])} for m in r["circumstance"]["dialogues"]]
        pairs.append(("\n\n".join(m["content"].strip("\n") for m in simulation), "\n\n".join(f"{m['character']}: {m['message']}".strip("\n") for m in reference)))
        stored.append(ev["cases"][f"{r['book_title']}-{r['i_p']}-{r['i_c']}"]["critique"]["rouge_l"])
    os.makedirs(args.work, exist_ok=True)
    pairs_path = os.path.join(args.work, "pairs.json")
    with open(pairs_path, "w", encoding="utf-8") as f:
        json.dump(pairs, f, ensure_ascii=False)
    runner = os.path.join(args.work, "runner.py")
    with open(runner, "w", encoding="utf-8") as f:
        f.write(RUNNER)
    results = {}
    print(f"stored rouge_l: {stored}")
    for v in VERSIONS:
        target = os.path.join(args.work, f"rouge-{v}")
        inst = subprocess.run(["uv", "pip", "install", "-q", "--python", sys.executable, "--target", target, "--no-deps", "--no-build-isolation", f"rouge=={v}", "six", "numpy"], capture_output=True, text=True)
        if inst.returncode != 0:
            results[v] = {"values": [], "equal": 0, "within_1e-4": 0, "install_failed": True}
            print(f"rouge {v}: could not be installed")
            continue
        r = subprocess.run([sys.executable, "-I", runner, target, pairs_path], capture_output=True, text=True)
        vals = json.loads(r.stdout) if r.returncode == 0 else [f"failed: {r.stderr.strip().splitlines()[-1]}"] * 3
        equal = sum(isinstance(a, float) and a == b for a, b in zip(vals, stored))
        close = sum(isinstance(a, float) and abs(a - b) < 1e-4 for a, b in zip(vals, stored))
        results[v] = {"values": vals, "equal": equal, "within_1e-4": close}
        print(f"rouge {v}: {vals}  equal {equal}/3, within 1e-4 {close}/3")
    if args.json:
        with open(args.json, "w", encoding="utf-8") as f:
            json.dump({"stored": stored, "versions": results}, f, indent=1)
    return 0


if __name__ == "__main__":
    sys.exit(main())