"""CoSER given-circumstance acting (protocol coser-gca-1.1): checks the platform's protocol module against the official code.

The platform module is worker/src/benchmarks/coser/gca (pure TypeScript); tools/experiments/coser_gca_dump.ts runs it on
the vectors this script writes. The Python side is tools/experiments/coser_reference.py: verbatim copies of the official
functions (Neph0s/CoSER at 624efc5, and 93c1474 where the paper-era code differs) run with scripted model outputs.
Settings and decisions: docs/复现规范/CoSER-核查矩阵.md (rows C01-C31, checks V01-V14; V06, V14 and the engine are checked by coser_e2e_check.py). One line per check; exit 1 on any
failure. No model is called.

    <python with nltk==3.9.1 and regex>/bin/python -I tools/experiments/coser_spec_check.py <CoSER repo> <nltk_data>

<CoSER repo> is a clone of github.com/Neph0s/CoSER containing commits 624efc5 and 93c1474 (read with `git show`, never
imported); <nltk_data> holds tokenizers/punkt_tab/english.
"""
import argparse
import copy
import hashlib
import importlib.util
import json
import math
import os
import random
import re
import subprocess
import sys
import tempfile
import types
import unicodedata
import warnings

HERE = os.path.dirname(os.path.abspath(__file__))
ROOT = os.path.abspath(os.path.join(HERE, "..", ".."))
GCA = os.path.join(ROOT, "worker", "src", "benchmarks", "coser", "gca")
TEST_SET_SHA256 = "eb175779d1d5adf99c893ff1f82de9b18ada912d3f385b3f5419195afdfb374a"
TEST_SET_BLOB = "edabd9251e2a86ab0e2a1860a0429479e4bfc5f7"
ENV_TS, NSP_TS = "openai/gpt-4o-2024-08-06", "openai/gpt-4o-mini-2024-07-18"
ENV_PY, NSP_PY = "gpt-4o-2024-08-06", "gpt-4o-mini"
# (family, platform model id, official-style model name that makes the official code take that family's branches)
ACTORS = [
    ("gpt", "openai/gpt-4o", "gpt-4o"),
    ("claude", "anthropic/claude-opus-5.5", "claude-3-5-sonnet-20240620"),
    ("o1", "openai/o1-mini", "o1-mini"),
    ("llama31-instruct", "meta-llama/llama-3.1-70b-instruct", "llama31-70-chat"),
    ("step", "stepfun/step-3", "step-1-8k"),
    ("coser", "neph0s/coser-llama-3.1-70b", "coser-70b"),
    ("other", "deepseek/deepseek-chat-v3.1", "qwen2.5-72b-instruct"),
]
FAMILY_IDS = {
    "openai/gpt-6-astra": "gpt", "openai/gpt-4o-2024-08-06": "gpt", "openai/gpt-4o-mini-2024-07-18": "gpt",
    "openai/chatgpt-4o-latest": "gpt", "openai/gpt-oss-120b": "gpt", "openai/o1": "o1", "openai/o1-preview": "o1",
    "openai/o3": "other", "openai/o4-mini": "other", "anthropic/claude-opus-5.5": "claude",
    "anthropic/claude-3.5-sonnet": "claude", "meta-llama/llama-3.1-8b-instruct": "llama31-instruct",
    "meta-llama/llama-3.1-70b-instruct": "llama31-instruct", "meta-llama/llama-3.1-405b-instruct": "other",
    "meta-llama/llama-3.3-70b-instruct": "other", "stepfun-ai/step-2-16k": "step", "neph0s/coser-llama-3.1-8b": "coser",
    "deepseek/deepseek-chat": "other", "qwen/qwen3-235b-a22b": "other", "google/gemini-2.5-pro": "other",
    "bedrock/anthropic.claude-3-haiku": "claude", "azure/gpt-4o": "gpt", "azure/o1-mini": "o1",
    "mistralai/mistral-7b-instruct": "other", "x-ai/grok-4": "other",
}

results = []


def check(cid, what, ok, detail=""):
    results.append(bool(ok))
    print(f"{'PASS' if ok else 'FAIL'} {cid} {what}")
    if not ok and detail:
        print("     " + str(detail)[:4000].replace("\n", "\n     "))


def first_diff(a, b, label=""):
    """A short description of where two JSON-like values differ."""
    if type(a) != type(b):
        return f"{label}: type {type(a).__name__} != {type(b).__name__}: {a!r:.300} vs {b!r:.300}"
    if isinstance(a, dict):
        for k in list(a) + [k for k in b if k not in a]:
            if k not in a or k not in b:
                return f"{label}.{k}: missing on one side"
            if a[k] != b[k]:
                return first_diff(a[k], b[k], f"{label}.{k}")
    if isinstance(a, list):
        if len(a) != len(b):
            for i, (x, y) in enumerate(zip(a, b)):
                if x != y:
                    return first_diff(x, y, f"{label}[{i}]")
            return f"{label}: length {len(a)} != {len(b)}"
        for i, (x, y) in enumerate(zip(a, b)):
            if x != y:
                return first_diff(x, y, f"{label}[{i}]")
    if isinstance(a, str) and a != b:
        i = next((i for i, (x, y) in enumerate(zip(a, b)) if x != y), min(len(a), len(b)))
        return f"{label}: strings differ at {i}: {a[max(0, i - 80):i + 80]!r} vs {b[max(0, i - 80):i + 80]!r}"
    return f"{label}: {a!r:.300} vs {b!r:.300}"


def compare_lists(cid, what, py, ts):
    bad = [i for i, (x, y) in enumerate(zip(py, ts)) if x != y]
    ok = len(py) == len(ts) and not bad
    detail = ""
    if len(py) != len(ts):
        detail = f"count {len(py)} (Python) vs {len(ts)} (platform)"
    elif bad:
        detail = f"{len(bad)} differ; first #{bad[0]}: " + first_diff(py[bad[0]], ts[bad[0]], "")
    check(cid, f"{what}: {len(py)} vectors, {len(bad)} differ", ok, detail)


def load_reference():
    spec = importlib.util.spec_from_file_location("coser_reference", os.path.join(HERE, "coser_reference.py"))
    mod = importlib.util.module_from_spec(spec)
    spec.loader.exec_module(mod)
    return mod


# ---------------------------------------------------------------------------------------------------------------------
# Vector generators (seeded)

def strip_prefix(s):
    return re.sub(r"^[^:\n]{1,40}:\s*", "", s, count=1)


def actor_pool(ref, test, sims):
    pool = []
    for f in sims:
        for r in f:
            for m in r["simulation"]:
                if m["role"] != "NSP":
                    pool += [m["content"], strip_prefix(m["content"])]
    for c in test[:60]:
        for d in c["dialogues"]:
            pool.append(d["message"])
    return pool


def synthetic_texts(rnd, names):
    words = "the a king fool sir my lord truth (smiles) [thinks] , . ! ? yes no never always storm night 's n't".split()
    out = []
    for _ in range(120):
        seq = [rnd.choice(words) for _ in range(rnd.randint(1, 8))]
        rep = " ".join(seq)
        k = rnd.choice([1, 4, 5, 6, 9])
        pre = " ".join(rnd.choice(words) for _ in range(rnd.randint(0, 12)))
        out.append((pre + " " + " ".join([rep] * k)).strip())
    for _ in range(60):  # non-consecutive repetition of 15-30 tokens
        seq = " ".join(rnd.choice(words) for _ in range(rnd.randint(14, 32)))
        mid = " ".join(rnd.choice(words) for _ in range(rnd.randint(0, 40)))
        out.append(f"{seq} {mid} {seq} tail")
    out += [
        "a" * 120 + " then " + "B" * 101 + " and " + "a" * 120,
        "x" * 150 + " " + "x" * 120,
        "x" * 120 + " " + "x" * 150,
        "Hello" * 30,
        "我很好" * 8 + "。" + "我很好" * 8,
        "こんにちは、" * 7,
        "数字 123 456 123 456 123 456 123 456 123 456 123 456",
        "",
        " ",
        "...",
        "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!",
        "Hi.\n\nHi.\n\nHi.\n\nHi.\n\nHi.\n\nHi.\n\nHi.\n\nHi.\n\nHi.\n\nHi.",
        "ending in digit 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5",
        "café naïve résumé " * 7,
        "_under_ score_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _",
        "́combining marks é é é é é é é é é é é",
        "²³ ½ ² ² ² ² ² ² ² ² ² ² ² ² ² ² ² ² ² ²",
        "emoji 😀 😀 😀 😀 😀 😀 😀 😀 😀 😀 😀 😀",
        "[thought] <role_think>hidden</role_think> (action) speech",
        "<system_thinking>\nmulti\nline\n</system_thinking>\nAfter.",
        "< _think >x</ _think > tail",
        "<think>kept</think> visible",
        "Line one [a thought\nspanning lines] end",
        "  [only thought]  ",
        "tabs\tand\x0bvertical\x0cfeeds\r\nwindows\x85next
sep　ideographic",
        "\x1cfile sep\x1d\x1e\x1f",
        "bom start",
    ]
    for n in names[:20]:
        out += [f"{n}: (nods) Indeed.", f"  {n}：全角冒号", f"Other: hi\n{n}: there", f"{n}:no space", f"[{n}: thinking]"]
    return out


def nsp_raw(rnd, names, current):
    n = rnd.choice(names + names + ["Nobody", "environment", current, ""])
    if rnd.random() < 0.6:  # the paper's prompt asks for a bare name
        return rnd.choice([n, n, n, f"{n}: their turn", f" {n} ", f"{n}."])
    pats = [
        "Next Speaker: {n}", "2. Next Speaker: {n}.", "**Next Speaker:** {n}", "**Next Speaker**: {n}",
        "1. Reasoning: {n} has been addressed: they must answer.\n2. Next Speaker: {n}", 'Next Speaker: "{n}"',
        "Next speaker is {n}", "{n}", "{n}: it is their turn", "<END CHAT>", "Next Speaker: <END CHAT>",
        'Next Speaker: "<END CHAT>"', "random", "", "NEXT SPEAKER - {n}!", "# Next Speaker\n{n}",
        "Reasoning: unclear.\nNext Speaker:", "Next Speaker: *{n}*", "The scene should end.\n\n2. Next Speaker: <END CHAT>",
    ]
    return rnd.choice(pats).format(n=n)


def nsp_vectors(names):
    out = []
    pats = [
        "Next Speaker: {n}", "2. Next Speaker: {n}.", "**Next Speaker:** {n}", "**Next Speaker**: {n}", "Next Speaker - {n}",
        "Next speaker is {n}", "next speaker: {n}!", "# Next Speaker\n{n}", "1. Reasoning: blah: blah\n2. Next Speaker: {n}",
        'Next Speaker: "{n}"', "Next Speaker: '{n}'", "Next Speaker: *{n}*", "{n}", "{n}: because", " {n} ",
        "NEXT SPEAKER: {n}", "Next  Speaker :  {n}", "Next Speaker: {n}\nNext Speaker: Environment", "The next speaker is {n}.",
        "Next Speakers: {n}", "Next Speaker—{n}", "**Next Speaker:**\n{n}", "Next Speaker: {n}, then Environment",
        "#Next Speaker# {n}", "Next Speaker:{n}", "Next\tSpeaker : {n}", "Next Speaker is: {n}", "Next Speaker isn't {n}",
        "Next Speaker　{n}", "**Next Speaker:** **{n}**", "Next Speaker: {n}...", "Next Speaker: {n}!\"'",
    ]
    for n in names:
        for p in pats:
            out.append(p.format(n=n))
    out += ["<END CHAT>", "Next Speaker: <END CHAT>", 'Next Speaker: "<END CHAT>"', "random", "Next Speaker: random", "", "   ",
            ":", "::", "Reasoning: x\nNext Speaker:", "Next Speaker:\n", "Next Speaker: ***", "Next Speaker: .,!", "a:b:c",
            " Elizabeth Bennet :", " Elizabeth Bennet ", "Elizabeth Bennet\n", "Elizabeth Bennet.", "Next Speaker: İstanbul",
            "next SPEAKER is Environment", "next speakeR iS Environment", "Next Speaker: Environment\x85"]
    return out


def judge_vectors():
    d = "Storyline Consistency"
    good = {d: {"flaws": [{"instance": "x", "type": "t", "severity": 3}, {"instance": "y", "type": "t", "severity": 2}]}}
    v = []

    def add(text, fix=None, dim=d, R=10):
        v.append({"text": text, "fix": fix if fix is not None else json.dumps(good), "dimension": dim, "R": R})

    add(json.dumps(good))
    add("```json\n" + json.dumps(good, indent=2) + "\n```")
    add("Here is my evaluation:\n" + json.dumps(good) + "\nThanks.")
    add('{"Storyline Consistency": {"flaws": [{"instance": "a\nb", "type": "t", "severity": 4}]}}')
    add('{"Storyline Consistency": {"flaws": [{"instance": "x", "severity": 3.0}, {"instance": "y", "severity": "3"}, {"severity": true}, {"severity": false}, {"instance": "z"}, {"severity": null}, {"severity": -2}, {"severity": 12}]}}')
    add('{"Storyline Consistency": {"flaws": []}}')
    add('{"Storyline Consistency": {"flaws": {}}}')
    add('{"Storyline Consistency": {"flaws": ""}}')
    add('{"Storyline Consistency": {"flaws": "x"}}')
    add('{"Storyline Consistency": {"flaws": {"a": 1}}}')
    add('{"Storyline Consistency": {"flaws": 3}}')
    add('{"Storyline Consistency": {"flaws": [1, 2]}}')
    add('{"Storyline Consistency": {"no_flaws": []}}')
    add('{"Storyline Consistency": ["flaws"]}')
    add('{"Storyline Consistency": "flaws here"}')
    add('{"Anthropomorphism": {"flaws": [{"severity": 2}]}}')
    add('{"Anthropomorphism": {"flaws": []}, "Storyline Consistency": {"flaws": [{"severity": 5}]}}')
    add('{"Storyline Consistency": {"flaws": [{"severity": 1}]}, "score": 50}')
    add('{"Storyline Consistency": {"flaws": [{"severity": 1}]}, "Storyline Consistency": {"flaws": [{"severity": 4}, {"severity": 4}]}}')
    add('[{"Storyline Consistency": {"flaws": []}}]')
    add("{}")
    add("[]")
    add("0")
    add("not json at all", fix="still not json")
    add("not json at all", fix=json.dumps(good))
    add('{"Storyline Consistency": {"flaws": [{"severity": 2},]}}', fix='{"Storyline Consistency": {"flaws": [{"severity": 2}]}}')
    add('{"Storyline Consistency": {"flaws": [{"instance": "he said "hi"", "severity": 2}]}}')
    add('{"Storyline Consistency": {"flaws": [{"severity": NaN}, {"severity": Infinity}, {"severity": 1e2}, {"severity": 7}]}}')
    add('{"Storyline Consistency": {"flaws": [{"severity": 2}]}} and {"Storyline Consistency": {"flaws": [{"severity": 2}, {"severity": 3}]}}')
    add('noise {"a": 1} 12345678901234567890 "a long string value here that is long" [1,2,3]')
    add('{"Storyline Consistency": {"flaws": [{"severity": 123456789012345678901234567890}]}}')
    add('{"Storyline Consistency": {"flaws": [{"severity": 9007199254740993}]}}', R=3)
    add('{"Storyline Consistency": {"flaws": [{"instance": "\\ud83d\\ude00 \\u00e9 \\ud800", "severity": 1}]}}')
    add('{"Storyline Consistency": {"flaws": [{"instance": "tab\there", "severity": 1}]}}')
    add('{"Storyline Consistency": {"flaws": [{"severity": 1}]}}')
    add('{"Storyline Consistency" : { "flaws" : [ { "severity" : 2 } ] } }   ')
    add('{"Storyline Consistency": {"flaws": [{"severity": 2}]}}}')
    add('{"Storyline Consistency": {"flaws": [{"severity": 01}]}}')
    add('{"Storyline Consistency": {"flaws": [{"severity": -0}, {"severity": 2.50}, {"severity": 1E+1}]}}')
    add('{"Character Fidelity": {"flaws": [{"severity": 4, "type": "Character Language"}]}}', dim="Character Fidelity", R=7)
    add('{"Storyline Quality": {"flaws": [{"severity": 10}, {"severity": 10}, {"severity": 10}]}}', dim="Storyline Quality", R=0)
    add('{"Storyline Quality": {"flaws": [{"severity": -10}]}}', dim="Storyline Quality", R=20)
    add('{"Storyline Quality": {"flaws": [{"severity": 3}]}}', dim="Storyline Quality", R=11)
    add('"just a string"', fix='"another string"')
    add('{"Storyline Consistency": {"flaws": [{"severity": 2}]}', fix='{"Storyline Consistency": {"flaws": [{"severity": 2}]}}')
    add('{"Storyline Consistency": {"flaws": [{"instance": "x\\ny", "severity": 2}]}}')
    add('{"Storyline Consistency": {"flaws": [{"instance": "a \\\\" b", "severity": 2}]}}')
    return v


def score_vectors(rnd):
    v = []
    for _ in range(300):
        flaws = []
        for _ in range(rnd.randint(0, 12)):
            kind = rnd.random()
            if kind < 0.7:
                sev = rnd.randint(-3, 12)
            elif kind < 0.8:
                sev = float(rnd.randint(1, 5))
            elif kind < 0.9:
                sev = str(rnd.randint(1, 5))
            else:
                sev = rnd.choice([True, False])
            flaws.append({"instance": "x", "severity": sev})
        v.append({"flaws": flaws, "R": rnd.randint(0, 20)})
    for S, R in [(20, 0), (0, 0), (21, 0), (-1, 0), (14, 20), (26, 20), (27, 20), (0, 20), (6, 20), (3, 11), (17, 1), (4, 15), (2, 12), (5, 20)]:
        v.append({"flaws": [{"severity": S}], "R": R})
    return v


def book_texts(repo, rnd, ref):
    texts = []
    names = [n for n in subprocess.run(["git", "-C", repo, "ls-tree", "-z", "--name-only", f"{ref.PINNED}:data/final"], capture_output=True, check=True).stdout.decode("utf-8").split("\0") if n]
    for name in sorted(names):
        d = json.loads(ref.git_show(repo, ref.PINNED, f"data/final/{name}"))
        plots = d["plots"]
        for p in rnd.sample(plots, min(12, len(plots))):
            t = p.get("text", "")
            if t:
                a = rnd.randint(0, max(0, len(t) - 1))
                texts.append(t[a:a + rnd.choice([80, 300, 1200, 4000])])
            texts.append(p.get("summary", ""))
            for c in p.get("conversation", [])[:2]:
                texts.append("\n\n".join(f"{x['character']}: {x['message']}" for x in c.get("dialogues", [])))
    return texts


TOKENIZER_EDGES = [
    "", " ", "\n", "Hello.", "Mr. Smith went to Washington. He arrived at 3 p.m. on Jan. 5th.", "e.g. this, i.e. that; etc.",
    "U.S.A. is big. The U.S. is too.", "Wait... What?! No!!! Yes?!", "He said, \"Come here.\" She didn't.", "'Tis the season; 'twas.",
    "I cannot, gonna, wanna go; gimme, lemme, gotta, d'ye, more'n.", "Whaddya want? Whatcha doing?", "(Sent1.) Sent2.",
    "\"Quote.\" Next one.", "--dash--and -- more -- --", "Prices: $3.88, 3,36 euros, 1.5e10, -.5, 12:30.", "a.b.c. d.e.f.",
    "Dr. J. R. R. Tolkien wrote. The end.", "The U.S. Army. The U.N. Council.", "“Smart quotes.” ‘Single.’ «Guillemets» „low“",
    "``Penn'' style quotes.''", "It's John's book, isn't it? They'll've done it.", "Line one.\nLine two\n\nParagraph two. Done",
    "No period at the end", "Ends with colon:", "Comma, end,", "ellipsis . . . spaced . . .", "A.\tB.\x0bC.\x0cD.",
    "End.
Next.　Ideographic space.", "Numbers 1. 2. 3. items", "St. Paul's is on 5th Ave. near the U.S.P.S. office.",
    "Hello!' she said.' Then 'quoted' words 'here'.", "*emphasis* and [brackets] {braces} <angles>", "#hashtag @at &amp; 100%",
    "Ünïcödé Çàfé. Straße. İstanbul. ΣΊΣΥΦΟΣ. DŽemal.", "Mixed CASE After. a lowercase start. 1st place. 2nd.",
    "J. Bach and S. Bach. K. Marx.", "Fig. 3 shows. Vol. II begins. No. 5 wins.", "?!", "....", ". . .", "a -- b --c d-- e",
    "(He paused.) \"Really?\" [Aside.] {Note.}", "'Hello,' he said. 'Goodbye.'", "can't won't shan't y'all o'clock rock'n'roll",
    "A sentence ending with a quote.\" Another starts here.", "Ask me? Why! Because.", "x.\n\ny.", "Jan.\nFeb. March.",
]


# ---------------------------------------------------------------------------------------------------------------------

def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("repo")
    ap.add_argument("nltk_data")
    ap.add_argument("--node", default="node")
    ap.add_argument("--keep", help="directory to keep the request and response JSON in")
    args = ap.parse_args()
    ref = load_reference()
    ref.load_critic_prompts(args.repo)
    rnd = random.Random(20261028)
    work = args.keep or tempfile.mkdtemp(prefix="coser_gca_")
    os.makedirs(work, exist_ok=True)

    # G: generated files and verbatim copies -----------------------------------------------------------------------
    gen = ref.render_strings(ref.extract_strings(args.repo))
    on_disk = open(os.path.join(GCA, "strings.gen.ts"), encoding="utf-8").read()
    check("G01", "strings.gen.ts equals the generator's output from the pinned sources (ast extraction, SHA-256 recorded)", gen == on_disk,
          first_diff(gen, on_disk, "strings.gen.ts"))
    punkt_dir = os.path.join(args.nltk_data, "tokenizers", "punkt_tab", "english")
    gen = ref.render_punkt(punkt_dir)
    on_disk = open(os.path.join(GCA, "punkt.gen.ts"), encoding="utf-8").read()
    check("G02", "punkt.gen.ts equals the generator's output from punkt_tab/english", gen == on_disk, first_diff(gen, on_disk, "punkt.gen.ts"))
    text = open(os.path.join(HERE, "coser_reference.py"), encoding="utf-8").read()
    blocks = re.findall(r"# >>> VERBATIM (\w+):(\S+) (\w+) \(lines \d+-\d+\)\n(.*?)\n# <<< VERBATIM", text, re.S)
    bad = []
    full = {"624efc5": ref.PINNED, "93c1474": ref.PAPER_ERA}
    import ast as _ast
    for c, path, name, body in blocks:
        src = ref.git_show(args.repo, full[c], path).decode("utf-8").replace("\r\n", "\n")  # utils.py, agent.py use CRLF
        node = ref.find_def(_ast.parse(src), name)
        if ref.source_block(src, node).rstrip("\n") != body:
            bad.append(f"{c}:{path} {name}")
    check("G03", f"coser_reference.py: {len(blocks)} verbatim copies equal their source blocks (CRLF read as LF, as Python reads source)", len(blocks) >= 16 and not bad, bad)

    # G05: the version in the protocol module, the adapter, the report and the matrix ------------------------------------
    read_text = lambda *parts: open(os.path.join(ROOT, *parts), encoding="utf-8").read()
    version = re.search(r"PROTOCOL_VERSION = '([^']+)'", read_text("worker/src/benchmarks/coser/gca/setup.ts")).group(1)
    stamp = re.search(r"version: `\$\{PROTOCOL_VERSION\} · (\d{4}-\d{2}-\d{2})`", read_text("worker/src/benchmarks/coser/index.ts")).group(1)
    full_version = f"{version} · {stamp}"
    check("G05", f"protocol version {full_version}: same in the protocol module, the adapter, this reference, the report and the matrix",
          ref.PROTOCOL == version and f"const PROTOCOL = '{full_version}';" in read_text("web/src/pages/CoserReport.tsx") and f"`{full_version}`" in read_text("docs/复现规范/CoSER-核查矩阵.md"))

    # V01: test set ------------------------------------------------------------------------------------------------
    blob = ref.git_show(args.repo, ref.PINNED, "data/test/test_set.json")
    git_blob = hashlib.sha1(b"blob %d\0" % len(blob) + blob).hexdigest()
    test = json.loads(blob)
    tid = json.loads(ref.git_show(args.repo, ref.PINNED, "test_data/test_circumstance_id.json"))
    tood = json.loads(ref.git_show(args.repo, ref.PINNED, "test_data/test_circumstance_ood.json"))
    strip = lambda s: {k: v for k, v in s.items() if k not in ("i_p", "tag")}
    fields_ok = all(
        isinstance(s.get("book"), str) and isinstance(s.get("i_c"), int) and isinstance(s.get("scenario"), str)
        and isinstance(s.get("plot", {}).get("summary"), str) and s.get("i_p") == s["plot"]["i_p"]
        and all(isinstance(k.get("name"), str) and isinstance(k.get("thought", ""), str) for k in s["key_characters"])
        and all(isinstance(v, str) for v in s["character_profiles"].values())
        and all(isinstance(x, str) and x for x in s["speaking_characters_w_env"]) and s["major_characters"]
        and all(set(d) == {"character", "message"} for d in s["dialogues"])
        for s in test)
    ids = [f"{s['book']}-{s['plot']['i_p']}-{s['i_c']}" for s in test]
    tags = [s["tag"] for s in test]
    ok = (hashlib.sha256(blob).hexdigest() == TEST_SET_SHA256 and git_blob == TEST_SET_BLOB and len(test) == 200
          and tags == ["id"] * 100 + ["ood"] * 100 and [strip(s) for s in test] == tid + tood and fields_ok and len(set(ids)) == 200)
    check("V01", f"test set: SHA-256 {hashlib.sha256(blob).hexdigest()[:8]}…, git blob {git_blob[:8]}, {len(test)} = "
          f"{tags.count('id')} id + {tags.count('ood')} ood, = test_data id+ood, fields complete, {len(set(ids))} unique ids", ok)
    test_path = os.path.join(work, "test_set.json")
    with open(test_path, "wb") as f:
        f.write(blob)
    sims = [json.loads(ref.git_show(args.repo, ref.PINNED, f"exp/simulation/{n}")) for n in
            ["test_set_gpt-4o.json", "test_set_gpt-4o-mini_rag=expr3_conv1.json", "test_set_gpt-4o_rag=expr3_conv1.json"]]
    evaluation = json.loads(ref.git_show(args.repo, ref.PINNED, "exp/evaluation/test_set_gpt-4o.json"))

    # Build the request -------------------------------------------------------------------------------------------
    req = {"testSet": test_path}
    req["setups"] = [{"sample": i, "actorModel": pid, "envModel": ENV_TS, "nspModel": NSP_TS} for i in range(200) for _, pid, _ in ACTORS]
    req["setups"] += [{"sample": i, "actorModel": "openai/gpt-4o", "envModel": ENV_TS, "nspModel": NSP_TS, "protocolVersion": "coser-gca-0.0-test"} for i in range(200)]
    req["families"] = list(FAMILY_IDS) + [pid for _, pid, _ in ACTORS]
    all_names = sorted({n for s in test for n in s["speaking_characters_w_env"]} | {"Environment"})
    special = [n for n in all_names if re.search(r"[^A-Za-z .\-]", n)]
    req["nsp"] = nsp_vectors(special + rnd.sample(all_names, 25) + ["Environment", "<END CHAT>"])
    decisions = []
    for i in rnd.sample(range(200), 60):
        sp = test[i]["speaking_characters_w_env"] + ([] if "Environment" in test[i]["speaking_characters_w_env"] else ["Environment"])
        for rd in [0, 4, 5, 6, 19]:
            for cur in sp:
                for _ in range(3):
                    decisions.append({"sample": i, "round": rd, "current": cur, "raw": nsp_raw(rnd, sp, cur),
                                      "actorModel": "openai/gpt-4o", "envModel": ENV_TS, "nspModel": NSP_TS})
    req["nspDecisions"] = decisions
    pool = actor_pool(ref, test, sims)
    synth = synthetic_texts(rnd, all_names)
    env_pool = [m["content"] for f in sims for r in f for m in r["simulation"] if m["role"] == "Environment"]
    env_pool += [strip_prefix(x) for x in env_pool] + [d["message"] for s in test for d in s["dialogues"] if d["character"] == "Environment"][:200]
    sim_specs = []
    for fam, pid, oname in ACTORS:
        samples = range(200) if fam == "gpt" else rnd.sample(range(200), 25)
        for i in samples:
            sp = test[i]["speaking_characters_w_env"] + ([] if "Environment" in test[i]["speaking_characters_w_env"] else ["Environment"])
            chars = [c for c in sp if c != "Environment"]

            def actor_out():
                x = rnd.random()
                if x < 0.06:
                    return ""
                if x < 0.16:
                    return rnd.choice(synth)
                if x < 0.3 and fam in ("llama31-instruct", "step"):
                    a, b = rnd.choice(chars), rnd.choice(sp)
                    return rnd.choice([f"{a}: (smiles) Hello there.\n{b}: Hi.\n{a}: More words.", f"{b}: Only someone else.",
                                       f"{a}: [thinking] fine.", "No prefix at all.", f"{a}:\n\nline\n{a}: again"])
                return rnd.choice(pool)

            scripts = {
                "actor": [actor_out() for _ in range(140)],
                "env": [rnd.choice(env_pool) if rnd.random() > 0.05 else "" for _ in range(80)],
                "nsp": [nsp_raw(rnd, sp, rnd.choice(sp)) for _ in range(80)],
            }
            sim_specs.append({"family": fam, "oname": oname, "sample": i, "actorModel": pid, "envModel": ENV_TS, "nspModel": NSP_TS, "scripts": scripts})
    req["sims"] = sim_specs
    texts7 = pool + synth + [d["message"] for s in test for d in s["dialogues"]]
    req["fixRep"] = texts7
    req["tokenizeWords"] = texts7
    req["removeThoughts"] = texts7
    speakers = all_names[:60] + special
    req["addSpeaker"] = [[rnd.choice(texts7), rnd.choice(speakers)] for _ in range(1500)]
    req["addSpeaker"] += [[f"{n}: hi", n] for n in speakers] + [[f"  {n}：hi", n] for n in speakers] + [[f"x\n {n}:y", n] for n in speakers]
    req["addSpeaker"] += [[f"{n}hi", n] for n in speakers] + [["", n] for n in speakers]
    pp = []
    for fam, pid, oname in ACTORS:
        for _ in range(120):
            name = rnd.choice(all_names[:80] + special)
            other = rnd.choice(all_names)
            raw = rnd.choice([rnd.choice(texts7), f"{name}: {rnd.choice(texts7)}", f"{other}: a\n{name}: b\n{other}: c\n{name}: d", "",
                              f"{name}:x", f" {name} : spaced", f"{name}: " + "again and again " * 8])
            pp.append({"family": fam, "oname": oname, "model": pid, "kind": "actor", "name": name, "raw": raw})
        for _ in range(10):
            pp.append({"family": fam, "oname": oname, "model": pid, "kind": "nsp", "name": "NSP", "raw": rnd.choice(texts7)})
    req["postprocess"] = pp
    req["repr"] = [[n] for n in all_names] + [special, ["it's", 'say "hi"', "both ' and \"", "back\\slash", "tab\tnew\nline", "\x00\x7f\x85\xa0é", "​
퟿", "😀\U000e0001"]]
    floats = [0.0, -0.0, 1.0, 0.1, 1e16, 1e15, 123456789012345.6, 1e-4, 9.99e-5, 1e-5, 0.3 * 11, 100 - (3 - 0.3 * 11) * 5, 2.5e-308, 5e-324,
              1.7976931348623157e308, 59.0, 66.16666666666667, -12.5, 1e22, 1.5e300]
    floats += [rnd.uniform(-1e3, 1e3) for _ in range(300)] + [rnd.random() * 10 ** rnd.randint(-30, 30) for _ in range(300)]
    req["floatRepr"] = [repr(x) for x in floats]
    jp = []
    for i, s in enumerate(test):
        recd = []
        for j, dlg in enumerate(s["dialogues"][:20]):
            msg = dlg["message"]
            if j % 7 == 3:
                msg = ""
            elif j % 7 == 5:
                msg = msg + " <role_think>inner</role_think>"
            recd.append({"role": dlg["character"], "content": ref.add_speaker_name(msg, dlg["character"])})
            recd.append({"role": "NSP", "content": "<END CHAT>" if j == 19 else rnd.choice(s["speaking_characters_w_env"])})
        jp.append({"sample": i, "record": recd})
    req["judgePrompts"] = jp
    req["judgeReads"] = judge_vectors()
    sv = score_vectors(rnd)
    req["scores"] = [{"flawsJson": json.dumps(x["flaws"]), "R": x["R"]} for x in sv]
    published = []
    for key, case in evaluation["cases"].items():
        R = len([m for m in case["simulation"] if m["role"] != "Environment"])
        for dim in ref.dimensions:
            published.append({"key": key, "dim": dim, "R": R, "flaws": case["critique"][dim]["flaws"], "stored": case["critique"][dim]["score"]})
    req["scores"] += [{"flawsJson": json.dumps(p["flaws"]), "R": p["R"]} for p in published]
    req["bleuRecords"] = [{"circumstance": r["circumstance"], "record": r["simulation"]} for r in sims[0]]
    btexts = book_texts(args.repo, rnd, ref) + TOKENIZER_EDGES + texts7[::7]
    req["wordTokenize"] = [t.lower() for t in btexts] + btexts
    refs = []
    for s in test:
        rs = "\n\n".join(f"{d['character']}: {ref.remove_inner_thoughts(d['message']) if d['character'] != 'Environment' else d['message']}".strip("\n") for d in s["dialogues"])
        refs.append(rs)
    pairs = []
    for k in range(700):
        r = refs[k % 200]
        x = rnd.random()
        if x < 0.4:
            hyp = refs[rnd.randrange(200)]
        elif x < 0.7:
            parts = r.split("\n\n")
            rnd.shuffle(parts)
            hyp = "\n\n".join(parts[: rnd.randint(1, len(parts))])
        elif x < 0.9:
            hyp = "\n\n".join(rnd.choice(pool) for _ in range(rnd.randint(1, 15)))
        else:
            hyp = rnd.choice(["", "the", "the king", "the king said no", r, r + " " + r, rnd.choice(btexts)])
        pairs.append({"ref": r, "sim": hyp})
    pairs += [{"ref": rnd.choice(btexts), "sim": rnd.choice(btexts)} for _ in range(300)]
    req["bleu"] = pairs
    logs = [repr(n / d) for d in range(1, 400, 7) for n in range(1, d + 1, 3)] + [repr(sys.float_info.min), repr(5e-324), repr(1e300), "0.5", "2.0", "1.0000000000000002", "0.9999999999999999"]
    logs += [repr(rnd.random() * 10 ** rnd.randint(-300, 300)) for _ in range(1500)]
    exps = [repr(rnd.uniform(-200, 5)) for _ in range(3000)] + ["0.0", "-708.0", "1e-20", "-1e-20", "709.0", "-744.0", repr(math.log(2))]
    req["math"] = {"log": logs, "exp": exps, "fsum": [[repr(rnd.uniform(-5, 0)) for _ in range(rnd.randint(0, 6))] for _ in range(500)] + [["1e100", "1.0", "-1e100"], ["0.1"] * 10]}
    rng_v = [{"protocol": "coser-gca-1.1", "sampleId": ids[i], "purpose": p, "n": n} for i in range(200) for p, n in [("environment-role", 4), ("nsp-fallback:3", 3), ("separator:X", 5), ("nsp-fallback:19", 7)]]
    req["rng"] = rng_v
    req["classes"] = True
    stat_rnd = random.Random(20261101)
    req["runStats"] = []
    for runs in (3, 3, 1, 4):
        rows = []
        for i in range(runs):
            for sample in range(12):
                dims = {k: stat_rnd.uniform(0, 110) for k in ("sc", "an", "cf", "sq")}
                if stat_rnd.random() < 0.8:
                    dims["bleu"] = stat_rnd.uniform(0, 20)
                rows.append({"sampleIdx": i, "split": "id" if sample < 6 else "ood", "dims": dims})
        req["runStats"].append(rows)

    req_path, resp_path = os.path.join(work, "request.json"), os.path.join(work, "response.json")
    with open(req_path, "w", encoding="utf-8") as f:
        json.dump(req, f, ensure_ascii=False)
    proc = subprocess.run([args.node, "--experimental-strip-types", "--no-warnings", os.path.join(HERE, "coser_gca_dump.ts"), req_path, resp_path], capture_output=True, text=True)
    if proc.returncode != 0:
        print(proc.stderr[-4000:])
        check("RUN", "coser_gca_dump.ts ran", False)
        return 1
    resp = json.load(open(resp_path, encoding="utf-8"))
    check("G04", "the module's SOURCES name the pinned blobs (test set SHA-256 included)",
          resp["meta"]["sources"] == ref.extract_strings(args.repo)["SOURCES"] and resp["meta"]["sources"]["test_set"]["sha256"] == TEST_SET_SHA256)

    # C19: family table ----------------------------------------------------------------------------------------------
    fam_ok = []
    for pid, f in zip(req["families"], resp["families"]):
        expect = FAMILY_IDS.get(pid) or next(a[0] for a in ACTORS if a[1] == pid)
        fam_ok.append(f["family"] == expect)
    rule_bad = []
    for fam, pid, oname in ACTORS:
        n = oname
        official = {  # what the official code does with the official-style name (paper-era name-prefix scope, C19)
            "systemRole": "user" if n.startswith("claude") or n.startswith("o1") else "system",
            "namePrefix": n in ["llama3p1-8b-instruct", "llama31-70-chat"] or "step" in n,
            "fixRepetition": not any(n.lower().startswith(t) for t in ["gpt", "claude"]),
            "outputExample": "coser" not in n.lower(),
            "systemToUserInRequest": n.startswith("claude"),
        }
        got = resp["families"][req["families"].index(pid)]
        if any(got[k] != v for k, v in official.items()):
            rule_bad.append((pid, got, official))
    check("C19", f"model-family table: {sum(fam_ok)}/{len(fam_ok)} ids in the expected family; rules equal the official name tests for {len(ACTORS) - len(rule_bad)}/{len(ACTORS)} families",
          all(fam_ok) and not rule_bad, rule_bad or [p for p, ok in zip(req["families"], fam_ok) if not ok])

    # V02: system prompts and first messages ------------------------------------------------------------------------
    n_cmp, n_bad, first, n_msgs = 0, 0, "", 0
    choices_by_sample = {}
    for k, c in enumerate(req["setups"][:1400]):
        ts = resp["setups"][k]
        fam, pid, oname = next(a for a in ACTORS if a[1] == c["actorModel"])
        decorators = {name: ch["value"] for name, ch in ts["choices"]["separators"].items()}
        built = ref.build_agents(test[c["sample"]], oname, ENV_PY, NSP_PY, "coser" not in oname.lower(), decorators, ts["choices"]["environmentRole"]["value"])
        py = {"speaking": built["speaking"], "involved": [list(x) for x in built["involved"].items()], "firstSpeaker": built["speaking"][0],
              "agents": [{"name": n, "messages": a.messages, "systemRole": a.system_role} for n, a in built["agents"].items()]}
        tsv = {"speaking": ts["speaking"], "involved": ts["involved"], "firstSpeaker": ts["firstSpeaker"],
               "agents": [{"name": a["name"], "messages": a["messages"], "systemRole": a["systemRole"]} for a in ts["agents"]]}
        n_cmp += 1
        n_msgs += sum(len(a["messages"]) for a in py["agents"])
        if py != tsv:
            n_bad += 1
            first = first or f"sample {c['sample']} {pid}: " + first_diff(py, tsv, "")
        choices_by_sample.setdefault(c["sample"], []).append(ts["choices"])
    special_samples = [i for i, s in enumerate(test) if any(n in special for n in s["speaking_characters_w_env"])]
    no_profile = sum(1 for k, c in enumerate(req["setups"][:1400:7]) for n in resp["setups"][k * 7]["speaking"] if n != "Environment" and n not in dict(resp["setups"][k * 7]["involved"]))
    check("V02", f"agent system prompts and first messages: {n_cmp} setups (200 samples × {len(ACTORS)} families), {n_msgs} messages, {n_bad} setups differ "
          f"(covers {len(special_samples)} samples with the {len(special)} special names, {no_profile} characters without a profile)",
          n_bad == 0, first)

    # V03: seeded choices -------------------------------------------------------------------------------------------
    same = all(all(ch == chs[0] for ch in chs) for chs in choices_by_sample.values())
    rng_bad = []
    for i in range(200):
        ch = choices_by_sample[i][0]
        for name, sel in ch["separators"].items():
            if sel["index"] != ref.platform_rng_index(ids[i], f"separator:{name}", 5):
                rng_bad.append((i, name))
        if ch["environmentRole"]["index"] != ref.platform_rng_index(ids[i], "environment-role", 4):
            rng_bad.append((i, "env"))
    other_version = sum(1 for i in range(200) if resp["setups"][1400 + i]["choices"] != choices_by_sample[i][0])
    rng_vec = [ref.platform_rng_index(r["sampleId"], r["purpose"], r["n"], protocol=r["protocol"]) for r in req["rng"]]
    check("V03", f"seeded choices: identical across {len(ACTORS)} actor models for all 200 samples (setup takes no run number); "
          f"equal to the Python construction ({len(rng_bad)} differ; {sum(a != b for a, b in zip(rng_vec, resp['rng']))}/{len(rng_vec)} raw draws differ); "
          f"another protocol version changes {other_version} samples", same and not rng_bad and rng_vec == resp["rng"] and other_version > 0, rng_bad[:5])

    # V04: NSP parsing and decisions ----------------------------------------------------------------------------------
    py_nsp = []
    for raw in req["nsp"]:
        next_actor = raw
        if ':' in next_actor:  # eval_reproduce.py:232-235 (93c1474)
            next_actor = next_actor.split(':')[0].strip()
        py_nsp.append({"parsed": next_actor})
    compare_lists("V04a", "NSP parsing (the text before the first colon, as at 93c1474)", py_nsp, resp["nsp"])
    py_dec, ts_dec = [], []
    for d, t in zip(req["nspDecisions"], resp["nspDecisions"]):
        s = test[d["sample"]]
        sp = s["speaking_characters_w_env"] + ([] if "Environment" in s["speaking_characters_w_env"] else ["Environment"])
        na, nxt, inv = ref.nsp_decide(d["raw"], d["round"], d["current"], sp, s["major_characters"], ids[d["sample"]])
        py_dec.append([na, nxt, inv])
        ts_dec.append([t["parsed"], t["next"], t["outcome"] == "fallback"])
    outcomes = {o: sum(1 for t in resp["nspDecisions"] if t["outcome"] == o) for o in ("next", "end", "fallback")}
    compare_lists("V04b", f"NSP decisions incl. END before/after round 5 and seeded fallback ({outcomes})", py_dec, ts_dec)

    # V05: simulations: every request each agent sends, and the record -------------------------------------------------
    n_bad, first, n_calls, fams = 0, "", 0, {}
    for spec, ts in zip(req["sims"], resp["sims"]):
        circ = test[spec["sample"]]
        tsetup = resp["setups"][spec["sample"] * len(ACTORS) + [a[1] for a in ACTORS].index(spec["actorModel"])]
        decorators = {name: ch["value"] for name, ch in tsetup["choices"]["separators"].items()}
        scripts = {spec["oname"]: spec["scripts"]["actor"], ENV_PY: spec["scripts"]["env"], NSP_PY: spec["scripts"]["nsp"]}
        out = ref.simulate(circ, spec["oname"], ENV_PY, NSP_PY, "coser" not in spec["oname"].lower(), decorators,
                           tsetup["choices"]["environmentRole"]["value"], scripts)
        py_calls = [{"agent": r["agent"], "messages": c["messages"], "temperature": c["temperature"], "max_tokens": c["max_tokens"]}
                    for r in out["requests"] for c in r["calls"]]
        ts_calls = [{k: c[k] for k in ("agent", "messages", "temperature", "max_tokens")} for c in ts["calls"]]
        left_py = {"actor": len(out["leftover"].get(spec["oname"], [])), "env": len(out["leftover"].get(ENV_PY, [])), "nsp": len(out["leftover"].get(NSP_PY, []))}
        ji = ref.judge_inputs(circ, out["simulation"], out["involved"])
        a = {"calls": py_calls, "record": out["simulation"], "leftover": left_py, "errors": out["errors"],
             "judge": {d: ref.judge_messages(circ, out["simulation"], out["involved"], d) for d in ref.dimensions},
             "bleuStrings": {"reference": ji["reference_str"], "simulation": ji["simulation_str"]}}
        b = {"calls": ts_calls, "record": ts["record"], "leftover": ts["leftover"], "errors": ts["errors"], "judge": ts["judge"], "bleuStrings": ts["bleuStrings"]}
        n_calls += len(py_calls)
        fams[spec["family"]] = fams.get(spec["family"], 0) + 1
        if a != b:
            n_bad += 1
            first = first or f"sample {spec['sample']} {spec['actorModel']}: " + first_diff(a, b, "")
    rounds = [sum(1 for t in s["transcript"] if t["kind"] == "speak") for s in resp["sims"]]
    check("V05", f"simulations {fams}: {len(req['sims'])} sessions, {n_calls} model requests (messages, temperature, max_tokens), the records, and the 4 judge prompts and BLEU strings built from each transcript; "
          f"{n_bad} sessions differ (speaker messages per session {min(rounds)}-{max(rounds)})", n_bad == 0, first)

    # V07: text functions ------------------------------------------------------------------------------------------
    compare_lists("V07a", "fix_repeation", [ref.fix_repeation(t) for t in req["fixRep"]], resp["fixRep"])
    compare_lists("V07b", "tokenize_words (regex module)", [ref.tokenize_words(t) for t in req["tokenizeWords"]], resp["tokenizeWords"])
    compare_lists("V07c", "remove_inner_thoughts (624efc5)", [ref.remove_inner_thoughts(t) for t in req["removeThoughts"]], resp["removeThoughts"])
    compare_lists("V07d", "add_speaker_name", [ref.add_speaker_name(d, s) for d, s in req["addSpeaker"]], resp["addSpeaker"])
    py_pp = []
    with ref.official_env():
        for p in req["postprocess"]:
            model = p["oname"]
            agent = ref.Agent(model, p["name"], None, system_prompt="x", retrieval_target=None)
            ref.SERVER.scripts = {model: [p["raw"]] * 6}
            ref.SERVER.requests, ref.SERVER.errors = [], []
            py_pp.append(agent.chat())
    compare_lists("V07e", "Agent.chat post-processing per family (own-utterance cut, fix_repeation, none for the NSP)", py_pp, resp["postprocess"])
    compare_lists("V07f", "Python repr of name lists", [str(x) for x in req["repr"]], resp["repr"])
    compare_lists("V07g", "repr(float)", req["floatRepr"], resp["floatRepr"])
    diff_rit = sum(1 for t in req["removeThoughts"] if ref.remove_inner_thoughts_624efc5(t) != ref.remove_inner_thoughts_93c1474(t))
    print(f"INFO V07 remove_inner_thoughts 624efc5 vs 93c1474 differ on {diff_rit}/{len(req['removeThoughts'])} inputs (the <…think> tags); the protocol uses 624efc5")

    # V08: judge prompts and judge reading ------------------------------------------------------------------------------
    n_bad, first = 0, ""
    for j, ts in zip(req["judgePrompts"], resp["judgePrompts"]):
        circ = test[j["sample"]]
        built = ref.build_agents(circ, "gpt-4o", ENV_PY, NSP_PY, True, {c: "" for c in circ["speaking_characters_w_env"]}, "a world model")
        py = {d: ref.judge_messages(circ, j["record"], built["involved"], d) for d in ref.dimensions}
        ji = ref.judge_inputs(circ, j["record"], built["involved"])
        a = {"messages": py, "actorRounds": ji["actor_rounds"], "bleuStrings": {"reference": ji["reference_str"], "simulation": ji["simulation_str"]}}
        if a != ts:
            n_bad += 1
            first = first or f"sample {j['sample']}: " + first_diff(a, ts, "")
    literal = all("(Only apply to the main characters: {major_characters})" in ts["messages"]["Character Fidelity"][0]["content"] for ts in resp["judgePrompts"])
    check("V08a", f"judge prompts: 200 samples × 4 dimensions (system + user), R and the BLEU strings; {n_bad} samples differ; literal {{major_characters}} kept in Character Fidelity: {literal}",
          n_bad == 0 and literal, first)
    py_reads = []
    for v in req["judgeReads"]:
        res, reqs = ref.read_judge(v["text"], ENV_PY, v["fix"])
        fix_prompt = reqs[0]["messages"][0]["content"] if reqs else None
        if isinstance(res, tuple):
            py_reads.append({"status": "crash", "fixPrompt": fix_prompt})
        elif res is False or not res:
            py_reads.append({"status": "retry", "fixPrompt": fix_prompt})
        elif v["dimension"] not in res:
            py_reads.append({"status": "missing-dimension", "fixPrompt": fix_prompt, "value": ref.tagged(res)})
        else:
            sc, unc, s, upper = ref.score(res, v["dimension"], v["R"])
            py_reads.append({"status": "ok", "fixPrompt": fix_prompt, "value": ref.tagged(res),
                             "score": {"score": repr(float(sc)), "upperClipped": repr(float(upper)), "unclipped": repr(float(unc)), "severitySum": str(s)}})
    statuses = {s: sum(1 for r in py_reads if r["status"] == s) for s in ("ok", "retry", "missing-dimension", "crash")}
    compare_lists("V08b", f"judge reading: extract_json (+ fix call), parse_response, severity types, score ({statuses}; fix calls {sum(1 for r in py_reads if r['fixPrompt'])})", py_reads, resp["judgeReads"])

    # V10: scores ------------------------------------------------------------------------------------------------------
    py_sc = []
    for x in sv:
        res = {"D": {"flaws": copy.deepcopy(x["flaws"])}}
        sc, unc, s, upper = ref.score(res, "D", x["R"])
        py_sc.append({"score": repr(float(sc)), "upperClipped": repr(float(upper)), "unclipped": repr(float(unc)), "severitySum": str(s)})
    over = sum(float(p["score"]) > 100 for p in py_sc)
    compare_lists("V10a", f"dimension score from constructed flaws (int/float/str/bool severities, floor at 0, no ceiling: {over} vectors above 100; the stored ceiling variant)", py_sc, resp["scores"][:len(sv)])
    pub_ts = resp["scores"][len(sv):]
    exact = [repr(float(p["stored"])) == t["score"] for p, t in zip(published, pub_ts)]
    py_pub = [repr(float(ref.score({p["dim"]: {"flaws": p["flaws"]}}, p["dim"], p["R"])[0])) == repr(float(p["stored"])) for p in published]
    case_ok = []
    for key, case in evaluation["cases"].items():
        vals = [float(t["score"]) for p, t in zip(published, pub_ts) if p["key"] == key]
        case_ok.append(sum(vals) / len(vals) == case["score"])
    check("V10", f"published exp/evaluation/test_set_gpt-4o.json: {sum(exact)}/{len(published)} dimension scores recomputed exactly "
          f"(Python line {sum(py_pub)}/{len(published)}), {sum(case_ok)}/{len(case_ok)} case scores", all(exact) and all(py_pub) and all(case_ok) and len(published) == 12,
          [(p["key"], p["dim"], p["stored"], t["score"]) for p, t, e in zip(published, pub_ts, exact) if not e])
    # V11: per-run means and their spread, per-split means (matrix C01, C27).
    py_stats = [py_run_stats(v) for v in req["runStats"]]
    compare_lists("V11", "run statistics: per-run means, their sample standard deviation (ddof=1), per-split means", py_stats, resp["runStats"])

    # V12: BLEU ------------------------------------------------------------------------------------------------------
    nltk = ref.nltk
    if nltk is None or nltk.__version__ != "3.9.1":
        check("V12", "nltk 3.9.1 available for the BLEU checks", False, f"nltk {getattr(nltk, '__version__', None)}")
    else:
        nltk.data.path[:] = [args.nltk_data]
        warnings.simplefilter("ignore")
        stored = [evaluation["cases"][f"{r['book_title']}-{r['i_p']}-{r['i_c']}"]["critique"]["bleu"] for r in sims[0]]
        py_pub = []
        for r in sims[0]:
            simulation = [m for m in r["simulation"] if m["role"] != "NSP"]
            simulation = [m if m["role"] == "Environment" else {**m, "content": ref.remove_inner_thoughts(m["content"])} for m in simulation]
            reference = [m if m["character"] == "Environment" else {**m, "message": ref.remove_inner_thoughts(m["message"])} for m in r["circumstance"]["dialogues"]]
            py_pub.append(ref.calculate_bleu_rouge(reference, simulation)[0])
        ts_pub = [b["bleu"] for b in resp["bleuRecords"]]
        check("V12a", f"published BLEU: platform {sum(repr(s) == t for s, t in zip(stored, ts_pub))}/3, Python reference {sum(s == p for s, p in zip(stored, py_pub))}/3 equal to the stored values "
              f"({', '.join(repr(s) for s in stored)})", [repr(s) for s in stored] == ts_pub and stored == py_pub)
        compare_lists("V12b", f"nltk 3.9.1 word_tokenize (Punkt + NLTKWordTokenizer) on book text, dialogues and edge cases (lower-cased and as written)",
                      [ref.word_tokenize(t) for t in req["wordTokenize"]], resp["wordTokenize"])
        # The platform computes math.log/math.exp correctly rounded. CPython calls the C library's log/exp, which on some
        # systems (this macOS libm included) is off by one ulp for a few inputs; nltk's BLEU is therefore also computed
        # with correctly rounded log/exp (decimal, 80 digits) and the platform must equal that on every vector.
        from nltk.translate import bleu_score
        tok = [(ref.word_tokenize(p["ref"].lower()), ref.word_tokenize(p["sim"].lower())) for p in req["bleu"]]
        libm = [repr(float(ref.sentence_bleu([r], h))) for r, h in tok]
        saved = bleu_score.math
        bleu_score.math = types.SimpleNamespace(log=cr_log, exp=cr_exp, fsum=math.fsum)
        try:
            crv = [repr(float(ref.sentence_bleu([r], h))) for r, h in tok]
        finally:
            bleu_score.math = saved
        n_libm = sum(a == b for a, b in zip(libm, resp["bleu"]))
        compare_lists("V12c", f"nltk 3.9.1 sentence_bleu(word_tokenize(ref.lower()), word_tokenize(sim.lower())) with correctly rounded log/exp, "
                      f"bit-exact (equal to this CPython's libm result: {n_libm}/{len(libm)})", crv, resp["bleu"])
        py_math = {"log": [repr(cr_log(float(x))) for x in req["math"]["log"]], "exp": [repr(cr_exp(float(x))) for x in req["math"]["exp"]],
                   "fsum": [repr(math.fsum(float(x) for x in xs)) for xs in req["math"]["fsum"]]}
        libm_math = {"log": [repr(math.log(float(x))) for x in req["math"]["log"]], "exp": [repr(math.exp(float(x))) for x in req["math"]["exp"]],
                     "fsum": py_math["fsum"]}
        for k in ("log", "exp", "fsum"):
            off = sum(a != b for a, b in zip(libm_math[k], py_math[k]))
            what = (f"math.{k}: platform equals the correctly rounded value (decimal, 80 digits); this CPython's libm differs from it on {off}"
                    if k != "fsum" else "math.fsum: platform port of CPython's msum")
            compare_lists(f"V12-{k}", what, py_math[k], resp["math"][k])

    # P01: character classes ------------------------------------------------------------------------------------------
    import regex as _regex
    tests = {
        "re_s": lambda c: re.match(r"\s", c), "re_w": lambda c: re.match(r"\w", c), "re_d": lambda c: re.match(r"\d", c),
        "regex_w": lambda c: _regex.match(r"\w", c), "regex_ps": lambda c: _regex.match(r"[\p{P}\p{S}]", c),
        "isupper": lambda c: c.isupper(), "islower": lambda c: c.islower(),
    }
    def to_set(ranges):
        out = set()
        for a, b in ranges:
            out.update(range(a, b + 1))
        return out
    assigned = [cp for cp in range(0x110000) if not 0xD800 <= cp <= 0xDFFF and unicodedata.category(chr(cp)) != "Cn"]
    mism = {}
    for k, f in tests.items():
        ts_set = to_set(resp["classes"][k])
        mism[k] = [cp for cp in assigned if bool(f(chr(cp))) != (cp in ts_set) and cp not in UNICODE_VERSION_DIFFS.get(k, ())]
    ts_pr = to_set(resp["classes"]["isprintable_nonascii"])
    mism["isprintable"] = [cp for cp in assigned if cp >= 0x80 and chr(cp).isprintable() != (cp in ts_pr)]
    check("P01", f"Python character classes on the {len(assigned)} code points assigned in this CPython's Unicode {unicodedata.unidata_version} "
          f"(platform JavaScript Unicode {process_unicode(args.node)}; known version differences {sorted(hex(c) for v in UNICODE_VERSION_DIFFS.values() for c in v)} excluded): mismatches "
          + ", ".join(f"{k} {len(v)}" for k, v in mism.items()),
          all(not v for v in mism.values()), {k: [hex(x) for x in v[:12]] for k, v in mism.items() if v})

    print(f"{sum(results)}/{len(results)} checks passed")
    return 0 if all(results) else 1


def py_run_stats(rows):
    """What eval_reproduce.py:577-579,685-693 (93c1474) computes per run and per split, in Python, from the same rows."""
    keys = ["sc", "an", "cf", "sq", "bleu", "rouge"]

    def means(sub):
        out = {}
        for k in keys:
            vals = [r["dims"][k] for r in sub if k in r["dims"]]
            if vals:
                out[k] = sum(vals) / len(vals)
        if all(g in out for g in keys[:4]):
            out["avg"] = sum(out[g] for g in keys[:4]) / 4
        return out

    runs = [{"sessions": sum(r["sampleIdx"] == i for r in rows), "means": means([r for r in rows if r["sampleIdx"] == i])} for i in sorted({r["sampleIdx"] for r in rows})]
    across = {}
    for k in keys + ["avg"]:
        vals = [r["means"][k] for r in runs if k in r["means"]]
        if vals:
            m = sum(vals) / len(vals)
            across[k] = {"mean": m, "sd": math.sqrt(sum((x - m) ** 2 for x in vals) / (len(vals) - 1)) if len(vals) > 1 else None}
    splits = {s: {"sessions": sum(r["split"] == s for r in rows), "means": means([r for r in rows if r["split"] == s])} for s in sorted({r["split"] for r in rows})}
    return {"runs": runs, "across": across, "splits": splits}


def cr_log(x):
    from decimal import Decimal, localcontext
    with localcontext() as ctx:
        ctx.prec = 80
        return float(Decimal(float(x)).ln())  # math.log(Fraction) converts with float() first


def cr_exp(x):
    from decimal import Decimal, localcontext
    with localcontext() as ctx:
        ctx.prec = 80
        return float(Decimal(float(x)).exp())


# Code points whose property differs between CPython's older Unicode (13.0 in 3.9, 14.0 in 3.11) and the JavaScript engine's
# newer Unicode: U+10FC, U+AB69 and U+A7F2-A7F4 (modifier letters) gained Other_Lowercase later. None occurs in the test set.
UNICODE_VERSION_DIFFS = {"islower": {0x10FC, 0xAB69, 0xA7F2, 0xA7F3, 0xA7F4}}


def process_unicode(node):
    try:
        return subprocess.run([node, "-p", "process.versions.unicode"], capture_output=True, text=True).stdout.strip()
    except Exception:
        return "?"


if __name__ == "__main__":
    sys.exit(main())