# python3 -I 02_exp.py <repo> <outdir>  -- recomputes the published evaluation file from its own critiques; counts only.
import sys, json, subprocess, re, collections, os
repo, out = sys.argv[1], sys.argv[2]
def show(c, p): return subprocess.run(['git','-C',repo,'show',f'{c}:{p}'],capture_output=True,check=True).stdout
ev=json.loads(show('624efc5','exp/evaluation/test_set_gpt-4o.json'))
print('eval keys',list(ev.keys()),'scores',ev['scores'])
cases=ev['cases']; print('n cases',len(cases))
DIM=['Storyline Consistency','Anthropomorphism','Character Fidelity','Storyline Quality']
bad=0; sevtypes=collections.Counter(); clipped=collections.Counter(); perdim={d:[] for d in DIM}; bl=[]; rl=[]; rounds=[]; casebad=0; minmax=[1e9,-1e9]
ndet=0; casescore=[]
for k,c in cases.items():
    R=len([m for m in c['simulation'] if m['role']!='Environment']); rounds.append(R)
    for d in DIM:
        cr=c['critique'][d]; S=0
        for f in cr['flaws']:
            sevtypes[type(f['severity']).__name__]+=1
            if isinstance(f['severity'],int): S+=f['severity']
        sc=max(0,min(100-(S-0.3*R)*5,100))
        if sc!=cr['score']: bad+=1
        un=100-(S-0.3*R)*5
        if un>100: clipped['>100']+=1
        if un<0: clipped['<0']+=1
        perdim[d].append(cr['score'])
    cs=sum(c['critique'][d]['score'] for d in DIM)/4
    if cs!=c['score']: casebad+=1
    casescore.append(cs)
    bl.append(c['critique']['bleu']); rl.append(c['critique']['rouge_l'])
print('dimension scores not reproduced by the formula:',bad,'case mean mismatches',casebad)
print('severity types',sevtypes,'clip events',clipped)
avg={d:sum(v)/len(v) for d,v in perdim.items()}
A=sum(avg.values())/4
print('recomputed',avg,'avg',A,'bleu',sum(bl)/len(bl),'rouge',sum(rl)/len(rl))
print('match published scores:',all(abs(ev['scores'][d]-avg[d])<1e-12 for d in DIM),abs(ev['scores']['avg']-A)<1e-12,abs(ev['scores']['bleu']-sum(bl)/len(bl))<1e-12,abs(ev['scores']['rouge_l']-sum(rl)/len(rl))<1e-12)
print('rounds R min/mean/max',min(rounds),sum(rounds)/len(rounds),max(rounds))
# simulation file
for name in ['exp/simulation/test_set_gpt-4o.json','exp/simulation/test_set_gpt-4o_rag=expr3_conv1.json','exp/simulation/test_set_gpt-4o-mini_rag=expr3_conv1.json']:
    sim=json.loads(show('624efc5',name))
    print(name,'n',len(sim),'keys',list(sim[0].keys()))
    stats=collections.Counter(); lines=[]; ends=collections.Counter(); nsp_vals=0
    for r in sim:
        c=r['circumstance']; sp=list(c['speaking_characters_w_env']); 
        if 'Environment' not in sp: sp.append('Environment')
        recs=r['simulation']; 
        stats['first_is_sp0']+= recs[0]['role']==sp[0]
        # alternating?
        speech=[m for m in recs if m['role']!='NSP']; lines.append(len(speech))
        pref=sum(1 for m in speech if any((l.strip().startswith(m['role']+':') or l.strip().startswith(m['role']+'：')) for l in m['content'].split('\n')))
        stats['speech_total']+=len(speech); stats['speech_with_own_prefix']+=pref
        stats['speech_start_prefix']+=sum(1 for m in speech if m['content'].startswith(m['role']+':'))
        for i,m in enumerate(recs):
            if m['role']!='NSP': continue
            nsp_vals+=1
            nxt=recs[i+1]['role'] if i+1<len(recs) else None; cur=recs[i-1]['role']
            v=m['content']
            if v=='<END CHAT>':
                if nxt is None: stats['end_ok']+=1
                else: stats['end_before_round5_fallback']+=1
            elif v in sp and v!=cur:
                stats['valid']+=1; stats['valid_follow_ok']+= (nxt==v or nxt is None)
            else:
                stats['invalid']+=1
                if v==cur: stats['invalid_same_as_current']+=1
                elif v=='random': stats['invalid_random']+=1
                elif v=='': stats['invalid_empty']+=1
                else: stats['invalid_other']+=1
                if nxt is None: stats['invalid_last']+=1
            if ':' in v: stats['nsp_has_colon']+=1
        stats['ended_by_token']+= recs[-1]['role']=='NSP' and recs[-1]['content']=='<END CHAT>'
        stats['reached_20']+= len(speech)==20
        if len(speech)<6: stats['lt6']+=1
    print(' lines min/mean/max',min(lines),sum(lines)/len(lines),max(lines),dict(stats),'nsp calls',nsp_vals)
    # distinct inner thoughts rate in actor lines
    a=[m for r in sim for m in r['simulation'] if m['role'] not in('NSP','Environment')]
    print(' actor lines',len(a),'with [..]',sum(1 for m in a if re.search(r'\[.*?\]',m['content'])),'with (..)',sum(1 for m in a if '(' in m['content']))
# dump JSON for the TS replay (outside the repo, not published)
os.makedirs(out,exist_ok=True)
sim=json.loads(show('624efc5','exp/simulation/test_set_gpt-4o.json'))
json.dump(sim,open(out+'/sim.json','w'))
json.dump(ev,open(out+'/eval.json','w'))
ts=json.loads(show('624efc5','data/test/test_set.json'))
json.dump(ts,open(out+'/test_set.json','w'))
# compare circumstance in sim with test_set
key=lambda d:(d['book'],d['plot']['i_p'],d['i_c'])
K={key(d):d for d in ts}
miss=0; diffkeys=collections.Counter(); inv=0
for r in sim:
    c=r['circumstance']; d=K.get(key(c))
    if d is None: miss+=1; continue
    for k in set(c)|set(d):
        if c.get(k)!=d.get(k): diffkeys[k]+=1
print('sim circumstances vs test_set: missing',miss,'differing fields',diffkeys)