# python3 -I 01_data.py <repo>  -- facts about data/test/test_set.json; prints counts only, never the text.
import sys, json, hashlib, subprocess, collections
repo = sys.argv[1]
def show(c, p):
    return subprocess.run(['git', '-C', repo, 'show', f'{c}:{p}'], capture_output=True, check=True).stdout
raw = show('624efc5', 'data/test/test_set.json')
print('test_set sha256', hashlib.sha256(raw).hexdigest(), 'bytes', len(raw))
ds = json.loads(raw)
print('n', len(ds))
print('history', subprocess.run(['git','-C',repo,'log','--format=%h %ad %s','--date=short','--','data/test/test_set.json'],capture_output=True,text=True).stdout.strip())
kc = collections.Counter(); 
print('top-level keys', sorted(ds[0].keys()))
tags = collections.Counter(d.get('tag') for d in ds)
print('tag', tags, 'first100', collections.Counter(d.get('tag') for d in ds[:100]))
print('books', len(set(d['book'] for d in ds)))
kcf = collections.Counter(); th_null=0; th_empty=0; th_nonstr=0; mot=0
for d in ds:
    for c in d['key_characters']:
        for k in c: kcf[k]+=1
        if 'motivation' in c: mot+=1
        if 'thought' in c:
            if c['thought'] is None: th_null+=1
            elif c['thought']=='' : th_empty+=1
            elif not isinstance(c['thought'],str): th_nonstr+=1
print('key_characters fields', kcf, 'thought null', th_null, 'empty', th_empty, 'motivation', mot)
pk=collections.Counter(); desc_null=0
for d in ds:
    for c in d['plot']['key_characters']:
        for k in c: pk[k]+=1
        if 'description' in c and not isinstance(c['description'],str): desc_null+=1
print('plot.key_characters fields', pk, 'nonstr description', desc_null)
# structural facts used by the speaker loop
major_not_speaking=[]; env_in_speaking=0; env_first=0; major_with_env=0; dup_speaking=0; dup_major=0; no_major=0
speak_no_agentname=0; no_profile=0; spk_lens=collections.Counter(); first_not_dialogue=0; dlg_lens=[]; only_env=0; nsp_named=0; prefix_ref=0
same_name_kc=0; dlg_speaker_missing=0; plotdup=0; profile_only_self=0
for i,d in enumerate(ds):
    sp=list(d['speaking_characters_w_env']); mj=d['major_characters']
    if 'Environment' in sp: env_in_speaking+=1
    else: sp.append('Environment')
    if sp[0]=='Environment': env_first+=1
    if 'Environment' in mj: major_with_env+=1
    if len(set(sp))!=len(sp): dup_speaking+=1
    if len(set(mj))!=len(mj): dup_major+=1
    if not mj: no_major+=1
    miss=[m for m in mj if m not in sp]
    if miss: major_not_speaking.append((i,len(miss)))
    if 'NSP' in sp: nsp_named+=1
    spk_lens[len([x for x in sp if x!='Environment'])]+=1
    dl=d['dialogues']; dlg_lens.append(len(dl))
    if dl and dl[0]['character']!=sp[0]: first_not_dialogue+=1
    if any(x['character'] not in sp for x in dl): dlg_speaker_missing+=1
    names=[c.get('name') for c in d['plot']['key_characters']]
    if len(set(names))!=len(names): plotdup+=1
    names=[c.get('name') for c in d['key_characters']]
    if len(set(names))!=len(names): same_name_kc+=1
print('Environment in speaking', env_in_speaking, 'speaking[0]==Environment', env_first, 'Environment in major', major_with_env)
print('duplicates speaking', dup_speaking, 'major', dup_major, 'empty major', no_major, 'NSP named', nsp_named)
print('samples with major_characters not in speaking:', len(major_not_speaking), major_not_speaking[:30])
print('speaking count dist', sorted(spk_lens.items()))
print('dialogue len min/mean/max', min(dlg_lens), sum(dlg_lens)/len(dlg_lens), max(dlg_lens))
print('first dialogue speaker != speaking[0]', first_not_dialogue, 'dialogue speaker not in speaking', dlg_speaker_missing)
print('plot key dup', plotdup, 'kc dup', same_name_kc)
# test_data files
for f in ['test_data/test_circumstance_id.json','test_data/test_circumstance_ood.json']:
    b=show('624efc5',f); j=json.loads(b); print(f, 'n', len(j), 'sha', hashlib.sha256(b).hexdigest()[:16], 'keys', sorted(j[0].keys())[:20] if isinstance(j,list) else list(j)[:5])
ids=json.loads(show('624efc5','test_data/test_circumstance_id.json')); ood=json.loads(show('624efc5','test_data/test_circumstance_ood.json'))
key=lambda d:(d['book'],d['plot']['i_p'],d['i_c'])
print('id+ood order equal to test_set', [key(d) for d in ids+ood]==[key(d) for d in ds], 'set equal', set(map(key,ids+ood))==set(map(key,ds)))
same=sum(1 for a,b in zip(ids+ood,ds) if a==b); print('entries identical', same)