import json, subprocess, sys, collections
repo=sys.argv[1]
def show(p,c='624efc5'): return subprocess.run(['git','-C',repo,'show',f'{c}:{p}'],capture_output=True,check=True).stdout
H=json.loads(show('data/test/held_out_plots.json')); T=json.loads(show('data/test/test_set.json'))
print('top keys',list(H.keys()))
books=[t['book'] for t in T]; bs=set(books)
allb=set()
for k,v in H.items():
    print(k,type(v).__name__,len(v))
    if isinstance(v,dict):
        print(' n keys',len(v),' sample types',collections.Counter(type(x).__name__ for x in v.values()))
        allb|=set(v.keys())
        print(' overlap',len(set(v.keys())&bs),' scenes',sum(b in v for b in books))
        for b in set(v)&bs:
            x=v[b]; L=len(x); ips=[p.get('i_p') for p in x] if isinstance(x,list) and x and isinstance(x[0],dict) else None
            sc=sorted(t['plot']['i_p'] for t in T if t['book']==b)
            print(' overlap book entry',type(x).__name__,L,'heldout i_p range',(min(ips),max(ips)) if ips and ips[0] is not None else None,'scene i_p',sc,'scene in heldout',sum(i in (ips or []) for i in sc))
    elif isinstance(v,list):
        print(' item',type(v[0]).__name__, list(v[0].keys())[:10] if isinstance(v[0],dict) else '')
        bb={x.get('book') for x in v if isinstance(x,dict)}; print(' books',len(bb),' overlap',len(bb&bs))
print('all books',len(allb))