# linguabase-spelling-game-prepare_words.py — build the level library
# for the anagram ladder game: three easy 4/5/6-letter words, then a
# harder 7-letter answer from the same tiles
# (worked example from linguabase.org: one pool of 7 tiles yields three
#  easy 4/5/6-letter words plus the 7-letter answer).
#
# NEEDS  the five Linguabase core tables (parquet), CC0, in the working
#        directory — download from linguabase.org/download.html:
#          vocabulary.core.parquet       word, core_rank (1 = most familiar)
#          scores_labels.core.parquet    content_rating, sensitivity
#          word_families.core.parquet    word -> relatives
#          clues.core.parquet            26 clue columns per word
#          definitions.core.parquet      one readable paragraph per word
#        Note: the shipped files are NOT rank-sorted; this script sorts
#        where order matters.
# RUN    python3 prepare_words.py            (requires pandas)
# OUT    levels.tsv    one row per level: pool, easy words, finale, clues
#        display.json  word -> meaning for every findable word
# TUNE   EASY_CAP and FINALE_CAP below — the two dials; at 50,000/70,000
#        this yields 1,510 levels (measured). Every downstream count moves
#        with them.
import pandas as pd, json
from collections import Counter

EASY_CAP, FINALE_CAP = 50_000, 70_000        # the two numbers you edit; all else follows

v   = pd.read_parquet('vocabulary.core.parquet')
s   = pd.read_parquet('scores_labels.core.parquet').set_index('word')
fam = pd.read_parquet('word_families.core.parquet').set_index('word').relatives
clu = pd.read_parquet('clues.core.parquet').set_index('word')
dfn = pd.read_parquet('definitions.core.parquet').set_index('word').definition

clean = v[~v.is_mwe & v.word.str.fullmatch('[a-z]{4,7}')          # 1 · the two lists
          & v.word.map(s.content_rating).isin(['G', 'PG'])
          & (v.word.map(s.sensitivity) > 3)].copy()
rank = dict(zip(clean.word, clean.core_rank))
kin  = lambda w: set(fam.get(w, []))
clean = clean[[not any(w in (b+'s', b+'es') and rank.get(b, 9e9) < n   # plurals only:
                       for b in kin(w))                               # ANSWERS goes,
               for w, n in zip(clean.word, clean.core_rank)]]         # ANSWER stays
finales = clean[(clean.word.str.len() == 7)
                & (clean.core_rank <= FINALE_CAP)].sort_values('core_rank')

sig     = lambda w: ''.join(sorted(w))
fits    = lambda w, pool: not Counter(w) - Counter(pool)
overlap = lambda a, b: sum((Counter(a) & Counter(b)).values())

def test(fin):                                   # 2 · the pool is the answer, sorted
    pool = sig(fin)                              # 3 · the validity test
    sp   = [w for w in clean.word if w != fin and fits(w, pool)]
    easy = {}
    for L in (4, 5, 6):                          # most familiar easy word of each length
        c = [w for w in sp if len(w) == L and rank[w] <= EASY_CAP
             and w not in kin(fin) and w not in fin]
        if not c:  return None                   # a length missing: throw it back
        easy[L] = min(c, key=rank.__getitem__)
    if set(pool) - set(''.join(easy.values())):  return None      # uncovered tile
    if any(len(w) == 7 and rank[w] < 2 * rank[fin] for w in sp):
        return None                              # a rival close enough to reach
    return [easy[4], easy[5], easy[6]], sp       # sp minus the easy words = the bonuses

def fresh(pool, history):                        # 4 · the overlap filter
    for age, old in enumerate(reversed(history)):
        n = overlap(pool, old)
        if n >= 6 or (n == 5 and age < 100) or (n == 4 and age < 10):
            return False
    return True

levels, display, history = [], {}, []
for fin in finales.word:
    got = test(fin)
    if not got or not fresh(sig(fin), history):  continue
    easy, sp = got
    lvl  = len(levels) + 1
    tier = 'expert' if lvl > 500 else 'learner'  # 5 · later levels take the expert angle
    main = clu.at[fin, 'clue_EXP' if tier == 'expert' else 'clue_LRN']
    hint = clu.at[fin, 'clue_LRN' if tier == 'expert' else 'clue_DEF']
    if any(r in main + hint for r in kin(fin)):  continue   # clue contains a relative of the answer
    history.append(sig(fin))
    levels.append((lvl, sig(fin).upper(), '|'.join(easy), fin, main, hint,
                   rank[fin], max(rank[w] for w in easy), tier))
    for w in [fin] + sp:
        display[w] = dfn[w]

cols = 'level_no pool easy_words finale clue_main clue_hint finale_rank easy_deepest clue_tier'.split()
pd.DataFrame(levels, columns=cols).to_csv('levels.tsv', sep='\t', index=False)
json.dump(display, open('display.json', 'w'), indent=1)
print(f'{len(levels):,} levels')