"""What a model receives for a word, and what counting its letters would need.

A word goes in as a few chunks, and each chunk is one id, a plain number. chunks() shows both sides:
the id the model receives and the spelling that id stands for. Counting a letter means looking inside
the chunks, and the ids alone do not say what is inside them. Words here are ASCII, so one byte is
one letter and a chunk's spelling is always readable.

The tokenizer is the toy of lesson 1 (bpe.py, trained on corpus.txt). The widget in ../widget.html
runs the same merges in JavaScript.
"""
from bpe import encode, load_corpus, train, vocab_bytes

DEFAULT_WORD = "strawberry"
DEFAULT_LETTER = "r"
EXAMPLES = {"strawberry": "r", "banana": "a", "Mississippi": "s", "tokenizer": "e", "unbelievable": "e"}   # word: the letter to count
MAX_WORD = 12        # letters the widget accepts; spelled out, 12 letters take 23 columns on screen
SEPARATOR = "-"      # what "spell it out" puts between the letters


def toy_merges():
    """The merges lesson 1 learns from corpus.txt: 512 of them, so ids 0 to 767."""
    return train(load_corpus())


def chunks(text, merges):
    """text as the model receives it: a list of (id, spelling), one pair per chunk. ASCII text only."""
    if not text.isascii():
        raise ValueError("chunks() reads ASCII text, so that one byte is one letter")
    spelling = vocab_bytes(merges)
    return [(i, spelling[i].decode("ascii")) for i in encode(text, merges)]


def count_inside(spelling, letter):
    """How many times the letter is in the spelling, capital or small."""
    return spelling.lower().count(letter.lower())


def count_from_chunks(text, letter, merges):
    """The count you get by looking inside every chunk and adding up. Doing this is what a model must
    learn: the chunk's spelling lives in the vocabulary table, not in its id."""
    return sum(count_inside(spelling, letter) for _, spelling in chunks(text, merges))


def count_ids_that_are_the_letter(text, letter, merges):
    """The count you get by reading only the ids and counting those that are the letter on its own
    (ids 0 to 255 are single bytes, so 114 is r). Letters inside bigger chunks are missed."""
    wanted = {ord(letter.lower()), ord(letter.upper())}
    return sum(1 for i, _ in chunks(text, merges) if i in wanted)


def spelled_out(word):
    """The word with a hyphen between its letters, the way a person spells it out."""
    return SEPARATOR.join(word)


def share_in_longer_chunks(words, merges):
    """The share of all letters of the words that sit in a chunk of two or more letters."""
    letters = hidden = 0
    for word in words:
        letters += len(word)
        hidden += sum(len(spelling) for _, spelling in chunks(word, merges) if len(spelling) > 1)
    return hidden / letters