"""Print the numbers that lesson.md quotes from the toy tokenizer. Run it from this folder:
python3 show_numbers.py
"""
import re

from bpe import NUM_MERGES, load_corpus
from letters import (DEFAULT_LETTER, DEFAULT_WORD, EXAMPLES, chunks, count_from_chunks, count_ids_that_are_the_letter,
                     count_inside, share_in_longer_chunks, spelled_out, toy_merges)

merges = toy_merges()
print(f"== The toy: 256 bytes + {len(merges)} merges = {256 + len(merges)} ids (NUM_MERGES = {NUM_MERGES})")

print("\n== Examples: letters, chunks, ids")
for word in EXAMPLES:
    got = chunks(word, merges)
    print(f"  {word:14}{len(word):>3} letters {len(got):>3} chunks  {' | '.join(s for _, s in got)}   ids {[i for i, _ in got]}")

word, letter = DEFAULT_WORD, DEFAULT_LETTER
got = chunks(word, merges)
holding = [s for _, s in got if count_inside(s, letter) and len(s) > 1]
print(f"\n== {word!r} and the letter {letter!r}")
print(f"  true count {count_inside(word, letter)}, adding up the chunks' spellings {count_from_chunks(word, letter, merges)},")
print(f"  counting ids that are the letter itself {count_ids_that_are_the_letter(word, letter, merges)}; chunks holding it with other letters: {holding}")

print("\n== Same letters, other writing")
for text in (word, word.capitalize(), word.upper(), " " + word):
    got = chunks(text, merges)
    print(f"  {text!r:16}{len(got):>3} chunks  {' | '.join(s.replace(' ', '_') for _, s in got)}   ids {[i for i, _ in got]}")

print("\n== Spelled out with hyphens")
text = spelled_out(word)
got = chunks(text, merges)
print(f"  {text!r}: {len(got)} chunks, most letters in one chunk {max(sum(c.isalpha() for c in s) for _, s in got)}")
print(f"  ids {[i for i, _ in got]}")

print("\n== Letters in chunks of two or more letters, on the toy's own training text")
words = re.findall(r"[A-Za-z]+", load_corpus())
n_chunks = sum(len(chunks(w, merges)) for w in words)
letters = sum(len(w) for w in words)
print(f"  {len(words):,} words, {letters:,} letters, {n_chunks:,} chunks: {letters / n_chunks:.2f} letters per chunk,"
      f" {100 * share_in_longer_chunks(words, merges):.1f}% of letters in a chunk of 2 or more")