Why does AI make things up?

The capital of France is Paris.
The capital of France is Paris.
The capital of France is Paris.
The capital of France is Paris.
The capital of France is Paris.
The capital of Japan is Tokyo.
The capital of Japan is Tokyo.
The capital of Japan is Tokyo.
The capital of Italy is Rome.
The capital of Italy is Rome.
The capital of Italy is Rome.
The capital of Spain is Madrid.
The capital of Spain is Madrid.
The capital of Germany is Berlin.
The capital of Germany is Berlin.
The capital of Egypt is Cairo.
The capital of Egypt is Cairo.
The capital of Canada is Ottawa.
The capital of Canada is Ottawa.
The capital of Greece is Athens.
The capital of Norway is Oslo.
The capital of Portugal is Lisbon.
The capital of Austria is Vienna.
The capital of Cuba is Havana.
The capital of Chile is Santiago.
The capital of Poland is Warsaw.
The capital of Turkey is Ankara.
The capital of Kenya is Nairobi.
The capital of Australia is Canberra.
The capital of China is Beijing.
Paris stands on the river Seine.
Tokyo is a very large city.
Rome has many old ruins.
Llamas live in the mountains of Peru.
Many people visit Peru to walk to Machu Picchu.
The ferry to Malta leaves from Sicily.
Laos lies between Thailand and Vietnam.

"""Keep the copy of the corpus and the questions inside ../widget.html equal to the ones here.

    python3 embed_widget.py          rewrite the two data blocks in widget.html
    python3 embed_widget.py --check  exit 1 if widget.html differs (test_widget.py runs the same check)
"""
import json
import re
import sys
from pathlib import Path

from prompts import PROMPTS
from toy_lm import load_corpus

WIDGET = Path(__file__).resolve().parent.parent / "widget.html"
CORPUS_BLOCK = re.compile(r'(<script type="text/plain" id="wmu-corpus">)(.*?)(</script>)', re.S)
PROMPTS_BLOCK = re.compile(r'(<script type="application/json" id="wmu-prompts">)(.*?)(</script>)', re.S)


def embed(html):
    """html with both data blocks rewritten from corpus.txt and prompts.py."""
    corpus = load_corpus()
    if "</script" in corpus.lower() or "<!--" in corpus:
        raise ValueError("the corpus would end or confuse the script element")
    prompts = json.dumps(PROMPTS, ensure_ascii=False, indent=1)
    html = CORPUS_BLOCK.sub(lambda m: m.group(1) + corpus + m.group(3), html, count=1)
    return PROMPTS_BLOCK.sub(lambda m: m.group(1) + "\n" + prompts + "\n" + m.group(3), html, count=1)


if __name__ == "__main__":
    current = WIDGET.read_text(encoding="utf-8")
    updated = embed(current)
    if "--check" in sys.argv:
        sys.exit(0 if updated == current else 1)
    WIDGET.write_text(updated, encoding="utf-8")
    print("widget.html data blocks now match corpus.txt and prompts.py")

"""The three questions the widget offers. None of the three capitals is in corpus.txt.

The true sentence is what the corpus would need to say for the toy to know the answer.
"""
PROMPTS = [
    {"name": "Peru", "prompt": "The capital of Peru is", "truth": "Lima"},
    {"name": "Laos", "prompt": "The capital of Laos is", "truth": "Vientiane"},
    {"name": "Malta", "prompt": "The capital of Malta is", "truth": "Valletta"},
]


def true_sentence(item):
    return f"{item['prompt']} {item['truth']}."

"""Print every number the lesson quotes. Run it from this folder: python3 show_numbers.py"""
from prompts import PROMPTS, true_sentence
from toy_lm import (IDK, answer, lines_of, load_corpus, model_with_extra, next_word_probs, ranked, smallest_k,
                    tokenize, train, with_temperature)

corpus = load_corpus()
lines = lines_of(corpus)
model = train(lines)
peru = PROMPTS[0]
PROMPT, TRUTH, SENTENCE = peru["prompt"], peru["truth"], true_sentence(peru)


def table(probs, n=5):
    return ", ".join(f"{w} {p:.1%}" for w, p in ranked(probs)[:n])


print("== The corpus (corpus.txt)")
capitals = [l for l in lines if l.startswith("The capital of")]
print(f"  {len(lines)} sentences, {sum(len(tokenize(l)) for l in lines)} tokens, {len(model['vocab'])} different words")
print(f"  {len(capitals)} are 'The capital of X is Y.' sentences about {len(set(capitals))} countries")
for country in ("France", "Japan", "Italy"):
    print(f"    The capital of {country} appears {sum(1 for l in lines if country in l and l.startswith('The capital'))} times")
print(f"  words after 'is': {sum(model['singles'][('is',)].values())} in all, {len(model['singles'][('is',)])} different")

print(f"\n== Prompt: {PROMPT!r}   (true word: {TRUTH})")
for item in PROMPTS:
    print(f"  {item['name']}: true word {item['truth']!r} in the corpus: {item['truth'] in model['vocab']}; "
          f"{item['prompt'].split()[-2]!r} in the corpus: {item['prompt'].split()[-2] in model['vocab']}")
probs = next_word_probs(model, PROMPT)
print(f"  probabilities add up to {sum(probs.values()):.12f}")
print(f"  top five: {table(probs)}")
print(f"  {TRUTH}: {probs.get(TRUTH, 0):.1%}")
seen = sum(model["singles"][("is",)].values())
kinds = len(model["singles"][("is",)])
trust = seen / (seen + kinds)
print(f"  by hand for Paris: trust {seen}/({seen}+{kinds}) = {trust:.2f}; "
      f"{trust:.2f} * 5/{seen} + {1 - trust:.2f} / {len(model['vocab'])} = {trust * 5 / seen + (1 - trust) / len(model['vocab']):.4f}")
print(f"  ('Peru', 'is') seen in the corpus: {('Peru', 'is') in model['pairs']}")
print(f"  flat guess 1/{len(model['vocab'])} = {1 / len(model['vocab']):.4f}")

one = model_with_extra(lines, SENTENCE, 1)
seen_is, kinds_is = sum(one["singles"][("is",)].values()), len(one["singles"][("is",)])
trust_is, trust_pair = seen_is / (seen_is + kinds_is), 1 / (1 + 1)
lower = trust_is * 1 / seen_is + (1 - trust_is) / len(one["vocab"])
print(f"  by hand for {TRUTH} after k = 1: 'is' now has {seen_is} followers, {kinds_is} different, trust {seen_is}/({seen_is}+{kinds_is}) = {trust_is:.4f}; "
      f"one-word estimate {trust_is:.4f} * 1/{seen_is} + {1 - trust_is:.4f} / {len(one['vocab'])} = {lower:.4f}; "
      f"('Peru', 'is') has 1 follower, 1 different, trust 1/(1+1) = {trust_pair}; "
      f"{trust_pair} * 1 + {1 - trust_pair} * {lower:.4f} = {trust_pair + (1 - trust_pair) * lower:.4f}")

print(f"\n== Adding k copies of {SENTENCE!r}")
for k in range(0, 9):
    probs = next_word_probs(model_with_extra(lines, SENTENCE, k), PROMPT)
    word, p = ranked(probs)[0]
    print(f"  k={k}: top {word} {p:.1%};  {TRUTH} {probs.get(TRUTH, 0):.1%};  runner-up {ranked(probs)[1][0]} {ranked(probs)[1][1]:.1%}")

print("\n== Smallest k")
for thr in (0.0, 0.5, 0.8, 0.9):
    print(f"  top word is {TRUTH} and has at least {thr:.0%}: k = {smallest_k(lines, PROMPT, SENTENCE, TRUTH, thr)}")
print(f"  other questions, no threshold: " + ", ".join(
    f"{i['name']} k = {smallest_k(lines, i['prompt'], true_sentence(i), i['truth'])}" for i in PROMPTS))

print("\n== A fact the corpus states 5 times")
probs = next_word_probs(model, "The capital of France is")
print(f"  top: {table(probs, 3)}")

print("\n== 'I don't know' below a threshold, k = 0")
probs = next_word_probs(model, PROMPT)
for thr in (0.05, 0.10, 0.11, 0.30, 0.50):
    print(f"  threshold {thr:.0%}: {answer(probs, thr)}")
print(f"  threshold 50%, k = 1: {answer(next_word_probs(model_with_extra(lines, SENTENCE, 1), PROMPT), 0.5)}")
print(f"  threshold 50%, France: {answer(next_word_probs(model, 'The capital of France is'), 0.5)}")

print("\n== Temperature, k = 0")
base = next_word_probs(model, PROMPT)
for t in (0.1, 0.2, 0.3, 0.5, 1.0, 1.5, 2.0):
    scaled = with_temperature(base, t)
    word, p = ranked(scaled)[0]
    print(f"  T={t}: top {word} {p:.1%}; {TRUTH} {scaled.get(TRUTH, 0):.1%}; at a 50% threshold the toy says: {answer(scaled, 0.5)}")

"""A toy language model that only counts which words follow which, using nothing but the standard library.

widget.html runs the same arithmetic in JavaScript; test_widget.py checks that the two agree.

The model reads the last two words of a prompt and asks three questions in turn, from most specific to
least: what followed these two words in the corpus, what followed the last word, and (when nothing at all
has been seen) any word of the corpus at random. The three answers are blended with the Witten-Bell
rule: the more different words have followed a context, the less that context is trusted.
"""
import re
from pathlib import Path

CORPUS = Path(__file__).resolve().parent / "corpus.txt"
START = "<s>"
TOKEN = re.compile(r"[A-Za-z0-9']+|[.,!?]")      # ASCII only, so Python and JavaScript split alike
IDK = "I don't know"


def load_corpus():
    return CORPUS.read_text(encoding="utf-8")


def tokenize(text):
    """Words and the marks . , ! ? as separate tokens. Capital letters count: 'Peru' is not 'peru'."""
    return TOKEN.findall(text)


def lines_of(text):
    """The non-empty lines of a corpus. Each line is one sentence."""
    return [line for line in text.split("\n") if line.strip()]


def train(lines):
    """Count what follows each pair of words and each single word. Every line starts after two START marks."""
    pairs, singles, vocab = {}, {}, set()
    for line in lines:
        words = [START, START] + tokenize(line)
        vocab.update(words[2:])
        for i in range(2, len(words)):
            for table, context in ((pairs, tuple(words[i - 2:i])), (singles, (words[i - 1],))):
                followers = table.setdefault(context, {})
                followers[words[i]] = followers.get(words[i], 0) + 1
    return {"pairs": pairs, "singles": singles, "vocab": sorted(vocab)}


def next_word_probs(model, prompt):
    """Probability of every word of the corpus coming next after the prompt. They add up to 1."""
    words = [START, START] + tokenize(prompt)
    vocab = model["vocab"]
    if not vocab:
        return {}
    probs = {w: 1 / len(vocab) for w in vocab}
    for table, context in ((model["singles"], tuple(words[-1:])), (model["pairs"], tuple(words[-2:]))):
        followers = table.get(context)
        if not followers:
            continue
        seen = sum(followers.values())
        trust = seen / (seen + len(followers))      # Witten-Bell: seen / (seen + kinds of follower)
        probs = {w: trust * followers.get(w, 0) / seen + (1 - trust) * probs[w] for w in vocab}
    return probs


def with_temperature(probs, temperature):
    """Raise every probability to 1/temperature and scale back to total 1. Below 1 it sharpens, above 1 it flattens."""
    powered = {w: p ** (1 / temperature) for w, p in probs.items()}
    total = sum(powered.values())
    return {w: p / total for w, p in powered.items()}


def ranked(probs):
    """(word, probability) from most to least likely. A tie goes to the word that sorts first."""
    return sorted(probs.items(), key=lambda item: (-item[1], item[0]))


def answer(probs, threshold=0.0):
    """The most likely word, or IDK when even that word has a probability below the threshold."""
    if not probs:
        return IDK
    word, p = ranked(probs)[0]
    return word if p >= threshold else IDK


def model_with_extra(lines, true_sentence, k):
    """The corpus with k more copies of one true sentence."""
    return train(lines + [true_sentence] * k)


def smallest_k(lines, prompt, true_sentence, truth, threshold=0.0, temperature=1.0, limit=50):
    """The fewest added copies of the true sentence after which the model answers `truth`, being at least
    `threshold` sure of it. None when `limit` copies are not enough."""
    for k in range(limit + 1):
        probs = with_temperature(next_word_probs(model_with_extra(lines, true_sentence, k), prompt), temperature)
        if answer(probs, threshold) == truth:
            return k
    return None

import pytest

from prompts import PROMPTS, true_sentence
from toy_lm import (IDK, answer, lines_of, load_corpus, model_with_extra, next_word_probs, ranked, smallest_k,
                    tokenize, train, with_temperature)

LINES = lines_of(load_corpus())
PERU = PROMPTS[0]
SENTENCE = true_sentence(PERU)


def probs_at(k, prompt=PERU["prompt"]):
    return next_word_probs(model_with_extra(LINES, SENTENCE, k), prompt)


@pytest.mark.parametrize("prompt", ["The capital of Peru is", "The capital of France is", "", "Peru", "zzz qqq", "is", "The capital of"])
@pytest.mark.parametrize("k", [0, 1, 7])
def test_probabilities_add_up_to_one(prompt, k):
    """Guards a backoff that leaks or double-counts probability (an unseen context, a one-word prompt, an empty one)."""
    probs = probs_at(k, prompt)
    assert sum(probs.values()) == pytest.approx(1, abs=1e-12)
    assert all(p >= 0 for p in probs.values())


@pytest.mark.parametrize("temperature", [0.1, 0.5, 1.0, 2.0])
def test_temperature_keeps_the_total_at_one(temperature):
    assert sum(with_temperature(probs_at(0), temperature).values()) == pytest.approx(1, abs=1e-12)


def test_thin_corpus_gives_a_fluent_wrong_answer():
    """The corpus never says what the capital of Peru, Laos or Malta is, yet the toy completes the sentence with
    a capital it has read about, and gives the true word no probability at all."""
    model = train(LINES)
    for item in PROMPTS:
        probs = next_word_probs(model, item["prompt"])
        top, p = ranked(probs)[0]
        assert top == "Paris" and top != item["truth"]
        assert item["truth"] not in model["vocab"] and probs.get(item["truth"], 0) == 0
        assert p == pytest.approx(0.1049, abs=5e-5)
        assert answer(probs) == "Paris"


def test_the_held_out_facts_are_not_in_the_corpus_in_any_wording():
    """Guards a corpus edit that quietly teaches the answer, which would make the demonstration dishonest."""
    text = load_corpus()
    for item in PROMPTS:
        assert item["truth"] not in text
        assert f"{item['name']} is" not in text            # no pair like ('Peru', 'is') to match the prompt
    assert ("Peru", "is") not in train(LINES)["pairs"]


def test_top_probability_matches_a_hand_calculation():
    """'is' is followed by 31 words, 19 of them different (30 capitals in 18 kinds, plus 'a' once). The trust in that
    context is 31/(31+19) = 0.62. Paris follows 'is' 5 times and the vocabulary has 77 words."""
    expected = 0.62 * 5 / 31 + 0.38 / 77
    assert next_word_probs(train(LINES), PERU["prompt"])["Paris"] == pytest.approx(expected, abs=1e-12)


def test_one_true_sentence_hand_calculation():
    """After one copy the pair ('Peru', 'is') was seen once, so its trust is 1/(1+1) = 0.5. The word 'is' now has 32
    followers, 20 of them different, so its trust is 32/52; Lima follows it once, and the vocabulary has 78 words."""
    single = (32 / 52) * (1 / 32) + (20 / 52) * (1 / 78)
    assert probs_at(1)["Lima"] == pytest.approx(0.5 + 0.5 * single, abs=1e-12)


def test_true_examples_take_over_and_the_code_finds_the_smallest_k():
    assert [ranked(probs_at(k))[0][0] for k in range(0, 4)] == ["Paris", "Lima", "Lima", "Lima"]
    assert smallest_k(LINES, PERU["prompt"], SENTENCE, PERU["truth"]) == 1
    for item in PROMPTS:
        assert smallest_k(LINES, item["prompt"], true_sentence(item), item["truth"]) == 1
    # Lima's share climbs with every copy
    shares = [probs_at(k)["Lima"] for k in range(1, 12)]
    assert shares == sorted(shares) and shares[0] == pytest.approx(0.5121, abs=5e-5)


@pytest.mark.parametrize("threshold, k", [(0.5, 1), (0.8, 4), (0.9, 8)])
def test_smallest_k_for_a_confidence_target(threshold, k):
    """Guards an off-by-one in the search: k-1 copies must fall short and k copies must reach the target."""
    assert smallest_k(LINES, PERU["prompt"], SENTENCE, PERU["truth"], threshold) == k
    assert ranked(probs_at(k))[0][1] >= threshold > ranked(probs_at(k - 1))[0][1]


def test_smallest_k_gives_up_at_the_limit():
    """Guards an off-by-one at the limit: 4 copies are needed at the 80% bar, so limit=4 finds them and limit=3 does not."""
    assert smallest_k(LINES, PERU["prompt"], SENTENCE, PERU["truth"], 0.8, limit=4) == 4
    assert smallest_k(LINES, PERU["prompt"], SENTENCE, PERU["truth"], 0.8, limit=3) is None


def test_abstaining_when_the_top_word_is_below_the_threshold():
    probs = probs_at(0)
    assert answer(probs, 0.5) == IDK
    assert answer(probs, 0.11) == IDK
    assert answer(probs, 0.10) == "Paris"                  # 10.5% clears a 10% bar: a low bar does not help
    assert answer(probs_at(1), 0.5) == "Lima"              # the same bar lets the right answer through
    assert answer(next_word_probs(train(LINES), "The capital of France is"), 0.5) == "Paris"   # and a well-read fact


def test_the_toy_gets_a_fact_it_has_read_often_right():
    """The control: the corpus states France's capital five times, so the same code answers it with 85%."""
    probs = next_word_probs(train(LINES), "The capital of France is")
    assert ranked(probs)[0][0] == "Paris" and ranked(probs)[0][1] == pytest.approx(0.8508, abs=5e-5)


def test_lower_temperature_makes_the_wrong_answer_surer_not_true():
    base = probs_at(0)
    shares = []
    for t in (2.0, 1.0, 0.5, 0.3, 0.1):
        scaled = with_temperature(base, t)
        top, p = ranked(scaled)[0]
        assert top == "Paris" and scaled.get("Lima", 0) == 0
        shares.append(p)
    assert shares == sorted(shares) and shares[-1] > 0.98
    # a 50% bar that makes the toy say "I don't know" at temperature 1 lets Paris through at 0.3
    assert answer(with_temperature(base, 1.0), 0.5) == IDK
    assert answer(with_temperature(base, 0.3), 0.5) == "Paris"


def test_temperature_one_changes_nothing():
    base = probs_at(0)
    scaled = with_temperature(base, 1.0)
    assert all(scaled[w] == pytest.approx(base[w], abs=1e-15) for w in base)


def test_a_tie_goes_to_the_word_that_sorts_first_whatever_the_corpus_order():
    """Guards the vocabulary order that ties rely on: it is sorted, so equal words rank alphabetically in either corpus order."""
    for lines in (["x b", "x a"], ["x a", "x b"]):
        probs = next_word_probs(train(lines), "x")
        assert probs["a"] == probs["b"]
        assert [w for w, _ in ranked(probs)] == ["a", "b", "x"]


def test_an_empty_corpus_has_nothing_to_say():
    assert next_word_probs(train([]), "anything") == {}
    assert answer({}, 0.0) == IDK


def test_tokenizer_keeps_ascii_words_and_marks_only():
    assert tokenize("Peru's capital, Lima! Café 2026?") == ["Peru's", "capital", ",", "Lima", "!", "Caf", "2026", "?"]
    assert lines_of("a\n\n  \nb\n") == ["a", "b"]

"""Guards for ../widget.html: its data blocks, its size, and that its model gives the numbers toy_lm.py gives."""
import json
import re
import subprocess

import embed_widget
from embed_widget import WIDGET, embed
from prompts import PROMPTS
from toy_lm import (answer, lines_of, load_corpus, model_with_extra, next_word_probs, ranked, smallest_k, tokenize, train,
                    with_temperature)

HTML = WIDGET.read_text(encoding="utf-8")

# Reads the model out of the widget between its marker comments, runs it with Node, and compares with the
# numbers Python wrote into the fixture: the same words, the same order, probabilities within 1e-12.
PARITY = r"""
import fs from 'node:fs';
const [fixture, widget] = process.argv.slice(2);
const fx = JSON.parse(fs.readFileSync(fixture, 'utf8'));
const html = fs.readFileSync(widget, 'utf8');
const js = html.slice(html.indexOf('/* ---------- the toy model'), html.indexOf('/* ---------- state'));
const api = new Function("'use strict';" + js + '; return {tokenize, linesOf, train, nextWordProbs, withTemperature, ranked, answer, smallestK};')();
const bad = [];
let checks = 0;
fx.tokens.forEach((t) => { checks++; if (JSON.stringify(api.tokenize(t[0])) !== JSON.stringify(t[1])) bad.push(['tokenize', t[0]]); });
fx.cases.forEach((c, i) => {
  const lines = api.linesOf(c.corpus);
  const model = api.train(lines);
  checks++;
  if (JSON.stringify(model.vocab) !== JSON.stringify(c.vocab)) { bad.push(['vocab', i]); return; }
  const probs = api.withTemperature(api.nextWordProbs(api.train(lines.concat(Array(c.k).fill(c.sentence))), c.prompt), c.temp);
  checks++;
  if (probs.size !== c.probs.length) { bad.push(['size', i]); return; }
  c.probs.forEach(([w, p]) => { if (!(Math.abs(probs.get(w) - p) <= 1e-12)) bad.push(['prob', i, w, probs.get(w), p]); });
  checks++;
  if (JSON.stringify(api.ranked(probs).map((e) => e[0])) !== JSON.stringify(c.ranked)) bad.push(['order', i]);
  checks++;
  if (api.answer(probs, c.threshold) !== c.answer) bad.push(['answer', i, api.answer(probs, c.threshold), c.answer]);
  if (c.truth !== null) {
    checks++;
    const k = api.smallestK(lines, c.prompt, c.sentence, c.truth, c.threshold, c.temp, 12);
    if (k !== c.smallest) bad.push(['smallest k', i, k, c.smallest]);
  }
});
console.log(JSON.stringify({ checks, bad: bad.slice(0, 5) }));
process.exit(bad.length ? 1 : 0);
"""

# Corpora a reader might paste: words that are also JavaScript object keys, non-ASCII text, an empty corpus.
CORPORA = {
    "lesson": load_corpus(),
    "odd words": "constructor is __proto__ .\nhasOwnProperty is toString !\nThe café is open, isn't it?\nNo. 2026 is 2026.\n\n\n",
    "one line": "The capital of Peru is Lima.\n",
    "empty": "",
    "one word": "x\n",                     # its only word has probability exactly 1, the edge of the answer-or-abstain check
}
PROMPT_CASES = [(p["prompt"], p["truth"]) for p in PROMPTS] + [
    ("The capital of France is", "Paris"), ("", "The"), ("constructor", "is"), ("the capital of peru is", "Lima"),
    ("is", "a"), ("zzz qqq", "Lima"), ("Café is", "open"),
]
TOKEN_CASES = ["Peru's capital, Lima! Café 2026?", "", "  \t ", "a_b-c", "éè", "end.", "x" * 40, "A.B,C!D?E"]


def test_widget_data_blocks_match_corpus_and_prompts():
    """Guards a corpus or question edit that was not copied into the widget (run embed_widget.py)."""
    assert embed(HTML) == HTML


def test_embed_rewrites_a_widget_whose_data_blocks_drifted():
    """Guards embed() skipping a block: a hand-edited corpus line or question must be put back."""
    drifted = HTML.replace("Tokyo.", "Kyoto.", 1).replace('"Lima"', '"Lime"', 1)
    assert drifted != HTML and embed(drifted) == HTML


def test_every_element_the_script_looks_up_exists():
    """Guards a widget that never starts: the parity harness runs only the model, so a misspelt id passed here
    and threw in the browser."""
    script = HTML[HTML.index("<script>"):]
    lookups = set(re.findall(r"\$\('([\w-]+)'\)", script))
    assert lookups and all(f'id="{i}"' in HTML for i in lookups), sorted(i for i in lookups if f'id="{i}"' not in HTML)


def test_corpus_is_safe_to_embed_in_a_script_element():
    corpus = load_corpus()
    assert "\r" not in corpus and "</script" not in corpus.lower() and "<!--" not in corpus


def test_widget_is_small_and_asks_for_nothing_online():
    assert len(HTML.encode("utf-8")) < 120 * 1024
    page = embed_widget.CORPUS_BLOCK.sub("", embed_widget.PROMPTS_BLOCK.sub("", HTML))
    for banned in ("http://", "https://", "fetch(", "XMLHttpRequest", "WebSocket", "sendBeacon", "import("):
        assert banned not in page, banned
    assert " src=" not in page and "@import" not in page


def test_widget_model_gives_the_numbers_of_the_python_code(tmp_path):
    """Guards the widget's JavaScript drifting from toy_lm.py: the lesson prints Python's numbers and the
    reader sees the widget's, so the two must agree for corpora with odd words, unseen prompts and every setting."""
    cases = []
    for name, corpus in CORPORA.items():
        lines = lines_of(corpus)
        vocab = train(lines)["vocab"]
        for prompt, truth in PROMPT_CASES:
            sentence = f"{prompt} {truth}."
            for k in (0, 1, 3):
                for temp in (0.3, 1.0, 1.7):
                    probs = with_temperature(next_word_probs(model_with_extra(lines, sentence, k), prompt), temp)
                    for threshold in (0.0, 0.5, 1.0):
                        cases.append({"name": name, "corpus": corpus, "prompt": prompt, "sentence": sentence, "truth": truth,
                                      "k": k, "temp": temp, "threshold": threshold, "vocab": vocab,
                                      "probs": [[w, p] for w, p in probs.items()], "ranked": [w for w, _ in ranked(probs)],
                                      "answer": answer(probs, threshold),
                                      "smallest": smallest_k(lines, prompt, sentence, truth, threshold, temp, limit=12)})
    fixture = {"tokens": [[t, tokenize(t)] for t in TOKEN_CASES], "cases": cases}
    (tmp_path / "fixture.json").write_text(json.dumps(fixture), encoding="utf-8")
    (tmp_path / "parity.mjs").write_text(PARITY, encoding="utf-8")
    run = subprocess.run(["node", str(tmp_path / "parity.mjs"), str(tmp_path / "fixture.json"), str(WIDGET)],
                         capture_output=True, text=True, timeout=120)
    assert run.returncode == 0, run.stdout + run.stderr
    result = json.loads(run.stdout)
    assert result["bad"] == [] and result["checks"] > 1000
    assert any(c["smallest"] == 1 for c in cases) and any(c["answer"] == "I don't know" for c in cases)