Why does the same question give different answers?

"""Keep the rows and the settings inside ../widget.html equal to the ones here.

    python3 embed_widget.py          rewrite the data block in widget.html
    python3 embed_widget.py --check  exit 1 if widget.html differs (test_widget.py runs the same check)
"""
import json
import re
import sys
from pathlib import Path

from sampler import DRAWS, ROWS, SLACK, WARMUP, as_chances, row_of, toy

WIDGET = Path(__file__).resolve().parent.parent / "widget.html"
CONFIG_BLOCK = re.compile(r'(<script type="application/json" id="wsd-config">)(.*?)(</script>)', re.S)


def config():
    """The rows the page offers, as the toy counted them, and the constants of the draw."""
    model = toy.train(toy.lines_of(toy.load_corpus()))
    rows = []
    for row in ROWS:
        ranked, total = row_of(model, row["context"])
        rows.append({"id": row["id"], "name": row["name"], "ranked": [[token, count] for token, count in ranked], "total": total})
    return {"rows": rows, "draws": DRAWS, "warmup": WARMUP, "slack": SLACK}


def embed(html):
    """html with the data block rewritten from sampler.py and lesson 4's toy."""
    settings = json.dumps(config(), ensure_ascii=False, separators=(", ", ": ")).replace("<", "\\u003c")
    return CONFIG_BLOCK.sub(lambda m: m.group(1) + "\n" + settings + "\n" + m.group(3), html, count=1)


if __name__ == "__main__":
    current = WIDGET.read_text(encoding="utf-8")
    updated = embed(current)
    if "--check" in sys.argv:
        sys.exit(0 if updated == current else 1)
    WIDGET.write_text(updated, encoding="utf-8")
    print("widget.html data block now matches sampler.py and lesson 4's toy")

"""Sampling on top of lesson 4's toy: temperature, top-p and a seeded draw, using nothing but the standard library.

Lesson 4's toy ends each pass with a list of possible next tokens and how often each followed the text it looked at.
This file turns such a list into chances, reshapes them with a temperature, cuts the unlikely tail with top-p, and
draws one token with a seeded generator. The draw is a step of its own, after the model has given its chances.
widget.html runs the same rules in JavaScript; test_widget.py checks that the two agree.

The order is temperature first, then top-p. A provider may order them differently.
"""
import importlib.util
import sys
from pathlib import Path

LESSON_4 = Path(__file__).resolve().parents[2] / "why-answers-appear-word-by-word" / "code"
SLACK = 1e-9           # rounding slack when a running total is compared with p, so Python and JavaScript cut alike
WARMUP = 4             # random numbers thrown away at the start: a small seed gives a tiny first number
DRAWS = 20             # how many picks one press of Draw 20 times makes


def _load_toy():
    """Lesson 4's toy_gen.py, loaded from its own folder so this lesson uses the very same corpus, tables and generator."""
    spec = importlib.util.spec_from_file_location("lesson4_toy_gen", LESSON_4 / "toy_gen.py")
    module = importlib.util.module_from_spec(spec)
    sys.modules[spec.name] = module
    spec.loader.exec_module(module)
    return module


toy = _load_toy()

# The rows the widget offers. A context is the run of earlier tokens whose followers the toy counted: three tokens
# for the two passes lesson 4 shows, one token for the other two.
ROWS = [
    {"id": "wait", "name": "After “Wait”", "context": ["Wait"]},
    {"id": "the", "name": "After “the”", "context": ["the"]},
    {"id": "pass1", "name": "Tea, pass 1: “make tea ?”", "context": ["make", "tea", "?"]},
    {"id": "pass11", "name": "Tea, pass 11: “bag in a”", "context": ["bag", "in", "a"]},
]


def row_of(model, context):
    """The followers of a run of 1 to 3 tokens, as the toy counted them: (ranked, total). Most to least common, a tie
    to the token that sorts first, which is the order `chances` in lesson 4 uses."""
    followers = model[len(context)][tuple(context)]
    return sorted(followers.items(), key=lambda item: (-item[1], item[0])), sum(followers.values())


def as_chances(ranked, total):
    """[(token, count)] and a total to [(token, chance)]: a token's chance is its count out of the total."""
    return [(token, count / total) for token, count in ranked]


def with_temperature(chances, temperature):
    """Raise each chance to the power 1 / temperature, then scale the chances back to a total of 1. The same rule as
    `with_temperature` in lesson 12. Below 1 it favours the top token, above 1 it evens the tokens out."""
    if temperature <= 0:
        raise ValueError("temperature must be above 0; use the top token for 'no randomness'")
    powered = [(token, chance ** (1 / temperature)) for token, chance in chances]
    total = 0.0
    for _, chance in powered:                       # one number at a time, in order, as widget.html adds them
        total += chance
    return [(token, chance / total) for token, chance in powered]


def top_p(chances, p):
    """Keep the fewest top tokens whose chances add up to at least p, then scale them back to a total of 1.
    `chances` must be ranked from the likeliest down. Tokens with equal chances keep the order they came in.
    A p of 1 keeps every token, even one whose chance is too small to show."""
    if p >= 1:
        return list(chances)
    kept, running = [], 0.0
    for token, chance in chances:
        kept.append((token, chance))
        running += chance
        if running >= p - SLACK:
            break
    return [(token, chance / running) for token, chance in kept]


def reshape(chances, temperature, p):
    """What the draw sees: temperature first, then top-p. Tokens top-p cuts are missing from the result."""
    return top_p(with_temperature(chances, temperature), p)


def choose(chances, u):
    """The token whose slice of 0 to 1 holds u, for u from 0 up to but below 1. Slices follow the order given."""
    running = 0.0
    for token, chance in chances:
        running += chance
        if u < running:
            return token
    return chances[-1][0]                           # u at or past the rounded-off total


def start_state(seed):
    """The generator state a seed starts from: lesson 4's `seed_state`, then WARMUP numbers thrown away.
    Without that, a small seed makes a first number near 0 and the first draw always takes the top token."""
    state = toy.seed_state(seed)
    for _ in range(WARMUP):
        state, _ = toy.seed_step(state)
    return state


def draw(chances, state, n=DRAWS):
    """n draws from the same chances. Returns (the picks, the generator state after them), so the next press goes on."""
    picks = []
    for _ in range(n):
        state, u = toy.seed_step(state)
        picks.append(choose(chances, u))
    return picks, state


def tally(picks, tokens):
    """How often each token in `tokens` came up in `picks`, in the order of `tokens`."""
    return [picks.count(token) for token in tokens]


def generate(model, prompt, state, temperature=1.0, p=1.0, cap=toy.CAP):
    """Write a reply with lesson 4's loop, but each pass draws from the reshaped chances. Returns (the tokens added,
    end mark included, and the generator state after them), so pressing regenerate goes on from where the last
    reply stopped. The same state and settings give the same reply."""
    tokens = toy.tokenize(prompt)
    reply = []
    for _ in range(cap):
        _, ranked, total = toy.chances(model, tokens)
        if not ranked:
            break
        state, u = toy.seed_step(state)
        token = choose(reshape(as_chances(ranked, total), temperature, p), u)
        reply.append(token)
        if token == toy.END:
            break
        tokens.append(token)
    return reply, state


def reply_text(reply):
    """The reply as a reader sees it (see `reply_of` in lesson 4, which takes whole passes)."""
    return toy.reply_of([{"token": token} for token in reply])


def first_split(a, b):
    """The position of the first token where two token lists differ, or None when one is the start of the other."""
    for i, (x, y) in enumerate(zip(a, b)):
        if x != y:
            return i
    return None if len(a) == len(b) else min(len(a), len(b))


def replies(model, prompt, count, seed, temperature=1.0, p=1.0, cap=toy.CAP):
    """`count` replies written one after another from one generator, as if the reader pressed regenerate that often."""
    state, out = start_state(seed), []
    for _ in range(count):
        reply, state = generate(model, prompt, state, temperature, p, cap)
        out.append(reply)
    return out


def corpus_replies(lines, prompt):
    """The reply part of every corpus line that starts with `prompt`, as text (a set, so a repeated line counts once)."""
    return {line[len(prompt):].strip() for line in lines if line.startswith(prompt)}

"""Print every number the lesson quotes. Run it from this folder: python3 show_numbers.py"""
from collections import Counter

import sampler as s

toy = s.toy
lines = toy.lines_of(toy.load_corpus())
model = toy.train(lines)
tea, plant = "How do I make tea?", "How do I water a plant?"
wait = s.as_chances(*s.row_of(model, ["Wait"]))
tokens = [t for t, _ in wait]


def pct(chances):
    return ", ".join(f"{t} {100 * c:.1f}%" for t, c in chances)


print("== The Wait row (what followed the token Wait in the 17 lines)")
ranked, total = s.row_of(model, ["Wait"])
print(f"  {total} times: " + ", ".join(f"{t} {c}" for t, c in ranked) + f"; {sum('Wait' in toy.tokenize(l) for l in lines)} of {len(lines)} lines have it")
for t in (0.1, 0.2, 0.5, 1.0, 2.0):
    print(f"  temperature {t}: {pct(s.with_temperature(wait, t))}")

print("\n== Top-p on the Wait row at temperature 1 (kept tokens, rescaled)")
for p in (0.5, 0.7, 0.8, 0.9, 1.0):
    print(f"  p {p}: {pct(s.reshape(wait, 1.0, p))}")
print("  temperature first, then top-p:")
for t, p in ((0.2, 0.7), (0.1, 0.7), (2.0, 0.7)):
    print(f"  temperature {t}, p {p}: {pct(s.reshape(wait, t, p))}")
print(f"  top-p first, then temperature 0.2 (p 0.7): {pct(s.with_temperature(s.top_p(wait, 0.7), 0.2))}")

print(f"\n== {s.DRAWS} draws from seed 1 (counts for {', '.join(tokens)})")
for t, p in ((1.0, 1.0), (0.2, 1.0), (0.1, 1.0), (2.0, 1.0), (1.0, 0.7)):
    picks, _ = s.draw(s.reshape(wait, t, p), s.start_state(1))
    print(f"  temperature {t}, p {p}: {s.tally(picks, tokens)}")
picks, state = s.draw(wait, s.start_state(1))
again, _ = s.draw(wait, state)
print(f"  temperature 1, p 1, seed 1, first press: {' '.join(picks)}")
print(f"  ... and the next press: {s.tally(again, tokens)}")
print(f"  same seed, again: {s.tally(s.draw(wait, s.start_state(1))[0], tokens)}; seed 2: {s.tally(s.draw(wait, s.start_state(2))[0], tokens)}")

print("\n== Worked example, pass 11 of tea (cup 3, mug 2)")
row = s.as_chances(*s.row_of(model, ["bag", "in", "a"]))
for t in (1.0, 0.5, 2.0):
    powered = [(tok, c ** (1 / t)) for tok, c in row]
    print(f"  temperature {t}: power {1 / t:g}: " + ", ".join(f"{tok} {c:.4f}" for tok, c in powered)
          + f"; total {sum(c for _, c in powered):.4f}; chances {pct(s.with_temperature(row, t))}")
for u in (0.5, 0.65):
    print(f"  u {u}: " + "; ".join(f"temperature {t} takes {s.choose(s.with_temperature(row, t), u)}" for t in (1.0, 0.5, 2.0)))
print("  tea, pass 1 (Boil 4, Heat 1): " + "; ".join(f"temperature {t}: {pct(s.with_temperature(s.as_chances(*s.row_of(model, ['make', 'tea', '?'])), t))}" for t in (0.5, 1.0, 2.0)))

print("\n== The generator")
raw = toy.seed_step(toy.seed_state(1))[1]
print(f"  first number for seed 1 with no warm-up: {raw:.4e}; with {s.WARMUP} thrown away: {toy.seed_step(s.start_state(1))[1]:.4f}")
heat = lambda start: sum(s.generate(model, tea, start(seed), cap=1)[0] == ["Heat"] for seed in range(1000))
print(f"  Heat as the first token over seeds 0-999: {heat(toy.seed_state)} of 1000 with no warm-up, {heat(s.start_state)} with it (its chance is 20%)")

print("\n== 1,000 replies from one seed (seed 1): different replies, of those not among the toy's own lines, top reply count")
for name, prompt in (("tea", tea), ("plant", plant)):
    known = s.corpus_replies(lines, prompt)
    for t, p in ((1.0, 1.0), (0.5, 1.0), (0.2, 1.0), (0.05, 1.0), (2.0, 1.0), (1.0, 0.7)):
        counted = Counter(s.reply_text(r) for r in s.replies(model, prompt, 1000, 1, t, p))
        print(f"  {name} temperature {t}, p {p}: {len(counted)} different, {sum(1 for x in counted if x not in known)} new, top reply {counted.most_common(1)[0][1]}")
    print(f"  {name}: {len(known)} different replies among its {sum(l.startswith(prompt) for l in lines)} lines")
print("  greedy tea: " + toy.reply_of(toy.generate(model, tea, 40)))
first = Counter(r[0] for r in s.replies(model, tea, 1000, 1))
print(f"  tea first token over those 1,000 replies: {dict(first)}")

print("\n== Two runs part ways (plant, seed 1, temperature 1)")
runs = s.replies(model, plant, 6, 1)
dry, wet = runs[0], runs[5]
fork = s.first_split(dry, wet)
print(f"  they share the first {fork} tokens, then {dry[fork]!r} ({len(dry) - fork - 1} tokens from there on, end mark not counted) and {wet[fork]!r} ({len(wet) - fork - 1})")
print(f"  next-token row at the fork: {s.row_of(model, ['the', 'soil', 'is'])}")
print("  one: " + s.reply_text(dry))
print("  two: " + s.reply_text(wet))
shared = set(s.reply_text(dry[fork:]).split(". ")) & set(s.reply_text(wet[fork:]).split(". "))
print(f"  sentences the two have in common after the fork: {sorted(shared)}")

print("\n== Frequency check: 100,000 draws from seed 1")
for t, p in ((1.0, 1.0), (0.5, 1.0), (2.0, 0.8)):
    chances = s.reshape(wait, t, p)
    picks, _ = s.draw(chances, s.start_state(1), 100000)
    seen = Counter(picks)
    worst = max(abs(seen[tok] / 100000 - c) for tok, c in chances)
    print(f"  temperature {t}, p {p}: largest gap between share drawn and chance {100 * worst:.2f} points; kept tokens {len(chances)}")

"""Tests for sampler.py. Each one names the mistake it would catch.

Run: PYTHONDONTWRITEBYTECODE=1 python3 -m pytest -q -p no:cacheprovider .
"""
from collections import Counter
from math import isclose

import pytest

import sampler as s

TOY = s.toy
LINES = TOY.lines_of(TOY.load_corpus())
MODEL = TOY.train(LINES)
TEA = "How do I make tea?"
PLANT = "How do I water a plant?"
WAIT = s.as_chances(*s.row_of(MODEL, ["Wait"]))        # the first row the widget opens on


def share(chances):
    return {token: chance for token, chance in chances}


def test_rows_are_lesson_4_numbers():
    """Lesson 5 must show the very chances lesson 4 showed: pass 1 of the tea question and pass 11."""
    assert s.row_of(MODEL, ["make", "tea", "?"]) == ([("Boil", 4), ("Heat", 1)], 5)
    assert s.row_of(MODEL, ["bag", "in", "a"]) == ([("cup", 3), ("mug", 2)], 5)
    greedy = TOY.generate(MODEL, TEA, 40)
    assert greedy[0]["ranked"] == [("Boil", 4), ("Heat", 1)]
    assert greedy[10]["ranked"] == [("cup", 3), ("mug", 2)]


def test_every_widget_row_exists_and_has_followers():
    """A row naming a context the toy never saw would crash the widget at start."""
    assert len({row["id"] for row in s.ROWS}) == len(s.ROWS)
    for row in s.ROWS:
        ranked, total = s.row_of(MODEL, row["context"])
        assert total == sum(count for _, count in ranked) > 0


def test_wait_row_is_what_the_lesson_says():
    """The lesson's bag of 13 marbles: three 4, ten 3, two 3, and one each for four, one, six."""
    ranked, total = s.row_of(MODEL, ["Wait"])
    assert total == 13
    assert ranked == [("three", 4), ("ten", 3), ("two", 3), ("four", 1), ("one", 1), ("six", 1)]


def test_generator_is_lesson_4s_park_miller():
    """Changing the factor or the modulus would change every draw in lesson 4 and here."""
    state = 1
    for _ in range(10000):
        state, _ = TOY.seed_step(state)
    assert state == 399268537


def test_choose_agrees_with_lesson_4_pick():
    """At temperature 1 and p = 1 the draw must be lesson 4's pick, or the two lessons would disagree on one seed."""
    for row in s.ROWS:
        ranked, total = s.row_of(MODEL, row["context"])
        chances = s.as_chances(ranked, total)
        state = s.start_state(7)
        for _ in range(2000):
            state, u = TOY.seed_step(state)
            assert s.choose(chances, u) == TOY.pick(ranked, total, u)


def test_choose_walks_the_slices_in_order():
    """u picks by position: the first slice starts at 0, and u past the rounded-off total takes the last token."""
    chances = [("a", 0.5), ("b", 0.25), ("c", 0.25)]
    assert [s.choose(chances, u) for u in (0.0, 0.49, 0.5, 0.74, 0.75, 0.99)] == ["a", "a", "b", "b", "c", "c"]
    assert s.choose([("a", 0.3), ("b", 0.3)], 0.7) == "b"


def test_temperature_one_changes_nothing():
    assert all(isclose(a, b) for (_, a), (_, b) in zip(WAIT, s.with_temperature(WAIT, 1.0)))


def test_temperature_below_one_sharpens_and_above_one_flattens():
    """The direction is the whole point: low favours the top token, high evens the tokens out."""
    base = WAIT[0][1]
    assert s.with_temperature(WAIT, 0.5)[0][1] > base > s.with_temperature(WAIT, 2.0)[0][1]
    assert s.with_temperature(WAIT, 0.05)[0][1] > 0.99
    spread = lambda chances: max(c for _, c in chances) - min(c for _, c in chances)
    assert spread(s.with_temperature(WAIT, 50.0)) < 0.01 < spread(WAIT)


def test_temperature_keeps_total_one_and_order():
    for t in (0.05, 0.2, 0.5, 1.0, 2.0):
        out = s.with_temperature(WAIT, t)
        assert isclose(sum(c for _, c in out), 1.0)
        assert [tok for tok, _ in out] == [tok for tok, _ in WAIT]


def test_temperature_is_lesson_12s_power_rule_and_softmax_of_scores_over_t():
    """Dividing the scores by T before softmax equals raising the chances to 1/T and rescaling (the Go deeper claim)."""
    from math import exp, log
    for t in (0.3, 0.7, 1.5):
        scores = [log(c) for _, c in WAIT]                       # scores whose softmax is the chances
        exps = [exp(x / t) for x in scores]
        soft = [e / sum(exps) for e in exps]
        assert all(isclose(a, c, rel_tol=1e-12) for a, (_, c) in zip(soft, s.with_temperature(WAIT, t)))


@pytest.mark.parametrize("bad", [0, 0.0, -1])
def test_temperature_zero_or_below_is_refused(bad):
    """Zero would divide by zero; the toy asks for the greedy choice some other way."""
    with pytest.raises(ValueError):
        s.with_temperature(WAIT, bad)


def test_top_p_keeps_the_smallest_top_set_and_rescales():
    kept = s.top_p(WAIT, 0.7)
    assert [t for t, _ in kept] == ["three", "ten", "two"]                    # 10 of 13 is 76.9%, the first total at or past 0.7
    assert [round(c, 3) for _, c in kept] == [0.4, 0.3, 0.3]
    assert isclose(sum(c for _, c in kept), 1.0)
    assert [t for t, _ in s.top_p(WAIT, 0.5)] == ["three", "ten"]


def test_top_p_one_keeps_everything_and_tiny_p_keeps_one():
    assert [t for t, _ in s.top_p(WAIT, 1.0)] == [t for t, _ in WAIT]
    assert s.top_p(WAIT, 0.001) == [("three", 1.0)]
    cold = s.with_temperature(WAIT, 0.05)                                  # four, one, six hold under 1e-9 between them
    assert len(s.top_p(cold, 1.0)) == 6 and len(s.top_p(cold, 0.9999)) < 6


def test_top_p_tie_goes_to_the_token_that_sorts_first():
    """At p = 0.8 only one of four, one and six fits, and which one is a rule, not luck: the one that sorts first."""
    assert [t for t, _ in s.top_p(WAIT, 0.8)] == ["three", "ten", "two", "four"]


def test_top_p_rounding_does_not_pull_in_one_more_token():
    """0.7 + 0.2 is 0.8999999999999999 in floating point; a cut at exactly 0.9 must still stop there."""
    assert [t for t, _ in s.top_p([("a", 0.7), ("b", 0.2), ("c", 0.1)], 0.9)] == ["a", "b"]


def test_reshape_applies_temperature_then_top_p():
    """The order matters: cutting first and sharpening after keeps three tokens at 0.2 and 0.7; this order keeps two."""
    assert [t for t, _ in s.reshape(WAIT, 0.2, 0.7)] == ["three", "ten"]
    assert [t for t, _ in s.with_temperature(s.top_p(WAIT, 0.7), 0.2)] == ["three", "ten", "two"]
    assert [t for t, _ in s.reshape(WAIT, 0.1, 0.7)] == ["three"]
    assert [t for t, _ in s.reshape(WAIT, 2.0, 0.7)] == ["three", "ten", "two", "four"]


def test_same_seed_same_draws_different_seed_different_draws():
    """The seed is what makes the toy repeatable; without it the lesson's 'same seed, same draws' would be false."""
    a, _ = s.draw(WAIT, s.start_state(1))
    b, _ = s.draw(WAIT, s.start_state(1))
    c, _ = s.draw(WAIT, s.start_state(2))
    assert a == b and a != c


def test_draw_goes_on_from_the_state_it_returns():
    """Pressing Draw 20 times again must continue the stream, not repeat the same 20 picks."""
    state = s.start_state(1)
    first, state = s.draw(WAIT, state)
    second, state = s.draw(WAIT, state)
    both, _ = s.draw(WAIT, s.start_state(1), 40)
    assert first + second == both and first != second


def test_the_opening_draws_the_lesson_quotes():
    """Seed 1, temperature 1, no cut: the 20 picks and counts the widget opens on."""
    picks, _ = s.draw(WAIT, s.start_state(1))
    tokens = [t for t, _ in WAIT]
    assert s.tally(picks, tokens) == [7, 4, 3, 2, 1, 3]
    assert len(picks) == 20


def test_small_seeds_do_not_always_start_with_the_top_token():
    """Without the warm-up a seed of 1 makes a first number near 0.00004 and the first draw is always the top token."""
    raw = TOY.seed_step(TOY.seed_state(1))[1]
    assert raw < 1e-4
    heat = lambda start: sum(s.generate(MODEL, TEA, start(seed), cap=1)[0] == ["Heat"] for seed in range(1000))
    assert heat(TOY.seed_state) == 0                                      # the lesson's 0 of 1,000 without the warm-up
    assert 150 < heat(s.start_state) < 250                                # Heat has a 20% chance on the first pass


def test_draws_follow_the_chances():
    """Frequency check: with a fixed seed, 100,000 draws land within a point of the chances (never flaky)."""
    for temperature, p in ((1.0, 1.0), (0.5, 1.0), (2.0, 0.8)):
        chances = s.reshape(WAIT, temperature, p)
        picks, _ = s.draw(chances, s.start_state(1), 100000)
        seen = Counter(picks)
        for token, chance in chances:
            assert abs(seen[token] / 100000 - chance) < 0.01
        assert set(seen) == {token for token, _ in chances}               # a token top-p cut never comes up


def test_low_temperature_replies_follow_the_greedy_reply():
    """At a very low temperature the toy writes lesson 4's greedy reply, for any seed, where the top token is unique."""
    greedy = [p["token"] for p in TOY.generate(MODEL, TEA, 40)]
    for seed in (1, 2, 3):
        reply, _ = s.generate(MODEL, TEA, s.start_state(seed), temperature=0.01)
        assert reply == greedy


def test_generate_is_repeatable_and_regenerate_continues_the_stream():
    one, state = s.generate(MODEL, PLANT, s.start_state(1))
    again, _ = s.generate(MODEL, PLANT, s.start_state(1))
    two, _ = s.generate(MODEL, PLANT, state)
    assert one == again
    assert s.replies(MODEL, PLANT, 2, 1) == [one, two]


def test_reply_counts_the_lesson_quotes():
    """1,000 tea replies from one seed: how many different ones, and how many are not among the toy's own lines."""
    known = s.corpus_replies(LINES, TEA)
    assert len(known) == 4
    texts = Counter(s.reply_text(r) for r in s.replies(MODEL, TEA, 1000, 1))
    assert (len(texts), sum(1 for t in texts if t not in known)) == (16, 12)
    cool = Counter(s.reply_text(r) for r in s.replies(MODEL, TEA, 1000, 1, temperature=0.05))
    assert len(cool) == 2 and cool.most_common(1)[0][1] == 999 and set(cool) <= known
    assert len({s.reply_text(r) for r in s.replies(MODEL, TEA, 1000, 1, temperature=0.01)}) == 1
    cut = {s.reply_text(r) for r in s.replies(MODEL, TEA, 1000, 1, p=0.7)}            # top-p 0.7 in 'How many different replies'
    assert (len(cut), len(cut - known)) == (4, 2)


def test_plant_runs_part_ways_at_dry_or_wet():
    """The branch the lesson shows: seed 1's first and sixth replies share 11 tokens, then one says dry and one wet."""
    runs = s.replies(MODEL, PLANT, 6, 1)
    dry, wet = runs[0], runs[5]
    fork = s.first_split(dry, wet)
    assert fork == 11
    assert (dry[fork], wet[fork]) == ("dry", "wet")
    assert (len(dry) - fork - 1, len(wet) - fork - 1) == (24, 12)           # tokens from dry or wet on, end mark not counted
    assert s.row_of(MODEL, ["the", "soil", "is"]) == ([("dry", 3), ("wet", 1)], 4)


def test_first_split():
    assert s.first_split(["a", "b"], ["a", "c"]) == 1
    assert s.first_split(["a"], ["a", "b"]) == 1
    assert s.first_split(["a", "b"], ["a", "b"]) is None

"""Guards for ../widget.html: its data block, its size, that its rules give the same chances and draws as sampler.py, and what a reader sees."""
import json
import re
import subprocess

import sampler as s
from embed_widget import CONFIG_BLOCK, WIDGET, embed

HTML = WIDGET.read_text(encoding="utf-8")
TOY = s.toy
MODEL = TOY.train(TOY.lines_of(TOY.load_corpus()))

# Reads the rules out of the widget between its marker comments, runs them with Node, and compares them with what
# Python wrote into the fixture: generator states, chances after temperature and top-p, and the picks drawn.
PARITY = r"""
import fs from 'node:fs';
const [fixture, widget] = process.argv.slice(2);
const fx = JSON.parse(fs.readFileSync(fixture, 'utf8'));
const html = fs.readFileSync(widget, 'utf8');
const cfg = JSON.parse(html.match(/<script type="application\/json" id="wsd-config">([\s\S]*?)<\/script>/)[1]);
const js = html.slice(html.indexOf('/* ---------- the rules ('), html.indexOf('/* ---------- state'));
const api = new Function('cfg', "'use strict';" + js + '; return {seedState, seedStep, startState, asChances, reshape, withTemperature, topP, choose, draw};')(cfg);
const bad = [];
let checks = 0;
const same = (a, b) => JSON.stringify(a) === JSON.stringify(b);
const close = (a, b) => a.length === b.length && a.every((e, i) => e[0] === b[i][0] && Math.abs(e[1] - b[i][1]) < 1e-12);
fx.seeds.forEach((e) => { checks += 2; if (api.seedState(e.seed) !== e.state) bad.push(['seedState', e.seed]); if (api.startState(e.seed) !== e.start) bad.push(['startState', e.seed]); });
fx.steps.forEach((e) => { checks++; if (!same(api.seedStep(e.state), { state: e.next, u: e.u })) bad.push(['seedStep', e.state]); });
fx.chances.forEach((e) => {
  const base = api.asChances({ ranked: e.ranked, total: e.total });
  checks += 3;
  if (!close(base, e.base)) bad.push(['asChances', e.row]);
  if (!close(api.withTemperature(base, e.t), e.hot)) bad.push(['withTemperature', e.row, e.t]);
  if (!close(api.reshape(base, e.t, e.p), e.shaped)) bad.push(['reshape', e.row, e.t, e.p, api.reshape(base, e.t, e.p), e.shaped]);
});
fx.chooses.forEach((e) => { checks++; if (api.choose(e.chances, e.u) !== e.token) bad.push(['choose', e.u]); });
fx.draws.forEach((e) => {
  const base = api.asChances({ ranked: e.ranked, total: e.total });
  const got = api.draw(api.reshape(base, e.t, e.p), e.state, e.n);
  checks += 2;
  if (!same(got.picks, e.picks)) bad.push(['picks', e.row, e.t, e.p, e.seed]);
  if (got.state !== e.after) bad.push(['state after', e.row, e.t, e.p, e.seed]);
});
console.log(JSON.stringify({ checks, bad: bad.slice(0, 5) }));
process.exit(bad.length ? 1 : 0);
"""

# Runs the page script against a small stand-in for the DOM, moves the controls, and reads back what a reader sees.
PAGE = r"""
import fs from 'node:fs';
const [widget, fixture] = process.argv.slice(2);
const html = fs.readFileSync(widget, 'utf8');
const fx = JSON.parse(fs.readFileSync(fixture, 'utf8'));
const block = (id) => html.match(new RegExp('<script type="[a-z/]+" id="' + id + '">([\\s\\S]*?)</script>'))[1];
const code = html.match(/<script>([\s\S]*?)<\/script>/)[1];

function el(tag) {
  const e = { tag, children: [], listeners: {}, className: '', hidden: false, value: '', style: {}, attrs: {}, _text: '' };
  e.setAttribute = (k, v) => { e.attrs[k] = v; };
  e.addEventListener = (type, f) => (e.listeners[type] = e.listeners[type] || []).push(f);
  e.appendChild = (c) => (e.children.push(c), c);
  Object.defineProperty(e, 'textContent', { get: () => e._text, set: (v) => { e._text = v; e.children = []; } });
  return e;
}
const shown = (e) => e._text + e.children.map((c) => ('text' in c ? c.text : shown(c))).join('');
function open() {
  const page = { root: el('html'), messages: [], parent: { other: true } };
  fx.ids.forEach((id) => { page[id] = el('x'); });
  page['wsd-config']._text = block('wsd-config');
  const doc = { getElementById: (id) => page[id], createElement: el, createTextNode: (text) => ({ text }), documentElement: page.root };
  const win = { addEventListener: (type, f) => page.messages.push(f) };
  new Function('document', 'window', 'parent', code)(doc, win, page.parent);
  return page;
}
const fire = (e, type) => (e.listeners[type || 'click'] || []).forEach((f) => f({ target: e }));

const bad = [];
const want = (what, got, expected) => { if (JSON.stringify(got) !== JSON.stringify(expected)) bad.push([what, got, expected]); };
function look(page, v, what) {                      // compares the page with one expected view
  want(what + ': temperature shown', page['wsd-temp-v']._text, v.t);
  want(what + ': top-p shown', page['wsd-p-v']._text, v.p);
  want(what + ': lead', page['wsd-lead']._text, v.lead);
  want(what + ': note', page['wsd-note']._text, v.note);
  want(what + ': picks', shown(page['wsd-picks']), v.picks);
  want(what + ': marked picks', page['wsd-picks'].children.filter((c) => c.tag === 'mark').map(shown), v.marked);
  const rows = page['wsd-list'].children;
  want(what + ': row count', rows.length, v.rows.length);
  rows.forEach((li, i) => {
    const r = v.rows[i], fill = li.children[1].children[0], tick = li.children[1].children[1];
    want(what + ': row ' + i + ' token', shown(li.children[0]), r.token);
    want(what + ': row ' + i + ' chance', shown(li.children[2]), r.chance);
    want(what + ': row ' + i + ' drawn', shown(li.children[3]), r.drawn);
    want(what + ': row ' + i + ' top mark', li.attrs['data-top'] === 'true', i === 0);
    want(what + ': row ' + i + ' cut mark', li.attrs['data-cut'] === 'true', r.cut);
    want(what + ': row ' + i + ' label', li.attrs['aria-label'], r.label);
    if (!(Math.abs(parseFloat(fill.style.width) - r.width) <= 0.051)) bad.push([what + ': row ' + i + ' bar', fill.style.width, r.width]);
    if (!(Math.abs(parseFloat(tick.style.left) - r.tick) <= 0.051)) bad.push([what + ': row ' + i + ' tick', tick.style.left, r.tick]);
  });
}
function settle(page, s) {                          // row, sliders and seed, as a reader would move them
  page['wsd-row'].value = String(s.row); fire(page['wsd-row'], 'change');
  page['wsd-temp'].value = s.t; fire(page['wsd-temp'], 'input');
  page['wsd-p'].value = s.p; fire(page['wsd-p'], 'input');
  page['wsd-seed'].value = String(s.seed); fire(page['wsd-seed'], 'change');
}

fx.scenarios.forEach((s, n) => {
  const page = open();
  settle(page, s);
  const name = 'scenario ' + n + ' (row ' + s.row + ', t ' + s.t + ', p ' + s.p + ', seed ' + s.seed + ')';
  s.views.forEach((v, k) => {
    if (k > 0) {
      fire(page['wsd-draw']);
      want(name + ': spoken after press ' + (k + 1), page['wsd-live']._text, v.spoken);
    }
    look(page, v, name + ' press ' + (k + 1));
  });
  fire(page['wsd-restart']);
  look(page, s.views[0], name + ' after Start over');
});

// the opening state is the one the lesson shows first; nothing is spoken until a press
const first = open();
look(first, fx.opening, 'opening state');
want('options', first['wsd-row'].children.map((o) => [o.value, o._text]), fx.options);
want('nothing spoken yet', first['wsd-live']._text, '');
want('seed written back', first['wsd-seed'].value, '1');

// choosing another row alone, with no slider or seed touched, shows that row from press 1
const swapped = open();
swapped['wsd-row'].value = '2'; fire(swapped['wsd-row'], 'change');
look(swapped, fx.swapped, 'row changed alone');

// a slider or the row moved after some presses starts that setting over at press 1
const moved = open();
fire(moved['wsd-draw']); fire(moved['wsd-draw']);
want('press count', moved['wsd-note']._text.slice(0, 8), 'Press 3:');
moved['wsd-temp'].value = '0.5'; fire(moved['wsd-temp'], 'input');
want('slider restarts', moved['wsd-note']._text.slice(0, 8), 'Press 1:');

// settings are kept inside their range and written back
const odd = open();
[['wsd-seed', '-5', '0'], ['wsd-seed', '1e12', '999999999'], ['wsd-seed', '', '1'], ['wsd-seed', '12.7', '12'], ['wsd-seed', 'x', '1']]
  .forEach(([id, typed, kept]) => { odd[id].value = typed; fire(odd[id], 'change'); want(id + ' typed ' + JSON.stringify(typed), odd[id].value, kept); });
[['wsd-temp', '0', '0.05'], ['wsd-temp', '9', '2'], ['wsd-temp', '', '1'], ['wsd-p', '0', '0.1'], ['wsd-p', '7', '1'], ['wsd-p', '', '1']]
  .forEach(([id, typed, kept]) => { odd[id].value = typed; fire(odd[id], 'input'); want(id + ' typed ' + JSON.stringify(typed), odd[id + '-v']._text, kept); });

// the host page theme: only the parent window may set it, and only to light or dark
const themed = open();
const send = (data, source) => themed.messages.forEach((f) => f({ data, source }));
send({ type: 'spicyshark:theme', theme: 'dark' }, { other: false });
send({ type: 'spicyshark:theme', theme: 'purple' }, themed.parent);
send({ type: 'other', theme: 'dark' }, themed.parent);
send(null, themed.parent);
want('ignored messages', themed.root.attrs, {});
send({ type: 'spicyshark:theme', theme: 'dark' }, themed.parent);
want('theme from the parent', themed.root.attrs, { 'data-theme': 'dark' });
send({ type: 'spicyshark:theme', theme: 'light' }, themed.parent);
want('light again', themed.root.attrs, { 'data-theme': 'light' });

console.log(JSON.stringify({ scenarios: fx.scenarios.length, bad: bad.slice(0, 6) }));
process.exit(bad.length ? 1 : 0);
"""

ROW_INDEX = {row["id"]: i for i, row in enumerate(s.ROWS)}
TEMPS = [float(f"{0.05 * k:.2f}") for k in range(1, 41)]            # the slider's steps: 0.05 to 2
PS = [float(f"{0.05 * k:.2f}") for k in range(2, 21)]               # 0.1 to 1
OPTIONS = [[str(i), row["name"]] for i, row in enumerate(s.ROWS)]


def plural(n, one, many):
    return f"{n} {one if n == 1 else many}"


def ranked_of(row):
    ranked, total = s.row_of(MODEL, row["context"])
    return ranked, total


def percent(chance):
    tenths = int(chance * 1000 + 0.5)
    return "<0.1%" if tenths < 1 else f"{tenths / 10:.1f}%"


def view(row_i, t, p, seed, presses):
    """What the page shows after `presses` presses (the first one is the setting itself)."""
    ranked, total = ranked_of(s.ROWS[row_i])
    base = s.as_chances(ranked, total)
    shaped = s.reshape(base, t, p)
    kept = dict(shaped)
    state, picks = s.start_state(seed), []
    for _ in range(presses):
        picks, state = s.draw(shaped, state)
    top = ranked[0][0]
    rows = []
    for token, chance in base:
        drawn = picks.count(token)
        c = kept.get(token)
        rows.append({"token": token, "chance": "cut" if c is None else percent(c), "drawn": str(drawn), "cut": c is None,
                     "width": 100 * (c or 0), "tick": 100 * chance,
                     "label": f"{token}: {'cut' if c is None else 'chance ' + percent(c)}, drawn {drawn}"})
    where = f"all {len(base)} tokens" if len(kept) == len(base) else f"the {plural(len(kept), 'token', 'tokens')} kept (of {len(base)})"
    note = f"Press {presses}: {s.DRAWS} picks from {where}."
    counts = ", ".join(f"{r['token']} {r['drawn']}" for r in rows if r["drawn"] != "0")
    return {"t": f"{t:g}", "p": f"{p:g}", "lead": f"Counted in the toy’s lines: {total} tokens followed this text.",
            "note": note, "picks": " ".join(picks), "marked": [x for x in picks if x == top], "rows": rows, "spoken": f"{note} {counts}."}


def scenario(row_i, t, p, seed, presses=3):
    return {"row": row_i, "t": f"{t:g}", "p": f"{p:g}", "seed": seed, "views": [view(row_i, t, p, seed, k) for k in range(1, presses + 1)]}


def run_node(tmp_path, script, fixture, args):
    (tmp_path / "fixture.json").write_text(json.dumps(fixture), encoding="utf-8")
    (tmp_path / "check.mjs").write_text(script, encoding="utf-8")
    run = subprocess.run(["node", str(tmp_path / "check.mjs"), *args(tmp_path)], capture_output=True, text=True, timeout=120)
    assert run.returncode == 0, run.stdout + run.stderr
    return json.loads(run.stdout)


def test_widget_data_block_matches_the_toy_and_sampler():
    """Guards a row or constant edit that was not copied into the widget (run embed_widget.py)."""
    assert embed(HTML) == HTML


def test_embed_rewrites_a_widget_whose_data_block_drifted():
    """Guards embed() skipping the block: a hand-edited count or constant must be put back."""
    drifted = HTML.replace('["three", 4]', '["three", 5]', 1).replace('"warmup": 4', '"warmup": 0', 1)
    assert drifted != HTML and embed(drifted) == HTML


def test_every_element_the_script_looks_up_exists_inside_the_root():
    """Guards a widget that never starts: the parity run covers only the rules, so a misspelt id passed there and threw in the browser."""
    root = HTML[HTML.index('<div class="wsd-root">'):HTML.index("<script")]
    script = HTML[HTML.index("<script>"):]
    classes = set(re.findall(r"\.(wsd-[\w-]+)", HTML[:HTML.index("<script")]))       # style classes the script gives to new elements
    ids = set(re.findall(r"'(wsd-[\w-]+)'", script)) - {"wsd-config"} - classes
    assert ids and all(f'id="{i}"' in root for i in ids), sorted(i for i in ids if f'id="{i}"' not in root)
    assert 'id="wsd-config"' in HTML and 'id="wsd-live"' in root


def test_config_is_safe_to_embed_and_names_the_rows_the_lesson_uses():
    block = CONFIG_BLOCK.search(HTML).group(2)
    assert "<" not in block and "</script" not in block.lower()
    config = json.loads(block)
    assert [r["id"] for r in config["rows"]] == ["wait", "the", "pass1", "pass11"]
    assert config["rows"][0]["ranked"][0] == ["three", 4] and config["rows"][0]["total"] == 13
    assert (config["draws"], config["warmup"]) == (20, 4)


def test_button_label_names_the_number_of_draws():
    """The label is plain text; it must not drift from the constant the draws use."""
    assert f">Draw {s.DRAWS} times</button>" in HTML


def test_widget_is_small_and_asks_for_nothing_online():
    assert len(HTML.splitlines()) < 300 and len(HTML.encode("utf-8")) < 60 * 1024
    page = CONFIG_BLOCK.sub("", HTML)
    for banned in ("http://", "https://", "fetch(", "XMLHttpRequest", "WebSocket", "sendBeacon", "import("):
        assert banned not in page, banned
    assert " src=" not in page and "@import" not in page


def test_widget_has_no_form_element():
    """Guards a dead button: the lesson page loads widget.html in a sandboxed frame, where a form cannot submit."""
    assert "<form" not in HTML and 'type="submit"' not in HTML


def test_widget_shows_at_most_five_controls_and_hides_the_rest():
    """Guards the first screen at 390 px: more than five controls push the bars off it. Extra settings stay in a closed details element."""
    root = HTML[HTML.index('<div class="wsd-root">'):HTML.index("<script")]
    outside = re.sub(r"<details>.*?</details>", "", root, flags=re.S)
    assert len(re.findall(r"<(?:select|button|input|textarea)\b", outside)) <= 5
    assert "<details><summary>More settings</summary>" in root and "<details open" not in root
    assert 'id="wsd-seed"' in re.search(r"<details><summary>More settings</summary>.*?</details>", root, re.S).group(0)


def test_widget_keeps_the_site_theme_listener_and_palette():
    assert "d.type !== 'spicyshark:theme'" in HTML and "e.source !== parent" in HTML and "parent === window" in HTML
    for pair in ("light-dark(#ffffff, #0a0f17)", "light-dark(#ebf0f6, #131b28)", "light-dark(#0a1626, #f4f7fb)",
                 "light-dark(#44566b, #a3b1c4)", "light-dark(#d2dae4, #1c2635)", "light-dark(#66768a, #7a8a9f)",
                 "light-dark(#f6f8fb, #05070c)", "--orange: light-dark(#b36200, #E69F00)",
                 ":root[data-theme=light]", ":root[data-theme=dark]"):
        assert pair in HTML, pair
    for banned in ("#0072B2", "#56B4E9", "#D55E00"):
        assert banned.lower() not in HTML.lower()


def test_widget_rules_give_the_chances_and_draws_of_the_python_sampler(tmp_path):
    """Guards the widget's JavaScript drifting from sampler.py: the lesson prints Python's numbers and the reader sees the
    widget's, so they must agree on the generator, on every chance the sliders can reach, on the cut, and on every pick."""
    seeds = [0, 1, 2, 3, 7, 12345, 999999999]
    steps = []
    for state in [s.start_state(x) for x in seeds] + [1, 2, 2147483646, 399268537]:
        nxt, u = TOY.seed_step(state)
        steps.append({"state": state, "next": nxt, "u": u})
    chances = []
    for i, row in enumerate(s.ROWS):
        ranked, total = ranked_of(row)
        base = s.as_chances(ranked, total)
        for t in TEMPS:
            for p in PS:
                chances.append({"row": i, "ranked": ranked, "total": total, "t": t, "p": p, "base": base,
                                "hot": s.with_temperature(base, t), "shaped": s.reshape(base, t, p)})
    chooses = [{"chances": [("a", 0.5), ("b", 0.25), ("c", 0.25)], "u": u, "token": s.choose([("a", 0.5), ("b", 0.25), ("c", 0.25)], u)}
               for u in (0.0, 0.49, 0.5, 0.74, 0.75, 0.9999999)]
    chooses.append({"chances": [("a", 0.3), ("b", 0.3)], "u": 0.7, "token": "b"})
    draws = []
    for i, row in enumerate(s.ROWS):
        ranked, total = ranked_of(row)
        base = s.as_chances(ranked, total)
        for t, p in ((1.0, 1.0), (0.2, 1.0), (2.0, 1.0), (1.0, 0.7), (0.05, 0.1), (0.35, 0.65)):
            for seed in (1, 2, 999999999):
                state = s.start_state(seed)
                picks, after = s.draw(s.reshape(base, t, p), state, 1000)
                draws.append({"row": i, "ranked": ranked, "total": total, "t": t, "p": p, "seed": seed, "state": state, "n": 1000, "picks": picks, "after": after})
    fixture = {"seeds": [{"seed": x, "state": TOY.seed_state(x), "start": s.start_state(x)} for x in seeds], "steps": steps,
               "chances": chances, "chooses": chooses, "draws": draws}
    result = run_node(tmp_path, PARITY, fixture, lambda d: [str(d / "fixture.json"), str(WIDGET)])
    assert result["bad"] == [] and result["checks"] > 9000
    assert any(len(set(d["picks"])) > 1 for d in draws)


def test_widget_page_shows_what_the_lesson_names(tmp_path):
    """Guards the part a reader sees: the opening state (Wait row, seed 1, 7 4 3 2 1 3), a new 20 picks per press, bars
    and counts for each setting, the cut tokens, the spoken summary, Start over, the settings' ranges and the theme."""
    scenarios = [scenario(0, 1.0, 1.0, 1), scenario(0, 0.2, 1.0, 1), scenario(0, 2.0, 1.0, 1), scenario(0, 1.0, 0.7, 1),
                 scenario(0, 0.1, 0.1, 1), scenario(0, 0.05, 1.0, 2), scenario(0, 2.0, 0.1, 999999999)]
    scenarios += [scenario(i, t, p, seed) for i in (1, 2, 3) for t, p, seed in ((1.0, 1.0, 1), (0.5, 0.8, 7), (1.5, 0.35, 0))]
    opening = view(0, 1.0, 1.0, 1, 1)
    assert [r["drawn"] for r in opening["rows"]] == ["7", "4", "3", "2", "1", "3"]
    assert [r["chance"] for r in opening["rows"]] == ["30.8%", "23.1%", "23.1%", "7.7%", "7.7%", "7.7%"]
    assert opening["note"] == "Press 1: 20 picks from all 6 tokens." and opening["picks"].split()[:3] == ["six", "ten", "three"]
    again = view(0, 1.0, 1.0, 1, 2)
    assert [r["drawn"] for r in again["rows"]] == ["6", "2", "5", "4", "3", "0"] and again["picks"] != opening["picks"]
    cut = view(0, 1.0, 0.7, 1, 1)
    assert [r["chance"] for r in cut["rows"]] == ["40.0%", "30.0%", "30.0%", "cut", "cut", "cut"]
    assert cut["note"] == "Press 1: 20 picks from the 3 tokens kept (of 6)."
    assert view(0, 0.1, 0.1, 1, 1)["note"] == "Press 1: 20 picks from the 1 token kept (of 6)."
    assert any(r["chance"] == "<0.1%" for r in view(0, 0.05, 1.0, 1, 1)["rows"])
    defaults = {i: re.search(rf'id="{i}"[^>]*? value="([^"]*)"', HTML).group(1) for i in ("wsd-temp", "wsd-p", "wsd-seed")}
    assert defaults == {"wsd-temp": "1", "wsd-p": "1", "wsd-seed": "1"}      # the page's own starting values, which the opening state assumes
    ids = sorted(set(re.findall(r'id="(wsd-[\w-]+)"', HTML)))
    fixture = {"ids": ids, "scenarios": scenarios, "opening": opening, "swapped": view(2, 1.0, 1.0, 1, 1), "options": OPTIONS}
    result = run_node(tmp_path, PAGE, fixture, lambda d: [str(WIDGET), str(d / "fixture.json")])
    assert result == {"scenarios": len(scenarios), "bad": []}