How do I check my AI's answers with a small test set?

"""Keep the data inside ../widget.html equal to the data in toy.py and evalset.py.

    python3 embed_widget.py          rewrite the data block in widget.html
    python3 embed_widget.py --check  exit 1 if widget.html differs (test_widget.py runs the same check)
"""
import json
import re
import sys
from pathlib import Path

from evalset import QUESTIONS, UNKNOWN_PHRASES, Z, describe, wobble
from toy import ALSO_SHARED, DONT_KNOW, GUESS, MIN_SHARED, PAGE, STOP

WIDGET = Path(__file__).resolve().parent.parent / "widget.html"
CONFIG_BLOCK = re.compile(r'(<script type="application/json" id="ev-config">)(.*?)(</script>)', re.S)


def config():
    return {"page": list(PAGE), "stop": sorted(STOP), "guess": GUESS, "dontKnow": DONT_KNOW, "minShared": MIN_SHARED,
            "alsoShared": ALSO_SHARED, "unknown": list(UNKNOWN_PHRASES), "z": Z,
            "wobble": ["".join("1" if flip else "0" for flip in row) for row in wobble()],
            "questions": [{"q": q[0], "right": q[1], "kind": q[2], "args": q[3], "check": describe(q)} for q in QUESTIONS]}


def dump(value):
    return json.dumps(value, ensure_ascii=False)


def embed(html):
    """html with the config block rewritten from toy.py and evalset.py (one key per line, one question per line)."""
    cfg = config()
    lines = [f" {dump(key)}: {dump(value)}" for key, value in cfg.items() if key != "questions"]
    lines.append(' "questions": [\n  ' + ",\n  ".join(dump(q) for q in cfg["questions"]) + "\n ]")
    block = "{\n" + ",\n".join(lines) + "\n}"
    return CONFIG_BLOCK.sub(lambda m: m.group(1) + "\n" + block + "\n" + m.group(3), html, count=1)


if __name__ == "__main__":
    current = WIDGET.read_text(encoding="utf-8")
    updated = embed(current)
    if "--check" in sys.argv:
        sys.exit(0 if updated == current else 1)
    WIDGET.write_text(updated, encoding="utf-8")
    print("widget.html data block now matches toy.py and evalset.py")

"""The test set (20 questions with known answers), four graders, the compare step and the Wilson interval.

Python standard library only. The questions are written for toy.py's help page; each has the right answer written
down first and a check that a program can run on the toy's answer.
"""
import math
import random
import re

from toy import answer

# kind: "contains" (every phrase in need appears as whole words, none in avoid), "numbers" (the numbers in the answer
# are exactly these), "unknown" (the answer says it does not know), "exact" (the answer is this sentence, ignoring
# capitals and punctuation).
QUESTIONS = (
    ("What time do you open on weekdays?", "9am to 6pm", "contains", {"need": ["9am", "6pm"]}),
    ("What are the opening hours on Saturday?", "10am to 4pm", "contains", {"need": ["10am", "4pm"], "avoid": ["9am"]}),
    ("How many days does a basic tune-up take?", "2 days", "numbers", {"numbers": [2]}),
    ("What does a full overhaul cost?", "$140", "contains", {"need": ["$140"]}),
    ("How much is the deposit on a rental?", "$100", "numbers", {"numbers": [100]}),
    ("What does a city bike cost to rent for a week?", "$60 a week", "contains", {"need": ["$60 a week"]}),
    ("How long is the warranty on repairs?", "90 days", "numbers", {"numbers": [90]}),
    ("When are the riding lessons?", "Saturdays at 11am", "contains", {"need": ["Saturdays", "11am"]}),
    ("Is the shop open on Sunday?", "No, closed", "exact", {"text": "Opening hours on Sunday: closed."}),
    ("What are your hours on Saturday and Sunday?", "10am to 4pm; closed Sunday", "contains", {"need": ["10am", "4pm", "closed"]}),
    ("What is the warranty on repairs and on a bike bought here?", "90 days; 2 years", "contains", {"need": ["90 day", "2 year"]}),
    ("How many months is the warranty on a bike bought here?", "24 months", "contains", {"need": ["24 month"]}),
    ("Do you deliver bikes?", "The page does not say", "unknown", {}),
    ("Do you sell helmets?", "The page does not say", "unknown", {}),
    ("Are you open on public holidays?", "The page does not say", "unknown", {}),
    ("Is there a student discount on a basic tune-up?", "The page does not say", "unknown", {}),
    ("Can I leave my bicycle by the door?", "Yes, free parking at the front door", "contains", {"need": ["parking"]}),
    ("What do I pay for a whole week?", "$60 a week", "contains", {"need": ["$60 a week"]}),
    ("Can I return a part after 40 days?", "No, only within 30 days", "contains", {"need": ["no", "30 days"]}),
    ("What does a bicycle rental cost for one day?", "$15 a day", "contains", {"need": ["$15 a day"]}),
)

UNKNOWN_PHRASES = ("i don't know", "i do not know", "does not say", "not sure")
NUMBER = re.compile(r"(?<![\w.])\d+(?:\.\d+)?(?!\w)")  # a whole number: "$60." gives 60, "9am" gives nothing
Z = 1.96  # the normal quantile for a 95% interval
WOBBLE_SEED, WOBBLE_RUNS, WOBBLE_CHANCE = 5, 5, 0.1


def norm(text):
    """Lower case, punctuation turned into spaces, one space between words ($ is kept)."""
    return " ".join(re.sub(r"[^a-z0-9$]+", " ", text.lower()).split())


def has(text, phrase):
    """True when phrase appears in text as whole words: "no" is not found in "I do not know"."""
    return f" {norm(phrase)} " in f" {norm(text)} "


def passes(question, text):
    """Grade one answer. question is one row of QUESTIONS."""
    _, _, kind, args = question
    if kind == "contains":
        return all(has(text, p) for p in args["need"]) and not any(has(text, p) for p in args.get("avoid", []))
    if kind == "numbers":
        return sorted({float(x) for x in NUMBER.findall(text)}) == sorted({float(n) for n in args["numbers"]})
    if kind == "unknown":
        return any(has(text, p) for p in UNKNOWN_PHRASES)
    if kind == "exact":
        return norm(text) == norm(args["text"])
    raise ValueError(kind)


def describe(question):
    """One line saying what the check looks for (the widget and the lesson's table show it)."""
    _, _, kind, args = question
    if kind == "contains":
        need = [f"“{p}”" for p in args["need"]]
        return "Answer has " + ", ".join(need[:-1]) + (" and " if len(need) > 1 else "") + need[-1] + "".join(f", not “{p}”" for p in args.get("avoid", []))
    if kind == "numbers":
        return "The numbers in the answer are exactly " + " and ".join(str(n) for n in args["numbers"])
    if kind == "unknown":
        return "Answer says it does not know"
    return f"Answer is exactly “{args['text']}”"


def wobble(seed=WOBBLE_SEED, runs=WOBBLE_RUNS, n=len(QUESTIONS), chance=WOBBLE_CHANCE):
    """runs x n table of True/False: in this run, this question's result comes out the other way."""
    rng = random.Random(seed)
    return [[rng.random() < chance for _ in range(n)] for _ in range(runs)]


def results(d, run=None):
    """[True/False per question] under instructions d. With run set, apply that row of the wobble table."""
    marks = [passes(q, answer(q[0], d)) for q in QUESTIONS]
    if run is None:
        return marks
    return [m != flip for m, flip in zip(marks, wobble()[run])]


def compare(before, after):
    """Per question: "fixed", "broke", "pass" (still passing) or "fail" (still failing)."""
    return [("fixed" if not b else "pass") if a else ("broke" if b else "fail") for b, a in zip(before, after)]


def wilson(passed, n, z=Z):
    """Wilson score interval for a proportion: (low, high) as fractions."""
    p = passed / n
    centre = p + z * z / (2 * n)
    spread = z * math.sqrt(p * (1 - p) / n + z * z / (4 * n * n))
    shrink = 1 + z * z / n
    return (centre - spread) / shrink, (centre + spread) / shrink

"""Print every number the lesson quotes. Run it from this folder: python3 show_numbers.py"""
import math
import random
from statistics import median

from evalset import QUESTIONS, WOBBLE_CHANCE, WOBBLE_RUNS, WOBBLE_SEED, Z, compare, results, wilson, wobble
from toy import FIELDS, SWITCH_NAMES, Instructions, all_settings, answer

OFF = Instructions()
base = results(OFF)


def count(marks):
    return sum(marks)


def pct(x):
    return f"{round(100 * x)}%"


def interval(passed, n=len(QUESTIONS)):
    low, high = wilson(passed, n)
    return f"{pct(low)} to {pct(high)}"


def name(d):
    on = [f for f in FIELDS if getattr(d, f)]
    return " + ".join(on) if on else "nothing on"


print("== The starting instructions: all three switches off")
print(f"  questions: {len(QUESTIONS)}; passing: {count(base)} ({pct(count(base) / len(QUESTIONS))}); failing: {len(QUESTIONS) - count(base)}")
print("  failing: " + "; ".join(f"Q{i + 1} {q[0]}" for i, q in enumerate(QUESTIONS) if not base[i]))
print(f"  \"Do you deliver bikes?\" answer: {answer('Do you deliver bikes?', OFF)}")

print("\n== Each setting against the starting instructions: fixed, broke, pass rate")
print("  switches | fixed | broke | passing | rate | 95% range")
for d in all_settings():
    after = results(d)
    c = compare(base, after)
    print(f"  {name(d)} | {c.count('fixed')} | {c.count('broke')} | {count(after)} of {len(QUESTIONS)} | {pct(count(after) / len(QUESTIONS))} | {interval(count(after))}")

print("\n== What each single switch fixes and breaks")
for field in FIELDS:
    d = Instructions(**{field: True})
    c = compare(base, results(d))
    fixed = [f"Q{i + 1}" for i, k in enumerate(c) if k == "fixed"]
    broke = [f"Q{i + 1}" for i, k in enumerate(c) if k == "broke"]
    print(f"  {SWITCH_NAMES[field]}: fixed {', '.join(fixed) or 'none'}; broke {', '.join(broke) or 'none'}")

print("\n== Questions that fail under all 8 settings")
always_fail = [i + 1 for i in range(len(QUESTIONS)) if not any(results(d)[i] for d in all_settings())]
always_pass = [i + 1 for i in range(len(QUESTIONS)) if all(results(d)[i] for d in all_settings())]
print(f"  always failing: {always_fail}; always passing: {always_pass}")

print("\n== The Wilson range (95%), worked for 13 of 20")
n, k = 20, 13
p = k / n
centre = p + Z * Z / (2 * n)
spread = Z * math.sqrt(p * (1 - p) / n + Z * Z / (4 * n * n))
shrink = 1 + Z * Z / n
print(f"  p = {p}; z = {Z}; centre = {centre:.4f}; spread = {spread:.4f}; shrink = {shrink:.4f}")
print(f"  low = {(centre - spread) / shrink:.4f}; high = {(centre + spread) / shrink:.4f}")
print("  12 of 20: " + interval(12) + "; 13 of 20: " + interval(13))
print("  range width by size at a 65% rate: " + "; ".join(
    f"{size} questions {pct(wilson(round(0.65 * size), size)[0])} to {pct(wilson(round(0.65 * size), size)[1])}" for size in (20, 100, 400)))

print(f"\n== Run-to-run wobble: the demo table (seed {WOBBLE_SEED}, {WOBBLE_RUNS} runs, each result flips with chance {WOBBLE_CHANCE})")
print("  run | starting instructions | don't-know switch | fixed | broke")
dk = Instructions(dont_know=True)
for r in range(WOBBLE_RUNS):
    b, a = results(OFF, r), results(dk, r)
    c = compare(b, a)
    print(f"  {r + 1} | {count(b)} | {count(a)} | {c.count('fixed')} | {c.count('broke')}")
spreads = []
worse = total = 0
new = results(dk)
for seed in range(1, 2001):
    counts = [(count(m != f for m, f in zip(base, row)), count(m != f for m, f in zip(new, row))) for row in wobble(seed)]
    spreads.append(max(b for b, _ in counts) - min(b for b, _ in counts))
    total += len(counts)
    worse += sum(a < b for b, a in counts)
print(f"  over 2000 seeds: median spread of the starting instructions across 5 runs = {median(spreads)} questions; "
      f"spread of 4 or more in {pct(sum(s >= 4 for s in spreads) / len(spreads))} of seeds")
print(f"  a single run shows the don't-know switch scoring lower than the starting instructions in {pct(worse / total)} of {total} runs")

"""A toy help-page assistant with three instruction switches. Python standard library only.

This is NOT a language model. It follows the fixed rules below, so the same question and the same switches
always give the same answer. A real model answers by probability, and the lesson says what the toy leaves out.

The scene: a small bike shop's help page, 15 one-fact sentences (made up). The assistant answers a customer's
question from the page. You can change its instructions with three switches; the lesson asks whether a change
helps or hurts across a whole set of questions.
"""
import re
from dataclasses import dataclass
from itertools import product

PAGE = (
    "Opening hours on weekdays are 9am to 6pm.",
    "Opening hours on Saturday are 10am to 4pm.",
    "Opening hours on Sunday: closed.",
    "A basic tune-up costs $60.",
    "A basic tune-up takes 2 days.",
    "A full overhaul costs $140.",
    "A full overhaul takes 5 days.",
    "Unused parts can be returned within 30 days with the receipt.",
    "Used parts cannot be returned.",
    "Repairs have a 90 day warranty.",
    "Bikes bought here have a 2 year warranty.",
    "A city bike rents for $15 a day or $60 a week.",
    "We hold a $100 deposit on your card during a rental and release it when the bike is back.",
    "Free bike parking is at the front door.",
    "Adult riding lessons are on Saturdays at 11am and cost $25.",
)

# Small words the toy ignores when it compares a question with a sentence.
STOP = frozenset(
    "a an the is are was be do does did can could will would i you your my me we our us it its of on in at to for by "
    "and or if with from as that this there here what which who how when where why much many long have has had get any".split())

GUESS = "Yes, we do."  # what the toy says when no sentence shares a word with the question and nothing stops it
DONT_KNOW = "I don't know. The help page does not say."
MIN_SHARED = 2  # the "Say you don't know" switch refuses when the best sentence shares fewer words than this
ALSO_SHARED = 2  # a second sentence is added when it shares at least this many words with the question

FIELDS = ("page_only", "dont_know", "one_sentence")
SWITCH_NAMES = {
    "page_only": "Answer only from the help page",
    "dont_know": "Say \"I don't know\" when the page doesn't say",
    "one_sentence": "Keep answers to one sentence",
}


@dataclass(frozen=True)
class Instructions:
    page_only: bool = False  # no own knowledge: no guess when nothing matches, no unit conversion
    dont_know: bool = False  # refuse when the best sentence shares fewer than MIN_SHARED words with the question
    one_sentence: bool = False  # say only the best sentence


def all_settings():
    """All 8 on/off settings, in a fixed order (the first switch changes slowest)."""
    return [Instructions(*flags) for flags in product((False, True), repeat=len(FIELDS))]


def stem(word):
    """Crude word ending removal, so that "opening" matches "open" and "returned" matches "return"."""
    if len(word) > 3 and word.endswith("s") and not word.endswith("ss"):
        word = word[:-1]
    for ending in ("ing", "ed"):
        if len(word) > 5 and word.endswith(ending):
            return word[: -len(ending)]
    return word


def words(text):
    """The set of words the toy compares: lower case, small words dropped, endings removed."""
    return {stem(w) for w in re.findall(r"[a-z0-9]+", text.lower()) if w not in STOP}


def ranked(question):
    """[(shared word count, sentence number)] best first; equal counts keep page order."""
    q = words(question)
    return sorted(((len(q & words(sentence)), i) for i, sentence in enumerate(PAGE)), key=lambda t: (-t[0], t[1]))


def answer(question, d):
    """What the toy says to one question under the instructions d."""
    (best, i), (next_best, j) = ranked(question)[:2]
    if best == 0:
        return DONT_KNOW if d.page_only or d.dont_know else GUESS
    if d.dont_know and best < MIN_SHARED:
        return DONT_KNOW
    said = [PAGE[i]]
    if next_best >= ALSO_SHARED and not d.one_sentence:
        said.append(PAGE[j])
    text = " ".join(said)
    if not d.page_only and "month" in words(question):
        text = re.sub(r"(\d+) year", lambda m: f"{int(m.group(1)) * 12} month", text)
    return text

"""Tests for toy.py and evalset.py. The toy is a fixed set of rules, so every number the lesson quotes can be checked exactly."""
import re
import textwrap
from pathlib import Path

import pytest

from evalset import QUESTIONS, compare, has, passes, results, wilson, wobble
from toy import DONT_KNOW, GUESS, PAGE, Instructions, all_settings, answer, ranked, stem, words

OFF = Instructions()


def Q(n):
    """Question number n as the lesson numbers it (1 to 20)."""
    return QUESTIONS[n - 1]


def numbers(marks, kind):
    return [i + 1 for i, m in enumerate(compare(results(OFF), marks)) if m == kind]


# An answer a person would accept for each of the 20 questions. Guards a check that no right answer can pass.
GOOD = {
    1: "We open at 9am and close at 6pm.", 2: "Saturday is 10am to 4pm.", 3: "2 days.", 4: "It costs $140.",
    5: "The deposit is $100.", 6: "A week costs $60 a week.", 7: "90 days.", 8: "On Saturdays at 11am.",
    9: "Opening hours on Sunday: closed", 10: "Saturday 10am to 4pm, Sunday closed.", 11: "Repairs 90 day, bikes 2 year.",
    12: "That is 24 month.", 13: "I don't know.", 14: "The page does not say.", 15: "I'm not sure.", 16: "I do not know.",
    17: "Yes, free parking is at the front door.", 18: "$60 a week.", 19: "No, parts only come back within 30 days.", 20: "$15 a day.",
}

# (question number, a wrong answer, why a looser check would pass it)
TEMPTING = [
    (7, "Repairs have a 190 day warranty.", "the text '90' is inside '190'"),
    (5, "A $1000 deposit is held.", "the text '100' is inside '1000'"),
    (3, "A basic tune-up takes 2 days. A full overhaul takes 5 days.", "the right number is there, with a wrong one beside it"),
    (19, "I do not know.", "the letters 'no' are inside 'not' and 'know'"),
    (2, "Opening hours on weekdays are 9am to 6pm. Opening hours on Saturday are 10am to 4pm.", "both sentences hold 'Saturday' hours"),
    (9, "Opening hours on Sunday: closed. Opening hours on Saturday are 10am to 4pm.", "the right sentence is there, with a second one"),
    (13, "Yes, we do.", "an answer that is neither a refusal nor a fact"),
    (6, "A city bike rents for $15 a day or $60 a day.", "the text '$60' is there, with the wrong period"),
    (1, "We open at 9am.", "the opening time is there, the closing time is not"),
    (8, "Lessons are on Saturdays.", "the day is there, the time is not"),
    (11, "Repairs have a 90 day warranty.", "one of the two warranties is there, the other is not"),
    (12, "The warranty is 24 years.", "the text '24' is there, with the wrong unit"),
    (17, "Please do not leave it by the door.", "the word 'door' is there, but no word about parking"),
    (18, "A city bike rents for $160 a week.", "the words 'a week' are there, with the wrong price"),
    (20, "A bicycle rental costs $15 a week.", "the text '$15' is there, with the wrong period"),
]


def test_each_check_accepts_a_right_answer():
    assert set(GOOD) == set(range(1, 21))
    wrong = [n for n, text in GOOD.items() if not passes(Q(n), text)]
    assert wrong == []


@pytest.mark.parametrize("n, text, note", TEMPTING)
def test_checks_reject_answers_a_looser_check_would_pass(n, text, note):
    assert not passes(Q(n), text), note


def test_phrases_match_whole_words_only():
    assert has("Used parts cannot be returned.", "used parts") and not has("Unused parts", "used parts")
    assert has("$60 a week.", "$60 a week") and not has("$160 a week", "$60 a week")


def test_the_starting_instructions_pass_12_of_20():
    marks = results(OFF)
    assert len(QUESTIONS) == 20 and sum(marks) == 12
    assert [i + 1 for i, m in enumerate(marks) if not m] == [2, 3, 13, 14, 15, 16, 19, 20]
    assert answer(Q(13)[0], OFF) == "Bikes bought here have a 2 year warranty."  # the lesson's hook: a made-up answer


def test_each_switch_fixes_some_questions_and_breaks_others():
    """The lesson's point: no switch only helps. Exact lists, so a rule edit that moves them fails here."""
    by_switch = {name: results(Instructions(**{name: True})) for name in ("page_only", "dont_know", "one_sentence")}
    assert {k: (numbers(v, "fixed"), numbers(v, "broke")) for k, v in by_switch.items()} == {
        "page_only": ([14], [12]),
        "dont_know": ([13, 14, 15], [17, 18]),
        "one_sentence": ([2, 3], [10, 11]),
    }
    assert [sum(v) for v in by_switch.values()] == [12, 13, 12]  # the same pass rate can hide different questions


def test_all_eight_settings_and_the_ones_that_never_change():
    scores = [sum(results(d)) for d in all_settings()]
    assert len(scores) == 8 and scores == [12, 12, 13, 13, 12, 12, 12, 12]
    never_pass = [n for n in range(1, 21) if not any(results(d)[n - 1] for d in all_settings())]
    always_pass = [n for n in range(1, 21) if all(results(d)[n - 1] for d in all_settings())]
    assert never_pass == [16, 19, 20] and always_pass == [1, 4, 5, 6, 7, 8, 9]


def test_toy_rules_behind_the_three_switches():
    assert (GUESS, DONT_KNOW) == ("Yes, we do.", "I don't know. The help page does not say.")  # the lesson quotes both
    # no shared word: a guess, or a refusal when either switch is on
    assert answer(Q(14)[0], OFF) == GUESS
    assert answer(Q(14)[0], Instructions(page_only=True)) == DONT_KNOW == answer(Q(14)[0], Instructions(dont_know=True))
    # one shared word is not enough for the don't-know switch; three (Q16) is too many for it to notice
    assert ranked(Q(17)[0])[0][0] == 1 and ranked(Q(16)[0])[0][0] == 3
    # the second sentence needs two shared words; one_sentence drops it
    assert answer(Q(11)[0], OFF) == "Bikes bought here have a 2 year warranty. Repairs have a 90 day warranty."
    assert answer(Q(11)[0], Instructions(one_sentence=True)) == "Bikes bought here have a 2 year warranty."
    # unit conversion happens unless page_only is on
    assert "24 month" in answer(Q(12)[0], OFF) and "24 month" not in answer(Q(12)[0], Instructions(page_only=True))


def test_word_matching_rules():
    assert stem("opening") == "open" and stem("returned") == "return" and stem("parts") == "part" and stem("class") == "class"
    assert words("Is the shop open on Sunday?") == {"shop", "open", "sunday"} and len(PAGE) == 15


def test_compare_names_each_question():
    assert compare([True, True, False, False], [True, False, True, False]) == ["pass", "broke", "fixed", "fail"]


def test_wilson_range_matches_the_nist_worked_value_and_the_edges():
    """NIST e-Handbook 7.2.4.1: 26 of 200 (13%), one-sided 95% (z = 1.645): lower limit 0.09577."""
    assert round(wilson(26, 200, z=1.645)[0], 5) == 0.09577
    low, high = wilson(13, 20)
    assert (round(low, 3), round(high, 3)) == (0.433, 0.819)
    assert [round(x, 3) for x in wilson(12, 20)] == [0.387, 0.781]
    assert wilson(0, 20)[0] == pytest.approx(0, abs=1e-12) and wilson(20, 20)[1] == pytest.approx(1)
    assert all(wilson(k, 20)[0] - 1e-12 <= k / 20 <= wilson(k, 20)[1] + 1e-12 for k in range(21))
    assert wilson(12, 20)[1] > wilson(13, 20)[0]  # 12 of 20 and 13 of 20 overlap: one question is inside the noise


def test_wobble_table_is_fixed_and_flips_results_the_same_way_for_every_instruction():
    table = wobble()
    assert len(table) == 5 and all(len(row) == 20 for row in table) and table == wobble()
    assert [sum(results(OFF, r)) for r in range(5)] == [10, 11, 10, 11, 14]  # the spread the lesson quotes: 4 questions
    d = Instructions(dont_know=True)
    assert [sum(results(d, r)) for r in range(5)] == [13, 12, 13, 12, 13]  # in run 5 the better change looks worse


def test_every_python_block_in_the_lesson_is_a_verbatim_excerpt_of_the_source():
    """The lesson quotes these files. If the code changes and the lesson does not, the reader sees code that was never tested."""
    here = Path(__file__).parent
    lesson = (here.parent / "lesson.md").read_text()
    blocks = re.findall(r"^```python\n(.*?)^```", lesson, flags=re.S | re.M)
    assert len(blocks) == 4, "answer, has, compare and wilson"
    for block in blocks:
        lines = block.rstrip("\n").splitlines()
        found = False
        for name in ("toy.py", "evalset.py"):
            source = (here / name).read_text().splitlines()
            windows = (textwrap.dedent("\n".join(source[i:i + len(lines)])) for i in range(len(source) - len(lines) + 1))
            found = found or textwrap.dedent("\n".join(lines)) in windows
        assert found, f"not in toy.py or evalset.py:\n{block}"

"""Guards for ../widget.html: its data block, what it may show, and that its rules give the same answers as toy.py and evalset.py."""
import json
import re
import subprocess

from embed_widget import WIDGET, embed
from evalset import NUMBER, QUESTIONS, WOBBLE_CHANCE, compare, describe, has, passes, results, wilson, wobble
from toy import FIELDS, Instructions, all_settings, answer

HTML = WIDGET.read_text(encoding="utf-8")
LESSON = WIDGET.parent
OFF = Instructions()
RUNS = [None, 0, 1, 2, 3, 4]

# Reads the rules out of the widget between its marker comments, runs them, and compares with the Python results.
PARITY = r"""
import fs from 'node:fs';
const [fixture, widget] = process.argv.slice(2);
const fx = JSON.parse(fs.readFileSync(fixture, 'utf8'));
const html = fs.readFileSync(widget, 'utf8');
const cfg = JSON.parse(html.match(/<script type="application\/json" id="ev-config">([\s\S]*?)<\/script>/)[1]);
const js = html.slice(html.indexOf("/* ---------- the toy's rules"), html.indexOf('/* ---------- the page'));
const api = new Function('cfg', "'use strict';" + js + '; return {answer, passes, results, compare, wilson, numbers, has, OFF};')(cfg);
const bad = [];
let checked = 0;
const same = (what, got, want) => { checked++; if (JSON.stringify(got) !== JSON.stringify(want)) bad.push([what, got, want]); };
fx.settings.forEach((s) => {
  same('answers ' + s.name, cfg.questions.map((q) => api.answer(q.q, s.d)), s.answers);
  fx.runs.forEach((run, k) => {
    same('results ' + s.name + ' run ' + run, api.results(s.d, run), s.results[k]);
    same('compare ' + s.name + ' run ' + run, api.compare(api.results(api.OFF, run), api.results(s.d, run)), s.compare[k]);
  });
});
fx.numbers.forEach(([text, want]) => same('numbers ' + text, api.numbers(text), want));
fx.has.forEach(([text, phrase, want]) => same('has ' + text + ' / ' + phrase, api.has(text, phrase), want));
fx.grades.forEach(([i, text, want]) => same('passes Q' + (i + 1) + ' ' + text, api.passes(cfg.questions[i], text), want));
fx.wilson.forEach(([k, lo, hi]) => {
  const w = api.wilson(k, 20);
  checked++;
  if (Math.abs(w[0] - lo) > 1e-12 || Math.abs(w[1] - hi) > 1e-12) bad.push(['wilson', k, w, lo, hi]);
});
console.log(JSON.stringify({ checked, bad: bad.slice(0, 5) }));
process.exit(bad.length ? 1 : 0);
"""


# Runs the page script (everything after the rules) against a small stand-in for the DOM, and reads back what a reader would see.
PAGE_TEST = r"""
import fs from 'node:fs';
const [widget, fixture] = process.argv.slice(2);
const html = fs.readFileSync(widget, 'utf8');
const { states, opening } = JSON.parse(fs.readFileSync(fixture, 'utf8'));
const cfgText = html.match(/<script type="application\/json" id="ev-config">([\s\S]*?)<\/script>/)[1];
const code = html.match(/<script>([\s\S]*?)<\/script>/)[1];

function el(tag) {
  const e = { tag, children: [], listeners: {}, className: '', checked: false, disabled: false, value: '0', attrs: {}, _text: '' };
  e.addEventListener = (type, f) => (e.listeners[type] = e.listeners[type] || []).push(f);
  e.appendChild = (c) => (e.children.push(c), c);
  e.setAttribute = (k, v) => (e.attrs[k] = v);
  Object.defineProperty(e, 'textContent', { get: () => e._text, set: (v) => { e._text = String(v); e.children = []; } });
  return e;
}
const controls = ['ev-page', 'ev-dk', 'ev-one', 'ev-wob', 'ev-run'];
const tagOf = (id) => html.match(new RegExp('<[a-z]+ [^>]*id="' + id + '"[^>]*>'))[0];
const value = (page, id) => (id === 'ev-run' ? page[id].value : page[id].checked);
function open() {
  const page = {};
  ['ev-live', 'ev-big', 'ev-cells', 'ev-detail', 'ev-rates', 'ev-runs', ...controls].forEach((id) => { page[id] = el('x'); });
  page['ev-config'] = el('script');
  page['ev-config']._text = cfgText;
  controls.forEach((id) => { page[id].checked = / checked[ >]/.test(tagOf(id)); page[id].disabled = / disabled[ >]/.test(tagOf(id)); });
  const doc = { getElementById: (id) => page[id], createElement: el, documentElement: { setAttribute() {} } };
  new Function('document', 'window', 'parent', code)(doc, { addEventListener() {} }, {});
  return page;
}
const fire = (e, type = 'change') => (e.listeners[type] || []).forEach((f) => f());
const cells = (page) => page['ev-cells'].children;
const detail = (page) => page['ev-detail'].children.map((p) => p._text);

const bad = [];
const want = (what, got, expected) => { if (JSON.stringify(got) !== JSON.stringify(expected)) bad.push([what, got, expected]); };

const start = open();
want('opening: headline', start['ev-big']._text, opening);
want('opening: 20 squares', cells(start).length, 20);
want('opening: don\'t-know switch is on, the other two off', [start['ev-page'].checked, start['ev-dk'].checked, start['ev-one'].checked], [false, true, false]);
want('opening: variation controls are off', [start['ev-wob'].checked, start['ev-run'].disabled, start['ev-runs']._text], [false, true, '']);
want('opening: first picked square is 13', cells(start).map((c) => c.attrs['aria-pressed']).indexOf('true'), 12);

states.forEach((st) => {
  const page = open(), was = controls.map((id) => value(page, id));
  page['ev-page'].checked = st.d[0]; page['ev-dk'].checked = st.d[1]; page['ev-one'].checked = st.d[2];
  page['ev-wob'].checked = st.run !== null;
  if (st.run !== null) page['ev-run'].value = String(st.run);
  (st.only ? [st.only] : controls.filter((id, k) => value(page, id) !== was[k])).forEach((id) => fire(page[id]));
  if (st.pick !== null) fire(cells(page)[st.pick], 'click');
  want(st.name + ': headline', page['ev-big']._text, st.headline);
  want(st.name + ': square words', cells(page).map((c) => c.children[1]._text), st.words);
  want(st.name + ': square classes', cells(page).map((c) => c.className), st.classes);
  want(st.name + ': detail', detail(page), st.detail);
  want(st.name + ': rates', page['ev-rates']._text, st.rates);
  want(st.name + ': runs line', page['ev-runs']._text, st.runs);
  want(st.name + ': run menu enabled only with variation on', page['ev-run'].disabled, st.run === null);
  want(st.name + ': screen reader line', page['ev-live']._text, st.live);
});
console.log(JSON.stringify({ states: states.length, bad: bad.slice(0, 5) }));
process.exit(bad.length ? 1 : 0);
"""


def flags(d):
    return [getattr(d, f) for f in FIELDS]


def js_settings(d):
    return {"pageOnly": d.page_only, "dontKnow": d.dont_know, "oneSentence": d.one_sentence}


def pct(x):
    return f"{round(100 * x)}%"


def span(k):
    lo, hi = wilson(k, 20)
    return f"{pct(lo)} to {pct(hi)}"


def expected_page(name, d, run, pick):
    """What the page should show, worked out from toy.py and evalset.py with none of the widget's code."""
    before, after = results(OFF, run), results(d, run)
    c = compare(before, after)
    kb, ka, fixed, broke = sum(before), sum(after), c.count("fixed"), c.count("broke")
    i = 12 if pick is None else pick
    q = QUESTIONS[i]
    flip = run is not None and wobble()[run][i]
    check = describe(q)
    gap = "" if kb == ka else " The ranges overlap, so a gap this size could come from which questions you picked." if wilson(kb, 20)[1] >= wilson(ka, 20)[0] and wilson(ka, 20)[1] >= wilson(kb, 20)[0] else " The ranges do not overlap."
    runs = [(sum(results(OFF, r)), sum(results(d, r))) for r in range(5)]
    low = lambda j: min(x[j] for x in runs)
    high = lambda j: max(x[j] for x in runs)
    headline = f"Nothing changed: still {pct(ka / 20)}." if fixed + broke == 0 else \
        f"This change fixed {fixed} question{'' if fixed == 1 else 's'} and broke {broke}: {pct(kb / 20)} to {pct(ka / 20)}."
    live = lambda v: ", ".join(f"Q{n + 1}" for n, x in enumerate(c) if x == v) or "none"
    return {"name": name, "d": flags(d), "run": run, "pick": pick, "headline": headline, "words": c,
            "classes": [f"ev-cell {x}" + (" sel" if n == i else "") for n, x in enumerate(c)],
            "detail": [f"Q{i + 1}. {q[0]}", f"Right answer: {q[1]}",
                       f"Starting instructions said: “{answer(q[0], OFF)}” ({'pass' if before[i] else 'fail'})",
                       f"This setting says: “{answer(q[0], d)}” ({'pass' if after[i] else 'fail'})",
                       "Check: " + check[0].lower() + check[1:] + "." + (" This run, both results came out the other way." if flip else "")],
            "rates": f"Starting instructions: {kb} of 20 ({span(kb)}, 95% range). This setting: {ka} of 20 ({span(ka)})." + gap,
            "runs": f"Across all 5 runs the starting instructions score {low(0)} to {high(0)} of 20; this setting scores {low(1)} to {high(1)}." if run is not None else "",
            "live": f"{headline} Fixed: {live('fixed')}. Broke: {live('broke')}."}


def run_node(tmp_path, script, fixture, args):
    (tmp_path / "script.mjs").write_text(script, encoding="utf-8")
    (tmp_path / "fixture.json").write_text(json.dumps(fixture), encoding="utf-8")
    result = subprocess.run(["node", str(tmp_path / "script.mjs"), *args(tmp_path / "fixture.json")], capture_output=True, text=True, timeout=60)
    assert result.returncode == 0, result.stdout + result.stderr
    return json.loads(result.stdout)


def test_widget_data_block_matches_toy():
    """Guards a toy.py or evalset.py edit that was not copied into the widget (run embed_widget.py)."""
    assert embed(HTML) == HTML


def test_every_element_the_script_looks_up_exists_inside_the_root():
    """Guards a widget that never starts: a lookup of an id that is not in the page throws in the browser."""
    root = HTML[HTML.index('<div class="ev-root">'):HTML.index("<script")]
    script = HTML[HTML.index("<script>"):]
    lookups = set(re.findall(r"\$\('([\w-]+)'\)", script))
    handlers = re.search(r"^\s*\['ev-page'.*$", script, re.M)
    assert handlers, "the handler list that starts with 'ev-page' has moved: update this test"
    lookups |= set(re.findall(r"'(ev-[\w-]+)'", handlers.group(0)))
    assert lookups and all(f'id="{i}"' in root for i in lookups), sorted(i for i in lookups if f'id="{i}"' not in root)


def test_widget_is_small_and_asks_for_nothing_online():
    assert len(HTML.splitlines()) <= 300 and len(HTML.encode("utf-8")) < 40 * 1024
    page = re.sub(r'<script type="application/json" id="ev-config">.*?</script>', "", HTML, flags=re.S)
    for banned in ("http://", "https://", "fetch(", "XMLHttpRequest", "WebSocket", "sendBeacon", "import("):
        assert banned not in page, banned
    assert " src=" not in page and "@import" not in page


def test_widget_has_no_form_element():
    """Guards a dead button: the lesson page loads widget.html in a sandboxed frame, where a form cannot submit."""
    assert "<form" not in HTML and 'type="submit"' not in HTML


def test_widget_keeps_the_site_theme_listener_and_palette():
    assert "d.type !== 'spicyshark:theme'" in HTML and "e.source !== parent" in HTML
    for pair in ("light-dark(#ffffff, #0a0f17)", "light-dark(#ebf0f6, #131b28)", "light-dark(#0a1626, #f4f7fb)",
                 "light-dark(#44566b, #a3b1c4)", "light-dark(#d2dae4, #1c2635)", "light-dark(#66768a, #7a8a9f)",
                 "--green: light-dark(#00855F, #009E73)", "--orange: light-dark(#b36200, #E69F00)"):
        assert pair in HTML, pair
    assert "1 in 10" in HTML and WOBBLE_CHANCE == 0.1  # the words in the widget say what evalset.py does


def test_no_blue_or_vermillion_anywhere_in_the_lesson():
    """Guards the site palette: ink, grey, orange and green only (no blue, no vermillion), in the page, the figures and the text."""
    files = [LESSON / "lesson.md", LESSON / "widget.html", *sorted((LESSON / "figs").glob("*.svg"))]
    found = {f.name: m for f in files if f.exists() for m in ("0072B2", "56B4E9", "D55E00") if m.lower() in f.read_text(encoding="utf-8").lower()}
    assert found == {}


def test_widget_rules_match_the_python_toy(tmp_path):
    """Guards the widget's JavaScript drifting from toy.py and evalset.py: same answers, marks, fixed/broke lists and ranges."""
    settings = []
    for d in all_settings():
        settings.append({"name": "+".join(f for f, on in zip(FIELDS, flags(d)) if on) or "off", "d": js_settings(d),
                         "answers": [answer(q[0], d) for q in QUESTIONS],
                         "results": [results(d, r) for r in RUNS], "compare": [compare(results(OFF, r), results(d, r)) for r in RUNS]})

    def nums(text):
        values = sorted({float(x) for x in NUMBER.findall(text)})
        return ",".join(str(int(v)) if v == int(v) else repr(v) for v in values)

    fixture = {"runs": RUNS, "settings": settings, "wilson": [[k, *wilson(k, 20)] for k in range(21)],
               "numbers": [[t, nums(t)] for t in ("$60.", "9am to 6pm", "a 190 day warranty, 2 days", "3.5 and 3.50 and 7", "$100 and $1000", "none", "1 2 2 1", "v2 5x 7")],
               "has": [[t, p, has(t, p)] for t, p in (("I do not know", "no"), ("No, only within 30 days", "no"), ("$160 a week", "$60 a week"),
                                                     ("Unused parts", "used parts"), ("Used parts.", "used parts"), ("Opening: 9AM", "9am"))],
               "grades": [[i, text, passes(QUESTIONS[i], text)] for i, text in
                          ((2, "2 days. 5 days."), (6, "a 190 day warranty"), (8, "Opening hours on Sunday: closed."), (18, "I do not know."), (18, "No, 30 days."),
                           (12, "Yes, we do."), (12, "The help page does not say."), (1, "10am to 4pm and 9am"))]}
    out = run_node(tmp_path, PARITY, fixture, lambda f: [str(f), str(WIDGET)])
    assert out["bad"] == [] and out["checked"] == 8 * (1 + 2 * len(RUNS)) + 8 + 6 + 8 + 21


def test_widget_page_shows_what_the_lesson_names(tmp_path):
    """Guards the part a reader sees: the headline sentence, the word in every square, the detail lines, the ranges, the switches the page opens with and the wiring of each control.

    The lesson names these states: the opening one (13 of 20: fixed 3, broke 2), one sentence (fixed 2, broke 2, same 60%), and run 5 of the variation (the new
    instructions score below the starting ones). The opening switches are read from widget.html's own attributes. Each control is also changed and fired on its
    own, so a control whose change handler is missing leaves the page unchanged and fails."""
    on = lambda *names: Instructions(**{n: True for n in names})
    states = [{**expected_page("opening", on("dont_know"), None, None), "live": ""},  # nothing is announced until a control changes
              expected_page("all off", OFF, None, 3),
              expected_page("one sentence", on("one_sentence"), None, 9),
              expected_page("page only", on("page_only"), None, 11),
              expected_page("all three", on("page_only", "dont_know", "one_sentence"), None, 15)]
    states += [expected_page(f"variation run {r + 1}", on("dont_know"), r, 17 if r == 2 else None) for r in range(5)]
    alone = [("ev-page", on("page_only", "dont_know"), None), ("ev-dk", OFF, None), ("ev-one", on("dont_know", "one_sentence"), None),
             ("ev-wob", on("dont_know"), 0), ("ev-run", on("dont_know"), 4)]
    states += [{**expected_page(f"{id} alone", d, run, None), "only": id} for id, d, run in alone]
    out = run_node(tmp_path, PAGE_TEST, {"states": states, "opening": states[0]["headline"]}, lambda f: [str(WIDGET), str(f)])
    assert out == {"states": len(states), "bad": []}
    assert states[0]["headline"] == "This change fixed 3 questions and broke 2: 60% to 65%."
    assert states[9]["headline"] == "This change fixed 2 questions and broke 3: 70% to 65%."  # run 5: the better change looks worse
    assert "<b>Vary the results</b>" in WIDGET.read_text()  # it flips results, not answers