Why is my AI app bill higher than I expected?

"""What an AI chat bill is made of, turn by turn, and a closed form for the total.

Python standard library only. Run `python3 show_numbers.py` to print every number the
lesson quotes.

The model: every turn the app sends the provider the system prompt, the earlier turns
(unless trimmed) and the new message, and the provider bills those as input tokens. It
then bills the reply, plus any hidden reasoning tokens, as output tokens. The two kinds
of token have two prices. Sizes are in tokens; prices are dollars per million tokens.
Reasoning tokens are assumed not to be sent back as history. Some providers do send
them back (the lesson cites which), and then the history part would be larger.
"""
from dataclasses import dataclass
from decimal import Decimal
from typing import Optional

PER = Decimal(1_000_000)
DAYS_PER_MONTH = 30


@dataclass(frozen=True)
class Chat:
    system: int                  # system prompt, tool definitions and pasted documents: sent on every call
    message: int                 # tokens in each new user message
    reply: int                   # visible tokens in each reply
    turns: int                   # user messages in the chat
    reasoning: int = 0           # hidden reasoning tokens per reply, billed as output
    keep: Optional[int] = None   # send only the last `keep` earlier turns; None sends them all

    def __post_init__(self):
        if self.turns < 1 or min(self.system, self.message, self.reply, self.reasoning) < 0:
            raise ValueError("turns must be at least 1 and sizes cannot be negative")
        if self.keep is not None and self.keep < 0:
            raise ValueError("keep cannot be negative")


@dataclass(frozen=True)
class Turn:
    system: int
    history: int
    message: int
    reply: int
    reasoning: int

    @property
    def input(self):
        return self.system + self.history + self.message

    @property
    def output(self):
        return self.reply + self.reasoning


def turn_parts(chat, k):
    """The tokens of turn k (counting from 1), split the way the widget's bar splits them."""
    earlier = k - 1 if chat.keep is None else min(k - 1, chat.keep)
    return Turn(chat.system, earlier * (chat.message + chat.reply), chat.message, chat.reply, chat.reasoning)


def simulate(chat):
    """Input and output tokens of the whole chat, found by walking through every turn."""
    history = []  # earlier message sizes, oldest first
    total_in = total_out = 0
    for _ in range(chat.turns):
        kept = history
        if chat.keep is not None:
            kept = history[max(0, len(history) - 2 * chat.keep):]
        total_in += chat.system + sum(kept) + chat.message
        total_out += chat.reply + chat.reasoning
        history += [chat.message, chat.reply]
    return total_in, total_out


def closed_form(chat):
    """The same two totals without the loop.

    Untrimmed, turn k re-sends k-1 earlier turns, and 0 + 1 + ... + (n-1) = n(n-1)/2.
    Trimmed to N turns, the first N+1 turns re-send 0, 1, ..., N earlier turns and every
    later turn re-sends N.
    """
    n = chat.turns
    t = n - 1 if chat.keep is None else min(n - 1, chat.keep)
    earlier_turns = t * (t + 1) // 2
    if chat.keep is not None:
        earlier_turns += chat.keep * (n - 1 - t)
    fixed = n * (chat.system + chat.message)
    total_in = fixed + (chat.message + chat.reply) * earlier_turns
    return total_in, n * (chat.reply + chat.reasoning)


@dataclass(frozen=True)
class Bill:
    input_tokens: int
    output_tokens: int
    input_cost: Decimal
    output_cost: Decimal
    retry_cost: Decimal

    @property
    def total(self):
        return self.input_cost + self.output_cost + self.retry_cost


def _price(value, name):
    value = Decimal(str(value))
    if value < 0:
        raise ValueError(f"{name} cannot be negative")
    return value


def turn_cost(chat, k, price_in, price_out):
    """Dollars for turn k: input at the input price plus output at the output price."""
    parts = turn_parts(chat, k)
    return (parts.input * _price(price_in, "price_in") + parts.output * _price(price_out, "price_out")) / PER


def bill(chat, price_in, price_out, retry_rate=0):
    """Dollars for the whole chat. Prices are dollars per million tokens.

    retry_rate is the average number of extra full calls per call: 0.1 means one call in ten
    is made twice and both are billed. A retry sends the same input again, so it adds
    retry_rate times the whole bill.
    """
    price_in, price_out, retry_rate = _price(price_in, "price_in"), _price(price_out, "price_out"), _price(retry_rate, "retry_rate")
    tokens_in, tokens_out = closed_form(chat)
    cost_in = tokens_in * price_in / PER
    cost_out = tokens_out * price_out / PER
    retry_cost = (cost_in + cost_out) * retry_rate
    return Bill(tokens_in, tokens_out, cost_in, cost_out, retry_cost)


def naive_bill(chat, price_in, price_out):
    """The estimate most people make first: what you type, and what comes back, each billed once."""
    tokens_in = chat.turns * chat.message
    tokens_out = chat.turns * (chat.reply + chat.reasoning)
    cost_in = tokens_in * _price(price_in, "price_in")
    cost_out = tokens_out * _price(price_out, "price_out")
    return (cost_in + cost_out) / PER


# The chat the lesson and its widget start from. The prices are example figures, not a quote.
EXAMPLE = Chat(system=500, message=60, reply=200, turns=10)
EXAMPLE_PRICES = (2, 10)         # dollars per million input tokens, dollars per million output tokens
EXAMPLE_KEEP = 3                 # the "last N turns" the widget's trim box starts at
EXAMPLE_CHATS_PER_DAY = 100

"""Keep the numbers inside ../widget.html equal to the ones in bill.py.

    python3 embed_widget.py          rewrite the data block in widget.html
    python3 embed_widget.py --check  exit 1 if widget.html differs (test_widget.py runs the same check)
"""
import json
import re
import sys
from pathlib import Path

from bill import DAYS_PER_MONTH, EXAMPLE, EXAMPLE_CHATS_PER_DAY, EXAMPLE_KEEP, EXAMPLE_PRICES

WIDGET = Path(__file__).resolve().parent.parent / "widget.html"
CONFIG_BLOCK = re.compile(r'(<script type="application/json" id="bil-config">)(.*?)(</script>)', re.S)
WORDS_PER_100_TOKENS = (60, 80)   # Google's tokens page: "100 tokens is equal to about 60-80 English words"


def config():
    return {"system": EXAMPLE.system, "turns": EXAMPLE.turns, "message": EXAMPLE.message, "reply": EXAMPLE.reply,
            "reasoning": EXAMPLE.reasoning, "chatsPerDay": EXAMPLE_CHATS_PER_DAY, "retryPercent": 0,
            "priceIn": EXAMPLE_PRICES[0], "priceOut": EXAMPLE_PRICES[1], "keep": EXAMPLE_KEEP,
            "daysPerMonth": DAYS_PER_MONTH, "wordsPer100Tokens": list(WORDS_PER_100_TOKENS)}


def embed(html):
    """html with the config block rewritten from bill.py."""
    block = json.dumps(config(), indent=1)
    return CONFIG_BLOCK.sub(lambda m: m.group(1) + "\n" + block + "\n" + m.group(3), html, count=1)


if __name__ == "__main__":
    current = WIDGET.read_text(encoding="utf-8")
    updated = embed(current)
    if "--check" in sys.argv:
        sys.exit(0 if updated == current else 1)
    WIDGET.write_text(updated, encoding="utf-8")
    print("widget.html data block now matches bill.py")

"""Print every number the lesson computes. Run it from this folder: python3 show_numbers.py"""
from dataclasses import replace
from decimal import Decimal

from bill import (DAYS_PER_MONTH, EXAMPLE, EXAMPLE_CHATS_PER_DAY, EXAMPLE_KEEP, EXAMPLE_PRICES as PRICES, bill, closed_form,
                  naive_bill, simulate, turn_cost, turn_parts)

P_IN, P_OUT = PRICES


def pct(part, whole):
    return f"{100 * Decimal(part) / Decimal(whole):.1f}%"


def money(x):
    text = f"{x.normalize():f}"
    whole, _, cents = text.partition(".")
    return f"${int(whole):,}.{cents.ljust(2, '0')}"


print(f"== The example chat: {EXAMPLE.turns} turns, system prompt {EXAMPLE.system}, each message {EXAMPLE.message}, "
      f"each reply {EXAMPLE.reply} tokens. Example prices ${P_IN} in, ${P_OUT} out per million tokens")
print("  turn | system | history | message | input | output | cost | running total")
running = Decimal(0)
for k in range(1, EXAMPLE.turns + 1):
    t, c = turn_parts(EXAMPLE, k), turn_cost(EXAMPLE, k, *PRICES)
    running += c
    print(f"  {k} | {t.system} | {t.history:,} | {t.message} | {t.input:,} | {t.output} | {money(c)} | {money(running)}")

b = bill(EXAMPLE, *PRICES)
loop, closed = simulate(EXAMPLE), closed_form(EXAMPLE)
print(f"\n== Whole chat: simulated {loop[0]:,} in / {loop[1]:,} out; closed form {closed[0]:,} in / {closed[1]:,} out")
print(f"  input {money(b.input_cost)} + output {money(b.output_cost)} = {money(b.total)}; output is {pct(b.output_cost, b.total)} of the money "
      f"and {pct(b.output_tokens, b.input_tokens + b.output_tokens)} of the tokens")
sys_tokens = EXAMPLE.turns * EXAMPLE.system
hist_tokens = sum(turn_parts(EXAMPLE, k).history for k in range(1, EXAMPLE.turns + 1))
new_tokens = EXAMPLE.turns * EXAMPLE.message
print(f"  input tokens by part: system {sys_tokens:,} ({pct(sys_tokens, b.input_tokens)}), history {hist_tokens:,} "
      f"({pct(hist_tokens, b.input_tokens)}), new messages {new_tokens:,} ({pct(new_tokens, b.input_tokens)})")
last = turn_parts(EXAMPLE, EXAMPLE.turns)
print(f"  turn {EXAMPLE.turns}: {last.input:,} input tokens to answer a {EXAMPLE.message}-token message "
      f"({last.input // EXAMPLE.message}x, rounded down)")

print("\n== Longer chats, same sizes (a 'fresh start' is a one-turn chat: system prompt + one message + one reply)")
fresh = bill(replace(EXAMPLE, turns=1), *PRICES).total
print(f"  one fresh start: {money(fresh)}")
print("  turns | input tokens | output tokens | cost | cost of that many fresh starts | ratio")
for n in (10, 20, 30):
    c = replace(EXAMPLE, turns=n)
    x = bill(c, *PRICES)
    print(f"  {n} | {x.input_tokens:,} | {x.output_tokens:,} | {money(x.total)} | {money(fresh * n)} | {x.total / (fresh * n):.2f}x")

print(f"\n== Trim to the last {EXAMPLE_KEEP} earlier turns")
for n in (10, 30):
    full, trimmed = bill(replace(EXAMPLE, turns=n), *PRICES), bill(replace(EXAMPLE, turns=n, keep=EXAMPLE_KEEP), *PRICES)
    print(f"  {n} turns: input {full.input_tokens:,} -> {trimmed.input_tokens:,}; cost {money(full.total)} -> {money(trimmed.total)} "
          f"({pct(full.total - trimmed.total, full.total)} less)")
kept = replace(EXAMPLE, turns=30, keep=EXAMPLE_KEEP)
print(f"  per-turn input tokens, trimmed 30-turn chat, turns 1-6: {[turn_parts(kept, k).input for k in range(1, 7)]}; "
      f"from turn 4 on every turn sends {turn_parts(kept, 4).input:,}")
print(f"  untrimmed turns 1-6: {[turn_parts(replace(EXAMPLE, turns=30), k).input for k in range(1, 7)]}; turn 30 sends "
      f"{turn_parts(replace(EXAMPLE, turns=30), 30).input:,}")

print("\n== Other levers, 10-turn chat, example prices")
levers = [("baseline", EXAMPLE), ("system prompt 2,000 (tools and pasted documents)", replace(EXAMPLE, system=2000)),
          ("system prompt 100", replace(EXAMPLE, system=100)), ("replies capped at 100 tokens", replace(EXAMPLE, reply=100)),
          ("hidden reasoning 800 tokens per reply", replace(EXAMPLE, reasoning=800))]
for name, c in levers:
    x = bill(c, *PRICES)
    print(f"  {name}: {x.input_tokens:,} in / {x.output_tokens:,} out = {money(x.total)} ({x.total / b.total:.2f}x baseline)")
think = bill(replace(EXAMPLE, reasoning=800), *PRICES)
print(f"  reasoning adds {money(think.total - b.total)}; billed output tokens {think.output_tokens:,} against "
      f"{EXAMPLE.turns * EXAMPLE.reply:,} visible ({think.output_tokens // (EXAMPLE.turns * EXAMPLE.reply)}x)")

print("\n== Retries: one call in ten is made twice")
r = bill(EXAMPLE, *PRICES, retry_rate="0.1")
print(f"  base {money(r.input_cost + r.output_cost)} + retries {money(r.retry_cost)} = {money(r.total)}")

print("\n== Smaller model for the easy steps (prices read 2026-10-19: Sonnet 5.5 $2 / $10, Haiku 5.5 $0.10 / $0.50 per million)")
big, small = bill(EXAMPLE, 2, 10), bill(EXAMPLE, "0.10", "0.50")
print(f"  10-turn chat: {money(big.total)} at $2 / $10 against {money(small.total)} at $0.10 / $0.50 = {big.total / small.total:.0f}x cheaper")

print("\n== The naive estimate: only what you type and what comes back, each billed once")
for turns in (10, 30):
    c = replace(EXAMPLE, turns=turns)
    naive, real = naive_bill(c, *PRICES), bill(c, *PRICES).total
    print(f"  {turns}-turn chat: naive {money(naive)}, real {money(real)}, real is {real / naive:.1f}x the estimate")

print(f"\n== A month: {EXAMPLE_CHATS_PER_DAY} chats a day, {DAYS_PER_MONTH} days")
chats = EXAMPLE_CHATS_PER_DAY * DAYS_PER_MONTH
for name, c in (("10-turn chat", EXAMPLE), (f"10-turn chat trimmed to {EXAMPLE_KEEP}", replace(EXAMPLE, keep=EXAMPLE_KEEP)),
                ("30-turn chat", replace(EXAMPLE, turns=30)), (f"30-turn chat trimmed to {EXAMPLE_KEEP}", replace(EXAMPLE, turns=30, keep=EXAMPLE_KEEP))):
    print(f"  {name}: {chats:,} chats x {money(bill(c, *PRICES).total)} = {money(chats * bill(c, *PRICES).total)}")

"""Guards for bill.py: the closed form must equal the turn-by-turn simulation, and trimming and
the reasoning and retry terms must add exactly what the lesson says."""
import itertools
from decimal import Decimal
from fractions import Fraction

import pytest

from bill import Chat, bill, closed_form, naive_bill, simulate, turn_cost, turn_parts

EXAMPLE = Chat(system=500, message=60, reply=200, turns=10)


def grid():
    for s, m, r, n, think, keep in itertools.product(
            (0, 500), (0, 60), (0, 200), range(1, 13), (0, 300), (None, 0, 1, 3, 20)):
        yield Chat(system=s, message=m, reply=r, turns=n, reasoning=think, keep=keep)


def test_closed_form_equals_simulating_every_turn():
    """Guards the algebra: a slip in n(n-1)/2 or in the trimmed sum shows up on some small chat."""
    count = 0
    for chat in grid():
        assert closed_form(chat) == simulate(chat), chat
        count += 1
    assert count == 2 * 2 * 2 * 12 * 2 * 5


def test_the_per_turn_parts_add_up_to_the_same_totals():
    for chat in grid():
        parts = [turn_parts(chat, k) for k in range(1, chat.turns + 1)]
        assert (sum(p.input for p in parts), sum(p.output for p in parts)) == closed_form(chat), chat


def test_example_chat_by_hand():
    """10 turns: input 500+60 at turn 1, then 260 more each turn; output 200 each turn."""
    assert [turn_parts(EXAMPLE, k).input for k in (1, 2, 3, 10)] == [560, 820, 1080, 2900]
    assert closed_form(EXAMPLE) == (5600 + 260 * 45, 2000) == (17300, 2000)
    b = bill(EXAMPLE, 2, 10)
    assert (b.input_cost, b.output_cost, b.total) == (Decimal("0.0346"), Decimal("0.02"), Decimal("0.0546"))


def test_untrimmed_input_grows_with_the_square_of_the_turns():
    """The first difference rises by m+r each turn, so the second difference is constant: a parabola."""
    def total(n):
        return closed_form(Chat(500, 60, 200, n))[0]
    firsts = [total(n) - total(n - 1) for n in range(2, 30)]
    assert firsts[0] == 500 + 60 + 260
    assert {b - a for a, b in zip(firsts, firsts[1:])} == {260}


def test_trimmed_cost_grows_linearly_once_the_window_is_full():
    """Guards the claim 'trim to N turns and the bill stops growing faster': every extra turn adds
    the same S + m + N(m+r) input tokens."""
    for keep in (0, 1, 3, 8):
        def total(n):
            return closed_form(Chat(500, 60, 200, n, keep=keep))[0]
        steps = {total(n) - total(n - 1) for n in range(keep + 2, 60)}
        assert steps == {500 + 60 + keep * 260}, keep


def test_trimming_changes_nothing_until_the_window_fills_and_never_costs_more():
    for n in range(1, 10):        # a window of 8 earlier turns holds every earlier turn of a chat up to 9 turns
        assert closed_form(Chat(500, 60, 200, n, keep=8)) == closed_form(Chat(500, 60, 200, n))
    for chat in grid():
        untrimmed = Chat(chat.system, chat.message, chat.reply, chat.turns, chat.reasoning)
        assert closed_form(chat)[0] <= closed_form(untrimmed)[0]
        assert closed_form(chat)[1] == closed_form(untrimmed)[1]


def test_reasoning_tokens_add_output_cost_only():
    """Guards the lesson sentence: hidden reasoning is billed as output, once per reply, and is
    not added to the input here (the model assumes it is not re-sent)."""
    plain, thinking = Chat(500, 60, 200, 10), Chat(500, 60, 200, 10, reasoning=800)
    a, b = bill(plain, 2, 10), bill(thinking, 2, 10)
    assert b.input_tokens == a.input_tokens
    assert b.output_tokens == a.output_tokens + 10 * 800
    assert b.total - a.total == Decimal(10 * 800) * 10 / 1_000_000 == Decimal("0.08")


def test_retries_add_their_share_of_every_call_again():
    base = bill(EXAMPLE, 2, 10)
    assert base.retry_cost == 0
    assert bill(EXAMPLE, 2, 10, retry_rate="0.1").retry_cost == base.total / 10
    assert bill(EXAMPLE, 2, 10, retry_rate="0.1").total == base.total * Decimal("1.1")


def test_retry_rate_is_the_expected_extra_cost_over_every_choice_of_retried_turns():
    """Guards 'the cost of retries is the retry rate times the bill': add up every way of choosing
    which turns of a 6-turn chat are made twice, each way weighted by its probability, exactly."""
    chat, q = Chat(500, 60, 200, 6, reasoning=100), Fraction(1, 4)
    per_turn = [Fraction(turn_cost(chat, k, 2, 10)) for k in range(1, 7)]
    expected = Fraction(0)
    for picked in itertools.product((False, True), repeat=6):
        p = Fraction(1)
        for flag in picked:
            p *= q if flag else 1 - q
        expected += p * sum(c * (2 if flag else 1) for c, flag in zip(per_turn, picked))
    assert expected == sum(per_turn) * (1 + q)
    assert Decimal(expected.numerator) / Decimal(expected.denominator) == bill(chat, 2, 10, retry_rate="0.25").total


def test_the_naive_estimate_misses_exactly_the_system_prompt_and_the_history():
    """Guards the lesson's opening claim: ignoring what rides along makes the estimate 2.6 times too low."""
    assert naive_bill(EXAMPLE, 2, 10) == Decimal("0.0212")
    long = Chat(system=500, message=60, reply=200, turns=30)
    assert naive_bill(long, 2, 10) == Decimal("0.0636")
    # hidden reasoning is output, so the naive estimate counts it: 10 turns x 600 more tokens at $10 per million
    assert naive_bill(Chat(system=500, message=60, reply=200, turns=10, reasoning=600), 2, 10) == Decimal("0.0812")
    for chat in (EXAMPLE, long, Chat(system=0, message=60, reply=200, turns=1)):
        tokens_in, _ = closed_form(chat)
        missed = Decimal(tokens_in - chat.turns * chat.message) * 2 / 1_000_000
        assert bill(chat, 2, 10).total - naive_bill(chat, 2, 10) == missed
    # a one-turn chat with no system prompt has nothing riding along, so the estimate is exact
    assert naive_bill(Chat(system=0, message=60, reply=200, turns=1), 2, 10) == bill(Chat(system=0, message=60, reply=200, turns=1), 2, 10).total


def test_the_check_yourself_answers():
    """Guards the numbers printed in the lesson's questions, so an edit to either side shows up here."""
    small = Chat(system=100, message=10, reply=20, turns=4)
    assert closed_form(small) == (4 * 110 + 30 * 6, 80) == (620, 80)
    assert closed_form(Chat(system=100, message=10, reply=20, turns=4, keep=1)) == (4 * 110 + 30 * 3, 80) == (530, 80)
    assert bill(small, 2, 10).total == Decimal("0.00204")
    thinker = Chat(system=0, message=0, reply=150, turns=1, reasoning=600)
    assert bill(thinker, 2, 10).output_cost == Decimal("0.0075") == 5 * bill(Chat(system=0, message=0, reply=150, turns=1), 2, 10).output_cost
    eight = Chat(system=200, message=30, reply=70, turns=8)
    assert closed_form(eight)[0] == 4640 and closed_form(Chat(system=200, message=30, reply=70, turns=8, keep=2))[0] == 3140


@pytest.mark.parametrize("kwargs", [
    dict(system=-1, message=1, reply=1, turns=1), dict(system=0, message=-1, reply=1, turns=1),
    dict(system=0, message=1, reply=-1, turns=1), dict(system=0, message=1, reply=1, turns=0),
    dict(system=0, message=1, reply=1, turns=1, reasoning=-1), dict(system=0, message=1, reply=1, turns=1, keep=-1),
])
def test_nonsense_sizes_are_refused(kwargs):
    with pytest.raises(ValueError):
        Chat(**kwargs)


def test_negative_prices_and_retry_rates_are_refused():
    for bad in (dict(price_in=-1, price_out=1), dict(price_in=1, price_out=-1), dict(price_in=1, price_out=1, retry_rate=-1)):
        with pytest.raises(ValueError):
            bill(EXAMPLE, **bad)

"""Guards for ../widget.html: its data block, its size, and that its bill arithmetic is bill.py's."""
import itertools
import json
import re
import shutil
import subprocess
from pathlib import Path

import pytest

from bill import DAYS_PER_MONTH, EXAMPLE, EXAMPLE_PRICES, Chat, bill, closed_form, turn_cost, turn_parts
import embed_widget
from embed_widget import WIDGET, embed

HTML = WIDGET.read_text(encoding="utf-8")

# Reads the bill functions out of the widget between its marker comments, runs them, and compares
# with the numbers Python wrote into the fixture.
PARITY = r"""
import fs from 'node:fs';
const [fixture, widget] = process.argv.slice(2);
const fx = JSON.parse(fs.readFileSync(fixture, 'utf8'));
const html = fs.readFileSync(widget, 'utf8');
const js = html.slice(html.indexOf('/* ---------- the bill'), html.indexOf('/* ---------- state'));
const api = new Function("'use strict';" + js + '; return {turnParts, totals, turnCost, bill};')();
const bad = [];
let checks = 0;
const close = (a, b) => Math.abs(a - b) <= 1e-12 * Math.max(1, Math.abs(b));
for (const f of fx) {
  const c = f.chat, [pIn, pOut] = f.prices;
  checks++;
  const t = api.totals(c);
  if (t.input !== f.input || t.output !== f.output) bad.push(['totals', c, t]);
  for (const [k, want] of f.parts) {
    if (JSON.stringify(api.turnParts(c, k)) !== JSON.stringify(want) || !close(api.turnCost(c, k, pIn, pOut), f.turnCosts[k - 1])) bad.push(['turn', c, k]);
  }
  const b = api.bill(c, pIn, pOut, f.retry);
  if (!close(b.inputCost, f.inputCost) || !close(b.outputCost, f.outputCost) || !close(b.retryCost, f.retryCost) || !close(b.total, f.total)) bad.push(['bill', c, b]);
}
console.log(JSON.stringify({ checks, bad: bad.slice(0, 5) }));
process.exit(bad.length ? 1 : 0);
"""


def test_widget_data_block_matches_bill_py():
    """Guards a change to the example chat or prices in bill.py that was not copied into the widget (run embed_widget.py)."""
    assert embed(HTML) == HTML


def test_every_element_the_script_looks_up_exists_where_it_looks():
    """Guards a widget that never starts: a lookup of an id that is not in the markup throws in the browser."""
    root = HTML[HTML.index('<div class="bil-root">'):HTML.index("<script")]
    script = HTML[HTML.index("<script>"):]
    ids = set(re.findall(r"\$\('([\w-]+)'\)", script)) | set(re.findall(r"id: '([\w-]+)'", script)) | set(re.findall(r"hint: '([\w-]+)'", script)) | set(re.findall(r"range: '([\w-]+)'", script))
    assert ids and all(f'id="{i}"' in root for i in ids), sorted(i for i in ids if f'id="{i}"' not in root)
    assert "$('bil-config')" not in script


def test_widget_is_small_and_asks_for_nothing_online():
    """The lesson page links the pricing pages; the widget, in a sandbox that cannot open links, holds no address."""
    assert len(HTML.encode("utf-8")) < 40 * 1024
    page = embed_widget.CONFIG_BLOCK.sub("", HTML)
    for banned in ("http://", "https://", "fetch(", "XMLHttpRequest", "WebSocket", "sendBeacon", "import("):
        assert banned not in page, banned
    assert " src=" not in page and "@import" not in page


def test_widget_has_no_form_element():
    """The lesson page loads the widget in a sandbox where a form cannot submit; inputs are read on change."""
    assert "<form" not in HTML


# Calls the widget's two sentence builders the way paint() does, for three chats, and prints what they say.
NOTES = r"""
import fs from 'node:fs';
const html = fs.readFileSync(process.argv[2], 'utf8');
const js = html.slice(html.indexOf('/* ---------- the bill'), html.indexOf('/* ---------- state'));
const api = new Function("'use strict';" + js + '; return {bill, turnCost, trendNote, trimNote};')();
const out = [];
for (const [turns, keep] of [[1, null], [1, 0], [10, 1], [10, 0], [10, null]]) {
  const mk = k => ({ system: 500, message: 60, reply: 200, turns, reasoning: 0, keep: k });
  const trim = keep !== null, chat = mk(keep), whole = mk(null);
  const b = api.bill(chat, 2, 10, 0), full = api.bill(whole, 2, 10, 0);
  out.push([api.trendNote(turns, api.turnCost(chat, 1, 2, 10), api.turnCost(chat, turns, 2, 10)),
            api.trimNote(trim, keep, trim && b.input !== full.input, b.total, full.total)]);
}
console.log(JSON.stringify(out));
"""

NOTES_WANT = [
    ["A one-turn chat has no history to re-send.", "Tick “Keep last” to compare."],
    ["A one-turn chat has no history to re-send.", "Your chat has no more than 1 turn, so nothing is cut yet."],
    ["Turn 1 costs $0.00312. Turn 10 costs $0.00364, 1.2 times as much.",
     "Sending only the last earlier turn: $0.03588 per chat instead of $0.0546, 34.3% less."],
    ["Turn 1 costs $0.00312. Turn 10 costs $0.00312, 1.0 times as much.",
     "Sending no earlier turns: $0.0312 per chat instead of $0.0546, 42.9% less."],
    ["Turn 1 costs $0.00312. Turn 10 costs $0.0078, 2.5 times as much.", "Tick “Keep last” to compare."],
]


def test_widget_sentences_read_right_for_one_turn_one_kept_turn_and_the_default(tmp_path):
    """Guards the plural slips in the trend and trim sentences ("last 1 earlier turns", "1 turns") and the
    wording of the sentences under the answer: one turn, one or no kept turns with a cut, and the lesson's default."""
    if shutil.which("node") is None:
        pytest.skip("Node.js is not installed")
    (tmp_path / "notes.mjs").write_text(NOTES, encoding="utf-8")
    run = subprocess.run(["node", str(tmp_path / "notes.mjs"), str(WIDGET)], capture_output=True, text=True, timeout=60)
    assert run.returncode == 0, run.stdout + run.stderr
    assert json.loads(run.stdout) == NOTES_WANT


def test_widget_bill_arithmetic_matches_the_python_code(tmp_path):
    """Guards the widget's JavaScript drifting from bill.py: its token counts must equal bill.py's exactly
    and its dollars must agree to rounding error, for trimmed, untrimmed, reasoning and retry cases."""
    if shutil.which("node") is None:
        pytest.skip("Node.js is not installed")
    chats = [Chat(s, m, r, n, think, keep) for s, m, r, n, think, keep in
             itertools.product((0, 500, 2000), (0, 60, 900), (0, 200), (1, 2, 3, 4, 10, 30, 60), (0, 800), (None, 0, 1, 3, 59))]
    chats.append(EXAMPLE)
    fixture = []
    for chat in chats:
        for prices, retry in (((2, 10), "0"), ((0.1, 0.5), "0.25"), ((10, 50), "1")):
            b = bill(chat, *prices, retry_rate=retry)
            fixture.append({
                "chat": {"system": chat.system, "message": chat.message, "reply": chat.reply, "turns": chat.turns,
                         "reasoning": chat.reasoning, "keep": chat.keep},
                "prices": list(prices), "retry": float(retry), "input": closed_form(chat)[0], "output": closed_form(chat)[1],
                "inputCost": float(b.input_cost), "outputCost": float(b.output_cost), "retryCost": float(b.retry_cost),
                "total": float(b.total),
                "parts": [[k, {"system": p.system, "history": p.history, "message": p.message, "reply": p.reply, "reasoning": p.reasoning}]
                          for k in {1, chat.turns, (chat.turns + 1) // 2} for p in [turn_parts(chat, k)]],
                "turnCosts": [float(turn_cost(chat, k, *prices)) for k in range(1, chat.turns + 1)],
            })
    (tmp_path / "fixture.json").write_text(json.dumps(fixture), encoding="utf-8")
    (tmp_path / "parity.mjs").write_text(PARITY, encoding="utf-8")
    run = subprocess.run(["node", str(tmp_path / "parity.mjs"), str(tmp_path / "fixture.json"), str(WIDGET)],
                         capture_output=True, text=True, timeout=120)
    assert run.returncode == 0, run.stdout + run.stderr
    assert json.loads(run.stdout)["checks"] == len(fixture) >= 1000


def test_widget_starts_from_the_lesson_example():
    """Guards the lesson text and the widget starting from different chats: the config is the lesson's example."""
    cfg = embed_widget.config()
    assert (cfg["system"], cfg["message"], cfg["reply"], cfg["turns"]) == (500, 60, 200, 10)
    assert (cfg["priceIn"], cfg["priceOut"]) == EXAMPLE_PRICES == (2, 10) and cfg["daysPerMonth"] == DAYS_PER_MONTH == 30