"""Run: PYTHONDONTWRITEBYTECODE=1 python3 -m pytest -q -p no:cacheprovider code/ -- each test guards a number the lesson, a figure or the widget shows a reader.""" import functools import math import re import statistics from pathlib import Path import capacity_plan as cp import make_figs LESSON = Path(__file__).resolve().parent.parent PEAK = (cp.USERS, cp.ACTIVE, cp.REPLIES) WEB_ADDRESS = re.compile("ht" + r"tps?://") # written in two pieces so that this file holds none either @functools.cache def two_gpus(): """The one load test the lesson quotes that is not in `opening_load_test`: 2 GPUs, where the gap passes and the wait fails.""" load = cp.opening_load_test() return cp.load_test(2, load["requests"], "usual", load["cap"]) @functools.cache def loose_cap(): """3 GPUs with the cap raised from 38 to 64: chats stop queueing, so the wait passes, and the batches grow past the speed target, so the gap fails.""" load = cp.opening_load_test() return cp.load_test(3, load["requests"], "usual", 64) @functools.cache def answers_load(): """The load test for the answers mix on the GPUs the plan gives it, which the lesson quotes beside the plan's chats in progress.""" p = cp.plan(cp.USERS, cp.ACTIVE, cp.MIXES["answers"]["each"], "answers", cp.TARGET_MS) return cp.load_test(p["n"], cp.arrivals(p["rate"], cp.LOAD_SECONDS, "answers"), "answers", cp.cap_for("answers")) def test_the_usual_chats_are_lesson_28s_chats(): """The memory limit of 200 and the speed table carry lesson 28's numbers on only if the 600 chats are the same ones.""" assert cp.chats_of("usual") == cp.L28.CHATS def test_every_speed_table_rises_and_reaches_every_target(): """A table that fell, or never reached a target, would make the speed limit jump or fail for one setting of the widget.""" for mix in cp.MIXES: rows = cp.step_table(mix) for a, b in zip(rows, rows[1:]): assert b[0] > a[0] and b[1] > a[1] and b[3] >= a[3], mix for target in cp.TARGETS: assert rows[0][3] < target < rows[-1][3] and rows[0][0] < cp.speed_limit(rows, target) < rows[-1][0], (mix, target) stays = [cp.stay(rows, row[0]) for row in rows] assert stays == sorted(stays), mix # a fuller GPU keeps each chat longer: the page's W grows with the load def test_chats_in_progress_is_the_arrival_rate_times_the_mean_time(): """Little's law as the lesson states it, one area counted twice. If the two sides came apart, the 109 of figure 2 would not be the 109.1 of the test.""" load = cp.opening_load_test() runs, plan = load["runs"], load["plan"] for run in runs.values(): assert math.isclose(run["in_progress"], run["rate"] * run["mean_time"], rel_tol=1e-9) spans = runs[plan["without"]]["spans"] # figure 2 draws the run whose GPUs are as full as the plan assumes curve = cp.in_progress_curve(spans, 600) for t in (0, 1, 60, 299, 599): # the sweep through the sorted events counts like a plain count assert curve[t] == sum(a <= t < d for a, d in spans) assert math.isclose(statistics.fmean(curve), runs[3]["in_progress"], rel_tol=0.03) # and its average is the 109.1, give or take the ramp at both ends assert math.isclose(runs[3]["in_progress"], plan["chats"], rel_tol=0.03) # and the plan's 110.5 is that same count read off the table def test_limits_gpu_counts_and_the_limit_that_binds_for_three_kinds_of_chat(): """The lesson's table: 1,000 users need 4, 55 or 6 GPUs, and which limit binds changes with the chats. `without` is the count with no headroom, which the load test refutes.""" want = {"usual": (200, 38, "speed", 1, 4, 4, 3), "documents": (14, 2, "speed", 9, 55, 55, 44), "answers": (85, 140, "memory", 6, 4, 6, 5)} assert [m["each"] for m in cp.MIXES.values()] == [120, 120, 30] for mix, (memory, speed, binding, n_memory, n_speed, n, without) in want.items(): p = cp.plan(cp.USERS, cp.ACTIVE, cp.MIXES[mix]["each"], mix, cp.TARGET_MS) assert (p["memory"], math.floor(p["speed"]), p["binding"], p["n_memory"], p["n_speed"], p["n"], p["without"]) == (memory, speed, binding, n_memory, n_speed, n, without), mix assert p["without"] < p["n"], mix # headroom costs a GPU in every kind of chat assert p["chats"] < cp.USERS * cp.ACTIVE // 100 and cp.MIXES[mix]["each"] * p["stay"] <= 3600, mix # a crowd that can exist: fewer chats than active users, each user mid-reply no more than all hour usual, ans = cp.plan(*PEAK, "usual", cp.TARGET_MS), cp.plan(*PEAK, "answers", cp.TARGET_MS) assert (usual["without"], usual["n"]) == (3, 4) and cp.peak_rate(*PEAK) == 20.0 assert ans["chats"] > cp.USERS * cp.ACTIVE // 100 and cp.REPLIES * ans["stay"] > 3600 # 120 replies each of 81 s would be impossible: the page uses 30 own = cp.plan(cp.USERS, cp.ACTIVE, cp.MIXES["answers"]["each"], "answers", cp.TARGET_MS) test = answers_load() assert test["passes"] and test["in_progress"] < 0.6 * own["chats"] and test["mean_time"] < 0.6 * own["stay"] # the plan's 405 chats at 81 s are design-point figures: the test on 6 GPUs sees about 231 at 47.5 s assert [cp.plan(*PEAK, "usual", t)["n"] for t in (30, 75)] == [5, 4] and [math.floor(cp.plan(*PEAK, "usual", t)["speed"]) for t in (30, 75)] == [15, 72] assert cp.gpus(0.56, 0.7) == 1 and cp.gpus(0.0, 5.0) == 1 and cp.gpus(110, 38) == 4 # 0.56 / (0.8 * 0.7) is 1.0000000000000002 in floating point and must not give 2 def test_dividing_the_rounded_numbers_the_page_prints_gives_the_pages_gpu_counts(): """A reader divides the numbers on the page: W to one decimal (whole above 10 s), the speed limit to one decimal, the memory limit as is. They must get the page's counts for every kind of chat.""" for mix, m in cp.MIXES.items(): p = cp.plan(cp.USERS, cp.ACTIVE, m["each"], mix, cp.TARGET_MS) seconds = round(p["stay"], 1) if p["stay"] < 10 else round(p["stay"]) chats, speed = p["rate"] * seconds, round(p["speed"], 1) assert (cp.gpus(chats, speed), cp.gpus(chats, p["memory"]), math.ceil(chats / min(speed, p["memory"]) - 1e-9)) == (p["n_speed"], p["n_memory"], p["without"]), mix def test_the_load_test_misses_at_3_gpus_and_passes_at_4_and_5(): """Three GPUs pass the arithmetic and miss under random arrivals. If the toy stopped missing, 'test before you sign' would have no example.""" load = cp.opening_load_test() runs, plan = load["runs"], load["plan"] assert (plan["without"], plan["n"], load["cap"], len(load["requests"])) == (3, 4, 38, 12000) assert [runs[n]["passes"] for n in (3, 4, 5)] == [False, True, True] assert runs[3]["gap"] > cp.TARGET_MS and runs[3]["wait"] > cp.WAIT_TARGET and runs[5]["gap"] < runs[4]["gap"] < cp.TARGET_MS assert not runs[plan["n"] - 1]["passes"] and runs[plan["n"]]["passes"] # one GPU down: 4 planned leaves 3 and misses, 5 planned leaves 4 and passes assert 3 * cp.x_at(cp.step_table("usual"), plan["speed"]) > plan["rate"] > 0.9 * 3 * cp.x_at(cp.step_table("usual"), plan["speed"]) # 3 GPUs are about 95% busy assert runs[3]["gap"] < 1.1 * cp.TARGET_MS and runs[3]["wait"] > 2 * cp.WAIT_TARGET # the lesson calls the gap a narrow miss and the wait a clear one two = two_gpus() # 2 GPUs: the gap passes and the wait fails, so only both conditions together reject it assert two["gap"] <= cp.TARGET_MS and two["wait"] > cp.WAIT_TARGET and not two["passes"] loose = loose_cap() # the other way round: the gap fails and the wait passes, so neither condition alone is the rule assert loose["gap"] > cp.TARGET_MS and loose["wait"] <= cp.WAIT_TARGET and not loose["passes"] assert (load["busiest"]["busiest"], round(load["busiest"]["average"])) == (1283, 1213) def test_a_model_too_big_for_one_gpu_takes_several_for_each_copy(): assert [cp.gpus_to_hold(m, 4) for m in ("8b", "70b", "405b")] == [4, 8, 44] def test_every_number_in_the_lesson_and_figures_comes_from_the_code(): """The prose and the four figures quote numbers by hand; a change to the constants or the server must show up here, not in a reader's check.""" prose = (LESSON / "lesson.md").read_text(encoding="utf-8") figures = {path.name: path.read_text(encoding="utf-8") for path in sorted((LESSON / "figs").glob("*.svg"))} assert set(figures) == set(re.findall(r"\]\(figs/([\w-]+\.svg)\)", prose)) == set(make_figs.FIGURES) and len(figures) == 4 for name, draw in make_figs.FIGURES.items(): # the files on disk are what make_figs.py draws now assert figures[name] == draw(), name rows, p = cp.step_table("usual"), cp.plan(*PEAK, "usual", cp.TARGET_MS) docs, ans, p30, p75 = (cp.plan(cp.USERS, cp.ACTIVE, cp.MIXES[mix]["each"], mix, t) for mix, t in (("documents", 50), ("answers", 50), ("usual", 30), ("usual", 75))) load = cp.opening_load_test() runs, busiest = load["runs"], load["busiest"] chats = cp.chats_of("usual") mean_prompt, mean_reply = statistics.fmean(chats["prompts"]), statistics.fmean(chats["replies"]) room = cp.L28.room(cp.SHARE, 8000) sweep = {row["cap"]: row for row in cp.L28.sweep(cp.SHARE, 8000)} speed, x_speed = math.floor(p["speed"]), cp.x_at(rows, p["speed"]) rate, floor, people = p["rate"], math.floor, cp.USERS * cp.ACTIVE // 100 s1, w1 = round(p["speed"], 1), round(p["stay"], 1) # the page writes the speed limit and W to one decimal page_chats = round(rate * w1) # 20 x 5.5 = 110, the number every division below starts from ans_w, ans_chats = round(ans["stay"]), round(ans["rate"] * round(ans["stay"])) # 81 s, and 5 x 81 = 405 documents = cp.MIXES["documents"] mean_document = statistics.fmean(cp.chats_of("documents")["prompts"]) answer_replies = cp.MIXES["answers"]["replies"] row = lambda r: f"| {r[0]:.1f} | {r[1]:.3f} | {r[3]:.1f} ms |" run_row = lambda n: f"| {n} | {runs[n]['gap']:.1f} ms | {runs[n]['wait']:.2f} s |" busy = rate / (3 * x_speed) * 100 q3 = 20 * 5 / (0.8 * 40) wanted = {"lesson.md": [ f"{cp.USERS:,} users, {people} of them active", f"each receiving {cp.REPLIES} replies", f"{round(rate * 3600):,} replies, or {rate:.0f} new replies a second", f"about {speed} chats at once", f"{cp.TARGET_MS} ms", f"{p['n']} GPUs plus a spare", "A100 chip with 80 GB", f"speed needs {p['n_speed']} and memory {p['n_memory']}", f"the replies each drop to {cp.MIXES['answers']['each']}, and memory, now needing {ans['n_memory']} GPUs to speed's {ans['n_speed']}, sets the limit", f"each receives one reply every 30 seconds: {rate:.0f} new replies a second", f"about {page_chats} chats, not {people} or {cp.USERS:,}", "reads the whole conversation again ([lesson 8]", f"holds {p['memory']} chats", f"at {p['speed']:.1f} chats ({speed} in the figure), the speed limit, long before memory runs out at {p['memory']}", f"At its speed limit, {p['speed']:.1f} chats, one GPU finishes {x_speed:.1f} replies a second, so each chat stays {p['speed']:.1f} / {x_speed:.1f} = {w1} s", f"about {rate:.0f} x {w1} = {page_chats} chats are in progress", f"on 3 GPUs, each near its limit, counts {round(runs[3]['in_progress'])} (Figure 2)", f"The solid line stays near {round(runs[3]['in_progress'])}.", f"the plan said {page_chats}.", f"Speed alone gives {page_chats} / {s1} = {page_chats / s1:.2f}, so {p['without']} GPUs", f"Speed then needs {page_chats} / (0.8 x {s1}) = {page_chats / (0.8 * s1):.2f}, so {p['n_speed']} GPUs", f"Memory needs {page_chats} / (0.8 x {p['memory']}) = {page_chats / (0.8 * p['memory']):.2f}, so {p['n_memory']}", f"One spare makes {p['n'] + 1}", f"{docs['memory']} chats but meets the target with only {floor(docs['speed'])}", f"the crowd needs {docs['n']} GPUs", f"{round(mean_document, -3):,.0f} tokens", f"replies of {answer_replies[0]:,} to {answer_replies[1]:,} tokens memory holds {ans['memory']} chats and speed {floor(ans['speed'])}", f"A chat on a GPU at its limit then stays {ans_w} s, so there a user gets about {floor(3600 / ans['stay'])} replies an hour, not {cp.REPLIES}", f"At {cp.MIXES['answers']['each']} each, {ans_chats} chats are in progress and need {ans['n']} GPUs", "so memory fills first at this target: with", f"With 3 GPUs the p99 time between words is {runs[3]['gap']:.1f} ms, a narrow miss of 50 ms, and the p99 wait for the first word", f"is {runs[3]['wait']:.2f} s, a clear miss of 2 s", f"With 4 it is {runs[4]['gap']:.1f} ms and {runs[4]['wait']:.2f} s", f"With 5, {runs[5]['gap']:.1f} ms and {runs[5]['wait']:.2f} s", f"Here {cp.USERS:,} users became {rate:.0f} replies", f"The same {cp.USERS:,} users need {p['n']} GPUs for usual chats and {docs['n']} for long documents; long answers, at {cp.MIXES['answers']['each']} replies each, need {ans['n']}", f"On 3 GPUs it gives {runs[3]['rate']:.2f} a second times {runs[3]['mean_time']:.2f} s, {runs[3]['in_progress']:.1f} chats, against the plan's {p['chats']:.1f}", f"The 3 GPUs hold {runs[3]['in_progress'] / 3:.0f} chats each, fewer than {p['speed']:.1f}", f"it gives {runs[4]['rate']:.2f} times {runs[4]['mean_time']:.2f} s, {runs[4]['in_progress']:.1f} chats", *(row(rows[i]) for i in (0, 3, 5, 7, 8, 9, 11)), f"at {rows[7][0]:.1f} chats, {rows[7][0] / rows[7][1]:.2f} s", f"`stay` reads the table at {p['speed']:.1f} chats: {p['speed']:.1f} / {x_speed:.2f} = {p['stay']:.2f} s", f"between {rows[7][0]:.1f} and {rows[8][0]:.1f}, at {p['speed']:.1f} chats", f"a cap of 64, with {sweep[64]['seconds_per_word'] * 1000:.1f} ms", f"at {rows[9][0]:.1f} chats the p99 is {rows[9][3]:.1f} ms", f"{cp.TARGETS[0]} ms gives {floor(p30['speed'])} chats and {p30['n']} GPUs, {cp.TARGETS[2]} ms gives {floor(p75['speed'])} and the same {p75['n']}", f"average {mean_prompt:,.0f} tokens of prompt and {mean_reply:.0f} of reply, {round(mean_prompt + mean_reply):,} in all, which is {cp.L28.blocks_for(round(mean_prompt + mean_reply))} blocks", f"{room['blocks']:,} blocks, and {room['blocks']:,} // {cp.L28.blocks_for(round(mean_prompt + mean_reply))} is {p['memory']}", f"`HEADROOM` is {cp.HEADROOM}", "def gpus(chats, limit):", "math.ceil(chats / (HEADROOM * limit) - 1e-9)", f"Without headroom the arithmetic gives {p['without']}", "| Chats | Limits | Replies each | GPUs, spare not counted |", f"| Usual | {p['memory']} and {speed} | {cp.MIXES['usual']['each']} | {p['n']} |", f"| Documents | {docs['memory']} and {floor(docs['speed'])} | {cp.MIXES['documents']['each']} | {docs['n']} |", f"| Answers | {ans['memory']} and {floor(ans['speed'])} | {cp.MIXES['answers']['each']} | {ans['n']} |", "Speed binds for the usual chats and the documents, memory for the answers.", f"For the answers W is {ans_w} s, so {ans['rate']:.0f} new replies a second keep {ans_chats} chats in progress: {ans_chats} / (0.8 x {ans['memory']}) = {ans_chats / (0.8 * ans['memory']):.2f}, so {ans['n_memory']} GPUs, " f"against {ans_chats} / (0.8 x {ans['speed']:.1f}) = {ans_chats / (0.8 * ans['speed']):.2f}, so {ans['n_speed']} for speed", f"For the documents W is {round(docs['stay'], 1)} s, so {round(docs['rate'] * round(docs['stay'], 1))} chats, and the speed limit is {round(docs['speed'], 1)}: " f"{round(docs['rate'] * round(docs['stay'], 1))} / (0.8 x {round(docs['speed'], 1)}) = {round(docs['rate'] * round(docs['stay'], 1)) / (0.8 * round(docs['speed'], 1)):.1f}, so {docs['n']} GPUs", f"on {ans['n']} GPUs the test averages {answers_load()['in_progress']:.0f} chats and {answers_load()['mean_time']:.1f} s a reply, against the plan's {ans_chats} and {ans_w} s", f"came to {runs[3]['in_progress']:.1f} on 3 GPUs, {runs[4]['in_progress']:.1f} on 4 and {runs[5]['in_progress']:.1f} on 5", f"{documents['prompts'][0]:,} to {documents['prompts'][1]:,} tokens, {mean_document:,.0f} on average", f"(seed {cp.LOAD_SEED})", f"seed {cp.CHAT_SEED}", f"a cap of {load['cap']} chats", *(run_row(n) for n in (3, 4, 5)), f"at most about {3 * x_speed:.0f} replies a second against {rate:.0f} arriving, so each is busy {busy:.0f}% of the time", f"with 2 GPUs the cap keeps the gap at {two_gpus()['gap']:.1f} ms, yet the wait is {two_gpus()['wait']:.0f} s", f"The busiest minute is {(busiest['ratio'] - 1) * 100:.1f}% above the average", f"the test passes at {runs[4]['gap']:.1f} ms", f"5 GPUs give {runs[5]['gap']:.1f} ms where 4 give {runs[4]['gap']:.1f} ms", f"{cp.L26.least_n('8b')} for the 8B model, {cp.L26.least_n('70b')} for the 70B and {cp.L26.least_n('405b')} for the 405B", f"4 copies take {cp.gpus_to_hold('8b', 4)}, {cp.gpus_to_hold('70b', 4)} and {cp.gpus_to_hold('405b', 4)} GPUs", f"3 GPUs run at {rate / (3 * x_speed) * 100:.0f}%, 4 at {rate / (4 * x_speed) * 100:.0f}% and 5 at {rate / (5 * x_speed) * 100:.0f}%", f"from {runs[4]['wait']:.2f} s to {runs[5]['wait']:.2f} s", f"Four GPUs at 80% finish about {4 * cp.HEADROOM * x_speed:.0f} replies a second against the {rate:.0f} needed", f"about {round(4 * cp.HEADROOM * x_speed / rate * cp.USERS, -2):,.0f} users", f"3 GPUs finish about {3 * cp.HEADROOM * x_speed:.0f} replies a second: about {round(3 * cp.HEADROOM * x_speed / rate * cp.USERS, -1):,.0f} users, short of {cp.USERS:,}, the same reason that test missed", f"{room['pool_tokens']:,} / 8,000 = {room['pool_tokens'] // 8000}", f"{people} x {cp.REPLIES} = {round(rate * 3600):,} replies in 3,600 seconds, which is {rate:.0f} a second", f"20 x 5 = 100 chats in progress; 100 / (0.8 x 40) = {q3:.1f}, which rounds up to {math.ceil(q3)}. With a spare, {math.ceil(q3) + 1}.", f"Memory, at the {cp.TARGET_MS} ms target. Replies of {answer_replies[0]:,} to {answer_replies[1]:,} tokens keep each chat open about {ans_w} s, so the pool fills at {ans['memory']} chats, before the speed limit of {floor(ans['speed'])}", f"the p99 wait is {runs[3]['wait']:.2f} s against the 2 s target, a clear miss, and the p99 gap {runs[3]['gap']:.1f} ms against 50 ms, a narrow one", "the \"stable\" version", "edited by Betsy Beyer, Google", ], "chats-in-progress.svg": [f"about {round(runs[3]['in_progress'])}", f"{cp.USERS:,} users", f"{people} active", "on 3 GPUs"], "two-limits.svg": [f"speed limit {speed} chats", f"memory holds {p['memory']}", f"target {cp.TARGET_MS} ms", f"{rows[0][3]:.0f} ms with 1 chat to {rows[-1][3]:.0f} ms with {rows[-1][0]:.0f}"], "which-limit-binds.svg": [f"{label}: memory {q['memory']}, speed {floor(q['speed'])}, {q['binding']} first" for label, q in (("Usual chats", p), ("Long documents, short answers", docs), ("Long answers", ans))] + [f"{q['memory'] if q['binding'] == 'memory' else floor(q['speed'])} binds" for q in (p, docs, ans)] + ["all on one scale"], "load-test.svg": [f"{runs[n]['gap']:.1f} ms" for n in (3, 4, 5)] + [f"{runs[n]['wait']:.2f} s" for n in (3, 4, 5)] + [f"12,000 random arrivals, {rate:.1f} a second", "target 50 ms", "target 2 s"]} widths = [float(w) for w in re.findall(r'" in text and "" in text and 'fill="#ffffff"' in text, name assert "