"""Print every number the lesson quotes. Run: python3 show_numbers.py""" from memory_calc import (ACTIVATION_TOKENS, BITS, FULL_BITS, GIB, GPUS, KV_VALUE_BYTES, MODELS, OVERHEAD_GIB, USABLE, activation_bytes, cards_needed, chats_that_fit, embedding_params, fits, head_dim, kv_bytes_per_token, kv_cache_bytes, load_config, memory_need, param_count, usable_bytes, weight_bytes) # Totals the model pages publish (Hugging Face "Model size", read 2026-10-19), to check the count against. PUBLISHED = {"1b": 1_235_814_400, "8b": 8_030_261_248, "70b": 70_553_706_496, "405b": 405_853_388_800} # Sizes of the .safetensors files of published copies (Hugging Face API, blobs=true, read 2026-10-19), to check the weights against. PUBLISHED_FILES = ( ("8B, 16-bit", "NousResearch/Meta-Llama-3.1-8B", "8b", 16, 16_060_556_376), ("8B, FP8", "RedHatAI/Meta-Llama-3.1-8B-Instruct-FP8", "8b", 8, 9_081_287_016), ("8B, AWQ 4-bit", "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4", "8b", 4, 5_727_938_576), ("8B, GPTQ 4-bit", "hugging-quants/Meta-Llama-3.1-8B-Instruct-GPTQ-INT4", "8b", 4, 5_735_719_888), ("70B, AWQ 4-bit", "hugging-quants/Meta-Llama-3.1-70B-Instruct-AWQ-INT4", "70b", 4, 39_767_996_256), ) # Parameters the model pages list as 16-bit (F16) and as packed 4-bit (I32) in the AWQ copies, whose config.json says # group_size 128 with zero_point on (Hugging Face, read 2026-10-19). Each group of 128 weights also stores one # 16-bit scale and one 4-bit zero point: 2.5 bytes. PUBLISHED_TENSORS = ( ("8B, AWQ 4-bit", "8b", 1_050_939_392, 6_979_321_856), ("70B, AWQ 4-bit", "70b", 2_102_665_216, 68_451_041_280), ) AWQ_GROUP = 128 def gib(n): return f"{n / GIB:.2f}" def naive_bytes(params, bits): """The common rule of thumb: the weights plus 20%.""" return params * bits // 8 * 1.2 def main(): print("== parameters, counted from the shape (published total alongside)") for key, (name, _) in MODELS.items(): cfg = load_config(key) n = param_count(cfg) print(f"{name}: {n:,} published {PUBLISHED[key]:,} equal={n == PUBLISHED[key]} " f"layers={cfg['num_hidden_layers']} hidden={cfg['hidden_size']} ffn={cfg['intermediate_size']} " f"kv_heads={cfg['num_key_value_heads']} head_dim={head_dim(cfg)} tied={bool(cfg.get('tie_word_embeddings'))}") print(f"== embeddings (token table, plus output matrix if untied), kept at {FULL_BITS} bits") for key, (name, _) in MODELS.items(): cfg = load_config(key) e = embedding_params(cfg) print(f"{name}: {e:,} weights = {e * FULL_BITS // 8:,} B = {gib(e * FULL_BITS // 8)} GiB; " f"share of all weights {e / param_count(cfg):.1%}; others {param_count(cfg) - e:,}") print("== weights in GiB by precision (embeddings at 16 bits, the rest at the stated bits)") for key, (name, _) in MODELS.items(): cfg = load_config(key) print(f"{name}: " + " ".join(f"{b}-bit {weight_bytes(cfg, b):,} B = {gib(weight_bytes(cfg, b))} GiB" for b in BITS)) print("== calculator against published files (GiB; gap = file minus calculator)") for label, repo, key, b, size in PUBLISHED_FILES: calc = weight_bytes(load_config(key), b) naive = param_count(load_config(key)) * b // 8 print(f"{label}: file {size:,} B = {gib(size)} GiB; calculator {calc:,} B = {gib(calc)} GiB; " f"gap {gib(size - calc)} GiB = {(size - calc) / size:.1%} of the file; " f"old formula (params x bits / 8) {gib(naive)} GiB, short by {gib(size - naive)} GiB = {(size - naive) / size:.1%}") print("== parameters the model page lists as 16-bit, against the embeddings and the norm weights") files = {label: size for label, _, _, _, size in PUBLISHED_FILES} for label, key, listed, quantized in PUBLISHED_TENSORS: cfg = load_config(key) e = embedding_params(cfg) norms = (2 * cfg["num_hidden_layers"] + 1) * cfg["hidden_size"] print(f"{label}: listed {listed:,}; embeddings {e:,}; difference {listed - e:,}; " f"norm weights (2 x layers + 1) x hidden {norms:,}") print("== the 4-bit gap: listed tensors plus one scale and one zero point per 128 weights") for label, key, listed, quantized in PUBLISHED_TENSORS: scales = quantized // AWQ_GROUP * 5 // 2 predicted = listed * 2 + quantized // 2 + scales size = files[label] print(f"{label}: 16-bit {listed:,} x 2 + 4-bit {quantized:,} x 0.5 + scales and zero points {scales:,} " f"= {predicted:,} B; file {size:,} B; file minus sum {size - predicted:,} B = {(size - predicted) / size:.4%}; " f"scales and zero points alone {gib(scales)} GiB = {scales / size:.1%} of the file") print("== attention cache per token and per chat") for key, (name, _) in MODELS.items(): cfg = load_config(key) per = kv_bytes_per_token(cfg) print(f"{name}: {per:,} B/token = {per / 1024:.0f} KiB; 8,192 tokens = {gib(kv_cache_bytes(cfg, 8192, 1))} GiB; " f"131,072 tokens = {gib(kv_cache_bytes(cfg, 131072, 1))} GiB") print(f"== working memory ({ACTIVATION_TOKENS} tokens at once) and fixed allowance") for key, (name, _) in MODELS.items(): print(f"{name}: activations {activation_bytes(load_config(key)):,} B = {gib(activation_bytes(load_config(key)))} GiB") print(f"overhead allowance {OVERHEAD_GIB} GiB; KV values {KV_VALUE_BYTES} bytes; usable share of a card {USABLE:.0%}") print("== usable memory per card") for g, label in GPUS: print(f"{g} GiB {label}: usable {gib(usable_bytes(g))} GiB") print("== Llama 3.1 8B, 16-bit, 8,192 tokens: parts as chats grow") for chats in (1, 4, 8, 32): m = memory_need("8b", 16, 8192, chats) row = " ".join(f"{g}:{'fits' if fits(m['total'], g) else 'no'}" for g, _ in GPUS) print(f"chats={chats}: weights {gib(m['weights'])} + cache {gib(m['kv_cache'])} + work {gib(m['activations'])} " f"+ fixed {gib(m['overhead'])} = {gib(m['total'])} GiB {row}") print("== every model and precision, 1 chat of 8,192 tokens: total GiB and cards of 80 GiB") for key, (name, _) in MODELS.items(): cells = [] for b in BITS: t = memory_need(key, b, 8192, 1)["total"] cells.append(f"{b}-bit {gib(t)} GiB ({cards_needed(t, 80)}x80, {cards_needed(t, 141)}x141)") print(f"{name}: " + "; ".join(cells)) print("== the 'weights plus 20%' rule against the full sum (8B, 16-bit)") weights_8b = weight_bytes(load_config("8b"), 16) print(f"rule: {gib(naive_bytes(param_count(load_config('8b')), 16))} GiB whatever the chats") for tokens, chats in ((8192, 1), (8192, 4), (8192, 32), (131072, 1)): m = memory_need("8b", 16, tokens, chats) print(f"{tokens:,} tokens x {chats} chats: full sum {gib(m['total'])} GiB = {m['total'] / weights_8b:.2f} x weights; " f"full sum minus rule = {(m['total'] - naive_bytes(param_count(load_config('8b')), 16)) / GIB:+.2f} GiB") print("== cards needed for 1 chat of 8,192 tokens (even split, ignores the cost of splitting)") print("columns: " + " ".join(f"{g} GiB" for g, _ in GPUS)) for key, b in (("8b", 16), ("8b", 4), ("70b", 16), ("70b", 4), ("405b", 16), ("405b", 4)): t = memory_need(key, b, 8192, 1)["total"] print(f"{MODELS[key][0]} {b}-bit ({gib(t)} GiB): " + " ".join(f"{g}:{cards_needed(t, g)}" for g, _ in GPUS)) print("== chats of 8,192 tokens that fit beside the weights (16-bit and 8-bit)") for key in ("8b", "70b"): for b in (16, 8): print(f"{MODELS[key][0]} {b}-bit: " + " ".join(f"{g} GiB:{chats_that_fit(key, b, 8192, g)}" for g, _ in GPUS)) print("== 8B 16-bit, one chat, as the context grows") cfg = load_config("8b") for tokens in (512, 8192, 32768, 131072): m = memory_need("8b", 16, tokens, 1) print(f"{tokens:,} tokens: cache {gib(m['kv_cache'])} GiB of {gib(m['total'])} GiB total = {m['kv_cache'] / m['total']:.0%}") print("== cache share when 32 chats of 8,192 tokens share the 8B") m = memory_need("8b", 16, 8192, 32) print(f"cache {gib(m['kv_cache'])} GiB, weights {gib(m['weights'])} GiB, total {gib(m['total'])} GiB") if __name__ == "__main__": main()