{
  "_source": "Llama 3 paper (arXiv 2407.21783), Table 3: 126 layers, model dimension 16,384, FFN dimension 53,248, 128 attention heads, 8 key/value heads. The vocabulary size (128,256) and the untied output layer are the same as the 8B and 70B files saved next to this one. The gated 405B config.json is not saved here.",
  "hidden_size": 16384,
  "intermediate_size": 53248,
  "max_position_embeddings": 131072,
  "model_type": "llama",
  "num_attention_heads": 128,
  "num_hidden_layers": 126,
  "num_key_value_heads": 8,
  "tie_word_embeddings": false,
  "torch_dtype": "bfloat16",
  "vocab_size": 128256
}