{"source":"AIMap","schema":"benchmark/v1","updated":"2026-10-22","benchmark":{"id":"mmlu","name":"MMLU","category":"knowledge","unit":"%","higherIsBetter":true,"status":"saturated","sources":["epoch"],"primaryVariant":null,"url":"https://github.com/hendrycks/test","year":2020,"desc":{"en":"About 14,000 multiple-choice questions over 57 subjects, from elementary maths to law and medicine.","zh-hans":"约 1.4 万道选择题，覆盖 57 个学科，从初等数学到法律、医学。"},"measures":{"en":"Broad academic knowledge (usually 5-shot exact match). Data compiled by Epoch AI from papers, model reports and HELM.","zh-hans":"广泛的学科知识（通常 5-shot 精确匹配）。数据由 Epoch AI 从论文、模型报告和 HELM 汇编。"},"limits":{"en":"Saturated at the top; studies found a share of mislabelled questions; prompt format changes scores by several points.","zh-hans":"头部已饱和；研究发现部分题目答案有误；提示格式不同可让分数差好几个点。"},"contamination":"high","contaminationNote":{"en":"Public since 2020 and widely copied on the web.","zh-hans":"2020 年公开，网上大量转载。"},"stats":{"results":198,"models":158,"latest":"2024-12-26","bySourceType":{"paper":111,"vendor":7,"official":80}}},"sources":[{"id":"epoch","name":"Epoch AI Benchmarking Hub","license":"CC BY 4.0","policy":"open","attribution":"Epoch AI, 'Capabilities & benchmarking', https://epoch.ai/benchmarks (CC BY 4.0)","site":"https://epoch.ai/benchmarks"}],"results":[{"model":"gpt-4o-nov-2024","modelName":"GPT-4o (Nov 2024)","provider":"OpenAI","rawName":"GPT-4o (Nov 2024)","variant":"compiled","setting":"Exact match; 5-shot; effort: Nov 2024","score":88.1,"unit":"%","rank":1,"rankComputedBySite":true,"date":"2024-11-20","sourceType":"paper","source":"epoch","sourceUrl":"http://arxiv.org/abs/2412.08905","collected":"2026-10-10"},{"model":"deepseek-v3","modelName":"DeepSeek-V3","provider":"DeepSeek","rawName":"DeepSeek-V3","variant":"compiled","setting":"Exact match; 5-shot","score":87.1,"unit":"%","rank":2,"rankComputedBySite":true,"date":"2024-12-26","sourceType":"paper","source":"epoch","sourceUrl":"https://arxiv.org/pdf/2412.19437","collected":"2026-10-10"},{"model":"gpt-4-mar-2023","modelName":"GPT-4 (Mar 2023)","provider":"OpenAI","rawName":"GPT-4 (Mar 2023)","variant":"compiled","setting":"Exact match; 5-shot; effort: Mar 2023","score":86.4,"unit":"%","rank":3,"rankComputedBySite":true,"date":"2023-03-14","sourceType":"paper","source":"epoch","sourceUrl":"https://arxiv.org/pdf/2303.08774","collected":"2026-10-10"},{"model":"llama-3-3-70b","modelName":"Llama 3.3 70B","provider":"Meta","rawName":"Llama 3.3 70B","variant":"compiled","setting":"Exact match; 5-shot","score":86.3,"unit":"%","rank":4,"rankComputedBySite":true,"date":"2024-12-06","sourceType":"paper","source":"epoch","sourceUrl":"http://arxiv.org/abs/2412.08905","collected":"2026-10-10"},{"model":"gemini-1-5-pro-may-2024","modelName":"Gemini 1.5 Pro (May 2024)","provider":"Google","rawName":"Gemini 1.5 Pro (May 2024)","variant":"compiled","setting":"Exact match; 5-shot","score":85.9,"unit":"%","rank":5,"rankComputedBySite":true,"date":"2024-05-14","sourceType":"vendor","source":"epoch","sourceUrl":"https://storage.googleapis.com/deepmind-media/gemini/gemini_v1_5_report.pdf","collected":"2026-10-10"},{"model":"qwen2-5-72b","modelName":"Qwen2.5-72B","provider":"Alibaba (Qwen)","rawName":"Qwen2.5-72B","variant":"compiled","setting":"Exact match; 5-shot","score":85,"unit":"%","rank":6,"rankComputedBySite":true,"date":"2024-09-19","sourceType":"paper","source":"epoch","sourceUrl":"https://arxiv.org/pdf/2412.19437","collected":"2026-10-10"},{"model":"phi-4","modelName":"Phi-4","provider":"Microsoft Research","rawName":"Phi-4","variant":"compiled","setting":"Exact match; 5-shot","score":84.8,"unit":"%","rank":7,"rankComputedBySite":true,"date":"2024-12-12","sourceType":"paper","source":"epoch","sourceUrl":"http://arxiv.org/abs/2412.08905","collected":"2026-10-10"},{"model":"llama-3-1-405b","modelName":"Llama 3.1-405B","provider":"Meta","rawName":"Llama 3.1-405B","variant":"compiled","setting":"Exact match; 5-shot","score":84.4,"unit":"%","rank":8,"rankComputedBySite":true,"date":"2024-07-23","sourceType":"paper","source":"epoch","sourceUrl":"https://arxiv.org/pdf/2412.19437","collected":"2026-10-10"},{"model":"gpt-4","modelName":"GPT-4","provider":"OpenAI","rawName":"GPT-4","variant":"compiled","setting":"Exact match; 5-shot","score":83.93,"unit":"%","rank":9,"rankComputedBySite":true,"date":"","sourceType":"paper","source":"epoch","sourceUrl":"http://arxiv.org/abs/2309.10305","collected":"2026-10-10"},{"model":"gemini-1-5-pro-feb-2024","modelName":"Gemini 1.5 Pro (Feb 2024)","provider":"Google","rawName":"Gemini 1.5 Pro (Feb 2024)","variant":"compiled","setting":"Exact match; 5-shot","score":81.9,"unit":"%","rank":10,"rankComputedBySite":true,"date":"2024-02-15","sourceType":"vendor","source":"epoch","sourceUrl":"https://storage.googleapis.com/deepmind-media/gemini/gemini_v1_5_report.pdf","collected":"2026-10-10"},{"model":"gpt-4o-mini","modelName":"GPT-4o mini","provider":"OpenAI","rawName":"GPT-4o mini","variant":"compiled","setting":"Exact match; 5-shot","score":81.8,"unit":"%","rank":11,"rankComputedBySite":true,"date":"2024-07-18","sourceType":"paper","source":"epoch","sourceUrl":"http://arxiv.org/abs/2412.08905","collected":"2026-10-10"},{"model":"qwen2-5-14b","modelName":"Qwen2.5-14B","provider":"Alibaba (Qwen)","rawName":"Qwen2.5-14B","variant":"compiled","setting":"Exact match; 5-shot","score":79.9,"unit":"%","rank":12,"rankComputedBySite":true,"date":"2024-09-19","sourceType":"paper","source":"epoch","sourceUrl":"http://arxiv.org/abs/2412.08905","collected":"2026-10-10"},{"model":"claude-2","modelName":"Claude 2","provider":"Anthropic","rawName":"Claude 2","variant":"compiled","setting":"Exact match; 5-shot","score":78.5,"unit":"%","rank":13,"rankComputedBySite":true,"date":"2023-07-11","sourceType":"vendor","source":"epoch","sourceUrl":"https://www-cdn.anthropic.com/5c49cc247484cecf107c699baf29250302e5da70/claude-2-model-card.pdf","collected":"2026-10-10"},{"model":"deepseek-v2-moe-236b-may-2024","modelName":"DeepSeek-V2 (MoE-236B, May 2024)","provider":"DeepSeek","rawName":"DeepSeek-V2 (MoE-236B, May 2024)","variant":"compiled","setting":"Exact match; 5-shot","score":78.4,"unit":"%","rank":14,"rankComputedBySite":true,"date":"2024-05-07","sourceType":"paper","source":"epoch","sourceUrl":"https://arxiv.org/pdf/2412.19437","collected":"2026-10-10"},{"model":"phi-3-medium-14b","modelName":"phi-3-medium 14B","provider":"Microsoft","rawName":"phi-3-medium 14B","variant":"compiled","setting":"Exact match; 5-shot","score":77.9,"unit":"%","rank":15,"rankComputedBySite":true,"date":"2024-04-23","sourceType":"paper","source":"epoch","sourceUrl":"http://arxiv.org/abs/2412.08905","collected":"2026-10-10"},{"model":"claude-1-3","modelName":"Claude 1.3","provider":"Anthropic","rawName":"Claude 1.3","variant":"compiled","setting":"Exact match; 5-shot","score":77,"unit":"%","rank":16,"rankComputedBySite":true,"date":"2023-04-18","sourceType":"vendor","source":"epoch","sourceUrl":"https://www-cdn.anthropic.com/5c49cc247484cecf107c699baf29250302e5da70/claude-2-model-card.pdf","collected":"2026-10-10"},{"model":"yi-34b","modelName":"Yi-34B","provider":"01.AI","rawName":"Yi-34B","variant":"compiled","setting":"Exact match; 5-shot","score":76.3,"unit":"%","rank":17,"rankComputedBySite":true,"date":"2023-11-02","sourceType":"paper","source":"epoch","sourceUrl":"http://arxiv.org/abs/2403.04652","collected":"2026-10-10"},{"model":"phi-3-small-7-4b","modelName":"phi-3-small 7.4B","provider":"Microsoft","rawName":"phi-3-small 7.4B","variant":"compiled","setting":"Exact match; 5-shot","score":75.7,"unit":"%","rank":18,"rankComputedBySite":true,"date":"2024-04-23","sourceType":"paper","source":"epoch","sourceUrl":"http://arxiv.org/abs/2404.14219","collected":"2026-10-10"},{"model":"qwen2-5-coder-14b","modelName":"Qwen2.5-Coder-14B","provider":"Alibaba (Qwen)","rawName":"Qwen2.5-Coder-14B","variant":"compiled","setting":"Exact match","score":75.2,"unit":"%","rank":19,"rankComputedBySite":true,"date":"2024-09-18","sourceType":"paper","source":"epoch","sourceUrl":"http://arxiv.org/abs/2409.12186","collected":"2026-10-10"},{"model":"claude-instant","modelName":"Claude Instant","provider":"Anthropic","rawName":"Claude Instant","variant":"compiled","setting":"Exact match; 5-shot","score":73.4,"unit":"%","rank":20,"rankComputedBySite":true,"date":"","sourceType":"vendor","source":"epoch","sourceUrl":"https://www-cdn.anthropic.com/5c49cc247484cecf107c699baf29250302e5da70/claude-2-model-card.pdf","collected":"2026-10-10"},{"model":"inflection-1","modelName":"Inflection-1","provider":"Inflection AI","rawName":"Inflection-1","variant":"compiled","setting":"Exact match; 5-shot","score":72.7,"unit":"%","rank":21,"rankComputedBySite":true,"date":"2023-06-22","sourceType":"vendor","source":"epoch","sourceUrl":"https://inflection.ai/blog/inflection-1","collected":"2026-10-10"},{"model":"gpt-3-5-turbo-nov-2023","modelName":"GPT-3.5 Turbo (Nov 2023)","provider":"OpenAI","rawName":"GPT-3.5 Turbo (Nov 2023)","variant":"compiled","setting":"Exact match; 5-shot; effort: Nov 2023","score":71.4,"unit":"%","rank":22,"rankComputedBySite":true,"date":"2023-11-06","sourceType":"paper","source":"epoch","sourceUrl":"http://arxiv.org/abs/2404.14219","collected":"2026-10-10"},{"model":"falcon-180b","modelName":"Falcon-180B","provider":"Technology Innovation Institute","rawName":"Falcon-180B","variant":"compiled","setting":"Exact match; 5-shot","score":70.6,"unit":"%","rank":23,"rankComputedBySite":true,"date":"2023-09-06","sourceType":"paper","source":"epoch","sourceUrl":"http://arxiv.org/abs/2311.16867","collected":"2026-10-10"},{"model":"mixtral-8x7b","modelName":"Mixtral 8x7B","provider":"Mistral AI","rawName":"Mixtral 8x7B","variant":"compiled","setting":"Exact match; 5-shot","score":70.6,"unit":"%","rank":23,"rankComputedBySite":true,"date":"2023-12-11","sourceType":"paper","source":"epoch","sourceUrl":"https://arxiv.org/pdf/2401.04088","collected":"2026-10-10"},{"model":"davinci-002","modelName":"davinci-002","provider":"OpenAI","rawName":"davinci-002","variant":"compiled","setting":"Exact match; 5-shot","score":70,"unit":"%","rank":25,"rankComputedBySite":true,"date":"2022-03-15","sourceType":"paper","source":"epoch","sourceUrl":"https://arxiv.org/pdf/2303.08774","collected":"2026-10-10"},{"model":"palm-540b","modelName":"PaLM (540B)","provider":"Google Research","rawName":"PaLM (540B)","variant":"compiled","setting":"Exact match; 5-shot","score":69.3,"unit":"%","rank":26,"rankComputedBySite":true,"date":"2022-04-04","sourceType":"paper","source":"epoch","sourceUrl":"http://arxiv.org/abs/2302.13971","collected":"2026-10-10"},{"model":"llama-2-70b","modelName":"Llama 2-70B","provider":"Meta","rawName":"Llama 2-70B","variant":"compiled","setting":"Exact match; 5-shot","score":68.9,"unit":"%","rank":27,"rankComputedBySite":true,"date":"2023-07-18","sourceType":"paper","source":"epoch","sourceUrl":"http://arxiv.org/abs/2307.09288","collected":"2026-10-10"},{"model":"phi-3-mini-3-8b","modelName":"phi-3-mini 3.8B","provider":"Microsoft","rawName":"phi-3-mini 3.8B","variant":"compiled","setting":"Exact match; 5-shot","score":68.8,"unit":"%","rank":28,"rankComputedBySite":true,"date":"2024-04-23","sourceType":"paper","source":"epoch","sourceUrl":"http://arxiv.org/abs/2404.14219","collected":"2026-10-10"},{"model":"stable-beluga-2","modelName":"Stable Beluga 2","provider":"Stability AI","rawName":"Stable Beluga 2","variant":"compiled","setting":"Exact match; 5-shot","score":68.6,"unit":"%","rank":29,"rankComputedBySite":true,"date":"2023-07-20","sourceType":"paper","source":"epoch","sourceUrl":"https://arxiv.org/pdf/2309.16609","collected":"2026-10-10"},{"model":"gpt-3-5-turbo","modelName":"GPT-3.5 Turbo","provider":"OpenAI","rawName":"GPT-3.5 Turbo","variant":"compiled","setting":"Exact match; 5-shot","score":68.54,"unit":"%","rank":30,"rankComputedBySite":true,"date":"","sourceType":"paper","source":"epoch","sourceUrl":"http://arxiv.org/abs/2309.10305","collected":"2026-10-10"},{"model":"yi-9b","modelName":"Yi-9B","provider":"01.AI","rawName":"Yi-9B","variant":"compiled","setting":"Exact match; 5-shot","score":68.4,"unit":"%","rank":31,"rankComputedBySite":true,"date":"2024-03-01","sourceType":"paper","source":"epoch","sourceUrl":"http://arxiv.org/abs/2403.04652","collected":"2026-10-10"},{"model":"qwen2-5-coder-7b","modelName":"Qwen2.5-Coder (7B)","provider":"Alibaba (Qwen)","rawName":"Qwen2.5-Coder (7B)","variant":"compiled","setting":"Exact match","score":68,"unit":"%","rank":32,"rankComputedBySite":true,"date":"2024-09-18","sourceType":"paper","source":"epoch","sourceUrl":"http://arxiv.org/abs/2409.12186","collected":"2026-10-10"},{"model":"yi-34b","modelName":"Yi-34B","provider":"01.AI","rawName":"Yi-34B","variant":"compiled","setting":"Exact match; 0-shot","score":67.6,"unit":"%","rank":33,"rankComputedBySite":true,"date":"2023-11-22","sourceType":"paper","source":"epoch","sourceUrl":"http://arxiv.org/abs/2403.04652","collected":"2026-10-10"},{"model":"chinchilla","modelName":"Chinchilla","provider":"Google","rawName":"Chinchilla","variant":"compiled","setting":"Exact match; 5-shot","score":67.5,"unit":"%","rank":34,"rankComputedBySite":true,"date":"2022-03-29","sourceType":"paper","source":"epoch","sourceUrl":"http://arxiv.org/abs/2302.13971","collected":"2026-10-10"},{"model":"internlm-104b","modelName":"InternLM (104B)","provider":"Shanghai AI Lab (InternLM)","rawName":"InternLM (104B)","variant":"compiled","setting":"Exact match; 5-shot","score":67.2,"unit":"%","rank":35,"rankComputedBySite":true,"date":"","sourceType":"vendor","source":"epoch","sourceUrl":"https://static.aminer.cn/upload/pdf/127/1564/656/6481884993eaf7045294a0c4_0.pdf","collected":"2026-10-10"},{"model":"aquilachat2-34b","modelName":"AquilaChat2 34B","provider":null,"rawName":"AquilaChat2 34B","variant":"compiled","setting":"Exact match; 5-shot","score":66.7,"unit":"%","rank":36,"rankComputedBySite":true,"date":"","sourceType":"paper","source":"epoch","sourceUrl":"http://arxiv.org/abs/2403.04652","collected":"2026-10-10"},{"model":"llama-3-8b","modelName":"Llama 3-8B","provider":"Meta","rawName":"Llama 3-8B","variant":"compiled","setting":"Exact match; 5-shot","score":66.5,"unit":"%","rank":37,"rankComputedBySite":true,"date":"2024-04-18","sourceType":"paper","source":"epoch","sourceUrl":"http://arxiv.org/abs/2404.14219","collected":"2026-10-10"},{"model":"qwen-14b","modelName":"Qwen-14B","provider":"Alibaba (Qwen)","rawName":"Qwen-14B","variant":"compiled","setting":"Exact match; 5-shot","score":66.3,"unit":"%","rank":38,"rankComputedBySite":true,"date":"2023-09-28","sourceType":"paper","source":"epoch","sourceUrl":"http://arxiv.org/abs/2402.16819","collected":"2026-10-10"},{"model":"aquilachat2-34b","modelName":"AquilaChat2 34B","provider":null,"rawName":"AquilaChat2 34B","variant":"compiled","setting":"Exact match; 0-shot","score":65.2,"unit":"%","rank":39,"rankComputedBySite":true,"date":"","sourceType":"paper","source":"epoch","sourceUrl":"http://arxiv.org/abs/2403.04652","collected":"2026-10-10"},{"model":"gemma-7b","modelName":"Gemma 7B","provider":"Google","rawName":"Gemma 7B","variant":"compiled","setting":"Exact match; 5-shot","score":64.3,"unit":"%","rank":40,"rankComputedBySite":true,"date":"2024-02-21","sourceType":"paper","source":"epoch","sourceUrl":"https://arxiv.org/pdf/2403.08295","collected":"2026-10-10"},{"model":"starcoder-2-15b","modelName":"StarCoder 2 15B","provider":"Hugging Face","rawName":"StarCoder 2 15B","variant":"compiled","setting":"Exact match","score":64.1,"unit":"%","rank":41,"rankComputedBySite":true,"date":"2024-02-20","sourceType":"paper","source":"epoch","sourceUrl":"http://arxiv.org/abs/2409.12186","collected":"2026-10-10"},{"model":"qwen-14b","modelName":"Qwen-14B","provider":"Alibaba (Qwen)","rawName":"Qwen-14B","variant":"compiled","setting":"Exact match; 0-shot","score":64,"unit":"%","rank":42,"rankComputedBySite":true,"date":"2023-09-24","sourceType":"paper","source":"epoch","sourceUrl":"http://arxiv.org/abs/2403.04652","collected":"2026-10-10"},{"model":"llama-65b","modelName":"LLaMA-65B","provider":"Meta","rawName":"LLaMA-65B","variant":"compiled","setting":"Exact match; 5-shot","score":63.4,"unit":"%","rank":43,"rankComputedBySite":true,"date":"2023-02-24","sourceType":"paper","source":"epoch","sourceUrl":"http://arxiv.org/abs/2307.09288","collected":"2026-10-10"},{"model":"yi-6b","modelName":"Yi 6B","provider":"01.AI","rawName":"Yi 6B","variant":"compiled","setting":"Exact match; 5-shot","score":63.2,"unit":"%","rank":44,"rankComputedBySite":true,"date":"2023-11-02","sourceType":"paper","source":"epoch","sourceUrl":"http://arxiv.org/abs/2403.04652","collected":"2026-10-10"},{"model":"llama-2-34b","modelName":"Llama 2-34B","provider":"Meta","rawName":"Llama 2-34B","variant":"compiled","setting":"Exact match; 5-shot","score":62.6,"unit":"%","rank":45,"rankComputedBySite":true,"date":"2023-07-18","sourceType":"paper","source":"epoch","sourceUrl":"http://arxiv.org/abs/2307.09288","collected":"2026-10-10"},{"model":"mistral-7b-v0-2","modelName":"Mistral 7B v0.2","provider":"Mistral AI","rawName":"Mistral 7B v0.2","variant":"compiled","setting":"Exact match; 5-shot","score":62.5,"unit":"%","rank":46,"rankComputedBySite":true,"date":"2023-12-11","sourceType":"paper","source":"epoch","sourceUrl":"https://arxiv.org/pdf/2403.08295","collected":"2026-10-10"},{"model":"mistral-7b-v0-1","modelName":"Mistral 7B v0.1","provider":"Mistral AI","rawName":"Mistral 7B v0.1","variant":"compiled","setting":"Exact match; 5-shot","score":62.5,"unit":"%","rank":46,"rankComputedBySite":true,"date":"2023-09-27","sourceType":"paper","source":"epoch","sourceUrl":"https://arxiv.org/pdf/2401.04088","collected":"2026-10-10"},{"model":"qwen2-5-coder-32b","modelName":"Qwen2.5-Coder-32B","provider":"Alibaba (Qwen)","rawName":"Qwen2.5-Coder-32B","variant":"compiled","setting":"Exact match","score":61.2,"unit":"%","rank":48,"rankComputedBySite":true,"date":"2024-09-18","sourceType":"paper","source":"epoch","sourceUrl":"http://arxiv.org/abs/2409.12186","collected":"2026-10-10"},{"model":"deepseek-coder-v2-lite-base","modelName":"DeepSeek-Coder-V2-Lite-Base","provider":"DeepSeek","rawName":"DeepSeek-Coder-V2-Lite-Base","variant":"compiled","setting":"Exact match","score":60.5,"unit":"%","rank":49,"rankComputedBySite":true,"date":"2024-06-13","sourceType":"paper","source":"epoch","sourceUrl":"http://arxiv.org/abs/2409.12186","collected":"2026-10-10"},{"model":"gopher-280b","modelName":"Gopher (280B)","provider":"Google","rawName":"Gopher (280B)","variant":"compiled","setting":"Exact match; 5-shot","score":60,"unit":"%","rank":50,"rankComputedBySite":true,"date":"2021-12-08","sourceType":"paper","source":"epoch","sourceUrl":"http://arxiv.org/abs/2112.11446","collected":"2026-10-10"},{"model":"llama-2-70b","modelName":"Llama 2-70B","provider":"Meta","rawName":"Llama 2-70B","variant":"compiled","setting":"Exact match; 0-shot","score":59.4,"unit":"%","rank":51,"rankComputedBySite":true,"date":"2023-07-18","sourceType":"paper","source":"epoch","sourceUrl":"http://arxiv.org/abs/2403.04652","collected":"2026-10-10"},{"model":"baichuan2-13b","modelName":"Baichuan2-13B","provider":"Baichuan","rawName":"Baichuan2-13B","variant":"compiled","setting":"Exact match; 5-shot","score":59.17,"unit":"%","rank":52,"rankComputedBySite":true,"date":"2023-09-06","sourceType":"paper","source":"epoch","sourceUrl":"http://arxiv.org/abs/2309.10305","collected":"2026-10-10"},{"model":"nemotron-4-15b","modelName":"Nemotron-4 15B","provider":"NVIDIA","rawName":"Nemotron-4 15B","variant":"compiled","setting":"Exact match; 5-shot","score":58.7,"unit":"%","rank":53,"rankComputedBySite":true,"date":"2024-02-26","sourceType":"paper","source":"epoch","sourceUrl":"http://arxiv.org/abs/2402.16819","collected":"2026-10-10"},{"model":"falcon-2-11b","modelName":"Falcon 2 11B","provider":"Technology Innovation Institute","rawName":"Falcon 2 11B","variant":"compiled","setting":"Exact match; 5-shot","score":58.4,"unit":"%","rank":54,"rankComputedBySite":true,"date":"2024-05-09","sourceType":"paper","source":"epoch","sourceUrl":"http://arxiv.org/abs/2407.14885","collected":"2026-10-10"},{"model":"yi-6b","modelName":"Yi 6B","provider":"01.AI","rawName":"Yi 6B","variant":"compiled","setting":"Exact match; 0-shot","score":58.2,"unit":"%","rank":55,"rankComputedBySite":true,"date":"2023-11-22","sourceType":"paper","source":"epoch","sourceUrl":"http://arxiv.org/abs/2403.04652","collected":"2026-10-10"},{"model":"llama-33b","modelName":"LLaMA-33B","provider":"Meta","rawName":"LLaMA-33B","variant":"compiled","setting":"Exact match; 5-shot","score":57.8,"unit":"%","rank":56,"rankComputedBySite":true,"date":"2023-02-24","sourceType":"paper","source":"epoch","sourceUrl":"http://arxiv.org/abs/2307.09288","collected":"2026-10-10"},{"model":"internlm-chat-20b","modelName":"internlm-chat-20b","provider":"Shanghai AI Lab (InternLM)","rawName":"internlm-chat-20b","variant":"compiled","setting":"Exact match; 5-shot","score":57.4,"unit":"%","rank":57,"rankComputedBySite":true,"date":"2023-09-17","sourceType":"paper","source":"epoch","sourceUrl":"http://arxiv.org/abs/2403.04652","collected":"2026-10-10"},{"model":"falcon-40b","modelName":"Falcon-40B","provider":"Technology Innovation Institute","rawName":"Falcon-40B","variant":"compiled","setting":"Exact match; 5-shot","score":56.9,"unit":"%","rank":58,"rankComputedBySite":true,"date":"2023-03-15","sourceType":"paper","source":"epoch","sourceUrl":"http://arxiv.org/abs/2407.14885","collected":"2026-10-10"},{"model":"phi-2","modelName":"Phi-2","provider":"Microsoft","rawName":"Phi-2","variant":"compiled","setting":"Exact match; 5-shot","score":56.3,"unit":"%","rank":59,"rankComputedBySite":true,"date":"2023-12-12","sourceType":"paper","source":"epoch","sourceUrl":"http://arxiv.org/abs/2404.14219","collected":"2026-10-10"},{"model":"internlm-chat-20b","modelName":"internlm-chat-20b","provider":"Shanghai AI Lab (InternLM)","rawName":"internlm-chat-20b","variant":"compiled","setting":"Exact match; 0-shot","score":55.6,"unit":"%","rank":60,"rankComputedBySite":true,"date":"2023-09-17","sourceType":"paper","source":"epoch","sourceUrl":"http://arxiv.org/abs/2403.04652","collected":"2026-10-10"},{"model":"xverse-13b","modelName":"XVERSE-13B","provider":null,"rawName":"XVERSE-13B","variant":"compiled","setting":"Exact match; 5-shot","score":55.21,"unit":"%","rank":61,"rankComputedBySite":true,"date":"","sourceType":"paper","source":"epoch","sourceUrl":"http://arxiv.org/abs/2309.10305","collected":"2026-10-10"},{"model":"baichuan2-13b-chat","modelName":"Baichuan2-13B-Chat","provider":"Baichuan","rawName":"Baichuan2-13B-Chat","variant":"compiled","setting":"Exact match; 0-shot","score":55.1,"unit":"%","rank":62,"rankComputedBySite":true,"date":"2023-09-06","sourceType":"paper","source":"epoch","sourceUrl":"http://arxiv.org/abs/2403.04652","collected":"2026-10-10"},{"model":"llama-2-13b","modelName":"Llama 2-13B","provider":"Meta","rawName":"Llama 2-13B","variant":"compiled","setting":"Exact match; 5-shot","score":54.8,"unit":"%","rank":63,"rankComputedBySite":true,"date":"2023-07-18","sourceType":"paper","source":"epoch","sourceUrl":"http://arxiv.org/abs/2307.09288","collected":"2026-10-10"},{"model":"baichuan-2-7b","modelName":"Baichuan 2-7B","provider":"Baichuan","rawName":"Baichuan 2-7B","variant":"compiled","setting":"Exact match; 5-shot","score":54.16,"unit":"%","rank":64,"rankComputedBySite":true,"date":"2023-09-20","sourceType":"paper","source":"epoch","sourceUrl":"http://arxiv.org/abs/2309.10305","collected":"2026-10-10"},{"model":"palm-62b","modelName":"PaLM 62B","provider":"Google","rawName":"PaLM 62B","variant":"compiled","setting":"Exact match; 5-shot","score":53.7,"unit":"%","rank":65,"rankComputedBySite":true,"date":"2022-04-04","sourceType":"paper","source":"epoch","sourceUrl":"http://arxiv.org/abs/2302.13971","collected":"2026-10-10"},{"model":"qwen2-5-coder-1-5b","modelName":"Qwen2.5-Coder (1.5B)","provider":"Alibaba (Qwen)","rawName":"Qwen2.5-Coder (1.5B)","variant":"compiled","setting":"Exact match","score":53.6,"unit":"%","rank":66,"rankComputedBySite":true,"date":"2024-09-18","sourceType":"paper","source":"epoch","sourceUrl":"http://arxiv.org/abs/2409.12186","collected":"2026-10-10"},{"model":"vicuna-13b","modelName":"Vicuna-13B","provider":"LMSYS","rawName":"Vicuna-13B","variant":"compiled","setting":"Exact match; 5-shot","score":52,"unit":"%","rank":67,"rankComputedBySite":true,"date":"","sourceType":"paper","source":"epoch","sourceUrl":"http://arxiv.org/abs/2309.10305","collected":"2026-10-10"},{"model":"baichuan-1-13b","modelName":"Baichuan 1-13B","provider":"Baichuan","rawName":"Baichuan 1-13B","variant":"compiled","setting":"Exact match; 5-shot","score":51.6,"unit":"%","rank":68,"rankComputedBySite":true,"date":"2023-07-11","sourceType":"paper","source":"epoch","sourceUrl":"http://arxiv.org/abs/2309.10305","collected":"2026-10-10"},{"model":"internlm-7b","modelName":"internlm-7b","provider":"Shanghai AI Lab (InternLM)","rawName":"internlm-7b","variant":"compiled","setting":"Exact match; 5-shot","score":51,"unit":"%","rank":69,"rankComputedBySite":true,"date":"2023-07-05","sourceType":"paper","source":"epoch","sourceUrl":"https://arxiv.org/pdf/2309.16609","collected":"2026-10-10"},{"model":"llama-2-13b","modelName":"Llama 2-13B","provider":"Meta","rawName":"Llama 2-13B","variant":"compiled","setting":"Exact match; 0-shot","score":50.9,"unit":"%","rank":70,"rankComputedBySite":true,"date":"2023-07-18","sourceType":"paper","source":"epoch","sourceUrl":"http://arxiv.org/abs/2403.04652","collected":"2026-10-10"},{"model":"baichuan2-13b-chat","modelName":"Baichuan2-13B-Chat","provider":"Baichuan","rawName":"Baichuan2-13B-Chat","variant":"compiled","setting":"Exact match; 5-shot","score":50.1,"unit":"%","rank":71,"rankComputedBySite":true,"date":"2023-09-06","sourceType":"paper","source":"epoch","sourceUrl":"http://arxiv.org/abs/2403.04652","collected":"2026-10-10"},{"model":"intellect-1","modelName":"INTELLECT-1","provider":"Prime Intellect","rawName":"INTELLECT-1","variant":"compiled","setting":"Exact match; 5-shot","score":49.89,"unit":"%","rank":72,"rankComputedBySite":true,"date":"2024-11-29","sourceType":"paper","source":"epoch","sourceUrl":"https://arxiv.org/abs/2412.01152","collected":"2026-10-10"},{"model":"chatglm2-6b","modelName":"chatglm2-6b","provider":"Zhipu AI (Z.ai)","rawName":"chatglm2-6b","variant":"compiled","setting":"Exact match; 5-shot","score":47.9,"unit":"%","rank":73,"rankComputedBySite":true,"date":"2023-06-24","sourceType":"paper","source":"epoch","sourceUrl":"https://arxiv.org/pdf/2309.16609","collected":"2026-10-10"},{"model":"mpt-30b","modelName":"MPT-30B","provider":"MosaicML","rawName":"MPT-30B","variant":"compiled","setting":"Exact match; 5-shot","score":47.9,"unit":"%","rank":73,"rankComputedBySite":true,"date":"2023-06-22","sourceType":"paper","source":"epoch","sourceUrl":"https://arxiv.org/pdf/2309.16609","collected":"2026-10-10"},{"model":"chatglm-2-6b-base","modelName":"ChatGLM 2-6B (base)*","provider":"Zhipu AI (Z.ai)","rawName":"ChatGLM 2-6B (base)*","variant":"compiled","setting":"Exact match; 5-shot","score":47.86,"unit":"%","rank":75,"rankComputedBySite":true,"date":"","sourceType":"paper","source":"epoch","sourceUrl":"http://arxiv.org/abs/2309.10305","collected":"2026-10-10"},{"model":"llama-13b","modelName":"LLaMA-13B","provider":"Meta","rawName":"LLaMA-13B","variant":"compiled","setting":"Exact match; 5-shot","score":46.9,"unit":"%","rank":76,"rankComputedBySite":true,"date":"2023-02-24","sourceType":"paper","source":"epoch","sourceUrl":"http://arxiv.org/abs/2307.09288","collected":"2026-10-10"},{"model":"llama-2-7b","modelName":"Llama 2-7B","provider":"Meta","rawName":"Llama 2-7B","variant":"compiled","setting":"Exact match; 5-shot","score":45.3,"unit":"%","rank":77,"rankComputedBySite":true,"date":"2023-07-18","sourceType":"paper","source":"epoch","sourceUrl":"http://arxiv.org/abs/2307.09288","collected":"2026-10-10"},{"model":"llama-2-7b","modelName":"Llama 2-7B","provider":"Meta","rawName":"Llama 2-7B","variant":"compiled","setting":"Exact match; 2-shot","score":45.3,"unit":"%","rank":77,"rankComputedBySite":true,"date":"2023-07-18","sourceType":"paper","source":"epoch","sourceUrl":"http://arxiv.org/abs/2309.05463","collected":"2026-10-10"},{"model":"qwen-7b","modelName":"Qwen-7B","provider":"Alibaba (Qwen)","rawName":"Qwen-7B","variant":"compiled","setting":"Exact match; 5-shot","score":45,"unit":"%","rank":79,"rankComputedBySite":true,"date":"2023-09-28","sourceType":"paper","source":"epoch","sourceUrl":"https://arxiv.org/pdf/2309.16609","collected":"2026-10-10"},{"model":"chinese-alpaca-plus-13b","modelName":"Chinese-Alpaca-Plus-13B","provider":"Community (Chinese-LLaMA)","rawName":"Chinese-Alpaca-Plus-13B","variant":"compiled","setting":"Exact match; 5-shot","score":43.9,"unit":"%","rank":80,"rankComputedBySite":true,"date":"","sourceType":"paper","source":"epoch","sourceUrl":"http://arxiv.org/abs/2309.10305","collected":"2026-10-10"},{"model":"instructgpt-175b","modelName":"InstructGPT 175B","provider":"OpenAI","rawName":"InstructGPT 175B","variant":"compiled","setting":"Exact match; 5-shot","score":43.9,"unit":"%","rank":80,"rankComputedBySite":true,"date":"2022-01-27","sourceType":"paper","source":"epoch","sourceUrl":"http://arxiv.org/abs/2112.11446","collected":"2026-10-10"},{"model":"baichuan1-7b","modelName":"Baichuan1-7B","provider":"Baichuan","rawName":"Baichuan1-7B","variant":"compiled","setting":"Exact match; 5-shot","score":42.3,"unit":"%","rank":82,"rankComputedBySite":true,"date":"2023-06-01","sourceType":"paper","source":"epoch","sourceUrl":"http://arxiv.org/abs/2309.10305","collected":"2026-10-10"},{"model":"gemma-2b","modelName":"Gemma 2B","provider":"Google","rawName":"Gemma 2B","variant":"compiled","setting":"Exact match; 5-shot","score":42.3,"unit":"%","rank":82,"rankComputedBySite":true,"date":"2024-02-21","sourceType":"paper","source":"epoch","sourceUrl":"https://arxiv.org/pdf/2403.08295","collected":"2026-10-10"},{"model":"qwen2-5-coder-0-5b","modelName":"Qwen2.5-Coder-0.5B","provider":"Alibaba (Qwen)","rawName":"Qwen2.5-Coder-0.5B","variant":"compiled","setting":"Exact match","score":42,"unit":"%","rank":84,"rankComputedBySite":true,"date":"2024-09-18","sourceType":"paper","source":"epoch","sourceUrl":"http://arxiv.org/abs/2409.12186","collected":"2026-10-10"},{"model":"codeqwen1-5-7b","modelName":"CodeQwen1.5-7B","provider":"Alibaba (Qwen)","rawName":"CodeQwen1.5-7B","variant":"compiled","setting":"Exact match","score":40.5,"unit":"%","rank":85,"rankComputedBySite":true,"date":"2024-04-15","sourceType":"paper","source":"epoch","sourceUrl":"http://arxiv.org/abs/2409.12186","collected":"2026-10-10"},{"model":"deepseek-coder-33b","modelName":"DeepSeek Coder 33B","provider":"DeepSeek","rawName":"DeepSeek Coder 33B","variant":"compiled","setting":"Exact match","score":39.4,"unit":"%","rank":86,"rankComputedBySite":true,"date":"2023-11-02","sourceType":"paper","source":"epoch","sourceUrl":"http://arxiv.org/abs/2409.12186","collected":"2026-10-10"},{"model":"starcoder-2-7b","modelName":"StarCoder 2 7B","provider":"Hugging Face","rawName":"StarCoder 2 7B","variant":"compiled","setting":"Exact match","score":38.8,"unit":"%","rank":87,"rankComputedBySite":true,"date":"2024-02-20","sourceType":"paper","source":"epoch","sourceUrl":"http://arxiv.org/abs/2409.12186","collected":"2026-10-10"},{"model":"phi-1-5","modelName":"Phi-1.5","provider":"Microsoft","rawName":"Phi-1.5","variant":"compiled","setting":"Exact match; 2-shot","score":37.6,"unit":"%","rank":88,"rankComputedBySite":true,"date":"2023-09-11","sourceType":"paper","source":"epoch","sourceUrl":"http://arxiv.org/abs/2309.05463","collected":"2026-10-10"},{"model":"starcoder-2-3b","modelName":"StarCoder 2 3B","provider":"Hugging Face","rawName":"StarCoder 2 3B","variant":"compiled","setting":"Exact match","score":36.6,"unit":"%","rank":89,"rankComputedBySite":true,"date":"2024-02-22","sourceType":"paper","source":"epoch","sourceUrl":"http://arxiv.org/abs/2409.12186","collected":"2026-10-10"},{"model":"deepseek-coder-6-7b","modelName":"DeepSeek Coder 6.7B","provider":"DeepSeek","rawName":"DeepSeek Coder 6.7B","variant":"compiled","setting":"Exact match","score":36.4,"unit":"%","rank":90,"rankComputedBySite":true,"date":"2023-11-02","sourceType":"paper","source":"epoch","sourceUrl":"http://arxiv.org/abs/2409.12186","collected":"2026-10-10"},{"model":"xgen-7b","modelName":"XGen-7B","provider":"Salesforce","rawName":"XGen-7B","variant":"compiled","setting":"Exact match; 5-shot","score":36.3,"unit":"%","rank":91,"rankComputedBySite":true,"date":"2023-06-27","sourceType":"paper","source":"epoch","sourceUrl":"http://arxiv.org/abs/2309.03450","collected":"2026-10-10"},{"model":"llama-7b","modelName":"LLaMA-7B","provider":"Meta","rawName":"LLaMA-7B","variant":"compiled","setting":"Exact match; 2-shot","score":35.2,"unit":"%","rank":92,"rankComputedBySite":true,"date":"2023-02-24","sourceType":"paper","source":"epoch","sourceUrl":"http://arxiv.org/abs/2309.05463","collected":"2026-10-10"},{"model":"llama-7b","modelName":"LLaMA-7B","provider":"Meta","rawName":"LLaMA-7B","variant":"compiled","setting":"Exact match; 5-shot","score":35.1,"unit":"%","rank":93,"rankComputedBySite":true,"date":"2023-02-24","sourceType":"paper","source":"epoch","sourceUrl":"http://arxiv.org/abs/2307.09288","collected":"2026-10-10"},{"model":"falcon-7b","modelName":"Falcon-7B","provider":"Technology Innovation Institute","rawName":"Falcon-7B","variant":"compiled","setting":"Exact match; 5-shot","score":35,"unit":"%","rank":94,"rankComputedBySite":true,"date":"2023-04-24","sourceType":"paper","source":"epoch","sourceUrl":"http://arxiv.org/abs/2407.14885","collected":"2026-10-10"},{"model":"gpt-neox-20b","modelName":"GPT-NeoX 20B","provider":"EleutherAI","rawName":"GPT-NeoX 20B","variant":"compiled","setting":"Exact match; 5-shot","score":33.6,"unit":"%","rank":95,"rankComputedBySite":true,"date":"","sourceType":"paper","source":"epoch","sourceUrl":"http://arxiv.org/abs/2302.13971","collected":"2026-10-10"},{"model":"xgen-7b","modelName":"XGen-7B","provider":"Salesforce","rawName":"XGen-7B","variant":"compiled","setting":"Exact match; 0-shot","score":32.1,"unit":"%","rank":96,"rankComputedBySite":true,"date":"2023-06-27","sourceType":"paper","source":"epoch","sourceUrl":"http://arxiv.org/abs/2309.03450","collected":"2026-10-10"},{"model":"llama-7b","modelName":"LLaMA-7B","provider":"Meta","rawName":"LLaMA-7B","variant":"compiled","setting":"Exact match; 0-shot","score":32,"unit":"%","rank":97,"rankComputedBySite":true,"date":"2023-02-24","sourceType":"paper","source":"epoch","sourceUrl":"http://arxiv.org/abs/2309.03450","collected":"2026-10-10"},{"model":"mpt-7b","modelName":"MPT-7B","provider":"MosaicML","rawName":"MPT-7B","variant":"compiled","setting":"Exact match; 5-shot","score":30.8,"unit":"%","rank":98,"rankComputedBySite":true,"date":"2023-05-05","sourceType":"paper","source":"epoch","sourceUrl":"https://arxiv.org/pdf/2309.16609","collected":"2026-10-10"},{"model":"open-llama-7b","modelName":"open_llama_7b","provider":null,"rawName":"open_llama_7b","variant":"compiled","setting":"Exact match; 5-shot","score":29.9,"unit":"%","rank":99,"rankComputedBySite":true,"date":"2023-06-07","sourceType":"paper","source":"epoch","sourceUrl":"http://arxiv.org/abs/2309.03450","collected":"2026-10-10"},{"model":"open-llama-7b","modelName":"open_llama_7b","provider":null,"rawName":"open_llama_7b","variant":"compiled","setting":"Exact match; 0-shot","score":28.6,"unit":"%","rank":100,"rankComputedBySite":true,"date":"2023-06-07","sourceType":"paper","source":"epoch","sourceUrl":"http://arxiv.org/abs/2309.03450","collected":"2026-10-10"},{"model":"qwen-1-8b","modelName":"Qwen-1_8B","provider":"Alibaba (Qwen)","rawName":"Qwen-1_8B","variant":"compiled","setting":"Exact match; 5-shot","score":28.2,"unit":"%","rank":101,"rankComputedBySite":true,"date":"2023-11-30","sourceType":"paper","source":"epoch","sourceUrl":"https://arxiv.org/pdf/2309.16609","collected":"2026-10-10"},{"model":"mpt-7b","modelName":"MPT-7B","provider":"MosaicML","rawName":"MPT-7B","variant":"compiled","setting":"Exact match; 0-shot","score":27.4,"unit":"%","rank":102,"rankComputedBySite":true,"date":"2023-05-05","sourceType":"paper","source":"epoch","sourceUrl":"http://arxiv.org/abs/2309.03450","collected":"2026-10-10"},{"model":"falcon-7b","modelName":"Falcon-7B","provider":"Technology Innovation Institute","rawName":"Falcon-7B","variant":"compiled","setting":"Exact match; 2-shot","score":26.9,"unit":"%","rank":103,"rankComputedBySite":true,"date":"2023-04-24","sourceType":"paper","source":"epoch","sourceUrl":"http://arxiv.org/abs/2309.05463","collected":"2026-10-10"},{"model":"mpt-7b","modelName":"MPT-7B","provider":"MosaicML","rawName":"MPT-7B","variant":"compiled","setting":"Exact match; 2-shot","score":26.8,"unit":"%","rank":104,"rankComputedBySite":true,"date":"2023-05-05","sourceType":"paper","source":"epoch","sourceUrl":"http://arxiv.org/abs/2309.05463","collected":"2026-10-10"},{"model":"redpajama-incite-7b-base","modelName":"RedPajama-INCITE-7B-Base","provider":null,"rawName":"RedPajama-INCITE-7B-Base","variant":"compiled","setting":"Exact match; 5-shot","score":26.3,"unit":"%","rank":105,"rankComputedBySite":true,"date":"2023-05-04","sourceType":"paper","source":"epoch","sourceUrl":"http://arxiv.org/abs/2309.03450","collected":"2026-10-10"},{"model":"cerebras-gpt-13b","modelName":"Cerebras-GPT-13B","provider":"Cerebras Systems","rawName":"Cerebras-GPT-13B","variant":"compiled","setting":"Exact match; 5-shot","score":26.2,"unit":"%","rank":106,"rankComputedBySite":true,"date":"2023-03-20","sourceType":"paper","source":"epoch","sourceUrl":"http://arxiv.org/abs/2309.03450","collected":"2026-10-10"},{"model":"dolly-2-0-12b","modelName":"Dolly 2.0-12b","provider":"Databricks","rawName":"Dolly 2.0-12b","variant":"compiled","setting":"Exact match; 5-shot","score":26.2,"unit":"%","rank":106,"rankComputedBySite":true,"date":"2023-04-11","sourceType":"paper","source":"epoch","sourceUrl":"http://arxiv.org/abs/2309.03450","collected":"2026-10-10"},{"model":"falcon-rw-1-3b","modelName":"Falcon-rw-1.3B","provider":"TII","rawName":"Falcon-rw-1.3B","variant":"compiled","setting":"Exact match; 2-shot","score":25.9,"unit":"%","rank":108,"rankComputedBySite":true,"date":"","sourceType":"paper","source":"epoch","sourceUrl":"http://arxiv.org/abs/2309.05463","collected":"2026-10-10"},{"model":"deepseek-coder-1-3b","modelName":"DeepSeek Coder 1.3B","provider":"DeepSeek","rawName":"DeepSeek Coder 1.3B","variant":"compiled","setting":"Exact match","score":25.8,"unit":"%","rank":109,"rankComputedBySite":true,"date":"2023-11-02","sourceType":"paper","source":"epoch","sourceUrl":"http://arxiv.org/abs/2409.12186","collected":"2026-10-10"},{"model":"redpajama-incite-7b-base","modelName":"RedPajama-INCITE-7B-Base","provider":null,"rawName":"RedPajama-INCITE-7B-Base","variant":"compiled","setting":"Exact match; 0-shot","score":25.8,"unit":"%","rank":109,"rankComputedBySite":true,"date":"2023-05-04","sourceType":"paper","source":"epoch","sourceUrl":"http://arxiv.org/abs/2309.03450","collected":"2026-10-10"},{"model":"gpt-j-6b","modelName":"GPT-J-6B","provider":"EleutherAI","rawName":"GPT-J-6B","variant":"compiled","setting":"Exact match; 0-shot","score":25.7,"unit":"%","rank":111,"rankComputedBySite":true,"date":"2021-08-05","sourceType":"paper","source":"epoch","sourceUrl":"http://arxiv.org/abs/2309.03450","collected":"2026-10-10"},{"model":"dolly-2-0-12b","modelName":"Dolly 2.0-12b","provider":"Databricks","rawName":"Dolly 2.0-12b","variant":"compiled","setting":"Exact match; 0-shot","score":25.4,"unit":"%","rank":112,"rankComputedBySite":true,"date":"2023-04-11","sourceType":"paper","source":"epoch","sourceUrl":"http://arxiv.org/abs/2309.03450","collected":"2026-10-10"},{"model":"palm-8b","modelName":"PaLM 8B","provider":"Google","rawName":"PaLM 8B","variant":"compiled","setting":"Exact match; 5-shot","score":25.4,"unit":"%","rank":112,"rankComputedBySite":true,"date":"","sourceType":"paper","source":"epoch","sourceUrl":"http://arxiv.org/abs/2302.13971","collected":"2026-10-10"},{"model":"gpt-j-6b","modelName":"GPT-J-6B","provider":"EleutherAI","rawName":"GPT-J-6B","variant":"compiled","setting":"Exact match; 5-shot","score":25.1,"unit":"%","rank":114,"rankComputedBySite":true,"date":"2021-08-05","sourceType":"paper","source":"epoch","sourceUrl":"http://arxiv.org/abs/2309.03450","collected":"2026-10-10"},{"model":"opt-13b","modelName":"opt-13b","provider":null,"rawName":"opt-13b","variant":"compiled","setting":"Exact match; 5-shot","score":25.1,"unit":"%","rank":114,"rankComputedBySite":true,"date":"2022-05-11","sourceType":"paper","source":"epoch","sourceUrl":"http://arxiv.org/abs/2309.03450","collected":"2026-10-10"},{"model":"cerebras-gpt-13b","modelName":"Cerebras-GPT-13B","provider":"Cerebras Systems","rawName":"Cerebras-GPT-13B","variant":"compiled","setting":"Exact match; 0-shot","score":24.6,"unit":"%","rank":116,"rankComputedBySite":true,"date":"2023-03-20","sourceType":"paper","source":"epoch","sourceUrl":"http://arxiv.org/abs/2309.03450","collected":"2026-10-10"},{"model":"opt-13b","modelName":"opt-13b","provider":null,"rawName":"opt-13b","variant":"compiled","setting":"Exact match; 0-shot","score":24.4,"unit":"%","rank":117,"rankComputedBySite":true,"date":"2022-05-11","sourceType":"paper","source":"epoch","sourceUrl":"http://arxiv.org/abs/2309.03450","collected":"2026-10-10"},{"model":"falcon-7b","modelName":"Falcon-7B","provider":"Technology Innovation Institute","rawName":"Falcon-7B","variant":"compiled","setting":"Exact match; 0-shot","score":23.9,"unit":"%","rank":118,"rankComputedBySite":true,"date":"2023-04-24","sourceType":"paper","source":"epoch","sourceUrl":"http://arxiv.org/abs/2309.03450","collected":"2026-10-10"},{"model":"claude-3-5-sonnet-october-2024","modelName":"Claude 3.5 Sonnet (October 2024)","provider":"Anthropic","rawName":"Claude 3.5 Sonnet (October 2024)","variant":"leaderboard","setting":"Exact match; 5-shot; effort: Oct 2024","score":87.3,"unit":"%","rank":1,"rankComputedBySite":true,"date":"2024-10-22","sourceType":"official","source":"epoch","sourceUrl":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","collected":"2026-10-10"},{"model":"deepseek-v3","modelName":"DeepSeek-V3","provider":"DeepSeek","rawName":"DeepSeek-V3","variant":"leaderboard","setting":"Exact match; 5-shot","score":87.2,"unit":"%","rank":2,"rankComputedBySite":true,"date":"2024-12-26","sourceType":"official","source":"epoch","sourceUrl":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","collected":"2026-10-10"},{"model":"gemini-1-5-pro-sept-2024","modelName":"Gemini 1.5 Pro (Sept 2024)","provider":"Google","rawName":"Gemini 1.5 Pro (Sept 2024)","variant":"leaderboard","setting":"Exact match; 5-shot","score":86.9,"unit":"%","rank":3,"rankComputedBySite":true,"date":"2024-09-24","sourceType":"official","source":"epoch","sourceUrl":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","collected":"2026-10-10"},{"model":"claude-3-5-sonnet","modelName":"Claude 3.5 Sonnet","provider":"Anthropic","rawName":"Claude 3.5 Sonnet","variant":"leaderboard","setting":"Exact match; 5-shot; effort: Jun 2024","score":86.5,"unit":"%","rank":4,"rankComputedBySite":true,"date":"2024-06-20","sourceType":"official","source":"epoch","sourceUrl":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","collected":"2026-10-10"},{"model":"claude-3-opus","modelName":"Claude 3 Opus","provider":"Anthropic","rawName":"Claude 3 Opus","variant":"leaderboard","setting":"Exact match; 5-shot","score":84.6,"unit":"%","rank":5,"rankComputedBySite":true,"date":"2024-02-29","sourceType":"official","source":"epoch","sourceUrl":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","collected":"2026-10-10"},{"model":"llama-3-1-405b","modelName":"Llama 3.1-405B","provider":"Meta","rawName":"Llama 3.1-405B","variant":"leaderboard","setting":"Exact match; 5-shot","score":84.5,"unit":"%","rank":6,"rankComputedBySite":true,"date":"2024-07-23","sourceType":"official","source":"epoch","sourceUrl":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","collected":"2026-10-10"},{"model":"gpt-4o-aug-2024","modelName":"GPT-4o (Aug 2024)","provider":"OpenAI","rawName":"GPT-4o (Aug 2024)","variant":"leaderboard","setting":"Exact match; 5-shot; effort: Aug 2024","score":84.3,"unit":"%","rank":7,"rankComputedBySite":true,"date":"2024-08-06","sourceType":"official","source":"epoch","sourceUrl":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","collected":"2026-10-10"},{"model":"gpt-4o-may-2024","modelName":"GPT-4o (May 2024)","provider":"OpenAI","rawName":"GPT-4o (May 2024)","variant":"leaderboard","setting":"Exact match; 5-shot; effort: May 2024","score":84.2,"unit":"%","rank":8,"rankComputedBySite":true,"date":"2024-05-13","sourceType":"official","source":"epoch","sourceUrl":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","collected":"2026-10-10"},{"model":"qwen2-5-72b","modelName":"Qwen2.5-72B","provider":"Alibaba (Qwen)","rawName":"Qwen2.5-72B","variant":"leaderboard","setting":"Exact match; 5-shot","score":83.4,"unit":"%","rank":9,"rankComputedBySite":true,"date":"2024-09-19","sourceType":"official","source":"epoch","sourceUrl":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","collected":"2026-10-10"},{"model":"gemini-1-5-pro-feb-2024","modelName":"Gemini 1.5 Pro (Feb 2024)","provider":"Google","rawName":"Gemini 1.5 Pro (Feb 2024)","variant":"leaderboard","setting":"Exact match; 5-shot","score":82.7,"unit":"%","rank":10,"rankComputedBySite":true,"date":"2024-02-15","sourceType":"official","source":"epoch","sourceUrl":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","collected":"2026-10-10"},{"model":"gpt-4-jun-2023","modelName":"GPT-4 (Jun 2023)","provider":"OpenAI","rawName":"GPT-4 (Jun 2023)","variant":"leaderboard","setting":"Exact match; 5-shot; effort: Jun 2023","score":82.4,"unit":"%","rank":11,"rankComputedBySite":true,"date":"2023-06-13","sourceType":"official","source":"epoch","sourceUrl":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","collected":"2026-10-10"},{"model":"qwen2-72b","modelName":"Qwen2-72B","provider":"Alibaba (Qwen)","rawName":"Qwen2-72B","variant":"leaderboard","setting":"Exact match; 5-shot","score":82.4,"unit":"%","rank":11,"rankComputedBySite":true,"date":"2024-06-07","sourceType":"official","source":"epoch","sourceUrl":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","collected":"2026-10-10"},{"model":"amazon-nova-pro","modelName":"Amazon Nova Pro","provider":"Amazon","rawName":"Amazon Nova Pro","variant":"leaderboard","setting":"Exact match; 5-shot","score":82,"unit":"%","rank":13,"rankComputedBySite":true,"date":"2024-12-03","sourceType":"official","source":"epoch","sourceUrl":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","collected":"2026-10-10"},{"model":"gpt-4-turbo-apr-2024","modelName":"GPT-4 Turbo (Apr 2024)","provider":"OpenAI","rawName":"GPT-4 Turbo (Apr 2024)","variant":"leaderboard","setting":"Exact match; 5-shot","score":81.3,"unit":"%","rank":14,"rankComputedBySite":true,"date":"2024-04-09","sourceType":"official","source":"epoch","sourceUrl":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","collected":"2026-10-10"},{"model":"gemini-1-5-pro-0409-preview","modelName":"Gemini 1.5 Pro (0409 preview)","provider":"Google","rawName":"Gemini 1.5 Pro (0409 preview)","variant":"leaderboard","setting":"Exact match; 5-shot","score":81,"unit":"%","rank":15,"rankComputedBySite":true,"date":"","sourceType":"official","source":"epoch","sourceUrl":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","collected":"2026-10-10"},{"model":"llama-3-2-90b","modelName":"Llama 3.2 90B","provider":"Meta","rawName":"Llama 3.2 90B","variant":"leaderboard","setting":"Exact match; 5-shot","score":80.3,"unit":"%","rank":16,"rankComputedBySite":true,"date":"2024-09-24","sourceType":"official","source":"epoch","sourceUrl":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","collected":"2026-10-10"},{"model":"llama-3-1-70b","modelName":"Llama 3.1-70B","provider":"Meta","rawName":"Llama 3.1-70B","variant":"leaderboard","setting":"Exact match; 5-shot","score":80.1,"unit":"%","rank":17,"rankComputedBySite":true,"date":"2024-07-23","sourceType":"official","source":"epoch","sourceUrl":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","collected":"2026-10-10"},{"model":"mistral-large-2-jul-2024","modelName":"Mistral Large 2 (Jul 2024)","provider":"Mistral AI","rawName":"Mistral Large 2 (Jul 2024)","variant":"leaderboard","setting":"Exact match; 5-shot","score":80,"unit":"%","rank":18,"rankComputedBySite":true,"date":"2024-07-24","sourceType":"official","source":"epoch","sourceUrl":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","collected":"2026-10-10"},{"model":"gemini-2-0-flash-dec-2024","modelName":"Gemini 2.0 Flash (Dec 2024)","provider":"Google","rawName":"Gemini 2.0 Flash (Dec 2024)","variant":"leaderboard","setting":"Exact match; 5-shot","score":79.7,"unit":"%","rank":19,"rankComputedBySite":true,"date":"2024-12-11","sourceType":"official","source":"epoch","sourceUrl":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","collected":"2026-10-10"},{"model":"gpt-4-turbo-nov-2023","modelName":"GPT-4 Turbo (Nov 2023)","provider":"OpenAI","rawName":"GPT-4 Turbo (Nov 2023)","variant":"leaderboard","setting":"Exact match; 5-shot","score":79.6,"unit":"%","rank":20,"rankComputedBySite":true,"date":"2023-11-06","sourceType":"official","source":"epoch","sourceUrl":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","collected":"2026-10-10"},{"model":"llama-3-70b","modelName":"Llama 3-70B","provider":"Meta","rawName":"Llama 3-70B","variant":"leaderboard","setting":"Exact match; 5-shot","score":79.3,"unit":"%","rank":21,"rankComputedBySite":true,"date":"2024-04-18","sourceType":"official","source":"epoch","sourceUrl":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","collected":"2026-10-10"},{"model":"yi-large","modelName":"Yi-Large","provider":"01.AI","rawName":"Yi-Large","variant":"leaderboard","setting":"Exact match; 5-shot","score":79.3,"unit":"%","rank":21,"rankComputedBySite":true,"date":"2024-05-13","sourceType":"official","source":"epoch","sourceUrl":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","collected":"2026-10-10"},{"model":"llama-3-3-70b","modelName":"Llama 3.3 70B","provider":"Meta","rawName":"Llama 3.3 70B","variant":"leaderboard","setting":"Exact match; 5-shot","score":79.1,"unit":"%","rank":23,"rankComputedBySite":true,"date":"2024-12-06","sourceType":"official","source":"epoch","sourceUrl":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","collected":"2026-10-10"},{"model":"palm-2-unicorn","modelName":"PaLM-2 (Unicorn)","provider":"Google","rawName":"PaLM-2 (Unicorn)","variant":"leaderboard","setting":"Exact match; 5-shot","score":78.6,"unit":"%","rank":24,"rankComputedBySite":true,"date":"","sourceType":"official","source":"epoch","sourceUrl":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","collected":"2026-10-10"},{"model":"palmyra-x-v3-72b","modelName":"Palmyra X V3 (72B)","provider":"Google","rawName":"Palmyra X V3 (72B)","variant":"leaderboard","setting":"Exact match; 5-shot","score":78.6,"unit":"%","rank":24,"rankComputedBySite":true,"date":"","sourceType":"official","source":"epoch","sourceUrl":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","collected":"2026-10-10"},{"model":"jamba-1-5-large","modelName":"Jamba 1.5 Large","provider":"AI21 Labs","rawName":"Jamba 1.5 Large","variant":"leaderboard","setting":"Exact match; 5-shot","score":78.2,"unit":"%","rank":26,"rankComputedBySite":true,"date":"","sourceType":"official","source":"epoch","sourceUrl":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","collected":"2026-10-10"},{"model":"gemini-1-5-flash-may-2024","modelName":"Gemini 1.5 Flash (May 2024)","provider":"Google","rawName":"Gemini 1.5 Flash (May 2024)","variant":"leaderboard","setting":"Exact match; 5-shot; effort: May 2024","score":77.9,"unit":"%","rank":27,"rankComputedBySite":true,"date":"2024-05-23","sourceType":"official","source":"epoch","sourceUrl":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","collected":"2026-10-10"},{"model":"gemini-1-5-flash-may-2024","modelName":"Gemini 1.5 Flash (May 2024)","provider":"Google","rawName":"Gemini 1.5 Flash (May 2024)","variant":"leaderboard","setting":"Exact match; 5-shot","score":77.8,"unit":"%","rank":28,"rankComputedBySite":true,"date":"2024-05-14","sourceType":"official","source":"epoch","sourceUrl":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","collected":"2026-10-10"},{"model":"mixtral-8x22b","modelName":"Mixtral 8x22B","provider":"Mistral AI","rawName":"Mixtral 8x22B","variant":"leaderboard","setting":"Exact match; 5-shot","score":77.8,"unit":"%","rank":28,"rankComputedBySite":true,"date":"2024-04-17","sourceType":"official","source":"epoch","sourceUrl":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","collected":"2026-10-10"},{"model":"solar-pro","modelName":"Solar Pro","provider":null,"rawName":"Solar Pro","variant":"leaderboard","setting":"Exact match; 5-shot","score":77.6,"unit":"%","rank":30,"rankComputedBySite":true,"date":"","sourceType":"official","source":"epoch","sourceUrl":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","collected":"2026-10-10"},{"model":"phi-3-medium-14b","modelName":"phi-3-medium 14B","provider":"Microsoft","rawName":"phi-3-medium 14B","variant":"leaderboard","setting":"Exact match; 5-shot","score":77.5,"unit":"%","rank":31,"rankComputedBySite":true,"date":"2024-04-23","sourceType":"official","source":"epoch","sourceUrl":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","collected":"2026-10-10"},{"model":"qwen1-5-72b","modelName":"Qwen1.5-72B","provider":"Alibaba (Qwen)","rawName":"Qwen1.5 (72B)","variant":"leaderboard","setting":"Exact match; 5-shot","score":77.4,"unit":"%","rank":32,"rankComputedBySite":true,"date":"","sourceType":"official","source":"epoch","sourceUrl":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","collected":"2026-10-10"},{"model":"amazon-nova-lite","modelName":"Amazon Nova Lite","provider":"Amazon","rawName":"Amazon Nova Lite","variant":"leaderboard","setting":"Exact match; 5-shot","score":77,"unit":"%","rank":33,"rankComputedBySite":true,"date":"2024-12-03","sourceType":"official","source":"epoch","sourceUrl":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","collected":"2026-10-10"},{"model":"qwen1-5-chat-110b","modelName":"Qwen1.5 Chat (110B)","provider":"Alibaba (Qwen)","rawName":"Qwen1.5 Chat (110B)","variant":"leaderboard","setting":"Exact match; 5-shot","score":76.8,"unit":"%","rank":34,"rankComputedBySite":true,"date":"","sourceType":"official","source":"epoch","sourceUrl":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","collected":"2026-10-10"},{"model":"gpt-4o-mini","modelName":"GPT-4o mini","provider":"OpenAI","rawName":"GPT-4o mini","variant":"leaderboard","setting":"Exact match; 5-shot","score":76.7,"unit":"%","rank":35,"rankComputedBySite":true,"date":"2024-07-18","sourceType":"official","source":"epoch","sourceUrl":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","collected":"2026-10-10"},{"model":"yi-34b","modelName":"Yi-34B","provider":"01.AI","rawName":"Yi-34B","variant":"leaderboard","setting":"Exact match; 5-shot","score":76.2,"unit":"%","rank":36,"rankComputedBySite":true,"date":"2023-11-02","sourceType":"official","source":"epoch","sourceUrl":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","collected":"2026-10-10"},{"model":"claude-3-sonnet","modelName":"Claude 3 Sonnet","provider":"Anthropic","rawName":"Claude 3 Sonnet","variant":"leaderboard","setting":"Exact match; 5-shot","score":75.9,"unit":"%","rank":37,"rankComputedBySite":true,"date":"2024-02-29","sourceType":"official","source":"epoch","sourceUrl":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","collected":"2026-10-10"},{"model":"gemma-2-27b","modelName":"Gemma 2 27B","provider":"Google","rawName":"Gemma 2 27B","variant":"leaderboard","setting":"Exact match; 5-shot","score":75.7,"unit":"%","rank":38,"rankComputedBySite":true,"date":"2024-06-24","sourceType":"official","source":"epoch","sourceUrl":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","collected":"2026-10-10"},{"model":"phi-3-7b","modelName":"Phi-3 (7B)","provider":"Microsoft","rawName":"Phi-3 (7B)","variant":"leaderboard","setting":"Exact match; 5-shot","score":75.7,"unit":"%","rank":38,"rankComputedBySite":true,"date":"","sourceType":"official","source":"epoch","sourceUrl":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","collected":"2026-10-10"},{"model":"qwen1-5-32b","modelName":"Qwen1.5-32B","provider":"Alibaba (Qwen)","rawName":"Qwen1.5-32B","variant":"leaderboard","setting":"Exact match; 5-shot","score":74.4,"unit":"%","rank":40,"rankComputedBySite":true,"date":"2024-02-04","sourceType":"official","source":"epoch","sourceUrl":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","collected":"2026-10-10"},{"model":"claude-3-5-haiku","modelName":"Claude 3.5 Haiku","provider":"Anthropic","rawName":"Claude 3.5 Haiku","variant":"leaderboard","setting":"Exact match; 5-shot; effort: Oct 2024","score":74.3,"unit":"%","rank":41,"rankComputedBySite":true,"date":"2024-10-22","sourceType":"official","source":"epoch","sourceUrl":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","collected":"2026-10-10"},{"model":"dbrx","modelName":"DBRX","provider":"Databricks","rawName":"DBRX Instruct","variant":"leaderboard","setting":"Exact match; 5-shot","score":74.1,"unit":"%","rank":42,"rankComputedBySite":true,"date":"","sourceType":"official","source":"epoch","sourceUrl":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","collected":"2026-10-10"},{"model":"gemini-1-5-flash-sep-2024","modelName":"Gemini 1.5 Flash (Sep 2024)","provider":"Google","rawName":"Gemini 1.5 Flash (Sep 2024)","variant":"leaderboard","setting":"Exact match; 5-shot","score":73.9,"unit":"%","rank":43,"rankComputedBySite":true,"date":"2024-09-24","sourceType":"official","source":"epoch","sourceUrl":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","collected":"2026-10-10"},{"model":"palmyra-x-004","modelName":"Palmyra-X-004","provider":"Writer","rawName":"Palmyra-X-004","variant":"leaderboard","setting":"Exact match; 5-shot","score":73.9,"unit":"%","rank":43,"rankComputedBySite":true,"date":"","sourceType":"official","source":"epoch","sourceUrl":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","collected":"2026-10-10"},{"model":"claude-3-haiku","modelName":"Claude 3 Haiku","provider":"Anthropic","rawName":"Claude 3 Haiku","variant":"leaderboard","setting":"Exact match; 5-shot","score":73.8,"unit":"%","rank":45,"rankComputedBySite":true,"date":"2024-03-07","sourceType":"official","source":"epoch","sourceUrl":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","collected":"2026-10-10"},{"model":"claude-2-1","modelName":"Claude 2.1","provider":"Anthropic","rawName":"Claude 2.1","variant":"leaderboard","setting":"Exact match; 5-shot","score":73.5,"unit":"%","rank":46,"rankComputedBySite":true,"date":"2023-11-21","sourceType":"official","source":"epoch","sourceUrl":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","collected":"2026-10-10"},{"model":"qwen2-5-7b","modelName":"Qwen2.5-7B","provider":"Alibaba (Qwen)","rawName":"Qwen2.5-7B","variant":"leaderboard","setting":"Exact match; 5-shot","score":72.9,"unit":"%","rank":47,"rankComputedBySite":true,"date":"2024-09-19","sourceType":"official","source":"epoch","sourceUrl":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","collected":"2026-10-10"},{"model":"deepseek-llm-chat-67b","modelName":"DeepSeek LLM Chat (67B)","provider":"DeepSeek","rawName":"DeepSeek LLM Chat (67B)","variant":"leaderboard","setting":"Exact match; 5-shot","score":72.5,"unit":"%","rank":48,"rankComputedBySite":true,"date":"","sourceType":"official","source":"epoch","sourceUrl":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","collected":"2026-10-10"},{"model":"gemma-2-9b","modelName":"Gemma 2 9B","provider":"Google","rawName":"Gemma 2 9B","variant":"leaderboard","setting":"Exact match; 5-shot","score":72.1,"unit":"%","rank":49,"rankComputedBySite":true,"date":"2024-06-24","sourceType":"official","source":"epoch","sourceUrl":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","collected":"2026-10-10"},{"model":"mixtral-8x7b-32k-seqlen","modelName":"Mixtral (8x7B 32K seqlen)","provider":"Mistral AI","rawName":"Mixtral (8x7B 32K seqlen)","variant":"leaderboard","setting":"Exact match; 5-shot","score":71.7,"unit":"%","rank":50,"rankComputedBySite":true,"date":"","sourceType":"official","source":"epoch","sourceUrl":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","collected":"2026-10-10"},{"model":"amazon-nova-micro","modelName":"Amazon Nova Micro","provider":"Amazon","rawName":"Amazon Nova Micro","variant":"leaderboard","setting":"Exact match; 5-shot","score":70.8,"unit":"%","rank":51,"rankComputedBySite":true,"date":"2024-12-03","sourceType":"official","source":"epoch","sourceUrl":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","collected":"2026-10-10"},{"model":"gemini-1-0-pro","modelName":"Gemini 1.0 Pro","provider":"Google","rawName":"Gemini 1.0 Pro","variant":"leaderboard","setting":"Exact match; 5-shot","score":70,"unit":"%","rank":52,"rankComputedBySite":true,"date":"2023-12-13","sourceType":"official","source":"epoch","sourceUrl":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","collected":"2026-10-10"},{"model":"jamba-1-5-mini","modelName":"Jamba 1.5 Mini","provider":"AI21 Labs","rawName":"Jamba 1.5 Mini","variant":"leaderboard","setting":"Exact match; 5-shot","score":69.9,"unit":"%","rank":53,"rankComputedBySite":true,"date":"","sourceType":"official","source":"epoch","sourceUrl":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","collected":"2026-10-10"},{"model":"llama-2-70b","modelName":"Llama 2-70B","provider":"Meta","rawName":"Llama 2-70B","variant":"leaderboard","setting":"Exact match; 5-shot","score":69.5,"unit":"%","rank":54,"rankComputedBySite":true,"date":"2023-07-18","sourceType":"official","source":"epoch","sourceUrl":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","collected":"2026-10-10"},{"model":"command-r","modelName":"Command R+","provider":"Cohere","rawName":"Command R+","variant":"leaderboard","setting":"Exact match; 5-shot","score":69.4,"unit":"%","rank":55,"rankComputedBySite":true,"date":"2024-08-30","sourceType":"official","source":"epoch","sourceUrl":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","collected":"2026-10-10"},{"model":"palm-2-bison","modelName":"PaLM-2 (Bison)","provider":"Google","rawName":"PaLM-2 (Bison)","variant":"leaderboard","setting":"Exact match; 5-shot","score":69.2,"unit":"%","rank":56,"rankComputedBySite":true,"date":"","sourceType":"official","source":"epoch","sourceUrl":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","collected":"2026-10-10"},{"model":"gpt-3-5-turbo-jun-2023","modelName":"GPT-3.5 Turbo (Jun 2023)","provider":"OpenAI","rawName":"GPT-3.5 Turbo (Jun 2023)","variant":"leaderboard","setting":"Exact match; 5-shot; effort: June 2023","score":68.9,"unit":"%","rank":57,"rankComputedBySite":true,"date":"2023-06-13","sourceType":"official","source":"epoch","sourceUrl":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","collected":"2026-10-10"},{"model":"claude-instant","modelName":"Claude Instant","provider":"Anthropic","rawName":"Claude Instant","variant":"leaderboard","setting":"Exact match; 5-shot","score":68.8,"unit":"%","rank":58,"rankComputedBySite":true,"date":"2023-08-09","sourceType":"official","source":"epoch","sourceUrl":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","collected":"2026-10-10"},{"model":"llama-3-8b","modelName":"Llama 3-8B","provider":"Meta","rawName":"Llama 3-8B","variant":"leaderboard","setting":"Exact match; 5-shot","score":68.8,"unit":"%","rank":58,"rankComputedBySite":true,"date":"2024-04-18","sourceType":"official","source":"epoch","sourceUrl":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","collected":"2026-10-10"},{"model":"mistral-large","modelName":"Mistral Large","provider":"Mistral AI","rawName":"Mistral Large","variant":"leaderboard","setting":"Exact match; 5-shot","score":68.8,"unit":"%","rank":58,"rankComputedBySite":true,"date":"2024-02-26","sourceType":"official","source":"epoch","sourceUrl":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","collected":"2026-10-10"},{"model":"mistral-small-2402","modelName":"mistral-small-2402","provider":"Mistral AI","rawName":"mistral-small-2402","variant":"leaderboard","setting":"Exact match; 5-shot","score":68.7,"unit":"%","rank":61,"rankComputedBySite":true,"date":"2024-02-26","sourceType":"official","source":"epoch","sourceUrl":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","collected":"2026-10-10"},{"model":"qwen1-5-14b","modelName":"Qwen1.5-14B","provider":"Alibaba (Qwen)","rawName":"Qwen1.5-14B","variant":"leaderboard","setting":"Exact match; 5-shot","score":68.6,"unit":"%","rank":62,"rankComputedBySite":true,"date":"2024-02-04","sourceType":"official","source":"epoch","sourceUrl":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","collected":"2026-10-10"},{"model":"arctic-instruct","modelName":"Arctic Instruct","provider":"Snowflake","rawName":"Arctic Instruct","variant":"leaderboard","setting":"Exact match; 5-shot","score":67.7,"unit":"%","rank":63,"rankComputedBySite":true,"date":"","sourceType":"official","source":"epoch","sourceUrl":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","collected":"2026-10-10"},{"model":"gpt-3-5-turbo-jan-2024","modelName":"GPT-3.5 Turbo (Jan 2024)","provider":"OpenAI","rawName":"GPT-3.5 Turbo (Jan 2024)","variant":"leaderboard","setting":"Exact match; 5-shot; effort: Jan 2024","score":67.3,"unit":"%","rank":64,"rankComputedBySite":true,"date":"2024-01-25","sourceType":"official","source":"epoch","sourceUrl":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","collected":"2026-10-10"},{"model":"gemma-7b","modelName":"Gemma 7B","provider":"Google","rawName":"Gemma 7B","variant":"leaderboard","setting":"Exact match; 5-shot","score":66.1,"unit":"%","rank":65,"rankComputedBySite":true,"date":"2024-02-21","sourceType":"official","source":"epoch","sourceUrl":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","collected":"2026-10-10"},{"model":"jamba-instruct","modelName":"Jamba Instruct","provider":"AI21 Labs","rawName":"Jamba Instruct","variant":"leaderboard","setting":"Exact match; 5-shot","score":65.9,"unit":"%","rank":66,"rankComputedBySite":true,"date":"","sourceType":"official","source":"epoch","sourceUrl":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","collected":"2026-10-10"},{"model":"mistral-nemo-2402","modelName":"Mistral NeMo (2402)","provider":"Mistral AI","rawName":"Mistral NeMo (2402)","variant":"leaderboard","setting":"Exact match; 5-shot","score":65.3,"unit":"%","rank":67,"rankComputedBySite":true,"date":"","sourceType":"official","source":"epoch","sourceUrl":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","collected":"2026-10-10"},{"model":"c4ai-command-r-08-2024","modelName":"c4ai-command-r-08-2024","provider":"Cohere","rawName":"c4ai-command-r-08-2024","variant":"leaderboard","setting":"Exact match; 5-shot","score":65.2,"unit":"%","rank":68,"rankComputedBySite":true,"date":"2024-08-30","sourceType":"official","source":"epoch","sourceUrl":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","collected":"2026-10-10"},{"model":"yi-6b","modelName":"Yi 6B","provider":"01.AI","rawName":"Yi 6B","variant":"leaderboard","setting":"Exact match; 5-shot","score":64,"unit":"%","rank":69,"rankComputedBySite":true,"date":"2023-11-02","sourceType":"official","source":"epoch","sourceUrl":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","collected":"2026-10-10"},{"model":"qwen1-5-7b","modelName":"Qwen1.5-7B","provider":"Alibaba (Qwen)","rawName":"Qwen1.5-7B","variant":"leaderboard","setting":"Exact match; 5-shot","score":62.6,"unit":"%","rank":70,"rankComputedBySite":true,"date":"2024-02-04","sourceType":"official","source":"epoch","sourceUrl":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","collected":"2026-10-10"},{"model":"palmyra-x-v2-33b","modelName":"Palmyra X V2 (33B)","provider":"Google","rawName":"Palmyra X V2 (33B)","variant":"leaderboard","setting":"Exact match; 5-shot","score":62.1,"unit":"%","rank":71,"rankComputedBySite":true,"date":"","sourceType":"official","source":"epoch","sourceUrl":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","collected":"2026-10-10"},{"model":"mistral-7b-v0-3","modelName":"Mistral 7B v0.3","provider":"Mistral AI","rawName":"Mistral 7B v0.3","variant":"leaderboard","setting":"Exact match; 5-shot","score":59.9,"unit":"%","rank":72,"rankComputedBySite":true,"date":"2024-05-27","sourceType":"official","source":"epoch","sourceUrl":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","collected":"2026-10-10"},{"model":"phi-2","modelName":"Phi-2","provider":"Microsoft","rawName":"Phi-2","variant":"leaderboard","setting":"Exact match; 5-shot","score":58.4,"unit":"%","rank":73,"rankComputedBySite":true,"date":"2023-12-12","sourceType":"official","source":"epoch","sourceUrl":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","collected":"2026-10-10"},{"model":"mistral-7b-v0-1","modelName":"Mistral 7B v0.1","provider":"Mistral AI","rawName":"Mistral 7B v0.1","variant":"leaderboard","setting":"Exact match; 5-shot","score":56.6,"unit":"%","rank":74,"rankComputedBySite":true,"date":"2023-09-27","sourceType":"official","source":"epoch","sourceUrl":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","collected":"2026-10-10"},{"model":"llama-3-2-11b","modelName":"Llama 3.2 11B","provider":"Meta","rawName":"Llama 3.2 11B","variant":"leaderboard","setting":"Exact match; 5-shot","score":56.5,"unit":"%","rank":75,"rankComputedBySite":true,"date":"2024-09-24","sourceType":"official","source":"epoch","sourceUrl":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","collected":"2026-10-10"},{"model":"llama-3-1-8b","modelName":"Llama 3.1-8B","provider":"Meta","rawName":"Llama 3.1-8B","variant":"leaderboard","setting":"Exact match; 5-shot","score":56.1,"unit":"%","rank":76,"rankComputedBySite":true,"date":"2024-07-23","sourceType":"official","source":"epoch","sourceUrl":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","collected":"2026-10-10"},{"model":"llama-2-13b","modelName":"Llama 2-13B","provider":"Meta","rawName":"Llama 2-13B","variant":"leaderboard","setting":"Exact match; 5-shot","score":55.4,"unit":"%","rank":77,"rankComputedBySite":true,"date":"2023-07-18","sourceType":"official","source":"epoch","sourceUrl":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","collected":"2026-10-10"},{"model":"olmo-1-7-7b","modelName":"OLMo 1.7 (7B)","provider":"Ai2","rawName":"OLMo 1.7 (7B)","variant":"leaderboard","setting":"Exact match; 5-shot","score":53.8,"unit":"%","rank":78,"rankComputedBySite":true,"date":"","sourceType":"official","source":"epoch","sourceUrl":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","collected":"2026-10-10"},{"model":"llama-2-7b","modelName":"Llama 2-7B","provider":"Meta","rawName":"Llama 2-7B","variant":"leaderboard","setting":"Exact match; 5-shot","score":45.8,"unit":"%","rank":79,"rankComputedBySite":true,"date":"2023-07-18","sourceType":"official","source":"epoch","sourceUrl":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","collected":"2026-10-10"},{"model":"olmo-7b","modelName":"OLMo (7B)","provider":"Ai2","rawName":"OLMo (7B)","variant":"leaderboard","setting":"Exact match; 5-shot","score":29.5,"unit":"%","rank":80,"rankComputedBySite":true,"date":"","sourceType":"official","source":"epoch","sourceUrl":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","collected":"2026-10-10"}]}