{"source":"AIMap","schema":"benchmark/v1","updated":"2026-10-22","benchmark":{"id":"swe-bench-verified","name":"SWE-bench Verified","category":"agent-coding","unit":"%","higherIsBetter":true,"status":"contested","sources":["swebench","epoch"],"primaryVariant":null,"url":"https://www.swebench.com/","year":2024,"desc":{"en":"500 human-validated GitHub issues from 12 Python repositories; the system must produce a patch that makes the hidden tests pass.","zh-hans":"来自 12 个 Python 仓库、经人工核验的 500 个 GitHub issue，系统需生成补丁让隐藏测试通过。"},"measures":{"en":"% of issues resolved by a model + agent scaffold. Rows from swebench.com are agent systems; rows from Epoch AI use Epoch's own scaffold.","zh-hans":"模型加 agent 框架解决 issue 的比例。swebench.com 的条目是各家 agent 系统；Epoch AI 的条目用其统一框架。"},"limits":{"en":"In February 2026 OpenAI stopped reporting SWE-bench Verified, saying it is contaminated and that 59.4% of 138 audited hard tasks had flawed tests or descriptions. Scores depend heavily on the agent scaffold.","zh-hans":"2026 年 2 月 OpenAI 宣布不再报告 SWE-bench Verified，理由是数据污染，且其审查的 138 道难题中 59.4% 的测试或描述有问题。分数也高度依赖 agent 框架。"},"contamination":"high","contaminationNote":{"en":"OpenAI found frontier models could reproduce gold patches from memory.","zh-hans":"OpenAI 发现前沿模型能凭记忆复现标准补丁。"},"stats":{"results":142,"models":81,"latest":"2026-06-25","bySourceType":{"independent":33,"official":109}}},"sources":[{"id":"swebench","name":"SWE-bench leaderboards (swebench.com)","license":"No explicit licence (NOASSERTION) – factual scores only, with citation","policy":"facts","attribution":"SWE-bench team, swebench.com leaderboards","site":"https://www.swebench.com/"},{"id":"epoch","name":"Epoch AI Benchmarking Hub","license":"CC BY 4.0","policy":"open","attribution":"Epoch AI, 'Capabilities & benchmarking', https://epoch.ai/benchmarks (CC BY 4.0)","site":"https://epoch.ai/benchmarks"}],"results":[{"model":"claude-opus-4-7","modelName":"Claude Opus 4.7","provider":"Anthropic","rawName":"Claude Opus 4.7","variant":"epoch","setting":"Epoch AI run (Epoch scaffold); effort: max","score":83.47,"unit":"%","rank":1,"rankComputedBySite":true,"date":"2026-04-20","sourceType":"independent","source":"epoch","sourceUrl":"https://epoch.ai/benchmarks","collected":"2026-10-10"},{"model":"gpt-5-5","modelName":"GPT-5.5","provider":"OpenAI","rawName":"GPT-5.5","variant":"epoch","setting":"Epoch AI run (Epoch scaffold); effort: xhigh","score":80.58,"unit":"%","rank":2,"rankComputedBySite":true,"date":"2026-04-24","sourceType":"independent","source":"epoch","sourceUrl":"https://epoch.ai/benchmarks","collected":"2026-10-10"},{"model":"gemini-3-5-flash","modelName":"Gemini 3.5 Flash","provider":"Google","rawName":"Gemini 3.5 Flash","variant":"epoch","setting":"Epoch AI run (Epoch scaffold)","score":79.34,"unit":"%","rank":3,"rankComputedBySite":true,"date":"2026-06-01","sourceType":"independent","source":"epoch","sourceUrl":"https://epoch.ai/benchmarks","collected":"2026-10-10"},{"model":"claude-opus-4-6","modelName":"Claude Opus 4.6","provider":"Anthropic","rawName":"Claude Opus 4.6","variant":"epoch","setting":"Epoch AI run (Epoch scaffold); effort: no thinking","score":78.72,"unit":"%","rank":4,"rankComputedBySite":true,"date":"2026-02-18","sourceType":"independent","source":"epoch","sourceUrl":"https://epoch.ai/benchmarks","collected":"2026-10-10"},{"model":"glm-5-2","modelName":"GLM-5.2","provider":"Zhipu AI (Z.ai)","rawName":"GLM-5.2","variant":"epoch","setting":"Epoch AI run (Epoch scaffold)","score":78.7,"unit":"%","rank":5,"rankComputedBySite":true,"date":"2026-06-25","sourceType":"independent","source":"epoch","sourceUrl":"https://epoch.ai/benchmarks","collected":"2026-10-10"},{"model":"deepseek-v4-pro","modelName":"DeepSeek-V4-Pro","provider":"DeepSeek","rawName":"DeepSeek-V4-Pro","variant":"epoch","setting":"Epoch AI run (Epoch scaffold); effort: max","score":77.64,"unit":"%","rank":6,"rankComputedBySite":true,"date":"2026-06-18","sourceType":"independent","source":"epoch","sourceUrl":"https://epoch.ai/benchmarks","collected":"2026-10-10"},{"model":"qwen3-7-max","modelName":"Qwen3.7-Max","provider":"Alibaba (Qwen)","rawName":"Qwen3.7-Max","variant":"epoch","setting":"Epoch AI run (Epoch scaffold)","score":77.27,"unit":"%","rank":7,"rankComputedBySite":true,"date":"2026-06-18","sourceType":"independent","source":"epoch","sourceUrl":"https://epoch.ai/benchmarks","collected":"2026-10-10"},{"model":"gpt-5-4","modelName":"GPT-5.4","provider":"OpenAI","rawName":"GPT-5.4","variant":"epoch","setting":"Epoch AI run (Epoch scaffold); effort: high","score":76.86,"unit":"%","rank":8,"rankComputedBySite":true,"date":"2026-03-06","sourceType":"independent","source":"epoch","sourceUrl":"https://epoch.ai/benchmarks","collected":"2026-10-10"},{"model":"qwen-3-6-max-preview","modelName":"Qwen 3.6 Max (Preview)","provider":"Alibaba (Qwen)","rawName":"Qwen 3.6 Max (Preview)","variant":"epoch","setting":"Epoch AI run (Epoch scaffold); effort: Preview","score":76.65,"unit":"%","rank":9,"rankComputedBySite":true,"date":"2026-05-28","sourceType":"independent","source":"epoch","sourceUrl":"https://epoch.ai/benchmarks","collected":"2026-10-10"},{"model":"kimi-k2-6","modelName":"Kimi K2.6","provider":"Moonshot AI","rawName":"Kimi K2.6","variant":"epoch","setting":"Epoch AI run (Epoch scaffold)","score":76.65,"unit":"%","rank":9,"rankComputedBySite":true,"date":"2026-05-08","sourceType":"independent","source":"epoch","sourceUrl":"https://epoch.ai/benchmarks","collected":"2026-10-10"},{"model":"claude-opus-4-5","modelName":"Claude Opus 4.5","provider":"Anthropic","rawName":"Claude Opus 4.5","variant":"epoch","setting":"Epoch AI run (Epoch scaffold); effort: no thinking","score":76.65,"unit":"%","rank":9,"rankComputedBySite":true,"date":"2026-02-05","sourceType":"independent","source":"epoch","sourceUrl":"https://epoch.ai/benchmarks","collected":"2026-10-10"},{"model":"gemini-3-1-pro","modelName":"Gemini 3.1 Pro","provider":"Google","rawName":"Gemini 3.1 Pro","variant":"epoch","setting":"Epoch AI run (Epoch scaffold)","score":75.62,"unit":"%","rank":12,"rankComputedBySite":true,"date":"2026-02-24","sourceType":"independent","source":"epoch","sourceUrl":"https://epoch.ai/benchmarks","collected":"2026-10-10"},{"model":"gemini-3-flash","modelName":"Gemini 3 Flash","provider":"Google","rawName":"Gemini 3 Flash","variant":"epoch","setting":"Epoch AI run (Epoch scaffold)","score":75.41,"unit":"%","rank":13,"rankComputedBySite":true,"date":"2026-02-18","sourceType":"independent","source":"epoch","sourceUrl":"https://epoch.ai/benchmarks","collected":"2026-10-10"},{"model":"claude-sonnet-4-6","modelName":"Claude Sonnet 4.6","provider":"Anthropic","rawName":"Claude Sonnet 4.6","variant":"epoch","setting":"Epoch AI run (Epoch scaffold); effort: no thinking","score":75.21,"unit":"%","rank":14,"rankComputedBySite":true,"date":"2026-02-21","sourceType":"independent","source":"epoch","sourceUrl":"https://epoch.ai/benchmarks","collected":"2026-10-10"},{"model":"gpt-5-3-codex","modelName":"GPT-5.3 Codex","provider":"OpenAI","rawName":"GPT-5.3 Codex","variant":"epoch","setting":"Epoch AI run (Epoch scaffold); effort: high","score":74.79,"unit":"%","rank":15,"rankComputedBySite":true,"date":"2026-02-25","sourceType":"independent","source":"epoch","sourceUrl":"https://epoch.ai/benchmarks","collected":"2026-10-10"},{"model":"glm-5-1","modelName":"GLM-5.1","provider":"Zhipu AI (Z.ai)","rawName":"GLM-5.1","variant":"epoch","setting":"Epoch AI run (Epoch scaffold)","score":74.17,"unit":"%","rank":16,"rankComputedBySite":true,"date":"2026-05-15","sourceType":"independent","source":"epoch","sourceUrl":"https://epoch.ai/benchmarks","collected":"2026-10-10"},{"model":"kimi-k2-5","modelName":"Kimi K2.5","provider":"Moonshot AI","rawName":"Kimi K2.5","variant":"epoch","setting":"Epoch AI run (Epoch scaffold)","score":73.76,"unit":"%","rank":17,"rankComputedBySite":true,"date":"2026-02-17","sourceType":"independent","source":"epoch","sourceUrl":"https://epoch.ai/benchmarks","collected":"2026-10-10"},{"model":"gpt-5-2","modelName":"GPT-5.2","provider":"OpenAI","rawName":"GPT-5.2","variant":"epoch","setting":"Epoch AI run (Epoch scaffold); effort: high","score":73.76,"unit":"%","rank":17,"rankComputedBySite":true,"date":"2026-02-12","sourceType":"independent","source":"epoch","sourceUrl":"https://epoch.ai/benchmarks","collected":"2026-10-10"},{"model":"gpt-5","modelName":"GPT-5","provider":"OpenAI","rawName":"GPT-5","variant":"epoch","setting":"Epoch AI run (Epoch scaffold); effort: high","score":73.55,"unit":"%","rank":19,"rankComputedBySite":true,"date":"2026-02-06","sourceType":"independent","source":"epoch","sourceUrl":"https://epoch.ai/benchmarks","collected":"2026-10-10"},{"model":"claude-opus-4-1","modelName":"Claude Opus 4.1","provider":"Anthropic","rawName":"Claude Opus 4.1","variant":"epoch","setting":"Epoch AI run (Epoch scaffold)","score":73.35,"unit":"%","rank":20,"rankComputedBySite":true,"date":"2026-02-11","sourceType":"independent","source":"epoch","sourceUrl":"https://epoch.ai/benchmarks","collected":"2026-10-10"},{"model":"gemini-3-pro","modelName":"Gemini 3 Pro","provider":"Google","rawName":"Gemini 3 Pro","variant":"epoch","setting":"Epoch AI run (Epoch scaffold)","score":72.93,"unit":"%","rank":21,"rankComputedBySite":true,"date":"2026-02-13","sourceType":"independent","source":"epoch","sourceUrl":"https://epoch.ai/benchmarks","collected":"2026-10-10"},{"model":"glm-5","modelName":"GLM-5","provider":"Zhipu AI (Z.ai)","rawName":"GLM-5","variant":"epoch","setting":"Epoch AI run (Epoch scaffold)","score":72.08,"unit":"%","rank":22,"rankComputedBySite":true,"date":"2026-02-15","sourceType":"independent","source":"epoch","sourceUrl":"https://epoch.ai/benchmarks","collected":"2026-10-10"},{"model":"gpt-5","modelName":"GPT-5","provider":"OpenAI","rawName":"GPT-5","variant":"epoch","setting":"Epoch AI run (Epoch scaffold); effort: medium","score":71.49,"unit":"%","rank":23,"rankComputedBySite":true,"date":"2026-02-05","sourceType":"independent","source":"epoch","sourceUrl":"https://epoch.ai/benchmarks","collected":"2026-10-10"},{"model":"claude-sonnet-4-5","modelName":"Claude Sonnet 4.5","provider":"Anthropic","rawName":"Claude Sonnet 4.5","variant":"epoch","setting":"Epoch AI run (Epoch scaffold); effort: no thinking","score":71.28,"unit":"%","rank":24,"rankComputedBySite":true,"date":"2026-02-05","sourceType":"independent","source":"epoch","sourceUrl":"https://epoch.ai/benchmarks","collected":"2026-10-10"},{"model":"claude-opus-4","modelName":"Claude Opus 4","provider":"Anthropic","rawName":"Claude Opus 4","variant":"epoch","setting":"Epoch AI run (Epoch scaffold)","score":70.66,"unit":"%","rank":25,"rankComputedBySite":true,"date":"2026-02-06","sourceType":"independent","source":"epoch","sourceUrl":"https://epoch.ai/benchmarks","collected":"2026-10-10"},{"model":"gpt-5-1","modelName":"GPT-5.1","provider":"OpenAI","rawName":"GPT-5.1","variant":"epoch","setting":"Epoch AI run (Epoch scaffold); effort: high","score":67.98,"unit":"%","rank":26,"rankComputedBySite":true,"date":"2026-02-18","sourceType":"independent","source":"epoch","sourceUrl":"https://epoch.ai/benchmarks","collected":"2026-10-10"},{"model":"gpt-5-mini","modelName":"GPT-5 mini","provider":"OpenAI","rawName":"GPT-5 mini","variant":"epoch","setting":"Epoch AI run (Epoch scaffold); effort: medium","score":64.67,"unit":"%","rank":27,"rankComputedBySite":true,"date":"2026-02-01","sourceType":"independent","source":"epoch","sourceUrl":"https://epoch.ai/benchmarks","collected":"2026-10-10"},{"model":"o3","modelName":"o3","provider":"OpenAI","rawName":"o3","variant":"epoch","setting":"Epoch AI run (Epoch scaffold); effort: medium","score":62.32,"unit":"%","rank":28,"rankComputedBySite":true,"date":"2026-02-12","sourceType":"independent","source":"epoch","sourceUrl":"https://epoch.ai/benchmarks","collected":"2026-10-10"},{"model":"claude-3-7-sonnet","modelName":"Claude 3.7 Sonnet","provider":"Anthropic","rawName":"Claude 3.7 Sonnet","variant":"epoch","setting":"Epoch AI run (Epoch scaffold)","score":60.95,"unit":"%","rank":29,"rankComputedBySite":true,"date":"2026-02-04","sourceType":"independent","source":"epoch","sourceUrl":"https://epoch.ai/benchmarks","collected":"2026-10-10"},{"model":"qwen-3-6-plus","modelName":"Qwen 3.6 Plus","provider":"Alibaba (Qwen)","rawName":"Qwen 3.6 Plus","variant":"epoch","setting":"Epoch AI run (Epoch scaffold); effort: 2026-04-02","score":57.85,"unit":"%","rank":30,"rankComputedBySite":true,"date":"2026-05-14","sourceType":"independent","source":"epoch","sourceUrl":"https://epoch.ai/benchmarks","collected":"2026-10-10"},{"model":"gemini-2-5-pro-jun-2025","modelName":"Gemini 2.5 Pro (Jun 2025)","provider":"Google","rawName":"Gemini 2.5 Pro (Jun 2025)","variant":"epoch","setting":"Epoch AI run (Epoch scaffold)","score":57.56,"unit":"%","rank":31,"rankComputedBySite":true,"date":"2026-02-13","sourceType":"independent","source":"epoch","sourceUrl":"https://epoch.ai/benchmarks","collected":"2026-10-10"},{"model":"gpt-4-1","modelName":"GPT-4.1","provider":"OpenAI","rawName":"GPT-4.1","variant":"epoch","setting":"Epoch AI run (Epoch scaffold)","score":48.54,"unit":"%","rank":32,"rankComputedBySite":true,"date":"2026-02-08","sourceType":"independent","source":"epoch","sourceUrl":"https://epoch.ai/benchmarks","collected":"2026-10-10"},{"model":"gpt-4o-nov-2024","modelName":"GPT-4o (Nov 2024)","provider":"OpenAI","rawName":"GPT-4o (Nov 2024)","variant":"epoch","setting":"Epoch AI run (Epoch scaffold); effort: Nov 2024","score":30.99,"unit":"%","rank":33,"rankComputedBySite":true,"date":"2026-02-11","sourceType":"independent","source":"epoch","sourceUrl":"https://epoch.ai/benchmarks","collected":"2026-10-10"},{"model":"claude-opus-4-5","modelName":"Claude Opus 4.5","provider":"Anthropic","rawName":"claude-opus-4-5","variant":"leaderboard","setting":"agent: Sonar Foundation Agent","score":79.2,"unit":"%","rank":1,"rankComputedBySite":true,"date":"2025-12-05","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"claude-opus-4-5","modelName":"Claude Opus 4.5","provider":"Anthropic","rawName":"claude-opus-4-5-20251101","variant":"leaderboard","setting":"agent: live-SWE-agent","score":79.2,"unit":"%","rank":1,"rankComputedBySite":true,"date":"2025-12-15","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"gemini-3-pro","modelName":"Gemini 3 Pro","provider":"Google","rawName":"gemini-3-pro-preview","variant":"leaderboard","setting":"agent: live-SWE-agent","score":77.4,"unit":"%","rank":3,"rankComputedBySite":true,"date":"2025-11-20","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"claude-sonnet-4","modelName":"Claude Sonnet 4","provider":"Anthropic","rawName":"claude-sonnet-4-20250514","variant":"leaderboard","setting":"agent: EPAM AI/Run Developer Agent","score":76.8,"unit":"%","rank":4,"rankComputedBySite":true,"date":"2025-08-04","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"claude-4-5-opus","modelName":"claude-4-5-opus","provider":"Anthropic","rawName":"claude-4-5-opus","variant":"leaderboard","setting":"agent: mini-SWE-agent","score":76.8,"unit":"%","rank":4,"rankComputedBySite":true,"date":"2026-02-17","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"gemini-3-flash","modelName":"Gemini 3 Flash","provider":"Google","rawName":"gemini-3-flash-preview","variant":"leaderboard","setting":"agent: mini-SWE-agent","score":75.8,"unit":"%","rank":6,"rankComputedBySite":true,"date":"2026-02-17","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"minimax-m2-5","modelName":"MiniMax-M2.5","provider":"MiniMax","rawName":"minimax-m2.5","variant":"leaderboard","setting":"agent: mini-SWE-agent","score":75.8,"unit":"%","rank":6,"rankComputedBySite":true,"date":"2026-02-17","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"claude-opus-4-6","modelName":"Claude Opus 4.6","provider":"Anthropic","rawName":"claude-opus-4-6","variant":"leaderboard","setting":"agent: mini-SWE-agent","score":75.6,"unit":"%","rank":8,"rankComputedBySite":true,"date":"2026-02-17","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"claude-sonnet-4","modelName":"Claude Sonnet 4","provider":"Anthropic","rawName":"claude-sonnet-4-20250514","variant":"leaderboard","setting":"agent: Harness AI","score":74.8,"unit":"%","rank":9,"rankComputedBySite":true,"date":"2025-07-31","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"claude-sonnet-4-5","modelName":"Claude Sonnet 4.5","provider":"Anthropic","rawName":"claude-sonnet-4-5","variant":"leaderboard","setting":"agent: Sonar Foundation Agent","score":74.8,"unit":"%","rank":9,"rankComputedBySite":true,"date":"2025-11-03","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"claude-4-sonnet","modelName":"claude-4-sonnet","provider":"Anthropic","rawName":"claude-4-sonnet-20250514","variant":"leaderboard","setting":"agent: Lingxi-v1.5","score":74.6,"unit":"%","rank":11,"rankComputedBySite":true,"date":"2025-07-20","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"gpt-5","modelName":"GPT-5","provider":"OpenAI","rawName":"gpt-5-2025-08-07","variant":"leaderboard","setting":"agent: Prometheus-v1.2.1","score":74.4,"unit":"%","rank":12,"rankComputedBySite":true,"date":"2025-10-15","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"claude-opus-4-5","modelName":"Claude Opus 4.5","provider":"Anthropic","rawName":"claude-opus-4-5-20251101","variant":"leaderboard","setting":"agent: mini-SWE-agent","score":74.4,"unit":"%","rank":12,"rankComputedBySite":true,"date":"2025-11-24","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"gemini-3-pro","modelName":"Gemini 3 Pro","provider":"Google","rawName":"gemini-3-pro-preview","variant":"leaderboard","setting":"agent: mini-SWE-agent","score":74.2,"unit":"%","rank":14,"rankComputedBySite":true,"date":"2025-11-18","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"claude-4-opus","modelName":"claude-4-opus","provider":"Anthropic","rawName":"claude-4-opus-20250514","variant":"leaderboard","setting":"agent: Tools","score":73.2,"unit":"%","rank":15,"rankComputedBySite":true,"date":"2025-05-22","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"glm-5","modelName":"GLM-5","provider":"Zhipu AI (Z.ai)","rawName":"glm-5","variant":"leaderboard","setting":"agent: mini-SWE-agent","score":72.8,"unit":"%","rank":16,"rankComputedBySite":true,"date":"2026-02-17","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"gpt-5-2","modelName":"GPT-5.2","provider":"OpenAI","rawName":"gpt-5-2","variant":"leaderboard","setting":"agent: mini-SWE-agent","score":72.8,"unit":"%","rank":16,"rankComputedBySite":true,"date":"2026-02-17","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"gpt-5-2-codex","modelName":"GPT-5.2 Codex","provider":"OpenAI","rawName":"gpt-5-2-codex","variant":"leaderboard","setting":"agent: mini-SWE-agent","score":72.8,"unit":"%","rank":16,"rankComputedBySite":true,"date":"2026-02-19","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"claude-4-sonnet","modelName":"claude-4-sonnet","provider":"Anthropic","rawName":"claude-4-sonnet-20250514","variant":"leaderboard","setting":"agent: Tools","score":72.4,"unit":"%","rank":19,"rankComputedBySite":true,"date":"2025-05-22","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"gpt-5","modelName":"GPT-5","provider":"OpenAI","rawName":"openai/gpt-5-2025-08-07","variant":"leaderboard","setting":"agent: OpenHands","score":71.8,"unit":"%","rank":20,"rankComputedBySite":true,"date":"2025-08-07","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"claude-sonnet-4-5","modelName":"Claude Sonnet 4.5","provider":"Anthropic","rawName":"claude-sonnet-4-5-20250929","variant":"leaderboard","setting":"agent: mini-SWE-agent","score":71.4,"unit":"%","rank":21,"rankComputedBySite":true,"date":"2026-02-17","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"claude-4-sonnet","modelName":"claude-4-sonnet","provider":"Anthropic","rawName":"claude-4-sonnet-20250514","variant":"leaderboard","setting":"agent: Bloop","score":71.2,"unit":"%","rank":22,"rankComputedBySite":true,"date":"2025-07-10","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"gpt-5","modelName":"GPT-5","provider":"OpenAI","rawName":"gpt-5-2025-08-07","variant":"leaderboard","setting":"agent: Prometheus-v1.2","score":71.2,"unit":"%","rank":22,"rankComputedBySite":true,"date":"2025-09-29","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"kimi-k2","modelName":"Kimi K2 Thinking","provider":"Moonshot AI","rawName":"kimi-k2-0905-preview","variant":"leaderboard","setting":"agent: Lingxi v1.5","score":71.2,"unit":"%","rank":22,"rankComputedBySite":true,"date":"2025-10-14","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"claude-4-sonnet","modelName":"claude-4-sonnet","provider":"Anthropic","rawName":"claude-4-sonnet-20250514","variant":"leaderboard","setting":"agent: Moatless Tools","score":70.8,"unit":"%","rank":25,"rankComputedBySite":true,"date":"2025-06-11","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"kimi-k2-5","modelName":"Kimi K2.5","provider":"Moonshot AI","rawName":"kimi-k2.5","variant":"leaderboard","setting":"agent: mini-SWE-agent","score":70.8,"unit":"%","rank":25,"rankComputedBySite":true,"date":"2026-02-17","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"claude-4-sonnet","modelName":"claude-4-sonnet","provider":"Anthropic","rawName":"claude-4-sonnet-20250514","variant":"leaderboard","setting":"agent: OpenHands","score":70.4,"unit":"%","rank":27,"rankComputedBySite":true,"date":"2025-05-24","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"deepseek-v3-2","modelName":"DeepSeek-V3.2","provider":"DeepSeek","rawName":"deepseek-v3.2","variant":"leaderboard","setting":"agent: mini-SWE-agent","score":70,"unit":"%","rank":28,"rankComputedBySite":true,"date":"2026-02-17","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"qwen3-coder-480b-a35b","modelName":"Qwen3-Coder-480B-A35B","provider":"Alibaba (Qwen)","rawName":"https://huggingface.co/Qwen/Qwen3-Coder-480B-A35B-Instruct","variant":"leaderboard","setting":"agent: OpenHands","score":69.6,"unit":"%","rank":29,"rankComputedBySite":true,"date":"2025-08-05","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"glm-4-6","modelName":"GLM-4.6","provider":"Zhipu AI (Z.ai)","rawName":"https://huggingface.co/zai-org/GLM-4.6","variant":"leaderboard","setting":"agent: Undisclosed","score":68.2,"unit":"%","rank":30,"rankComputedBySite":true,"date":"2025-09-30","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"claude-4-opus","modelName":"claude-4-opus","provider":"Anthropic","rawName":"claude-4-opus-20250514","variant":"leaderboard","setting":"agent: mini-SWE-agent","score":67.6,"unit":"%","rank":31,"rankComputedBySite":true,"date":"2025-08-02","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"claude-4-sonnet","modelName":"claude-4-sonnet","provider":"Anthropic","rawName":"claude-4-sonnet-20250514","variant":"leaderboard","setting":"agent: SWE-agent","score":66.6,"unit":"%","rank":32,"rankComputedBySite":true,"date":"2025-05-22","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"claude-haiku-4-5","modelName":"Claude Haiku 4.5","provider":"Anthropic","rawName":"claude-haiku-4-5-20251001","variant":"leaderboard","setting":"agent: mini-SWE-agent","score":66.6,"unit":"%","rank":32,"rankComputedBySite":true,"date":"2026-02-17","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"claude-3-7-sonnet","modelName":"Claude 3.7 Sonnet","provider":"Anthropic","rawName":"claude-3-7-sonnet-20250219","variant":"leaderboard","setting":"agent: Aime-coder v1","score":66.4,"unit":"%","rank":34,"rankComputedBySite":true,"date":"2025-05-14","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"gpt-5-1","modelName":"GPT-5.1","provider":"OpenAI","rawName":"gpt-5.1-2025-11-13","variant":"leaderboard","setting":"agent: mini-SWE-agent","score":66,"unit":"%","rank":35,"rankComputedBySite":true,"date":"2025-11-20","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"gpt-5-1-codex","modelName":"GPT-5.1-Codex","provider":"OpenAI","rawName":"gpt-5.1-codex","variant":"leaderboard","setting":"agent: mini-SWE-agent","score":66,"unit":"%","rank":35,"rankComputedBySite":true,"date":"2025-11-24","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"kimi-k2-jul-2025","modelName":"Kimi K2 (Jul 2025)","provider":"Moonshot AI","rawName":"moonshot/kimi-k2-0711-preview","variant":"leaderboard","setting":"agent: OpenHands","score":65.4,"unit":"%","rank":37,"rankComputedBySite":true,"date":"2025-07-16","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"gpt-5","modelName":"GPT-5","provider":"OpenAI","rawName":"gpt-5-2025-08-07","variant":"leaderboard","setting":"agent: mini-SWE-agent","score":65,"unit":"%","rank":38,"rankComputedBySite":true,"date":"2025-08-07","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"claude-4-sonnet","modelName":"claude-4-sonnet","provider":"Anthropic","rawName":"claude-4-sonnet-20250514","variant":"leaderboard","setting":"agent: mini-SWE-agent","score":64.93,"unit":"%","rank":39,"rankComputedBySite":true,"date":"2025-07-26","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"o1-preview","modelName":"o1-preview","provider":"OpenAI","rawName":"o1-preview","variant":"leaderboard","setting":"agent: W&B Programmer O1 crosscheck5","score":64.6,"unit":"%","rank":40,"rankComputedBySite":true,"date":"2025-01-17","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"o4-mini","modelName":"o4-mini","provider":"OpenAI","rawName":"o4-mini","variant":"leaderboard","setting":"agent: PatchPilot-v1.1","score":64.6,"unit":"%","rank":40,"rankComputedBySite":true,"date":"2025-05-03","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"glm-4-5","modelName":"GLM-4.5","provider":"Zhipu AI (Z.ai)","rawName":"https://huggingface.co/zai-org/GLM-4.5","variant":"leaderboard","setting":"agent: Undisclosed","score":64.2,"unit":"%","rank":42,"rankComputedBySite":true,"date":"2025-07-28","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"claude-3-5-sonnet","modelName":"Claude 3.5 Sonnet","provider":"Anthropic","rawName":"claude-3-5-sonnet-20241022","variant":"leaderboard","setting":"agent: AgentScope","score":63.4,"unit":"%","rank":43,"rankComputedBySite":true,"date":"2025-02-06","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"kimi-k2","modelName":"Kimi K2 Thinking","provider":"Moonshot AI","rawName":"Kimi-K2-Thinking","variant":"leaderboard","setting":"agent: mini-SWE-agent","score":63.4,"unit":"%","rank":43,"rankComputedBySite":true,"date":"2025-12-10","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"claude-3-5-sonnet","modelName":"Claude 3.5 Sonnet","provider":"Anthropic","rawName":"claude-3-5-sonnet-20241022","variant":"leaderboard","setting":"agent: EPAM AI/Run Developer Agent","score":62.8,"unit":"%","rank":45,"rankComputedBySite":true,"date":"2025-02-28","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"claude-3-7-sonnet","modelName":"Claude 3.7 Sonnet","provider":"Anthropic","rawName":"claude-3-7-sonnet-20250219","variant":"leaderboard","setting":"agent: SWE-agent","score":62.4,"unit":"%","rank":46,"rankComputedBySite":true,"date":"2025-02-25","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"claude-3-5-sonnet","modelName":"Claude 3.5 Sonnet","provider":"Anthropic","rawName":"claude-3-5-sonnet-20241022","variant":"leaderboard","setting":"agent: CodeStory Midwit Agent","score":62.2,"unit":"%","rank":47,"rankComputedBySite":true,"date":"2024-12-21","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"minimax-m2","modelName":"MiniMax-M2","provider":"MiniMax","rawName":"minimax-m2","variant":"leaderboard","setting":"agent: mini-SWE-agent","score":61,"unit":"%","rank":48,"rankComputedBySite":true,"date":"2025-11-24","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"qwen3-coder-30b-a3b-instruct","modelName":"Qwen3-Coder-30B-A3B-Instruct","provider":"Alibaba (Qwen)","rawName":"Qwen3-Coder-30B-A3B-Instruct","variant":"leaderboard","setting":"agent: EntroPO + R2E","score":60.4,"unit":"%","rank":49,"rankComputedBySite":true,"date":"2025-09-01","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"claude-3-5-sonnet","modelName":"Claude 3.5 Sonnet","provider":"Anthropic","rawName":"claude-3-5-sonnet-20241022","variant":"leaderboard","setting":"agent: Learn-by-interact","score":60.2,"unit":"%","rank":50,"rankComputedBySite":true,"date":"2025-01-10","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"deepseek-v3-2-reasoner","modelName":"deepseek-v3.2-reasoner","provider":"DeepSeek","rawName":"deepseek-v3.2-reasoner","variant":"leaderboard","setting":"agent: mini-SWE-agent","score":60,"unit":"%","rank":51,"rankComputedBySite":true,"date":"2025-12-01","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"gpt-5-mini","modelName":"GPT-5 mini","provider":"OpenAI","rawName":"gpt-5-mini-2025-08-07","variant":"leaderboard","setting":"agent: mini-SWE-agent","score":59.8,"unit":"%","rank":52,"rankComputedBySite":true,"date":"2025-08-07","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"https-huggingface-co-agentica-org-deepswe-preview","modelName":"/huggingface.co/agentica-org/DeepSWE-Preview","provider":"OpenAI","rawName":"https://huggingface.co/agentica-org/DeepSWE-Preview","variant":"leaderboard","setting":"agent: DeepSWE-Preview","score":58.8,"unit":"%","rank":53,"rankComputedBySite":true,"date":"2025-06-29","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"o3","modelName":"o3","provider":"OpenAI","rawName":"o3-20250416","variant":"leaderboard","setting":"agent: mini-SWE-agent","score":58.4,"unit":"%","rank":54,"rankComputedBySite":true,"date":"2025-07-26","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"claude-sonnet-4","modelName":"Claude Sonnet 4","provider":"Anthropic","rawName":"claude-sonnet-4-20250514","variant":"leaderboard","setting":"agent: Artemis Agent v2","score":57,"unit":"%","rank":55,"rankComputedBySite":true,"date":"2025-09-24","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"devstral-small-2","modelName":"Devstral Small 2","provider":"Mistral AI","rawName":"devstral-small-2512","variant":"leaderboard","setting":"agent: mini-SWE-agent","score":56.4,"unit":"%","rank":56,"rankComputedBySite":true,"date":"2025-12-09","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"qwen3-coder-480b-a35b","modelName":"Qwen3-Coder-480B-A35B","provider":"Alibaba (Qwen)","rawName":"Qwen3-Coder-480B-A35B-Instruct","variant":"leaderboard","setting":"agent: mini-SWE-agent","score":55.4,"unit":"%","rank":57,"rankComputedBySite":true,"date":"2025-08-02","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"glm-4-6","modelName":"GLM-4.6","provider":"Zhipu AI (Z.ai)","rawName":"glm-4.6","variant":"leaderboard","setting":"agent: mini-SWE-agent","score":55.4,"unit":"%","rank":57,"rankComputedBySite":true,"date":"2025-12-01","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"claude-3-5-sonnet","modelName":"Claude 3.5 Sonnet","provider":"Anthropic","rawName":"claude-3-5-sonnet-20241022","variant":"leaderboard","setting":"agent: devlo","score":54.2,"unit":"%","rank":59,"rankComputedBySite":true,"date":"2024-11-08","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"glm-4-5","modelName":"GLM-4.5","provider":"Zhipu AI (Z.ai)","rawName":"GLM-4.5","variant":"leaderboard","setting":"agent: mini-SWE-agent","score":54.2,"unit":"%","rank":59,"rankComputedBySite":true,"date":"2025-08-22","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"devstral-2512","modelName":"devstral-2512","provider":"Mistral AI","rawName":"devstral-2512","variant":"leaderboard","setting":"agent: mini-SWE-agent","score":53.8,"unit":"%","rank":61,"rankComputedBySite":true,"date":"2025-12-09","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"frogboss-32b-2510","modelName":"FrogBoss-32B-2510","provider":null,"rawName":"FrogBoss-32B-2510","variant":"leaderboard","setting":"agent: FrogBoss-32B-2510","score":53.6,"unit":"%","rank":62,"rankComputedBySite":true,"date":"2025-11-10","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"gemini-2-5-pro-jun-2025","modelName":"Gemini 2.5 Pro (Jun 2025)","provider":"Google","rawName":"gemini-2.5-pro","variant":"leaderboard","setting":"agent: mini-SWE-agent","score":53.6,"unit":"%","rank":62,"rankComputedBySite":true,"date":"2025-07-26","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"kimi-k2-jul-2025","modelName":"Kimi K2 (Jul 2025)","provider":"Moonshot AI","rawName":"kimi-k2-instruct","variant":"leaderboard","setting":"agent: CodeSweep - SWE-agent","score":53.4,"unit":"%","rank":64,"rankComputedBySite":true,"date":"2025-08-04","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"claude-3-7-sonnet","modelName":"Claude 3.7 Sonnet","provider":"Anthropic","rawName":"claude-3-7-sonnet-20250219","variant":"leaderboard","setting":"agent: mini-SWE-agent","score":52.8,"unit":"%","rank":65,"rankComputedBySite":true,"date":"2025-07-20","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"claude-3-5-sonnet","modelName":"Claude 3.5 Sonnet","provider":"Anthropic","rawName":"claude-3-5-sonnet-20241022","variant":"leaderboard","setting":"agent: Engine Labs","score":51.8,"unit":"%","rank":66,"rankComputedBySite":true,"date":"2024-11-25","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"claude-3-5-sonnet","modelName":"Claude 3.5 Sonnet","provider":"Anthropic","rawName":"claude-3-5-sonnet-20241022","variant":"leaderboard","setting":"agent: AutoCodeRover-v2.1","score":51.6,"unit":"%","rank":67,"rankComputedBySite":true,"date":"2025-01-22","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"https-huggingface-co-qwen-qwen3-coder-30b-a3b-instruct","modelName":"/huggingface.co/Qwen/Qwen3-Coder-30B-A3B-Instruct","provider":"Alibaba (Qwen)","rawName":"https://huggingface.co/Qwen/Qwen3-Coder-30B-A3B-Instruct","variant":"leaderboard","setting":"agent: OpenHands","score":51.6,"unit":"%","rank":67,"rankComputedBySite":true,"date":"2025-08-05","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"claude-3-5-sonnet","modelName":"Claude 3.5 Sonnet","provider":"Anthropic","rawName":"claude-3-5-sonnet-20241022","variant":"leaderboard","setting":"agent: Agentless-1.5","score":50.8,"unit":"%","rank":69,"rankComputedBySite":true,"date":"2024-12-02","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"qwen2-5-coder-32b-instruct","modelName":"Qwen2.5-Coder-32B-Instruct","provider":"Alibaba (Qwen)","rawName":"Qwen2.5-Coder-32B-Instruct","variant":"leaderboard","setting":"agent: Skywork-SWE-32B","score":47,"unit":"%","rank":70,"rankComputedBySite":true,"date":"2025-06-16","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"devstral-small","modelName":"Devstral-Small-2505","provider":"Mistral AI","rawName":"mistralai/Devstral-Small-2505","variant":"leaderboard","setting":"agent: OpenHands","score":46.8,"unit":"%","rank":71,"rankComputedBySite":true,"date":"2025-05-20","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"claude-3-5-sonnet","modelName":"Claude 3.5 Sonnet","provider":"Anthropic","rawName":"claude-3-5-sonnet-20241022","variant":"leaderboard","setting":"agent: AutoCodeRover-v2.0","score":46.2,"unit":"%","rank":72,"rankComputedBySite":true,"date":"2024-11-08","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"frogmini-14b-2510","modelName":"FrogMini-14B-2510","provider":null,"rawName":"FrogMini-14B-2510","variant":"leaderboard","setting":"agent: FrogMini-14B-2510","score":45,"unit":"%","rank":73,"rankComputedBySite":true,"date":"2025-11-10","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"o4-mini","modelName":"o4-mini","provider":"OpenAI","rawName":"o4-mini-20250416","variant":"leaderboard","setting":"agent: mini-SWE-agent","score":45,"unit":"%","rank":73,"rankComputedBySite":true,"date":"2025-07-26","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"gemini-2-0-flash-dec-2024","modelName":"Gemini 2.0 Flash (Dec 2024)","provider":"Google","rawName":"gemini-2.0-flash-exp","variant":"leaderboard","setting":"agent: CodeShellAgent","score":44.2,"unit":"%","rank":75,"rankComputedBySite":true,"date":"2025-01-18","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"kimi-k2-jul-2025","modelName":"Kimi K2 (Jul 2025)","provider":"Moonshot AI","rawName":"Kimi-K2-Instruct","variant":"leaderboard","setting":"agent: mini-SWE-agent","score":43.8,"unit":"%","rank":76,"rankComputedBySite":true,"date":"2025-08-07","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"o3-mini","modelName":"o3-mini","provider":"OpenAI","rawName":"o3-mini","variant":"leaderboard","setting":"agent: Agentless Lite","score":42.4,"unit":"%","rank":77,"rankComputedBySite":true,"date":"2025-02-14","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"amazon-nova-premier-v1-0","modelName":"amazon.nova-premier-v1:0","provider":"Amazon","rawName":"amazon.nova-premier-v1:0","variant":"leaderboard","setting":"agent: Amazon Nova Premier 1.0","score":42.4,"unit":"%","rank":77,"rankComputedBySite":true,"date":"2025-05-27","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"https-huggingface-co-agentica-org-deepswe-preview","modelName":"/huggingface.co/agentica-org/DeepSWE-Preview","provider":"OpenAI","rawName":"https://huggingface.co/agentica-org/DeepSWE-Preview","variant":"leaderboard","setting":"agent: R2E-Gym","score":42.2,"unit":"%","rank":79,"rankComputedBySite":true,"date":"2025-06-29","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"deepseek-v3-mar-2025","modelName":"DeepSeek-V3 (Mar 2025)","provider":"DeepSeek","rawName":"DeepSeek-V3-0324","variant":"leaderboard","setting":"agent: SWE-Exp","score":42,"unit":"%","rank":80,"rankComputedBySite":true,"date":"2025-08-06","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"llama3-swe-rl-70b","modelName":"Llama3-SWE-RL-70B","provider":"Meta","rawName":"Llama3-SWE-RL-70B","variant":"leaderboard","setting":"agent: Agentless Mini","score":41.2,"unit":"%","rank":81,"rankComputedBySite":true,"date":"2025-02-26","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"claude-3-haiku","modelName":"Claude 3 Haiku","provider":"Anthropic","rawName":"claude-3-haiku-20240307","variant":"leaderboard","setting":"agent: Tools","score":40.6,"unit":"%","rank":82,"rankComputedBySite":true,"date":"2024-10-22","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"qwen-2-5","modelName":"Qwen 2.5","provider":"Alibaba (Qwen)","rawName":"Qwen 2.5","variant":"leaderboard","setting":"agent: SWE-agent","score":40.2,"unit":"%","rank":83,"rankComputedBySite":true,"date":"2025-05-11","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"gpt-4-1","modelName":"GPT-4.1","provider":"OpenAI","rawName":"gpt-4.1-20250414","variant":"leaderboard","setting":"agent: mini-SWE-agent","score":39.58,"unit":"%","rank":84,"rankComputedBySite":true,"date":"2025-07-26","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"gpt-4o-may-2024","modelName":"GPT-4o (May 2024)","provider":"OpenAI","rawName":"gpt-4o-2024-05-13","variant":"leaderboard","setting":"agent: Agentless-1.5","score":38.8,"unit":"%","rank":85,"rankComputedBySite":true,"date":"2024-10-28","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"gpt-4o-may-2024","modelName":"GPT-4o (May 2024)","provider":"OpenAI","rawName":"gpt-4o-2024-05-13","variant":"leaderboard","setting":"agent: AutoCodeRover","score":38.4,"unit":"%","rank":86,"rankComputedBySite":true,"date":"2024-06-28","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"devstral-small-2507","modelName":"devstral-small-2507","provider":"Mistral AI","rawName":"devstral-small-2507","variant":"leaderboard","setting":"agent: SWE-agent","score":38,"unit":"%","rank":87,"rankComputedBySite":true,"date":"2025-07-25","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"gpt-5-nano","modelName":"GPT-5 nano","provider":"OpenAI","rawName":"gpt-5-nano-2025-08-07","variant":"leaderboard","setting":"agent: mini-SWE-agent","score":34.8,"unit":"%","rank":88,"rankComputedBySite":true,"date":"2025-08-07","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"claude-3-5-sonnet","modelName":"Claude 3.5 Sonnet","provider":"Anthropic","rawName":"claude-3-5-sonnet-20241022","variant":"leaderboard","setting":"agent: SWE-agent","score":33.6,"unit":"%","rank":89,"rankComputedBySite":true,"date":"2024-06-20","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"qwen-2-5","modelName":"Qwen 2.5","provider":"Alibaba (Qwen)","rawName":"Qwen 2.5","variant":"leaderboard","setting":"agent: SWE-Fixer","score":32.8,"unit":"%","rank":90,"rankComputedBySite":true,"date":"2025-03-06","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"gpt-4o-may-2024","modelName":"GPT-4o (May 2024)","provider":"OpenAI","rawName":"gpt-4o-2024-08-06","variant":"leaderboard","setting":"agent: MASAI","score":32.6,"unit":"%","rank":91,"rankComputedBySite":true,"date":"2024-06-12","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"gemini-2-5-flash-jun-2025","modelName":"Gemini 2.5 Flash (Jun 2025)","provider":"Google","rawName":"gemini-2.5-flash","variant":"leaderboard","setting":"agent: mini-SWE-agent","score":28.73,"unit":"%","rank":92,"rankComputedBySite":true,"date":"2025-07-26","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"gpt-4o-may-2024","modelName":"GPT-4o (May 2024)","provider":"OpenAI","rawName":"gpt-4o-2024-08-06","variant":"leaderboard","setting":"agent: EPAM AI/Run Developer Agent","score":27,"unit":"%","rank":93,"rankComputedBySite":true,"date":"2024-10-16","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"gpt-4o-may-2024","modelName":"GPT-4o (May 2024)","provider":"OpenAI","rawName":"gpt-4o-2024-05-13","variant":"leaderboard","setting":"agent: AppMap Navie","score":26.2,"unit":"%","rank":94,"rankComputedBySite":true,"date":"2024-06-15","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"gpt-oss-120b","modelName":"gpt-oss-120b","provider":"OpenAI","rawName":"gpt-oss-120b","variant":"leaderboard","setting":"agent: mini-SWE-agent","score":26,"unit":"%","rank":95,"rankComputedBySite":true,"date":"2025-08-07","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"gpt-4-1-mini","modelName":"GPT-4.1 mini","provider":"OpenAI","rawName":"gpt-4.1-mini-20250414","variant":"leaderboard","setting":"agent: mini-SWE-agent","score":23.94,"unit":"%","rank":96,"rankComputedBySite":true,"date":"2025-07-20","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"gpt-4o-may-2024","modelName":"GPT-4o (May 2024)","provider":"OpenAI","rawName":"gpt-4o-2024-05-13","variant":"leaderboard","setting":"agent: SWE-agent","score":23.2,"unit":"%","rank":97,"rankComputedBySite":true,"date":"2024-07-28","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"mcts-refine-7b","modelName":"MCTS-Refine-7B","provider":null,"rawName":"MCTS-Refine-7B","variant":"leaderboard","setting":"agent: MCTS-Refine-7B","score":23.2,"unit":"%","rank":97,"rankComputedBySite":true,"date":"2025-06-27","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"gpt-4-turbo-nov-2023","modelName":"GPT-4 Turbo (Nov 2023)","provider":"OpenAI","rawName":"gpt-4-1106-preview","variant":"leaderboard","setting":"agent: SWE-agent","score":22.4,"unit":"%","rank":99,"rankComputedBySite":true,"date":"2024-04-02","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"gpt-4o-may-2024","modelName":"GPT-4o (May 2024)","provider":"OpenAI","rawName":"gpt-4o-20241120","variant":"leaderboard","setting":"agent: mini-SWE-agent","score":21.62,"unit":"%","rank":100,"rankComputedBySite":true,"date":"2025-07-20","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"llama-4-maverick-instruct","modelName":"llama-4-maverick-instruct","provider":"Meta","rawName":"llama-4-maverick-instruct","variant":"leaderboard","setting":"agent: mini-SWE-agent","score":21.04,"unit":"%","rank":101,"rankComputedBySite":true,"date":"2025-07-20","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"claude-3-opus","modelName":"Claude 3 Opus","provider":"Anthropic","rawName":"claude-3-opus-20240229","variant":"leaderboard","setting":"agent: SWE-agent","score":15.8,"unit":"%","rank":102,"rankComputedBySite":true,"date":"2024-04-02","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"gemini-2-0-flash","modelName":"gemini-2.0-flash","provider":"Google","rawName":"gemini-2.0-flash","variant":"leaderboard","setting":"agent: mini-SWE-agent","score":13.52,"unit":"%","rank":103,"rankComputedBySite":true,"date":"2025-07-26","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"llama-4-scout-instruct","modelName":"llama-4-scout-instruct","provider":"Meta","rawName":"llama-4-scout-instruct","variant":"leaderboard","setting":"agent: mini-SWE-agent","score":9.06,"unit":"%","rank":104,"rankComputedBySite":true,"date":"2025-07-20","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"qwen2-5-coder-32b-instruct","modelName":"Qwen2.5-Coder-32B-Instruct","provider":"Alibaba (Qwen)","rawName":"Qwen2.5-Coder-32B-Instruct","variant":"leaderboard","setting":"agent: mini-SWE-agent","score":9,"unit":"%","rank":105,"rankComputedBySite":true,"date":"2025-08-03","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"claude-3-opus","modelName":"Claude 3 Opus","provider":"Anthropic","rawName":"claude-3-opus-20240229","variant":"leaderboard","setting":"agent: RAG baseline","score":7,"unit":"%","rank":106,"rankComputedBySite":true,"date":"2024-04-02","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"claude-2","modelName":"Claude 2","provider":"Anthropic","rawName":"claude-2","variant":"leaderboard","setting":"agent: RAG baseline","score":4.4,"unit":"%","rank":107,"rankComputedBySite":true,"date":"2023-10-10","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"gpt-4-turbo-nov-2023","modelName":"GPT-4 Turbo (Nov 2023)","provider":"OpenAI","rawName":"gpt-4-1106-preview","variant":"leaderboard","setting":"agent: RAG baseline","score":2.8,"unit":"%","rank":108,"rankComputedBySite":true,"date":"2024-04-02","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"},{"model":"swe-llama","modelName":"SWE-Llama","provider":"Meta","rawName":"SWE-Llama","variant":"leaderboard","setting":"agent: RAG baseline","score":1.4,"unit":"%","rank":109,"rankComputedBySite":true,"date":"2023-10-10","sourceType":"official","source":"swebench","sourceUrl":"https://www.swebench.com/#verified","collected":"2026-10-10"}]}