{"source":"AIMap","schema":"benchmark/v1","updated":"2026-10-22","benchmark":{"id":"tau2-bench","name":"τ²-bench","category":"agent-tool","unit":"%","higherIsBetter":true,"status":"active","sources":["tau2"],"primaryVariant":"retail","url":"https://taubench.com/","year":2025,"desc":{"en":"Sierra's customer-service agent benchmark: the agent follows a policy and uses tools while an LLM-simulated user talks to it (retail, airline, telecom, banking).","zh-hans":"Sierra 的客服 agent 基准：agent 要遵守业务规则并调用工具，对面是由大模型模拟的用户（零售、航空、电信、银行）。"},"measures":{"en":"pass^1 per domain (pass^k in the source measures reliability over k repeats).","zh-hans":"各领域 pass^1（来源中的 pass^k 衡量重复 k 次都成功的可靠性）。"},"limits":{"en":"Many rows are submitted by the model developer (labelled 'vendor'); prompt addenda and user-simulator model differ.","zh-hans":"不少条目由模型厂商自己提交（标为“厂商自报”）；提示补充和模拟用户所用模型各不相同。"},"contamination":"medium","contaminationNote":{"en":"Tasks are public in the repo.","zh-hans":"任务在仓库中公开。"},"stats":{"results":100,"models":43,"latest":"2026-08-03","bySourceType":{"official":72,"vendor":28}}},"sources":[{"id":"tau2","name":"τ²-bench leaderboard submissions","license":"MIT (repo)","policy":"open","attribution":"Sierra Research, τ²-bench","site":"https://taubench.com/"}],"results":[{"model":"claude-opus-4-5","modelName":"Claude Opus 4.5","provider":"Anthropic","rawName":"Claude Opus 4.5","variant":"airline","setting":"airline domain, pass^1; effort: high; submitted by Sierra; v1.0.1","score":84,"unit":"%","rank":1,"rankComputedBySite":true,"date":"2026-05-05","sourceType":"official","source":"tau2","sourceUrl":"https://taubench.com/","collected":"2026-10-10"},{"model":"gpt-5-2","modelName":"GPT-5.2","provider":"OpenAI","rawName":"GPT-5.2","variant":"airline","setting":"airline domain, pass^1; effort: high; submitted by Sierra; v1.0.1","score":83,"unit":"%","rank":2,"rankComputedBySite":true,"date":"2026-05-05","sourceType":"official","source":"tau2","sourceUrl":"https://taubench.com/","collected":"2026-10-10"},{"model":"gemini-3-flash","modelName":"Gemini 3 Flash","provider":"Google","rawName":"Gemini 3 Flash","variant":"airline","setting":"airline domain, pass^1; effort: high; submitted by Sierra; v1.0.1","score":82.5,"unit":"%","rank":3,"rankComputedBySite":true,"date":"2026-03-02","sourceType":"official","source":"tau2","sourceUrl":"https://taubench.com/","collected":"2026-10-10"},{"model":"glm-5","modelName":"GLM-5","provider":"Zhipu AI (Z.ai)","rawName":"GLM-5","variant":"airline","setting":"airline domain, pass^1; effort: enabled; submitted by Sierra; v1.0.1","score":82.5,"unit":"%","rank":3,"rankComputedBySite":true,"date":"2026-02-27","sourceType":"official","source":"tau2","sourceUrl":"https://taubench.com/","collected":"2026-10-10"},{"model":"qwen3-5-397b-a17b","modelName":"Qwen3.5 397B-A17B","provider":"Alibaba (Qwen)","rawName":"Qwen3.5-397B-A17B","variant":"airline","setting":"airline domain, pass^1; effort: enabled; submitted by Sierra; v1.0.1","score":81.5,"unit":"%","rank":5,"rankComputedBySite":true,"date":"2026-02-27","sourceType":"official","source":"tau2","sourceUrl":"https://taubench.com/","collected":"2026-10-10"},{"model":"gemini-3-pro","modelName":"Gemini 3 Pro","provider":"Google","rawName":"Gemini 3 Pro","variant":"airline","setting":"airline domain, pass^1; effort: high; submitted by Sierra; v1.0.1","score":80.5,"unit":"%","rank":6,"rankComputedBySite":true,"date":"2026-03-02","sourceType":"official","source":"tau2","sourceUrl":"https://taubench.com/","collected":"2026-10-10"},{"model":"gemini-3-0-pro","modelName":"Gemini 3.0 Pro","provider":"Google","rawName":"Gemini 3.0 Pro","variant":"airline","setting":"airline domain, pass^1; submitted by Sierra; vv0.1.3","score":73,"unit":"%","rank":7,"rankComputedBySite":true,"date":"2025-11-18","sourceType":"official","source":"tau2","sourceUrl":"https://taubench.com/","collected":"2026-10-10"},{"model":"claude-sonnet-4-5","modelName":"Claude Sonnet 4.5","provider":"Anthropic","rawName":"Claude Sonnet 4.5","variant":"airline","setting":"airline domain, pass^1; effort: enabled; submitted by Sierra; v1.0.1","score":72,"unit":"%","rank":8,"rankComputedBySite":true,"date":"2026-02-24","sourceType":"official","source":"tau2","sourceUrl":"https://taubench.com/","collected":"2026-10-10"},{"model":"qwen3-max-thinking-preview","modelName":"Qwen3-Max-Thinking-Preview","provider":"Alibaba (Qwen)","rawName":"Qwen3-Max-Thinking-Preview","variant":"airline","setting":"airline domain, pass^1; submitted by Qwen; vv0.1.3","score":71,"unit":"%","rank":9,"rankComputedBySite":true,"date":"2025-10-30","sourceType":"vendor","source":"tau2","sourceUrl":"https://qwenlm.github.io/","collected":"2026-10-10"},{"model":"claude-sonnet-4-5","modelName":"Claude Sonnet 4.5","provider":"Anthropic","rawName":"Claude-Sonnet-4.5","variant":"airline","setting":"airline domain, pass^1; submitted by Anthropic; v0.1.3","score":70,"unit":"%","rank":10,"rankComputedBySite":true,"date":"2025-09-29","sourceType":"vendor","source":"tau2","sourceUrl":"https://www.anthropic.com/news/claude-sonnet-4-5","collected":"2026-10-10"},{"model":"qwen3-max","modelName":"Qwen3-Max","provider":"Alibaba (Qwen)","rawName":"Qwen3-Max-Thinking","variant":"airline","setting":"airline domain, pass^1; submitted by Qwen; vv0.1.3","score":69,"unit":"%","rank":11,"rankComputedBySite":true,"date":"2026-01-23","sourceType":"vendor","source":"tau2","sourceUrl":"https://qwen.ai/blog?id=qwen3-max-thinking","collected":"2026-10-10"},{"model":"claude-3-7-sonnet","modelName":"Claude 3.7 Sonnet","provider":"Anthropic","rawName":"Claude-3.7-Sonnet","variant":"airline","setting":"airline domain, pass^1; submitted by Sierra; v0.1.3","score":64.2,"unit":"%","rank":12,"rankComputedBySite":true,"date":"2025-06-09","sourceType":"official","source":"tau2","sourceUrl":"https://taubench.com/","collected":"2026-10-10"},{"model":"deepseek-v3-2","modelName":"DeepSeek-V3.2","provider":"DeepSeek","rawName":"DeepSeek-V3.2","variant":"airline","setting":"airline domain, pass^1; submitted by DeepSeek; vv0.1.3","score":63.8,"unit":"%","rank":13,"rankComputedBySite":true,"date":"2025-12-01","sourceType":"vendor","source":"tau2","sourceUrl":"https://huggingface.co/deepseek-ai/DeepSeek-V3.2/resolve/main/assets/paper.pdf","collected":"2026-10-10"},{"model":"gpt-5","modelName":"GPT-5","provider":"OpenAI","rawName":"GPT-5","variant":"airline","setting":"airline domain, pass^1; submitted by Sierra; v0.1.3","score":62.5,"unit":"%","rank":14,"rankComputedBySite":true,"date":"2025-08-09","sourceType":"official","source":"tau2","sourceUrl":"https://taubench.com/","collected":"2026-10-10"},{"model":"claude-sonnet-4","modelName":"Claude Sonnet 4","provider":"Anthropic","rawName":"Claude Sonnet 4","variant":"airline","setting":"airline domain, pass^1; submitted by Anthropic; voriginal tau-bench","score":60,"unit":"%","rank":15,"rankComputedBySite":true,"date":"2025-05-22","sourceType":"vendor","source":"tau2","sourceUrl":"https://www.anthropic.com/news/claude-4","collected":"2026-10-10"},{"model":"claude-opus-4","modelName":"Claude Opus 4","provider":"Anthropic","rawName":"Claude Opus 4","variant":"airline","setting":"airline domain, pass^1; submitted by Anthropic; voriginal tau-bench","score":59.6,"unit":"%","rank":16,"rankComputedBySite":true,"date":"2025-05-22","sourceType":"vendor","source":"tau2","sourceUrl":"https://www.anthropic.com/news/claude-4","collected":"2026-10-10"},{"model":"qwen3-max","modelName":"Qwen3-Max","provider":"Alibaba (Qwen)","rawName":"Qwen3-Max","variant":"airline","setting":"airline domain, pass^1; submitted by Alibaba; v0.1.3","score":59.5,"unit":"%","rank":17,"rankComputedBySite":true,"date":"2024-09-23","sourceType":"official","source":"tau2","sourceUrl":"https://taubench.com/","collected":"2026-10-10"},{"model":"kimi-k2","modelName":"Kimi K2 Thinking","provider":"Moonshot AI","rawName":"Kimi-k2","variant":"airline","setting":"airline domain, pass^1; submitted by Moonshot AI; v0.1.3","score":56.5,"unit":"%","rank":18,"rankComputedBySite":true,"date":"2025-07-28","sourceType":"vendor","source":"tau2","sourceUrl":"https://arxiv.org/abs/2507.20534","collected":"2026-10-10"},{"model":"claude-opus-4-1","modelName":"Claude Opus 4.1","provider":"Anthropic","rawName":"Claude Opus 4.1","variant":"airline","setting":"airline domain, pass^1; submitted by Anthropic; voriginal tau-bench","score":56,"unit":"%","rank":19,"rankComputedBySite":true,"date":"2025-08-05","sourceType":"vendor","source":"tau2","sourceUrl":"https://www.anthropic.com/news/claude-opus-4-1","collected":"2026-10-10"},{"model":"gpt-4-1","modelName":"GPT-4.1","provider":"OpenAI","rawName":"GPT-4.1","variant":"airline","setting":"airline domain, pass^1; submitted by Sierra; v0.1.3","score":56,"unit":"%","rank":19,"rankComputedBySite":true,"date":"2025-06-09","sourceType":"official","source":"tau2","sourceUrl":"https://taubench.com/","collected":"2026-10-10"},{"model":"nemotron-orchestrator-8b","modelName":"Nemotron-Orchestrator-8B","provider":"NVIDIA","rawName":"Nemotron-Orchestrator-8B","variant":"airline","setting":"airline domain, pass^1; submitted by NVIDIA; v0.1.3","score":56,"unit":"%","rank":19,"rankComputedBySite":true,"date":"2025-09-15","sourceType":"vendor","source":"tau2","sourceUrl":"https://arxiv.org/abs/2511.21689","collected":"2026-10-10"},{"model":"gpt-5-2","modelName":"GPT-5.2","provider":"OpenAI","rawName":"GPT-5.2","variant":"airline","setting":"airline domain, pass^1; effort: none; submitted by Sierra; v1.0.1","score":52.5,"unit":"%","rank":22,"rankComputedBySite":true,"date":"2026-05-26","sourceType":"official","source":"tau2","sourceUrl":"https://taubench.com/","collected":"2026-10-10"},{"model":"o4-mini","modelName":"o4-mini","provider":"OpenAI","rawName":"o4-mini","variant":"airline","setting":"airline domain, pass^1; submitted by Sierra; v0.1.3","score":52.1,"unit":"%","rank":23,"rankComputedBySite":true,"date":"2025-06-09","sourceType":"official","source":"tau2","sourceUrl":"https://taubench.com/","collected":"2026-10-10"},{"model":"o3","modelName":"o3","provider":"OpenAI","rawName":"o3","variant":"airline","setting":"airline domain, pass^1; submitted by OpenAI; v0.1.3","score":52,"unit":"%","rank":24,"rankComputedBySite":true,"date":"2025-04-16","sourceType":"vendor","source":"tau2","sourceUrl":"https://openai.com/index/introducing-o3-and-o4-mini/","collected":"2026-10-10"},{"model":"gpt-4-1-mini","modelName":"GPT-4.1 mini","provider":"OpenAI","rawName":"GPT-4.1-mini","variant":"airline","setting":"airline domain, pass^1; submitted by Sierra; v0.1.3","score":48.7,"unit":"%","rank":25,"rankComputedBySite":true,"date":"2025-06-09","sourceType":"official","source":"tau2","sourceUrl":"https://taubench.com/","collected":"2026-10-10"},{"model":"qwen-3-8-max","modelName":"Qwen 3.8 Max","provider":"Alibaba (Qwen)","rawName":"Qwen 3.8 Max","variant":"banking","setting":"banking_knowledge domain, pass^1; effort: xhigh; submitted by Sierra; v1.0.1","score":55.2,"unit":"%","rank":1,"rankComputedBySite":true,"date":"2026-08-03","sourceType":"official","source":"tau2","sourceUrl":"https://taubench.com/","collected":"2026-10-10"},{"model":"claude-opus-5","modelName":"Claude Opus 5","provider":"Anthropic","rawName":"Claude Opus 5","variant":"banking","setting":"banking_knowledge domain, pass^1; effort: max; submitted by Sierra; v1.0.1","score":48.7,"unit":"%","rank":2,"rankComputedBySite":true,"date":"2026-08-03","sourceType":"official","source":"tau2","sourceUrl":"https://taubench.com/","collected":"2026-10-10"},{"model":"grok-4-5","modelName":"Grok 4.5","provider":"xAI","rawName":"Grok 4.5","variant":"banking","setting":"banking_knowledge domain, pass^1; effort: high; submitted by Sierra; v1.0.1","score":47.9,"unit":"%","rank":3,"rankComputedBySite":true,"date":"2026-07-24","sourceType":"official","source":"tau2","sourceUrl":"https://taubench.com/","collected":"2026-10-10"},{"model":"gpt-5-6-sol","modelName":"GPT-5.6 Sol","provider":"OpenAI","rawName":"GPT-5.6-sol","variant":"banking","setting":"banking_knowledge domain, pass^1; effort: xhigh; submitted by Sierra; v1.0.1","score":46.9,"unit":"%","rank":4,"rankComputedBySite":true,"date":"2026-07-22","sourceType":"official","source":"tau2","sourceUrl":"https://taubench.com/","collected":"2026-10-10"},{"model":"gpt-5-5","modelName":"GPT-5.5","provider":"OpenAI","rawName":"GPT-5.5","variant":"banking","setting":"banking_knowledge domain, pass^1; effort: xhigh; submitted by Sierra; v1.0.1","score":44.6,"unit":"%","rank":5,"rankComputedBySite":true,"date":"2026-07-22","sourceType":"official","source":"tau2","sourceUrl":"https://taubench.com/","collected":"2026-10-10"},{"model":"muse-spark-1-1","modelName":"Muse Spark 1.1","provider":"Meta","rawName":"Muse Spark 1.1","variant":"banking","setting":"banking_knowledge domain, pass^1; effort: xhigh; submitted by Sierra; v1.0.1","score":40.5,"unit":"%","rank":6,"rankComputedBySite":true,"date":"2026-07-23","sourceType":"official","source":"tau2","sourceUrl":"https://taubench.com/","collected":"2026-10-10"},{"model":"claude-opus-4-7","modelName":"Claude Opus 4.7","provider":"Anthropic","rawName":"Claude Opus 4.7","variant":"banking","setting":"banking_knowledge domain, pass^1; effort: max; submitted by Sierra; v1.0.1","score":40.2,"unit":"%","rank":7,"rankComputedBySite":true,"date":"2026-07-23","sourceType":"official","source":"tau2","sourceUrl":"https://taubench.com/","collected":"2026-10-10"},{"model":"claude-fable-5","modelName":"Claude Fable 5","provider":"Anthropic","rawName":"Claude Fable 5","variant":"banking","setting":"banking_knowledge domain, pass^1; effort: max; submitted by Sierra; v1.0.1","score":39.7,"unit":"%","rank":8,"rankComputedBySite":true,"date":"2026-07-23","sourceType":"official","source":"tau2","sourceUrl":"https://taubench.com/","collected":"2026-10-10"},{"model":"claude-opus-4-8","modelName":"Claude Opus 4.8","provider":"Anthropic","rawName":"Claude Opus 4.8","variant":"banking","setting":"banking_knowledge domain, pass^1; effort: max; submitted by Sierra; v1.0.1","score":39.7,"unit":"%","rank":8,"rankComputedBySite":true,"date":"2026-07-23","sourceType":"official","source":"tau2","sourceUrl":"https://taubench.com/","collected":"2026-10-10"},{"model":"gpt-5-4","modelName":"GPT-5.4","provider":"OpenAI","rawName":"GPT-5.4","variant":"banking","setting":"banking_knowledge domain, pass^1; effort: xhigh; submitted by Sierra; v1.0.1","score":39.4,"unit":"%","rank":10,"rankComputedBySite":true,"date":"2026-05-06","sourceType":"official","source":"tau2","sourceUrl":"https://taubench.com/","collected":"2026-10-10"},{"model":"glm-5-2","modelName":"GLM-5.2","provider":"Zhipu AI (Z.ai)","rawName":"GLM-5.2","variant":"banking","setting":"banking_knowledge domain, pass^1; effort: xhigh; submitted by Sierra; v1.0.1","score":37.1,"unit":"%","rank":11,"rankComputedBySite":true,"date":"2026-07-24","sourceType":"official","source":"tau2","sourceUrl":"https://taubench.com/","collected":"2026-10-10"},{"model":"kimi-k3","modelName":"Kimi K3","provider":"Moonshot AI","rawName":"Kimi K3","variant":"banking","setting":"banking_knowledge domain, pass^1; effort: max; submitted by Sierra; v1.0.1","score":37.1,"unit":"%","rank":11,"rankComputedBySite":true,"date":"2026-07-24","sourceType":"official","source":"tau2","sourceUrl":"https://taubench.com/","collected":"2026-10-10"},{"model":"gpt-5-2","modelName":"GPT-5.2","provider":"OpenAI","rawName":"GPT-5.2","variant":"banking","setting":"banking_knowledge domain, pass^1; effort: high; submitted by Sierra; v1.0.1","score":32.2,"unit":"%","rank":13,"rankComputedBySite":true,"date":"2026-05-05","sourceType":"official","source":"tau2","sourceUrl":"https://taubench.com/","collected":"2026-10-10"},{"model":"distyl-buttonagent","modelName":"Distyl ButtonAgent","provider":"Distyl AI","rawName":"Distyl ButtonAgent","variant":"banking","setting":"banking_knowledge domain, pass^1; effort: high; submitted by Distyl AI; v0.2.1-dev","score":31.2,"unit":"%","rank":14,"rankComputedBySite":true,"date":"2026-03-25","sourceType":"vendor","source":"tau2","sourceUrl":"https://www.mixedbread.com/docs/stores/overview","collected":"2026-10-10"},{"model":"claude-opus-4-6","modelName":"Claude Opus 4.6","provider":"Anthropic","rawName":"Claude Opus 4.6","variant":"banking","setting":"banking_knowledge domain, pass^1; effort: max; submitted by Sierra; v1.0.1","score":27.3,"unit":"%","rank":15,"rankComputedBySite":true,"date":"2026-05-06","sourceType":"official","source":"tau2","sourceUrl":"https://taubench.com/","collected":"2026-10-10"},{"model":"gemini-3-flash","modelName":"Gemini 3 Flash","provider":"Google","rawName":"Gemini 3 Flash","variant":"banking","setting":"banking_knowledge domain, pass^1; effort: high; submitted by Sierra; v1.0.1","score":27.3,"unit":"%","rank":15,"rankComputedBySite":true,"date":"2026-03-02","sourceType":"official","source":"tau2","sourceUrl":"https://taubench.com/","collected":"2026-10-10"},{"model":"gemini-3-1-pro","modelName":"Gemini 3.1 Pro","provider":"Google","rawName":"Gemini 3.1 Pro Preview","variant":"banking","setting":"banking_knowledge domain, pass^1; effort: high; submitted by Sierra; v1.0.1","score":26,"unit":"%","rank":17,"rankComputedBySite":true,"date":"2026-05-05","sourceType":"official","source":"tau2","sourceUrl":"https://taubench.com/","collected":"2026-10-10"},{"model":"claude-sonnet-4-5","modelName":"Claude Sonnet 4.5","provider":"Anthropic","rawName":"Claude Sonnet 4.5","variant":"banking","setting":"banking_knowledge domain, pass^1; effort: enabled; submitted by Sierra; v1.0.1","score":25.3,"unit":"%","rank":18,"rankComputedBySite":true,"date":"2026-02-24","sourceType":"official","source":"tau2","sourceUrl":"https://taubench.com/","collected":"2026-10-10"},{"model":"inkling","modelName":"Inkling","provider":"Thinking Machines","rawName":"Inkling","variant":"banking","setting":"banking_knowledge domain, pass^1; effort: max; submitted by Sierra; v1.0.1","score":25,"unit":"%","rank":19,"rankComputedBySite":true,"date":"2026-07-24","sourceType":"official","source":"tau2","sourceUrl":"https://taubench.com/","collected":"2026-10-10"},{"model":"claude-opus-4-5","modelName":"Claude Opus 4.5","provider":"Anthropic","rawName":"Claude Opus 4.5","variant":"banking","setting":"banking_knowledge domain, pass^1; effort: high; submitted by Sierra; v1.0.1","score":24.7,"unit":"%","rank":20,"rankComputedBySite":true,"date":"2026-05-05","sourceType":"official","source":"tau2","sourceUrl":"https://taubench.com/","collected":"2026-10-10"},{"model":"gemini-3-pro","modelName":"Gemini 3 Pro","provider":"Google","rawName":"Gemini 3 Pro","variant":"banking","setting":"banking_knowledge domain, pass^1; effort: high; submitted by Sierra; v1.0.1","score":18,"unit":"%","rank":21,"rankComputedBySite":true,"date":"2026-03-02","sourceType":"official","source":"tau2","sourceUrl":"https://taubench.com/","collected":"2026-10-10"},{"model":"grok-4-2","modelName":"Grok 4.2","provider":"xAI","rawName":"Grok 4.2","variant":"banking","setting":"banking_knowledge domain, pass^1; effort: high; submitted by Sierra; v1.0.1","score":18,"unit":"%","rank":21,"rankComputedBySite":true,"date":"2026-05-05","sourceType":"official","source":"tau2","sourceUrl":"https://taubench.com/","collected":"2026-10-10"},{"model":"grok-4-fast","modelName":"Grok 4 fast","provider":"xAI","rawName":"Grok 4 fast","variant":"banking","setting":"banking_knowledge domain, pass^1; effort: high; submitted by Sierra; v1.0.1","score":15.7,"unit":"%","rank":23,"rankComputedBySite":true,"date":"2026-05-05","sourceType":"official","source":"tau2","sourceUrl":"https://taubench.com/","collected":"2026-10-10"},{"model":"gemini-2-5-pro-jun-2025","modelName":"Gemini 2.5 Pro (Jun 2025)","provider":"Google","rawName":"Gemini 2.5 Pro","variant":"banking","setting":"banking_knowledge domain, pass^1; effort: high; submitted by Sierra; v1.0.1","score":13.7,"unit":"%","rank":24,"rankComputedBySite":true,"date":"2026-05-05","sourceType":"official","source":"tau2","sourceUrl":"https://taubench.com/","collected":"2026-10-10"},{"model":"grok-4-1-fast","modelName":"Grok 4.1 fast","provider":"xAI","rawName":"Grok 4.1 fast","variant":"banking","setting":"banking_knowledge domain, pass^1; effort: high; submitted by Sierra; v1.0.1","score":13.1,"unit":"%","rank":25,"rankComputedBySite":true,"date":"2026-05-05","sourceType":"official","source":"tau2","sourceUrl":"https://taubench.com/","collected":"2026-10-10"},{"model":"gpt-5-2","modelName":"GPT-5.2","provider":"OpenAI","rawName":"GPT-5.2","variant":"banking","setting":"banking_knowledge domain, pass^1; effort: none; submitted by Sierra; v1.0.1","score":12.6,"unit":"%","rank":26,"rankComputedBySite":true,"date":"2026-05-26","sourceType":"official","source":"tau2","sourceUrl":"https://taubench.com/","collected":"2026-10-10"},{"model":"glm-5","modelName":"GLM-5","provider":"Zhipu AI (Z.ai)","rawName":"GLM-5","variant":"banking","setting":"banking_knowledge domain, pass^1; effort: enabled; submitted by Sierra; v1.0.1","score":9.8,"unit":"%","rank":27,"rankComputedBySite":true,"date":"2026-02-27","sourceType":"official","source":"tau2","sourceUrl":"https://taubench.com/","collected":"2026-10-10"},{"model":"qwen3-5-397b-a17b","modelName":"Qwen3.5 397B-A17B","provider":"Alibaba (Qwen)","rawName":"Qwen3.5-397B-A17B","variant":"banking","setting":"banking_knowledge domain, pass^1; effort: enabled; submitted by Sierra; v1.0.1","score":9.8,"unit":"%","rank":27,"rankComputedBySite":true,"date":"2026-02-27","sourceType":"official","source":"tau2","sourceUrl":"https://taubench.com/","collected":"2026-10-10"},{"model":"claude-sonnet-4-5","modelName":"Claude Sonnet 4.5","provider":"Anthropic","rawName":"Claude-Sonnet-4.5","variant":"retail","setting":"retail domain, pass^1; submitted by Anthropic; v0.1.3","score":86.2,"unit":"%","rank":1,"rankComputedBySite":true,"date":"2025-09-29","sourceType":"vendor","source":"tau2","sourceUrl":"https://www.anthropic.com/news/claude-sonnet-4-5","collected":"2026-10-10"},{"model":"gemini-3-0-pro","modelName":"Gemini 3.0 Pro","provider":"Google","rawName":"Gemini 3.0 Pro","variant":"retail","setting":"retail domain, pass^1; submitted by Sierra; vv0.1.3","score":85.3,"unit":"%","rank":2,"rankComputedBySite":true,"date":"2025-11-18","sourceType":"official","source":"tau2","sourceUrl":"https://taubench.com/","collected":"2026-10-10"},{"model":"qwen3-5-397b-a17b","modelName":"Qwen3.5 397B-A17B","provider":"Alibaba (Qwen)","rawName":"Qwen3.5-397B-A17B","variant":"retail","setting":"retail domain, pass^1; effort: enabled; submitted by Sierra; v1.0.1","score":84.4,"unit":"%","rank":3,"rankComputedBySite":true,"date":"2026-02-27","sourceType":"official","source":"tau2","sourceUrl":"https://taubench.com/","collected":"2026-10-10"},{"model":"nemotron-orchestrator-8b","modelName":"Nemotron-Orchestrator-8B","provider":"NVIDIA","rawName":"Nemotron-Orchestrator-8B","variant":"retail","setting":"retail domain, pass^1; submitted by NVIDIA; v0.1.3","score":84.2,"unit":"%","rank":4,"rankComputedBySite":true,"date":"2025-09-15","sourceType":"vendor","source":"tau2","sourceUrl":"https://arxiv.org/abs/2511.21689","collected":"2026-10-10"},{"model":"raft-30b-a3b","modelName":"RAFT-30B-A3B","provider":"Northeastern University HAI Lab","rawName":"RAFT-30B-A3B","variant":"retail","setting":"retail domain, pass^1; submitted by Northeastern University HAI Lab","score":82.5,"unit":"%","rank":5,"rankComputedBySite":true,"date":"2026-04-30","sourceType":"vendor","source":"tau2","sourceUrl":"https://raft.hailab.io/","collected":"2026-10-10"},{"model":"claude-opus-4-1","modelName":"Claude Opus 4.1","provider":"Anthropic","rawName":"Claude Opus 4.1","variant":"retail","setting":"retail domain, pass^1; submitted by Anthropic; voriginal tau-bench","score":82.4,"unit":"%","rank":6,"rankComputedBySite":true,"date":"2025-08-05","sourceType":"vendor","source":"tau2","sourceUrl":"https://www.anthropic.com/news/claude-opus-4-1","collected":"2026-10-10"},{"model":"gpt-5-2","modelName":"GPT-5.2","provider":"OpenAI","rawName":"GPT-5.2","variant":"retail","setting":"retail domain, pass^1; effort: high; submitted by Sierra; v1.0.1","score":81.6,"unit":"%","rank":7,"rankComputedBySite":true,"date":"2026-05-05","sourceType":"official","source":"tau2","sourceUrl":"https://taubench.com/","collected":"2026-10-10"},{"model":"gpt-5","modelName":"GPT-5","provider":"OpenAI","rawName":"GPT-5","variant":"retail","setting":"retail domain, pass^1; submitted by Sierra; v0.1.3","score":81.6,"unit":"%","rank":7,"rankComputedBySite":true,"date":"2025-08-09","sourceType":"official","source":"tau2","sourceUrl":"https://taubench.com/","collected":"2026-10-10"},{"model":"claude-opus-4","modelName":"Claude Opus 4","provider":"Anthropic","rawName":"Claude Opus 4","variant":"retail","setting":"retail domain, pass^1; submitted by Anthropic; voriginal tau-bench","score":81.4,"unit":"%","rank":9,"rankComputedBySite":true,"date":"2025-05-22","sourceType":"vendor","source":"tau2","sourceUrl":"https://www.anthropic.com/news/claude-4","collected":"2026-10-10"},{"model":"deepseek-v3-2","modelName":"DeepSeek-V3.2","provider":"DeepSeek","rawName":"DeepSeek-V3.2","variant":"retail","setting":"retail domain, pass^1; submitted by DeepSeek; vv0.1.3","score":81.1,"unit":"%","rank":10,"rankComputedBySite":true,"date":"2025-12-01","sourceType":"vendor","source":"tau2","sourceUrl":"https://huggingface.co/deepseek-ai/DeepSeek-V3.2/resolve/main/assets/paper.pdf","collected":"2026-10-10"},{"model":"claude-sonnet-4","modelName":"Claude Sonnet 4","provider":"Anthropic","rawName":"Claude Sonnet 4","variant":"retail","setting":"retail domain, pass^1; submitted by Anthropic; voriginal tau-bench","score":80.5,"unit":"%","rank":11,"rankComputedBySite":true,"date":"2025-05-22","sourceType":"vendor","source":"tau2","sourceUrl":"https://www.anthropic.com/news/claude-4","collected":"2026-10-10"},{"model":"claude-opus-4-5","modelName":"Claude Opus 4.5","provider":"Anthropic","rawName":"Claude Opus 4.5","variant":"retail","setting":"retail domain, pass^1; effort: high; submitted by Sierra; v1.0.1","score":79.6,"unit":"%","rank":12,"rankComputedBySite":true,"date":"2026-05-05","sourceType":"official","source":"tau2","sourceUrl":"https://taubench.com/","collected":"2026-10-10"},{"model":"qwen3-max","modelName":"Qwen3-Max","provider":"Alibaba (Qwen)","rawName":"Qwen3-Max-Thinking","variant":"retail","setting":"retail domain, pass^1; submitted by Qwen; vv0.1.3","score":79.4,"unit":"%","rank":13,"rankComputedBySite":true,"date":"2026-01-23","sourceType":"vendor","source":"tau2","sourceUrl":"https://qwen.ai/blog?id=qwen3-max-thinking","collected":"2026-10-10"},{"model":"gemini-3-flash","modelName":"Gemini 3 Flash","provider":"Google","rawName":"Gemini 3 Flash","variant":"retail","setting":"retail domain, pass^1; effort: high; submitted by Sierra; v1.0.1","score":76.8,"unit":"%","rank":14,"rankComputedBySite":true,"date":"2026-03-02","sourceType":"official","source":"tau2","sourceUrl":"https://taubench.com/","collected":"2026-10-10"},{"model":"gemini-3-pro","modelName":"Gemini 3 Pro","provider":"Google","rawName":"Gemini 3 Pro","variant":"retail","setting":"retail domain, pass^1; effort: high; submitted by Sierra; v1.0.1","score":75.9,"unit":"%","rank":15,"rankComputedBySite":true,"date":"2026-03-02","sourceType":"official","source":"tau2","sourceUrl":"https://taubench.com/","collected":"2026-10-10"},{"model":"qwen3-max-thinking-preview","modelName":"Qwen3-Max-Thinking-Preview","provider":"Alibaba (Qwen)","rawName":"Qwen3-Max-Thinking-Preview","variant":"retail","setting":"retail domain, pass^1; submitted by Qwen; vv0.1.3","score":75.4,"unit":"%","rank":16,"rankComputedBySite":true,"date":"2025-10-30","sourceType":"vendor","source":"tau2","sourceUrl":"https://qwenlm.github.io/","collected":"2026-10-10"},{"model":"gpt-5-2","modelName":"GPT-5.2","provider":"OpenAI","rawName":"GPT-5.2","variant":"retail","setting":"retail domain, pass^1; effort: none; submitted by Sierra; v1.0.1","score":75,"unit":"%","rank":17,"rankComputedBySite":true,"date":"2026-05-26","sourceType":"official","source":"tau2","sourceUrl":"https://taubench.com/","collected":"2026-10-10"},{"model":"gpt-4-1","modelName":"GPT-4.1","provider":"OpenAI","rawName":"GPT-4.1","variant":"retail","setting":"retail domain, pass^1; submitted by Sierra; v0.1.3","score":74,"unit":"%","rank":18,"rankComputedBySite":true,"date":"2025-06-09","sourceType":"official","source":"tau2","sourceUrl":"https://taubench.com/","collected":"2026-10-10"},{"model":"o3","modelName":"o3","provider":"OpenAI","rawName":"o3","variant":"retail","setting":"retail domain, pass^1; submitted by OpenAI; v0.1.3","score":73.9,"unit":"%","rank":19,"rankComputedBySite":true,"date":"2025-04-16","sourceType":"vendor","source":"tau2","sourceUrl":"https://openai.com/index/introducing-o3-and-o4-mini/","collected":"2026-10-10"},{"model":"glm-5","modelName":"GLM-5","provider":"Zhipu AI (Z.ai)","rawName":"GLM-5","variant":"retail","setting":"retail domain, pass^1; effort: enabled; submitted by Sierra; v1.0.1","score":73.7,"unit":"%","rank":20,"rankComputedBySite":true,"date":"2026-02-27","sourceType":"official","source":"tau2","sourceUrl":"https://taubench.com/","collected":"2026-10-10"},{"model":"claude-sonnet-4-5","modelName":"Claude Sonnet 4.5","provider":"Anthropic","rawName":"Claude Sonnet 4.5","variant":"retail","setting":"retail domain, pass^1; effort: enabled; submitted by Sierra; v1.0.1","score":72.4,"unit":"%","rank":21,"rankComputedBySite":true,"date":"2026-02-24","sourceType":"official","source":"tau2","sourceUrl":"https://taubench.com/","collected":"2026-10-10"},{"model":"qwen3-max","modelName":"Qwen3-Max","provider":"Alibaba (Qwen)","rawName":"Qwen3-Max","variant":"retail","setting":"retail domain, pass^1; submitted by Alibaba; v0.1.3","score":72.2,"unit":"%","rank":22,"rankComputedBySite":true,"date":"2024-09-23","sourceType":"official","source":"tau2","sourceUrl":"https://taubench.com/","collected":"2026-10-10"},{"model":"claude-3-7-sonnet","modelName":"Claude 3.7 Sonnet","provider":"Anthropic","rawName":"Claude-3.7-Sonnet","variant":"retail","setting":"retail domain, pass^1; submitted by Sierra; v0.1.3","score":72.1,"unit":"%","rank":23,"rankComputedBySite":true,"date":"2025-06-09","sourceType":"official","source":"tau2","sourceUrl":"https://taubench.com/","collected":"2026-10-10"},{"model":"kimi-k2","modelName":"Kimi K2 Thinking","provider":"Moonshot AI","rawName":"Kimi-k2","variant":"retail","setting":"retail domain, pass^1; submitted by Moonshot AI; v0.1.3","score":70.6,"unit":"%","rank":24,"rankComputedBySite":true,"date":"2025-07-28","sourceType":"vendor","source":"tau2","sourceUrl":"https://arxiv.org/abs/2507.20534","collected":"2026-10-10"},{"model":"o4-mini","modelName":"o4-mini","provider":"OpenAI","rawName":"o4-mini","variant":"retail","setting":"retail domain, pass^1; submitted by Sierra; v0.1.3","score":68.3,"unit":"%","rank":25,"rankComputedBySite":true,"date":"2025-06-09","sourceType":"official","source":"tau2","sourceUrl":"https://taubench.com/","collected":"2026-10-10"},{"model":"gpt-4-1-mini","modelName":"GPT-4.1 mini","provider":"OpenAI","rawName":"GPT-4.1-mini","variant":"retail","setting":"retail domain, pass^1; submitted by Sierra; v0.1.3","score":61.4,"unit":"%","rank":26,"rankComputedBySite":true,"date":"2025-06-09","sourceType":"official","source":"tau2","sourceUrl":"https://taubench.com/","collected":"2026-10-10"},{"model":"qwen3-max","modelName":"Qwen3-Max","provider":"Alibaba (Qwen)","rawName":"Qwen3-Max-Thinking","variant":"telecom","setting":"telecom domain, pass^1; submitted by Qwen; vv0.1.3","score":98.2,"unit":"%","rank":1,"rankComputedBySite":true,"date":"2026-01-23","sourceType":"vendor","source":"tau2","sourceUrl":"https://qwen.ai/blog?id=qwen3-max-thinking","collected":"2026-10-10"},{"model":"claude-sonnet-4-5","modelName":"Claude Sonnet 4.5","provider":"Anthropic","rawName":"Claude-Sonnet-4.5","variant":"telecom","setting":"telecom domain, pass^1; submitted by Anthropic; v0.1.3","score":98,"unit":"%","rank":2,"rankComputedBySite":true,"date":"2025-09-29","sourceType":"vendor","source":"tau2","sourceUrl":"https://www.anthropic.com/news/claude-sonnet-4-5","collected":"2026-10-10"},{"model":"gemini-3-0-pro","modelName":"Gemini 3.0 Pro","provider":"Google","rawName":"Gemini 3.0 Pro","variant":"telecom","setting":"telecom domain, pass^1; submitted by Sierra; vv0.1.3","score":98,"unit":"%","rank":2,"rankComputedBySite":true,"date":"2025-11-18","sourceType":"official","source":"tau2","sourceUrl":"https://taubench.com/","collected":"2026-10-10"},{"model":"qwen3-5-397b-a17b","modelName":"Qwen3.5 397B-A17B","provider":"Alibaba (Qwen)","rawName":"Qwen3.5-397B-A17B","variant":"telecom","setting":"telecom domain, pass^1; effort: enabled; submitted by Sierra; v1.0.1","score":97.8,"unit":"%","rank":4,"rankComputedBySite":true,"date":"2026-02-27","sourceType":"official","source":"tau2","sourceUrl":"https://taubench.com/","collected":"2026-10-10"},{"model":"deepseek-v3-2","modelName":"DeepSeek-V3.2","provider":"DeepSeek","rawName":"DeepSeek-V3.2","variant":"telecom","setting":"telecom domain, pass^1; submitted by DeepSeek; vv0.1.3","score":96.2,"unit":"%","rank":5,"rankComputedBySite":true,"date":"2025-12-01","sourceType":"vendor","source":"tau2","sourceUrl":"https://huggingface.co/deepseek-ai/DeepSeek-V3.2/resolve/main/assets/paper.pdf","collected":"2026-10-10"},{"model":"gpt-5","modelName":"GPT-5","provider":"OpenAI","rawName":"GPT-5","variant":"telecom","setting":"telecom domain, pass^1; submitted by Sierra; v0.1.3","score":95.8,"unit":"%","rank":6,"rankComputedBySite":true,"date":"2025-08-09","sourceType":"official","source":"tau2","sourceUrl":"https://taubench.com/","collected":"2026-10-10"},{"model":"qwen3-max-thinking-preview","modelName":"Qwen3-Max-Thinking-Preview","provider":"Alibaba (Qwen)","rawName":"Qwen3-Max-Thinking-Preview","variant":"telecom","setting":"telecom domain, pass^1; submitted by Qwen; vv0.1.3","score":95.8,"unit":"%","rank":6,"rankComputedBySite":true,"date":"2025-10-30","sourceType":"vendor","source":"tau2","sourceUrl":"https://qwenlm.github.io/","collected":"2026-10-10"},{"model":"claude-opus-4-5","modelName":"Claude Opus 4.5","provider":"Anthropic","rawName":"Claude Opus 4.5","variant":"telecom","setting":"telecom domain, pass^1; effort: high; submitted by Sierra; v1.0.1","score":92.3,"unit":"%","rank":8,"rankComputedBySite":true,"date":"2026-05-05","sourceType":"official","source":"tau2","sourceUrl":"https://taubench.com/","collected":"2026-10-10"},{"model":"gemini-3-flash","modelName":"Gemini 3 Flash","provider":"Google","rawName":"Gemini 3 Flash","variant":"telecom","setting":"telecom domain, pass^1; effort: high; submitted by Sierra; v1.0.1","score":91.2,"unit":"%","rank":9,"rankComputedBySite":true,"date":"2026-03-02","sourceType":"official","source":"tau2","sourceUrl":"https://taubench.com/","collected":"2026-10-10"},{"model":"gemini-3-pro","modelName":"Gemini 3 Pro","provider":"Google","rawName":"Gemini 3 Pro","variant":"telecom","setting":"telecom domain, pass^1; effort: high; submitted by Sierra; v1.0.1","score":91,"unit":"%","rank":10,"rankComputedBySite":true,"date":"2026-03-02","sourceType":"official","source":"tau2","sourceUrl":"https://taubench.com/","collected":"2026-10-10"},{"model":"gpt-5-2","modelName":"GPT-5.2","provider":"OpenAI","rawName":"GPT-5.2","variant":"telecom","setting":"telecom domain, pass^1; effort: high; submitted by Sierra; v1.0.1","score":89.7,"unit":"%","rank":11,"rankComputedBySite":true,"date":"2026-05-05","sourceType":"official","source":"tau2","sourceUrl":"https://taubench.com/","collected":"2026-10-10"},{"model":"nemotron-orchestrator-8b","modelName":"Nemotron-Orchestrator-8B","provider":"NVIDIA","rawName":"Nemotron-Orchestrator-8B","variant":"telecom","setting":"telecom domain, pass^1; submitted by NVIDIA; v0.1.3","score":88.6,"unit":"%","rank":12,"rankComputedBySite":true,"date":"2025-09-15","sourceType":"vendor","source":"tau2","sourceUrl":"https://arxiv.org/abs/2511.21689","collected":"2026-10-10"},{"model":"glm-5","modelName":"GLM-5","provider":"Zhipu AI (Z.ai)","rawName":"GLM-5","variant":"telecom","setting":"telecom domain, pass^1; effort: enabled; submitted by Sierra; v1.0.1","score":86.8,"unit":"%","rank":13,"rankComputedBySite":true,"date":"2026-02-27","sourceType":"official","source":"tau2","sourceUrl":"https://taubench.com/","collected":"2026-10-10"},{"model":"claude-sonnet-4-5","modelName":"Claude Sonnet 4.5","provider":"Anthropic","rawName":"Claude Sonnet 4.5","variant":"telecom","setting":"telecom domain, pass^1; effort: enabled; submitted by Sierra; v1.0.1","score":84.9,"unit":"%","rank":14,"rankComputedBySite":true,"date":"2026-02-24","sourceType":"official","source":"tau2","sourceUrl":"https://taubench.com/","collected":"2026-10-10"},{"model":"qwen3-max","modelName":"Qwen3-Max","provider":"Alibaba (Qwen)","rawName":"Qwen3-Max","variant":"telecom","setting":"telecom domain, pass^1; submitted by Alibaba; v0.1.3","score":84.2,"unit":"%","rank":15,"rankComputedBySite":true,"date":"2024-09-23","sourceType":"official","source":"tau2","sourceUrl":"https://taubench.com/","collected":"2026-10-10"},{"model":"kimi-k2","modelName":"Kimi K2 Thinking","provider":"Moonshot AI","rawName":"Kimi-k2","variant":"telecom","setting":"telecom domain, pass^1; submitted by Moonshot AI; v0.1.3","score":65.8,"unit":"%","rank":16,"rankComputedBySite":true,"date":"2025-07-28","sourceType":"vendor","source":"tau2","sourceUrl":"https://arxiv.org/abs/2507.20534","collected":"2026-10-10"},{"model":"gpt-5-2","modelName":"GPT-5.2","provider":"OpenAI","rawName":"GPT-5.2","variant":"telecom","setting":"telecom domain, pass^1; effort: none; submitted by Sierra; v1.0.1","score":57.2,"unit":"%","rank":17,"rankComputedBySite":true,"date":"2026-05-26","sourceType":"official","source":"tau2","sourceUrl":"https://taubench.com/","collected":"2026-10-10"},{"model":"o4-mini","modelName":"o4-mini","provider":"OpenAI","rawName":"o4-mini","variant":"telecom","setting":"telecom domain, pass^1; submitted by Sierra; v0.1.3","score":50.2,"unit":"%","rank":18,"rankComputedBySite":true,"date":"2025-06-09","sourceType":"official","source":"tau2","sourceUrl":"https://taubench.com/","collected":"2026-10-10"},{"model":"claude-3-7-sonnet","modelName":"Claude 3.7 Sonnet","provider":"Anthropic","rawName":"Claude-3.7-Sonnet","variant":"telecom","setting":"telecom domain, pass^1; submitted by Sierra; v0.1.3","score":49,"unit":"%","rank":19,"rankComputedBySite":true,"date":"2025-06-09","sourceType":"official","source":"tau2","sourceUrl":"https://taubench.com/","collected":"2026-10-10"},{"model":"gpt-4-1-mini","modelName":"GPT-4.1 mini","provider":"OpenAI","rawName":"GPT-4.1-mini","variant":"telecom","setting":"telecom domain, pass^1; submitted by Sierra; v0.1.3","score":48.9,"unit":"%","rank":20,"rankComputedBySite":true,"date":"2025-06-09","sourceType":"official","source":"tau2","sourceUrl":"https://taubench.com/","collected":"2026-10-10"},{"model":"gpt-4-1","modelName":"GPT-4.1","provider":"OpenAI","rawName":"GPT-4.1","variant":"telecom","setting":"telecom domain, pass^1; submitted by Sierra; v0.1.3","score":34,"unit":"%","rank":21,"rankComputedBySite":true,"date":"2025-06-09","sourceType":"official","source":"tau2","sourceUrl":"https://taubench.com/","collected":"2026-10-10"}]}