{"generated":"2026-11-18","index_version":"2.0","tests":{"aider":{"name":"Aider Polyglot","tester":"Aider","area":"coding","scored":false,"url":"https://aider.chat/docs/leaderboards/","reason":"Gauthier runs models on 225 public Exercism exercises graded by unit tests, one harness. Edit format varies per model (diff/whole/diff-fenced), and 'architect' rows pair two models. Public static set, widely contaminated; leaderboard last updated Nov 2025 so no recent models. Whether community members contribute rows is unknown.","difficulty":0.015,"slope":4.494},"aime":{"name":"AIME-style maths (OTIS mock)","tester":"Epoch AI","area":"maths","scored":false,"url":"https://epoch.ai/benchmarks/about","reason":"Epoch-run, answer-extractor plus exact match against the key. The exams and answer key are public PDFs (static, 45 problems). 38 of 297 rows have no logs; one row (grok-4-0709) has no log and no run date.","difficulty":-0.056,"slope":6.105},"apex":{"name":"APEX-Agents","tester":"Mercor","area":"agents","scored":true,"url":"https://www.mercor.com/apex/apex-agents-leaderboard/","reason":"Mercor runs models on a private task set (public sample only) with LLM-judge over expert rubrics. Leaderboard shows both a Loop and a ReAct harness and the file does not say which; a GitHub issue reports reproduction gaps. Mercor sells data to labs (minor conflict).","difficulty":0.733,"slope":1.898},"arc:arc_agi_1":{"name":"ARC-AGI-1","tester":"ARC Prize","area":"novel","scored":false,"url":"https://arcprize.org/leaderboard","reason":"ARC Prize verifies every score itself on a semi-private set; standard harness only.","difficulty":0.245,"slope":5.417},"arc:arc_agi_2":{"name":"ARC-AGI-2","tester":"ARC Prize","area":"novel","scored":true,"url":"https://arcprize.org/leaderboard","reason":"ARC Prize verifies every score itself on a semi-private set; standard harness only.","difficulty":0.614,"slope":6.274},"arc:arc_agi_3":{"name":"ARC-AGI-3","tester":"ARC Prize","area":"novel","scored":false,"url":"https://arcprize.org/leaderboard","reason":"ARC Prize verifies every score itself on a semi-private set; standard harness only.","difficulty":1.28,"slope":6.931},"fm":{"name":"FrontierMath (tiers 1-3)","tester":"Epoch AI","area":"maths","scored":true,"url":"https://epoch.ai/benchmarks/frontiermath","reason":"Epoch-run, Python-function answers graded by exact match, 1M token cap. Private problem set; OpenAI funded it and has access to most problems (Epoch holds out a set), so OpenAI model rows are not perfectly independent. Logs are private by design, so checkability is partial. 23 of 114 rows have no logs link.","difficulty":0.506,"slope":3.341},"gpqa":{"name":"GPQA Diamond (Epoch AI)","tester":"Epoch AI","area":"reasoning","scored":true,"url":"https://epoch.ai/benchmarks/about","reason":"Epoch runs every model itself with one Inspect task (zero-shot CoT, 'ANSWER: X' parsing). Set is the public 198-question GPQA Diamond (canary string only), so contamination is a standing risk. 36 of 319 rows (all 2026 frontier rows) have no log link; one older row (grok-4-0709) has no log and no start date, so its provenance is unknown.","difficulty":-0.053,"slope":3.686},"hle":{"name":"Humanity's Last Exam","tester":"Scale AI","area":"reasoning","scored":true,"url":"https://labs.scale.com/leaderboard/humanitys_last_exam","reason":"Scale runs all models on the public 2,500 questions with one prompt and an o3-mini extractor/judge; a private held-out set exists but is not reported. Epoch's review marks HLE Flawed (46% of a sampled set had defects), so labels are noisy. File has no Source column; attribution to Scale is from Epoch's page and the matching columns.","difficulty":1.149,"slope":2.139},"livebench:agentic-coding":{"name":"LiveBench Agentic Coding","tester":"LiveBench","area":"coding","scored":false,"url":"https://livebench.ai","reason":"Fresh questions every release, graded against answers, run by the LiveBench team.","difficulty":0.585,"slope":1.004},"livebench:coding":{"name":"LiveBench Coding","tester":"LiveBench","area":"coding","scored":false,"url":"https://livebench.ai","reason":"Fresh questions every release, graded against answers, run by the LiveBench team.","difficulty":-0.666,"slope":0.932},"livebench:data-analysis":{"name":"LiveBench Data Analysis","tester":"LiveBench","area":"reasoning","scored":false,"url":"https://livebench.ai","reason":"Fresh questions every release, graded against answers, run by the LiveBench team.","difficulty":-0.607,"slope":0.848},"livebench:if":{"name":"LiveBench IF","tester":"LiveBench","area":"reasoning","scored":false,"url":"https://livebench.ai","reason":"Fresh questions every release, graded against answers, run by the LiveBench team.","difficulty":-0.258,"slope":0.743},"livebench:language":{"name":"LiveBench Language","tester":"LiveBench","area":"reasoning","scored":false,"url":"https://livebench.ai","reason":"Fresh questions every release, graded against answers, run by the LiveBench team.","difficulty":-0.627,"slope":1.019},"livebench:mathematics":{"name":"LiveBench Mathematics","tester":"LiveBench","area":"maths","scored":false,"url":"https://livebench.ai","reason":"Fresh questions every release, graded against answers, run by the LiveBench team.","difficulty":-1.035,"slope":1.241},"livebench:reasoning":{"name":"LiveBench Reasoning","tester":"LiveBench","area":"reasoning","scored":false,"url":"https://livebench.ai","reason":"Fresh questions every release, graded against answers, run by the LiveBench team.","difficulty":-0.617,"slope":1.316},"math5":{"name":"MATH (level 5)","tester":"Epoch AI","area":"maths","scored":false,"url":"https://epoch.ai/benchmarks/about","reason":"Epoch-run, same harness for all (8 runs). Public 2021 dataset, effectively saturated (~98%) and likely contaminated. Log viewer is withheld (copyright), though a Logs link exists for every row.","difficulty":-0.365,"slope":5.523},"metr":{"name":"METR time horizon","tester":"METR","area":"long","scored":true,"url":"https://metr.org/time-horizons/","reason":"METR runs the models itself on its own task suite with auto-scored tasks. Scaffold is chosen per model (ReAct, Triframe, Claude Code, Codex) and the suite changed (v1.0 vs v1.1; 23 rows have no version), so conditions are not perfectly uniform. Part of the suite is public (HCAST/RE-Bench), part private.","difficulty":2.252,"slope":0.611},"mmlu":{"name":"MMLU (HELM runs)","tester":"Stanford CRFM","area":"reasoning","scored":false,"url":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","reason":"HELM Lite runs MMLU itself, 5-shot, same prompts, published predictions. All other rows come from developers' papers (lab-reported). Public static set, saturated, last HELM Lite models are Dec 2024.","difficulty":-0.843,"slope":2.271},"scale:swe_bench_pro":{"name":"SWE-bench Pro (public set)","tester":"Scale AI","area":"coding","scored":false,"url":"https://labs.scale.com/leaderboard/swe_bench_pro","reason":"Scale AI runs every model itself with the same agent setup.","difficulty":0.7,"slope":1.465},"swe":{"name":"SWE-bench Verified","tester":"Epoch AI","area":"coding","scored":true,"url":"https://epoch.ai/benchmarks/swe-bench-verified","reason":"Epoch-run with one bash/editor/patch scaffold; unit-test grading. Public set, heavily contaminated per OpenAI's audit; Epoch's Benchmark Review marks it Flawed (>20% broken tasks, reward hack). Epoch upgraded the scaffold in Feb 2026, so older and newer rows are not strictly same-conditions.","difficulty":-0.176,"slope":1.416},"tb":{"name":"Terminal-Bench 2.0 (Terminus 2 runs)","tester":"Terminal-Bench team","area":"coding","scored":false,"url":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","reason":"Leaderboard entries are model + submitter-chosen agent (Codex CLI, Claude Code, Droid, ~45 custom agents), so most rows fail same-conditions. Terminus 2 is the benchmark's neutral reference agent and has 53 rows (many duplicated across two source labels), but they stop around Feb 2026, so newest models (GPT-5.4/5.5, Opus 4.7) have no Terminus 2 row. Public 89-task set; Epoch's review of the later 4.0 version marks it Flawed.","difficulty":0.528,"slope":2.533},"vals:biomysterybench":{"name":"BioMysteryBench","tester":"Vals AI","area":"agents","scored":false,"url":"https://www.vals.ai/benchmarks/biomysterybench","reason":"Vals AI runs every model itself with one harness; rows on a vendor's own harness are dropped.","difficulty":0.195,"slope":1.174},"vals:code-migration":{"name":"Code Migration","tester":"Vals AI","area":"coding","scored":false,"url":"https://www.vals.ai/benchmarks/code-migration","reason":"Vals AI runs every model itself with one harness; rows on a vendor's own harness are dropped.","difficulty":1.025,"slope":2.28},"vals:corp_fin_v2":{"name":"CorpFin v2","tester":"Vals AI","area":"work","scored":false,"url":"https://www.vals.ai/benchmarks/corp_fin_v2","reason":"Vals AI runs every model itself with one harness; rows on a vendor's own harness are dropped.","difficulty":-0.238,"slope":0.83},"vals:cyber":{"name":"CyberBench v1.1","tester":"Vals AI","area":"coding","scored":false,"url":"https://www.vals.ai/benchmarks/cyber","reason":"Vals AI runs every model itself with one harness; rows on a vendor's own harness are dropped.","difficulty":-0.163,"slope":0.741},"vals:emb":{"name":"EMB","tester":"Vals AI","area":"work","scored":false,"url":"https://www.vals.ai/benchmarks/emb","reason":"Vals AI runs every model itself with one harness; rows on a vendor's own harness are dropped.","difficulty":0.564,"slope":1.764},"vals:fabv2":{"name":"Finance Agent (v2)","tester":"Vals AI","area":"work","scored":true,"url":"https://www.vals.ai/benchmarks/fabv2","reason":"Vals AI runs every model itself with one harness; rows on a vendor's own harness are dropped.","difficulty":0.689,"slope":0.991},"vals:gpqa":{"name":"GPQA Diamond (Vals AI)","tester":"Vals AI","area":"reasoning","scored":false,"url":"https://www.vals.ai/benchmarks/gpqa","reason":"Vals AI runs every model itself with one harness; rows on a vendor's own harness are dropped.","difficulty":-0.151,"slope":2.453},"vals:hlab":{"name":"HLAB-Vals-1.1","tester":"Vals AI","area":"work","scored":false,"url":"https://www.vals.ai/benchmarks/hlab","reason":"Vals AI runs every model itself with one harness; rows on a vendor's own harness are dropped.","difficulty":3.336,"slope":0.857},"vals:ioi":{"name":"IOI","tester":"Vals AI","area":"coding","scored":false,"url":"https://www.vals.ai/benchmarks/ioi","reason":"Vals AI runs every model itself with one harness; rows on a vendor's own harness are dropped.","difficulty":0.734,"slope":3.168},"vals:lcb":{"name":"LiveCodeBench","tester":"Vals AI","area":"coding","scored":false,"url":"https://www.vals.ai/benchmarks/lcb","reason":"Vals AI runs every model itself with one harness; rows on a vendor's own harness are dropped.","difficulty":-0.293,"slope":2.3},"vals:legal_bench":{"name":"LegalBench","tester":"Vals AI","area":"work","scored":false,"url":"https://www.vals.ai/benchmarks/legal_bench","reason":"Vals AI runs every model itself with one harness; rows on a vendor's own harness are dropped.","difficulty":-1.309,"slope":0.898},"vals:legal_research":{"name":"Legal Research Bench","tester":"Vals AI","area":"work","scored":false,"url":"https://www.vals.ai/benchmarks/legal_research","reason":"Vals AI runs every model itself with one harness; rows on a vendor's own harness are dropped.","difficulty":1.31,"slope":1.22},"vals:math500":{"name":"MATH 500","tester":"Vals AI","area":"maths","scored":false,"url":"https://www.vals.ai/benchmarks/math500","reason":"Vals AI runs every model itself with one harness; rows on a vendor's own harness are dropped.","difficulty":-1.324,"slope":1.496},"vals:medcode":{"name":"MedCode","tester":"Vals AI","area":"work","scored":false,"url":"https://www.vals.ai/benchmarks/medcode","reason":"Vals AI runs every model itself with one harness; rows on a vendor's own harness are dropped.","difficulty":0.884,"slope":0.787},"vals:medscribe":{"name":"MedScribe","tester":"Vals AI","area":"work","scored":false,"url":"https://www.vals.ai/benchmarks/medscribe","reason":"Vals AI runs every model itself with one harness; rows on a vendor's own harness are dropped.","difficulty":-0.856,"slope":1.029},"vals:mgsm":{"name":"MGSM","tester":"Vals AI","area":"maths","scored":false,"url":"https://www.vals.ai/benchmarks/mgsm","reason":"Vals AI runs every model itself with one harness; rows on a vendor's own harness are dropped.","difficulty":-2.137,"slope":1.016},"vals:mmlu_pro":{"name":"MMLU Pro","tester":"Vals AI","area":"reasoning","scored":false,"url":"https://www.vals.ai/benchmarks/mmlu_pro","reason":"Vals AI runs every model itself with one harness; rows on a vendor's own harness are dropped.","difficulty":-0.806,"slope":1.378},"vals:mmmu":{"name":"MMMU Pro","tester":"Vals AI","area":"reasoning","scored":false,"url":"https://www.vals.ai/benchmarks/mmmu","reason":"Vals AI runs every model itself with one harness; rows on a vendor's own harness are dropped.","difficulty":-0.332,"slope":1.231},"vals:mortgage_tax":{"name":"MortgageTax","tester":"Vals AI","area":"work","scored":false,"url":"https://www.vals.ai/benchmarks/mortgage_tax","reason":"Vals AI runs every model itself with one harness; rows on a vendor's own harness are dropped.","difficulty":-0.435,"slope":0.672},"vals:mysterymechanism":{"name":"MysteryMechanism","tester":"Vals AI","area":"novel","scored":false,"url":"https://www.vals.ai/benchmarks/mysterymechanism","reason":"Vals AI runs every model itself with one harness; rows on a vendor's own harness are dropped.","difficulty":1.49,"slope":1.366},"vals:programbench":{"name":"ProgramBench","tester":"Vals AI","area":"coding","scored":false,"url":"https://www.vals.ai/benchmarks/programbench","reason":"Vals AI runs every model itself with one harness; rows on a vendor's own harness are dropped.","difficulty":4.187,"slope":1.071},"vals:proof_bench":{"name":"ProofBench v1.1","tester":"Vals AI","area":"maths","scored":false,"url":"https://www.vals.ai/benchmarks/proof_bench","reason":"Vals AI runs every model itself with one harness; rows on a vendor's own harness are dropped.","difficulty":0.712,"slope":4.236},"vals:public-benefits-bench":{"name":"Public Benefits Bench v1.1","tester":"Vals AI","area":"work","scored":false,"url":"https://www.vals.ai/benchmarks/public-benefits-bench","reason":"Vals AI runs every model itself with one harness; rows on a vendor's own harness are dropped.","difficulty":0.144,"slope":0.873},"vals:sage":{"name":"SAGE","tester":"Vals AI","area":"work","scored":false,"url":"https://www.vals.ai/benchmarks/sage","reason":"Vals AI runs every model itself with one harness; rows on a vendor's own harness are dropped.","difficulty":-0.699,"slope":1.384},"vals:skillsbench":{"name":"SkillsBench","tester":"Vals AI","area":"agents","scored":false,"url":"https://www.vals.ai/benchmarks/skillsbench","reason":"Vals AI runs every model itself with one harness; rows on a vendor's own harness are dropped.","difficulty":0.553,"slope":0.973},"vals:srebench":{"name":"SRE Bench","tester":"Vals AI","area":"agents","scored":false,"url":"https://www.vals.ai/benchmarks/srebench","reason":"Vals AI runs every model itself with one harness; rows on a vendor's own harness are dropped.","difficulty":1.732,"slope":2.056},"vals:swebench":{"name":"SWE-bench","tester":"Vals AI","area":"coding","scored":false,"url":"https://www.vals.ai/benchmarks/swebench","reason":"Vals AI runs every model itself with one harness; rows on a vendor's own harness are dropped.","difficulty":-0.402,"slope":1.204},"vals:tax_agent_bench":{"name":"Tax Agent Bench","tester":"Vals AI","area":"work","scored":false,"url":"https://www.vals.ai/benchmarks/tax_agent_bench","reason":"Vals AI runs every model itself with one harness; rows on a vendor's own harness are dropped.","difficulty":1.687,"slope":1.191},"vals:tax_eval_v2":{"name":"TaxEval v2","tester":"Vals AI","area":"work","scored":false,"url":"https://www.vals.ai/benchmarks/tax_eval_v2","reason":"Vals AI runs every model itself with one harness; rows on a vendor's own harness are dropped.","difficulty":-1.018,"slope":0.718},"vals:terminal-bench-4":{"name":"Terminal-Bench 4.0","tester":"Vals AI","area":"coding","scored":true,"url":"https://www.vals.ai/benchmarks/terminal-bench-4","reason":"Vals AI runs every model itself with one harness; rows on a vendor's own harness are dropped.","difficulty":1.204,"slope":2.686},"vals:terminal-bench-science":{"name":"Terminal-Bench-Science","tester":"Vals AI","area":"agents","scored":false,"url":"https://www.vals.ai/benchmarks/terminal-bench-science","reason":"Vals AI runs every model itself with one harness; rows on a vendor's own harness are dropped.","difficulty":1.364,"slope":3.357},"vals:vcb-1-100":{"name":"Vibe Code Bench 1-100","tester":"Vals AI","area":"coding","scored":false,"url":"https://www.vals.ai/benchmarks/vcb-1-100","reason":"Vals AI runs every model itself with one harness; rows on a vendor's own harness are dropped.","difficulty":2.296,"slope":1.031},"vals:vibe-code":{"name":"Vibe Code Bench v1.1","tester":"Vals AI","area":"coding","scored":false,"url":"https://www.vals.ai/benchmarks/vibe-code","reason":"Vals AI runs every model itself with one harness; rows on a vendor's own harness are dropped.","difficulty":0.598,"slope":4.378},"x:Balrog":{"name":"BALROG","tester":"BALROG team","area":"agents","scored":false,"url":"https://balrogai.com/","reason":"Default zero-shot BALROG harness across six game environments, progress scored by the games. Leaderboard also accepts 'agentic strategy' submissions (not model rows); 11 old rows have no trajectories. Environments are public but procedurally generated.","difficulty":0.669,"slope":1.307},"x:CL-bench":{"name":"CL-bench","tester":"Tencent Hunyuan / Fudan","area":"reasoning","scored":false,"url":"https://github.com/Tencent-Hunyuan/CL-bench","reason":"Authors run all models; task passes only if all expert rubrics pass, judged by an LLM. Public dataset. Tencent is a model lab but none of its models are in the rows I saw. Run conditions not fully verified.","difficulty":2.062,"slope":0.91},"x:CadEval":{"name":"CadEval","tester":"Will Patrick","area":"coding","scored":false,"url":"https://github.com/wgpatrick/cadeval","reason":"Individual author runs models through one OpenSCAD pipeline; pass/fail via render and geometry checks (Chamfer, Hausdorff, volume). Public prompts. Small, 14 rows, mostly 2025 models. Creator identified from the repo, not confirmed by Epoch.","difficulty":0.073,"slope":2.544},"x:Chess Puzzles":{"name":"Chess puzzles","tester":"Epoch AI","area":"maths","scored":false,"url":"https://epoch.ai/benchmarks/chess-puzzles","reason":"Epoch generated 100 novel unpublished positions; answer extracted by a model, then exact match to Stockfish's single best move. Good anti-gaming. 39 of 227 rows (recent) have no logs.","difficulty":0.954,"slope":2.398},"x:DTBench":{"name":"DTBench","tester":"Redwood Research","area":"agents","scored":false,"url":"https://alignment.anthropic.com/2026/conceptual-reasoning-index/","reason":"Redwood runs all models at max effort/tokens; 407 expert-checked multiple-choice decision-theory questions. Built in collaboration with Anthropic, whose models lead; the conflict is real but Anthropic is not the tester. Dataset public/private status not confirmed.","difficulty":-0.05,"slope":2.648},"x:DeepResearch Bench":{"name":"DeepResearch Bench","tester":"FutureSearch","area":"agents","scored":false,"url":"https://futuresearch.ai/deep-research-bench/","reason":"FutureSearch runs models in a ReAct agent over a frozen web snapshot (RetroSearch); 91 tasks, most held out; scored by binary/F1 against curated answers. One row is a commercial agent product (Agent == True).","difficulty":0.703,"slope":0.606},"x:DeepSWE":{"name":"DeepSWE","tester":"Datacurve","area":"coding","scored":false,"url":"https://deepswe.datacurve.ai/","reason":"Datacurve runs all models in mini-swe-agent at stated reasoning effort; hand-written verifiers, tasks written from scratch (canary GUID, so public). Epoch's Benchmark Review marks v1.1 Flawed (>20% of tasks have verifier/spec issues). Datacurve sells data to labs, a minor conflict.","difficulty":0.662,"slope":2.517},"x:EBR-bench":{"name":"EBR-bench","tester":"Epoch AI","area":"reasoning","scored":false,"url":"https://epoch.ai/benchmarks/ebr-bench","reason":"Epoch-created Earthborne Rangers game benchmark, run by Epoch with a ReAct harness; scored on in-game objectives (programmatic). Eight learning playthroughs then two scored. No logs for any of 24 rows (transcripts unpublished). Held-out status not stated.","difficulty":1.077,"slope":3.352},"x:Fiction.LiveBench":{"name":"Fiction.LiveBench","tester":"Fiction.live","area":"reasoning","scored":false,"url":"https://epoch.ai/benchmarks/fictionlivebench","reason":"Single operator runs every model on 36 questions about 30 stories at several context lengths. I could not determine whether questions are private or how answers are graded (assumed answer-key comprehension questions).","difficulty":-0.139,"slope":3.276},"x:FrontierMath-2025-02-28-Private":{"name":"FrontierMath (2025 set)","tester":"Epoch AI","area":"maths","scored":false,"url":"https://epoch.ai/benchmarks/frontiermath","reason":"Epoch-run, exact-answer grading. Private problem set; OpenAI funded it and has access to most problems (Epoch holds out a set), so OpenAI model rows are not perfectly independent. Logs are private by design, so checkability is partial. Superseded by Tiers 1-3 v2, which Epoch says fixed answer errors in 42% of problems, so v1 scores are noisier.","difficulty":0.847,"slope":2.977},"x:FrontierMath-Tier-4-2025-07-01-Private":{"name":"FrontierMath tier 4 (2025 set)","tester":"Epoch AI","area":"maths","scored":false,"url":"https://epoch.ai/benchmarks/frontiermath","reason":"Epoch-run, exact-answer grading. Private problem set; OpenAI funded it and has access to most problems (Epoch holds out a set), so OpenAI model rows are not perfectly independent. Logs are private by design, so checkability is partial. Superseded by Tier 4 v2.","difficulty":1.301,"slope":2.668},"x:FrontierMath-Tier-4-v2-Private":{"name":"FrontierMath (tier 4)","tester":"Epoch AI","area":"maths","scored":false,"url":"https://epoch.ai/benchmarks/frontiermath","reason":"Epoch-run, exact-answer grading. Private problem set; OpenAI funded it and has access to most problems (Epoch holds out a set), so OpenAI model rows are not perfectly independent. Logs are private by design, so checkability is partial. Epoch reports Tier 4 v2 as saturated (every problem solved at least once). Contains a Google DeepMind multi-agent system row that Epoch ran blind through the UI.","difficulty":0.834,"slope":4.778},"x:FrontierSWE":{"name":"FrontierSWE","tester":"Proximal","area":"coding","scored":false,"url":"https://www.frontierswe.com/","reason":"Proximal runs every model in its own minimal 'proximus' harness, 5 trials, 20 h per task, mean score. Whether tasks are private and whether grading is tests or judge is not documented on the pages I could read. One model used a fallback model on content-filtered tasks.","difficulty":1.17,"slope":2.868},"x:Furniture Assembly":{"name":"Furniture assembly","tester":"Epoch AI","area":"agents","scored":false,"url":"https://epoch.ai/benchmarks/furniture-assembly","reason":"Epoch-created, 60 public IKEA photos, answer key plus an LLM grader (GPT-5.6 Sol, an OpenAI model) for describing mistakes. Public static set; no logs on any of 29 rows. Brand new (Sept 2026).","difficulty":1.181,"slope":4.442},"x:GSM8K":{"name":"GSM8K (HELM runs)","tester":"Stanford CRFM","area":"maths","scored":false,"url":"https://crfm.stanford.edu/helm/classic/latest/#/groups/gsm","reason":"Only the 11 HELM Classic rows (models up to mid-2023) are independent and same-harness; all others are lab papers. Public, saturated, contaminated.","difficulty":-0.599,"slope":6.068},"x:GSO-Bench":{"name":"GSO-Bench","tester":"GSO team","area":"coding","scored":false,"url":"https://gso-bench.github.io/index.html","reason":"All models in the OpenHands scaffold, graded by correctness tests plus speedup versus the human commit; hack detector (adjusted scores available). 102 public tasks. Unknown whether Berkeley runs every model or accepts submissions; the leaderboard says 'unless otherwise specified'.","difficulty":0.878,"slope":4.388},"x:GeoBench":{"name":"GeoBench","tester":"GeoBench team","area":"reasoning","scored":false,"url":"https://epoch.ai/benchmarks/geobench","reason":"Same prompt and settings, scored by distance to the true location; maps are public GeoGuessr community maps, so static and contaminable. Two rows used search tools.","difficulty":-0.798,"slope":0.803},"x:LMCA":{"name":"LMCA","tester":"Redwood Research","area":"agents","scored":false,"url":"https://alignment.anthropic.com/2026/conceptual-reasoning-index/","reason":"Score is correlation between a model's argument ratings and human expert ratings (noisy ceiling ~91). Dataset access is restricted, which is good for anti-gaming but limits third-party checks. Same Anthropic collaboration caveat as DTBench; Fable 5 used a fallback model on refusals.","difficulty":0.759,"slope":1.565},"x:MirrorCode":{"name":"MirrorCode","tester":"Epoch AI","area":"coding","scored":false,"url":"https://epoch.ai/benchmarks/mirrorcode","reason":"Co-developed with METR, run by Epoch with a simple Inspect ReAct scaffold; graded by tests, with ~34% of tests withheld from agents and 3 of 25 programs kept private. All 9 rows have no logs.","difficulty":0.871,"slope":2.158},"x:Mystery Game Puzzles":{"name":"Mystery game puzzles","tester":"Epoch AI","area":"agents","scored":false,"url":"https://epoch.ai/benchmarks/mystery-game-puzzles","reason":"Epoch-created, game identity deliberately secret, 100 programmatically generated positions graded against an answer key. Prompts/transcripts withheld by design, so third parties cannot check; 43 of 135 rows have no logs.","difficulty":1.067,"slope":3.477},"x:SimpleBench":{"name":"SimpleBench","tester":"SimpleBench team","area":"reasoning","scored":false,"url":"https://simple-bench.com/","reason":"Same prompt, 5 runs per question at fixed temperature; the large majority of the 213 questions are private. LM Council rows (25 in the index) are re-hosted by the same curator, not an independent source. Because questions are private, third parties cannot audit.","difficulty":0.559,"slope":2.224},"x:SimpleQA Verified":{"name":"SimpleQA Verified","tester":"Epoch AI","area":"reasoning","scored":false,"url":"https://epoch.ai/benchmarks/simpleqa-verified","reason":"Epoch runs Google's SimpleQA Verified with an LLM grader against the reference answer. Public 1,000-question set. Epoch's own Benchmark Reviews list it as Flawed (second-hand report). 22 recent rows have no logs.","difficulty":0.738,"slope":1.742},"x:Surface Evolver Bench":{"name":"Surface Evolver Bench","tester":"Yann Henon","area":"maths","scored":false,"url":"https://epoch.ai/benchmarks/surface-evolver-bench","reason":"Deterministic grading with hidden reference simulation and physical quantities; one run per task so noisy. Whether the author ran every model in one harness is not stated. New (June 2026).","difficulty":0.465,"slope":2.924},"x:VPCT":{"name":"VPCT","tester":"Chase Brower","area":"reasoning","scored":false,"url":"https://epoch.ai/benchmarks/vpct","reason":"Single prompt, public code, 100 ramp problems graded as right or wrong; run counts vary per model (1 to 3). Public problem set; whether a hidden split exists is unknown. One row comes from a creator tweet.","difficulty":0.535,"slope":5.458},"x:WeirdML":{"name":"WeirdML","tester":"Havard T. Ihle","area":"coding","scored":false,"url":"https://htihle.github.io/weirdml.html","reason":"Single author runs all models on the same Docker/GPU pipeline, 5 iterations with feedback, accuracy measured on the task's test data. Tasks are in a public repo; whether any are private is unknown. v2 only (v3 is not in the index).","difficulty":0.294,"slope":2.027}},"areas":{"reasoning":"Reasoning & knowledge","maths":"Maths","coding":"Coding","agents":"Agents","work":"Professional work","novel":"Novel problems","long":"Long tasks"},"excluded_lab_reported":337,"method":{"version":"2.0","cap":2,"validation":{"by_cap":{"2":{"held_out_testers_mae":6.93,"median":5.43,"n":1470,"mae_2026_models":6.69,"by_tester":{"ARC Prize":[8.09,76],"Aider":[9.97,48],"BALROG team":[5.71,33],"Chase Brower":[6.56,26],"Datacurve":[8.81,26],"Epoch AI":[9.26,208],"Fiction.live":[10.42,52],"FutureSearch":[5.46,23],"GSO team":[6.19,30],"GeoBench team":[10.85,44],"Havard T. Ihle":[6.28,129],"LiveBench":[3.6,45],"METR":[1.91,36],"Mercor":[7.21,62],"Proximal":[5.76,25],"Redwood Research":[3.76,176],"Scale AI":[6.53,51],"SimpleBench team":[6.94,106],"Stanford CRFM":[7.21,49],"Tencent Hunyuan / Fudan":[2.77,19],"Terminal-Bench team":[7.43,33],"Vals AI":[6.23,133],"Will Patrick":[7.11,14],"Yann Henon":[11.7,26]}},"3":{"held_out_testers_mae":6.89,"median":5.29,"n":1470,"mae_2026_models":6.62,"by_tester":{"ARC Prize":[7.88,76],"Aider":[10.16,48],"BALROG team":[5.68,33],"Chase Brower":[6.39,26],"Datacurve":[8.69,26],"Epoch AI":[9.17,208],"Fiction.live":[10.49,52],"FutureSearch":[5.5,23],"GSO team":[6.19,30],"GeoBench team":[10.83,44],"Havard T. Ihle":[6.26,129],"LiveBench":[3.57,45],"METR":[1.91,36],"Mercor":[7.22,62],"Proximal":[5.54,25],"Redwood Research":[3.76,176],"Scale AI":[6.45,51],"SimpleBench team":[6.9,106],"Stanford CRFM":[7.29,49],"Tencent Hunyuan / Fudan":[2.82,19],"Terminal-Bench team":[7.4,33],"Vals AI":[6.09,133],"Will Patrick":[7.23,14],"Yann Henon":[11.84,26]}},"4":{"held_out_testers_mae":6.88,"median":5.34,"n":1470,"mae_2026_models":6.58,"by_tester":{"ARC Prize":[7.8,76],"Aider":[10.33,48],"BALROG team":[5.69,33],"Chase Brower":[6.36,26],"Datacurve":[8.64,26],"Epoch AI":[9.13,208],"Fiction.live":[10.58,52],"FutureSearch":[5.51,23],"GSO team":[6.11,30],"GeoBench team":[10.81,44],"Havard T. Ihle":[6.29,129],"LiveBench":[3.57,45],"METR":[1.92,36],"Mercor":[7.17,62],"Proximal":[5.42,25],"Redwood Research":[3.77,176],"Scale AI":[6.39,51],"SimpleBench team":[6.88,106],"Stanford CRFM":[7.38,49],"Tencent Hunyuan / Fudan":[2.86,19],"Terminal-Bench team":[7.32,33],"Vals AI":[6.01,133],"Will Patrick":[7.42,14],"Yann Henon":[11.9,26]}},"6":{"held_out_testers_mae":6.88,"median":5.34,"n":1470,"mae_2026_models":6.55,"by_tester":{"ARC Prize":[7.7,76],"Aider":[10.5,48],"BALROG team":[5.71,33],"Chase Brower":[6.34,26],"Datacurve":[8.51,26],"Epoch AI":[9.13,208],"Fiction.live":[10.64,52],"FutureSearch":[5.56,23],"GSO team":[6.15,30],"GeoBench team":[10.81,44],"Havard T. Ihle":[6.35,129],"LiveBench":[3.58,45],"METR":[1.95,36],"Mercor":[7.1,62],"Proximal":[5.34,25],"Redwood Research":[3.8,176],"Scale AI":[6.3,51],"SimpleBench team":[6.91,106],"Stanford CRFM":[7.51,49],"Tencent Hunyuan / Fudan":[2.88,19],"Terminal-Bench team":[7.29,33],"Vals AI":[5.94,133],"Will Patrick":[7.59,14],"Yann Henon":[11.95,26]}},"10":{"held_out_testers_mae":6.9,"median":5.38,"n":1470,"mae_2026_models":6.55,"by_tester":{"ARC Prize":[7.67,76],"Aider":[10.59,48],"BALROG team":[5.75,33],"Chase Brower":[6.39,26],"Datacurve":[8.38,26],"Epoch AI":[9.13,208],"Fiction.live":[10.77,52],"FutureSearch":[5.58,23],"GSO team":[6.26,30],"GeoBench team":[10.8,44],"Havard T. Ihle":[6.39,129],"LiveBench":[3.6,45],"METR":[1.96,36],"Mercor":[7.01,62],"Proximal":[5.2,25],"Redwood Research":[3.82,176],"Scale AI":[6.23,51],"SimpleBench team":[6.96,106],"Stanford CRFM":[7.6,49],"Tencent Hunyuan / Fudan":[2.91,19],"Terminal-Bench team":[7.38,33],"Vals AI":[5.9,133],"Will Patrick":[7.6,14],"Yann Henon":[12.05,26]}},"1000":{"held_out_testers_mae":6.92,"median":5.32,"n":1470,"mae_2026_models":6.57,"by_tester":{"ARC Prize":[7.77,76],"Aider":[10.63,48],"BALROG team":[5.82,33],"Chase Brower":[6.54,26],"Datacurve":[8.27,26],"Epoch AI":[9.16,208],"Fiction.live":[10.77,52],"FutureSearch":[5.59,23],"GSO team":[6.14,30],"GeoBench team":[10.83,44],"Havard T. Ihle":[6.45,129],"LiveBench":[3.61,45],"METR":[1.96,36],"Mercor":[6.91,62],"Proximal":[5.12,25],"Redwood Research":[3.86,176],"Scale AI":[6.15,51],"SimpleBench team":[7.02,106],"Stanford CRFM":[7.7,49],"Tencent Hunyuan / Fudan":[2.94,19],"Terminal-Bench team":[7.5,33],"Vals AI":[5.88,133],"Will Patrick":[7.6,14],"Yann Henon":[12.14,26]}}},"chosen_cap":2,"held_out_results_mae":6.12,"held_out_results_median":4.32},"testers":{"ARC Prize":3,"Aider":1,"BALROG team":1,"Chase Brower":1,"Datacurve":1,"Epoch AI":14,"Fiction.live":1,"FutureSearch":1,"GSO team":1,"GeoBench team":1,"Havard T. Ihle":1,"LiveBench":7,"METR":1,"Mercor":1,"Proximal":1,"Redwood Research":2,"Scale AI":2,"SimpleBench team":1,"Stanford CRFM":2,"Tencent Hunyuan / Fudan":1,"Terminal-Bench team":1,"Vals AI":33,"Will Patrick":1,"Yann Henon":1},"sources_reviewed":"2026-10-05","rule":["Independent: someone other than the model's lab ran or verified it.","Same conditions for every model: one published setup; self-submitted leaderboards don't count.","Checkable: per-model results are published.","Hard to game: private or refreshed questions where possible.","Measures correctness: graded against answers, tests or task success, not preference."],"excluded":[{"name":"ANLI","tester":"Model developers' own papers/model cards","reason":"All 9 rows come from the Phi-3 report; lab-reported."},{"name":"ARC AI2","tester":"Model developers' own papers/model cards","reason":"All rows are numbers from developers' papers (Qwen, Falcon, Llama 2, Phi, etc.); no independent runs. Old and saturated."},{"name":"ARC-AGI-1 (Epoch's copy)","tester":"ARC Prize","reason":"ARC Prize's own results are read directly from its leaderboard; Epoch's copy would count them twice."},{"name":"ARC-AGI-2 (Epoch's copy)","tester":"ARC Prize","reason":"ARC Prize's own results are read directly from its leaderboard; Epoch's copy would count them twice."},{"name":"BBH","tester":"Model developers' own papers/model cards","reason":"All rows are developer papers (Qwen, Yi, Baichuan, Llama 2, DeepSeek, Nemotron, Gemma, Phi). Public and saturated."},{"name":"FrontierCode","tester":"Cognition","reason":"Each model is scored in its own vendor harness (claude-code, codex, grok-build, devin, chisel, cursor-cli), so harness and model are confounded. Cognition also makes models on the board (SWE-1.6, 1.7, 2) and the Devin harness. Tasks private; grading is a mix of tests and maintainer rubrics. Only a mini-swe-agent subset (open-weight models only) would be same-conditions, and it has no frontier anchor."},{"name":"GDPval","tester":"OpenAI","reason":"Created and graded by OpenAI (blinded expert pairwise vs a human deliverable), so the tester is a lab with models on the board; only 11 rows, last mid-model-era (GPT-5.2). Pairwise vs human, not an absolute answer key. Judgment call: independence is the main failure; grading by experts is otherwise sound."},{"name":"HellaSwag","tester":"Model developers' own papers/model cards","reason":"In-index rows are all developer-paper numbers (HELM rows exist in the file but none are in the index). Public, saturated."},{"name":"LAMBADA","tester":"Model developers' own papers/model cards","reason":"Rows from Qwen and Falcon reports and similar papers; lab-reported, saturated."},{"name":"Lech Mazur Writing","tester":"Lech Mazur","reason":"Scores are LLM-panel judgments of story quality on a 10-element creative brief; there is no right answer, so it is closer to preference than correctness. Frozen V1 leaderboard (Aug 2025). Judgment call; the rubric exists, so it could be argued either way."},{"name":"OSWorld","tester":"Submitting teams (agent builders); OSWorld maintainers verify","reason":"Each leaderboard row is a team's own agent plus model, with step budgets varying (15/50/100); several rows are open agents with no named base model and one is from an Anthropic announcement. Public static 369-task set. Only 8 rows are in the index."},{"name":"OpenBookQA","tester":"Model developers' own papers/model cards","reason":"Rows from developers' papers; lab-reported, saturated."},{"name":"PIQA","tester":"Model developers' own papers/model cards","reason":"Rows from developers' papers; lab-reported, saturated."},{"name":"Poker Agent (Vals AI)","tester":"Vals AI","reason":"Scored against the other models at the table, so the opposition changes with who is entered: not the same conditions for every model."},{"name":"ProofBench (Epoch's copy)","tester":"Vals AI","reason":"Vals AI's own ProofBench results are read directly from Vals; Epoch's copy would count them twice."},{"name":"Public Benefits Bench v1 (Vals AI)","tester":"Vals AI","reason":"Superseded by its newer version."},{"name":"Remote Labor Index","tester":"Scale AI + CAIS","reason":"Scale/CAIS run it with human expert graders on a private 230-project set (good), but agents differ per row (OpenHands CLI, computer-use, vendor agents like Codex or CoWork) and the board reports each agent's best environment. Row names confirm a mix. Could be reconsidered if Scale states a single harness."},{"name":"ScienceQA","tester":"Leaderboard submitters / developers' papers","reason":"Rows are submissions by model builders to the ScienceQA board or numbers from papers (Phi-3, Falcon2, VisionLLM v2); no common harness or independent tester. Only 6 rows in the index."},{"name":"Terminal-Bench 2.0 (Vals AI run)","tester":"Vals AI","reason":"Superseded by Terminal-Bench 2.1 and 4.0."},{"name":"TriviaQA","tester":"Model developers' own papers/model cards","reason":"Rows from Llama 2, Gemma, Phi-3 and similar papers plus one PapersWithCode row; lab-reported."},{"name":"Voice Code Bench (Vals AI)","tester":"Vals AI","reason":"Mixes speech recognition into the result; out of scope for a text capability index."},{"name":"Winogrande","tester":"Model developers' own papers/model cards","reason":"Rows from developers' papers; lab-reported, saturated."}],"too_few_models":["CL-bench Life (14)","CUA-bench (3)","Cybench (12)","ExploitBench (9)","OSWorld 2.0 (9)","PostTrainBench (11)","The Agent Company (11)","Time Horizon Index: KSP (13)"],"excluded_lab_reported":337},"models":[{"id":"chatgpt","name":"ChatGPT","lab":"OpenAI","date":"2022-11-30","type":"flagship","major":true,"open":false,"release":{"title":"ChatGPT (GPT-3.5)","source":"https://openai.com/index/chatgpt/","verified":true,"confidence":"high"}},{"id":"llama","name":"LLaMA","lab":"Meta","date":"2023-02-24","type":"open-weights","major":true,"open":true,"release":{"title":"LLaMA (1)","source":"https://en.wikipedia.org/wiki/Llama_(language_model)","verified":false,"confidence":"high"},"index":{"score":97,"lo":80.9,"hi":113.1,"se":2.61,"solve":11.9,"status":"insufficient","areas":["maths"],"core_areas":[],"testers":["Stanford CRFM"],"tests":{"x:GSM8K":{"value":0.466,"norm":0.466,"version":"LLaMA-65B"}},"variant":"LLaMA-65B","match":"family"}},{"id":"llama-13b","name":"LLaMA-13B","lab":"Meta","date":"2023-02-24","type":"open-weights","major":false,"open":true,"index":{"score":84.6,"lo":67.2,"hi":101.9,"se":4.74,"solve":9.6,"status":"insufficient","areas":["maths"],"core_areas":[],"testers":["Stanford CRFM"],"tests":{"x:GSM8K":{"value":0.154,"norm":0.154,"version":"LLaMA-13B"}},"variant":"LLaMA-13B","match":"self"},"date_from":"epoch"},{"id":"llama-33b","name":"LLaMA-33B","lab":"Meta","date":"2023-02-24","type":"open-weights","major":false,"open":true,"index":{"score":92,"lo":75.7,"hi":108.2,"se":2.97,"solve":10.9,"status":"insufficient","areas":["maths"],"core_areas":[],"testers":["Stanford CRFM"],"tests":{"x:GSM8K":{"value":0.32,"norm":0.32,"version":"LLaMA-33B"}},"variant":"LLaMA-33B","match":"self"},"date_from":"epoch"},{"id":"llama-7b","name":"LLaMA-7B","lab":"Meta","date":"2023-02-24","type":"open-weights","major":false,"open":true,"index":{"score":79.7,"lo":60,"hi":99.4,"se":7.37,"solve":8.9,"status":"insufficient","areas":["maths"],"core_areas":[],"testers":["Stanford CRFM"],"tests":{"x:GSM8K":{"value":0.08,"norm":0.08,"version":"LLaMA-7B"}},"variant":"LLaMA-7B","match":"self"},"date_from":"epoch"},{"id":"chatglm-6b","name":"ChatGLM-6B","lab":"Zhipu","date":"2023-03-14","type":"open-weights","major":true,"open":true,"release":{"title":"ChatGLM-6B","source":"https://huggingface.co/zai-org/chatglm-6b","verified":true,"confidence":"high"}},{"id":"claude","name":"Claude","lab":"Anthropic","date":"2023-03-14","type":"flagship","major":true,"open":false,"release":{"title":"Claude (1)","source":"https://www.anthropic.com/news/introducing-claude","verified":true,"confidence":"high"}},{"id":"gpt-4","name":"GPT-4","lab":"OpenAI","date":"2023-03-14","type":"flagship","major":true,"open":false,"release":{"title":"GPT-4","source":"https://openai.com/index/gpt-4-research/","verified":true,"confidence":"high"},"index":{"score":100,"lo":100,"hi":100,"se":9.42,"solve":12.6,"status":"rated","areas":["long","maths","reasoning"],"core_areas":["long","reasoning"],"testers":["Epoch AI","METR"],"tests":{"metr":{"value":0.361067,"norm":0.1834,"version":"gpt-4-0314"},"aime":{"value":0.005555555555555556,"norm":0.0056,"version":"gpt-4-0314"},"gpqa":{"value":0.3573232323232323,"norm":0.3573,"version":"gpt-4-0314"}},"variant":"GPT-4 (Mar 2023)","match":"exact"}},{"id":"qwen","name":"Qwen","lab":"Alibaba","date":"2023-04-11","type":"flagship","major":true,"open":false,"release":{"title":"Qwen / Tongyi Qianwen (beta)","source":"https://en.wikipedia.org/wiki/Qwen","verified":false,"confidence":"high"}},{"id":"tongyi-qianwen","name":"Tongyi Qianwen","lab":"Alibaba","date":"2023-04-11","type":"flagship","major":true,"open":false,"release":{"title":"Qwen / Tongyi Qianwen (beta)","source":"https://en.wikipedia.org/wiki/Qwen","verified":false,"confidence":"high"}},{"id":"palm-2","name":"PaLM 2","lab":"Google","date":"2023-05-10","type":"flagship","major":true,"open":false,"release":{"title":"PaLM 2","source":"https://blog.google/innovation-and-ai/products/google-palm-2-ai-large-language-model/","verified":true,"confidence":"high"}},{"id":"gpt-3-5-turbo-jun-2023","name":"GPT-3.5 Turbo (Jun 2023)","lab":"OpenAI","date":"2023-06-13","type":"flagship","major":false,"open":false,"index":{"score":96.9,"lo":80.8,"hi":112.9,"se":2.53,"solve":11.9,"status":"provisional","areas":["maths","reasoning"],"core_areas":[],"testers":["Stanford CRFM"],"tests":{"mmlu":{"value":0.689,"norm":0.689,"version":"gpt-3.5-turbo-0613"},"x:GSM8K":{"value":0.469,"norm":0.469,"version":"gpt-3.5-turbo-0613"}},"variant":"GPT-3.5 Turbo (Jun 2023)","match":"self"},"date_from":"epoch"},{"id":"gpt-4-jun-2023","name":"GPT-4 (Jun 2023)","lab":"OpenAI","date":"2023-06-13","type":"flagship","major":false,"open":false,"index":{"score":101.8,"lo":85.1,"hi":118.5,"se":3.78,"solve":13.1,"status":"rated","areas":["agents","coding","maths","reasoning"],"core_areas":["reasoning"],"testers":["Epoch AI","Havard T. Ihle","Redwood Research","Stanford CRFM"],"tests":{"x:LMCA":{"value":17.06,"norm":0.1706,"version":"gpt-4-0613"},"mmlu":{"value":0.824,"norm":0.824,"version":"gpt-4-0613"},"x:Mystery Game Puzzles":{"value":0.12,"norm":0.12,"version":"gpt-4-0613"},"gpqa":{"value":0.30650252525252525,"norm":0.3065,"version":"gpt-4-0613"},"math5":{"value":0.22970166163141995,"norm":0.2297,"version":"gpt-4-0613"},"x:DTBench":{"value":0.6267,"norm":0.6267,"version":"gpt-4-0613"},"aime":{"value":0.011111111111111112,"norm":0.0111,"version":"gpt-4-0613"},"x:WeirdML":{"value":0.1236,"norm":0.1236,"version":"gpt-4-0613"}},"variant":"GPT-4 (Jun 2023)","match":"self"},"api":{"or_id":"openai/gpt-4","in":30,"out":60,"context":8191,"hf":null,"created":"2023-05-28"},"date_from":"epoch"},{"id":"claude-2","name":"Claude 2","lab":"Anthropic","date":"2023-07-11","type":"flagship","major":true,"open":false,"release":{"title":"Claude 2","source":"https://www.anthropic.com/news/claude-2","verified":true,"confidence":"high"},"index":{"score":94.1,"lo":75.7,"hi":112.5,"se":6.05,"solve":11.3,"status":"rated","areas":["agents","maths","reasoning"],"core_areas":["reasoning"],"testers":["Epoch AI","Redwood Research"],"tests":{"math5":{"value":0.1172583081570997,"norm":0.1173,"version":"claude-2.0"},"aime":{"value":0.025,"norm":0.025,"version":"claude-2.0"},"gpqa":{"value":0.3465909090909091,"norm":0.3466,"version":"claude-2.0"},"x:DTBench":{"value":0.5185000000000001,"norm":0.5185,"version":"claude-2.0"}},"variant":"Claude 2","match":"exact"}},{"id":"llama-2","name":"Llama 2","lab":"Meta","date":"2023-07-18","type":"open-weights","major":true,"open":true,"release":{"title":"Llama 2","source":"https://ai.meta.com/research/publications/llama-2-open-foundation-and-fine-tuned-chat-models/","verified":true,"confidence":"high"},"index":{"score":95.2,"lo":79.3,"hi":111.2,"se":2.34,"solve":11.6,"status":"rated","areas":["agents","maths","reasoning"],"core_areas":["reasoning"],"testers":["Epoch AI","Redwood Research","Stanford CRFM"],"tests":{"gpqa":{"value":0.26325757575757575,"norm":0.2633,"version":"Llama-2-70b-chat-hf"},"math5":{"value":0.032854984894259816,"norm":0.0329,"version":"Llama-2-70b-chat-hf"},"x:DTBench":{"value":0.4165,"norm":0.4165,"version":"Llama-2-70b-chat"},"x:GSM8K":{"value":0.484,"norm":0.484,"version":"Llama-2-70b-hf "},"aime":{"value":0,"norm":0,"version":"Llama-2-70b-chat-hf"},"mmlu":{"value":0.695,"norm":0.695,"version":"Llama-2-70b-hf "}},"variant":"Llama 2-70B","match":"family"}},{"id":"llama-2-13b","name":"Llama 2-13B","lab":"Meta","date":"2023-07-18","type":"open-weights","major":false,"open":true,"index":{"score":87.1,"lo":70.5,"hi":103.7,"se":3.57,"solve":10,"status":"rated","areas":["agents","maths","reasoning"],"core_areas":[],"testers":["Redwood Research","Stanford CRFM"],"tests":{"mmlu":{"value":0.554,"norm":0.554,"version":"Llama-2-13b"},"x:DTBench":{"value":0.42240000000000005,"norm":0.4224,"version":"Llama-2-13b-chat"},"x:GSM8K":{"value":0.245,"norm":0.245,"version":"Llama-2-13b"}},"variant":"Llama 2-13B","match":"self"},"date_from":"epoch"},{"id":"llama-2-7b","name":"Llama 2-7B","lab":"Meta","date":"2023-07-18","type":"open-weights","major":false,"open":true,"index":{"score":69.2,"lo":45.8,"hi":92.6,"se":10.67,"solve":7.6,"status":"insufficient","areas":["reasoning"],"core_areas":[],"testers":["Stanford CRFM"],"tests":{"mmlu":{"value":0.458,"norm":0.458,"version":"Llama-2-7b"}},"variant":"Llama 2-7B","match":"self"},"date_from":"epoch"},{"id":"qwen-7b","name":"Qwen-7B","lab":"Alibaba","date":"2023-08-03","type":"open-weights","major":true,"open":true,"release":{"title":"Qwen-7B (first open weights)","source":null,"verified":false,"confidence":"high"}},{"id":"claude-instant","name":"Claude Instant","lab":"Anthropic","date":"2023-08-09","type":"flagship","major":false,"open":false,"index":{"score":93.2,"lo":72.3,"hi":114.1,"se":8.54,"solve":11.1,"status":"provisional","areas":["agents","reasoning"],"core_areas":[],"testers":["Redwood Research","Stanford CRFM"],"tests":{"mmlu":{"value":0.688,"norm":0.688,"version":"claude-instant-1.2"},"x:DTBench":{"value":0.45780000000000004,"norm":0.4578,"version":"claude-instant-1.2"}},"variant":"Claude Instant","match":"self"},"date_from":"epoch"},{"id":"claude-instant-1-2","name":"Claude Instant 1.2","lab":"Anthropic","date":"2023-08-09","type":"small","major":true,"open":false,"release":{"title":"Claude Instant 1.2","source":"https://www.anthropic.com/news/releasing-claude-instant-1-2","verified":true,"confidence":"high"}},{"id":"code-llama","name":"Code Llama","lab":"Meta","date":"2023-08-24","type":"open-weights","major":true,"open":true,"release":{"title":"Code Llama","source":"https://ai.meta.com/blog/code-llama-large-language-model-coding/","verified":true,"confidence":"high"}},{"id":"gpt-3-5-turbo-instruct","name":"GPT-3.5 Turbo Instruct","lab":"OpenAI","date":"2023-09-18","type":"flagship","major":false,"open":false,"index":{"score":111.9,"lo":36.2,"hi":187.7,"se":45.05,"solve":16.1,"status":"insufficient","areas":["long"],"core_areas":["long"],"testers":["METR"],"tests":{"metr":{"value":0.214611,"norm":0.1352,"version":"gpt-3.5-turbo-instruct"}},"variant":"GPT-3.5 Turbo Instruct","match":"self"},"date_from":"epoch"},{"id":"mistral-7b","name":"Mistral 7B","lab":"Mistral","date":"2023-09-27","type":"open-weights","major":true,"open":true,"release":{"title":"Mistral 7B","source":"https://mistral.ai/news/announcing-mistral-7b/","verified":true,"confidence":"high"},"index":{"score":92.9,"lo":76.8,"hi":109,"se":2.74,"solve":11.1,"status":"provisional","areas":["maths","reasoning"],"core_areas":[],"testers":["Stanford CRFM"],"tests":{"mmlu":{"value":0.566,"norm":0.566,"version":"Mistral-7B-v0.1"},"x:GSM8K":{"value":0.381,"norm":0.381,"version":"Mistral-7B-v0.1"}},"variant":"Mistral 7B v0.1","match":"family"}},{"id":"kimi-chat","name":"Kimi Chat","lab":"Moonshot","date":"2023-10-09","type":"flagship","major":true,"open":false,"release":{"title":"Kimi Chat (debut; 200K-char context)","source":"https://www.kimi.com/en/help/others/chat-issues","verified":false,"confidence":"med"}},{"id":"ernie-4-0","name":"ERNIE 4.0","lab":"Baidu","date":"2023-10-17","type":"flagship","major":true,"open":false,"release":{"title":"ERNIE 4.0","source":null,"verified":false,"confidence":"high"}},{"id":"deepseek-coder","name":"DeepSeek Coder","lab":"DeepSeek","date":"2023-11-02","type":"open-weights","major":true,"open":true,"release":{"title":"DeepSeek Coder","source":null,"verified":false,"confidence":"high"}},{"id":"grok-1","name":"Grok-1","lab":"xAI","date":"2023-11-04","type":"flagship","major":true,"open":true,"release":{"title":"Grok-1","source":"https://en.wikipedia.org/wiki/Grok_(chatbot)","verified":false,"confidence":"high"},"open_from":"2024-03-17","events":[{"date":"2024-03-17","title":"Grok-1 open weights","source":"https://x.ai/news/grok-os"}],"aliases":["grok-1-2024-03"]},{"id":"gpt-3-5-turbo-nov-2023","name":"GPT-3.5 Turbo (Nov 2023)","lab":"OpenAI","date":"2023-11-06","type":"flagship","major":false,"open":false,"index":{"score":94.4,"lo":76.7,"hi":112.1,"se":5.19,"solve":11.4,"status":"provisional","areas":["maths","reasoning"],"core_areas":["reasoning"],"testers":["Epoch AI"],"tests":{"gpqa":{"value":0.2803030303030303,"norm":0.2803,"version":"gpt-3.5-turbo-1106"},"math5":{"value":0.15889350453172205,"norm":0.1589,"version":"gpt-3.5-turbo-1106"}},"variant":"GPT-3.5 Turbo (Nov 2023)","match":"self"},"date_from":"epoch"},{"id":"gpt-4-turbo","name":"GPT-4 Turbo","lab":"OpenAI","date":"2023-11-06","type":"flagship","major":true,"open":false,"release":{"title":"GPT-4 Turbo","source":"https://developers.openai.com/api/docs/models/gpt-4-turbo","verified":false,"confidence":"high"},"index":{"score":106.8,"lo":90.7,"hi":122.9,"se":2.62,"solve":14.4,"status":"rated","areas":["agents","long","maths","reasoning"],"core_areas":["long","reasoning"],"testers":["Epoch AI","METR","Redwood Research","Stanford CRFM"],"tests":{"x:DTBench":{"value":0.616,"norm":0.616,"version":"gpt-4-turbo"},"math5":{"value":0.40020770392749244,"norm":0.4002,"version":"gpt-4-1106-preview"},"x:LMCA":{"value":9.78,"norm":0.0978,"version":"gpt-4-turbo"},"metr":{"value":0.351726,"norm":0.181,"version":"gpt-4-0125-preview"},"mmlu":{"value":0.796,"norm":0.796,"version":"gpt-4-turbo"},"gpqa":{"value":0.4236111111111111,"norm":0.4236,"version":"gpt-4-1106-preview"}},"variant":"GPT-4 Turbo (Nov 2023)","match":"exact"}},{"id":"claude-2-1","name":"Claude 2.1","lab":"Anthropic","date":"2023-11-21","type":"flagship","major":true,"open":false,"release":{"title":"Claude 2.1","source":"https://www.anthropic.com/news/claude-2-1","verified":true,"confidence":"high"},"index":{"score":96.3,"lo":77.5,"hi":115.2,"se":6.53,"solve":11.8,"status":"rated","areas":["agents","coding","maths","reasoning"],"core_areas":["reasoning"],"testers":["Epoch AI","Havard T. Ihle","Redwood Research","Stanford CRFM"],"tests":{"aime":{"value":0.019444444444444445,"norm":0.0194,"version":"claude-2.1"},"gpqa":{"value":0.32954545454545453,"norm":0.3295,"version":"claude-2.1"},"x:WeirdML":{"value":0.0706,"norm":0.0706,"version":"claude-2.1"},"x:DTBench":{"value":0.5095000000000001,"norm":0.5095,"version":"claude-2.1"},"mmlu":{"value":0.735,"norm":0.735,"version":"claude-2.1"}},"variant":"Claude 2.1","match":"exact"}},{"id":"deepseek-llm-67b","name":"DeepSeek LLM 67B","lab":"DeepSeek","date":"2023-11-29","type":"open-weights","major":true,"open":true,"release":{"title":"DeepSeek LLM 67B","source":null,"verified":false,"confidence":"high"},"index":{"score":87.2,"lo":62.9,"hi":111.6,"se":11.41,"solve":10,"status":"provisional","areas":["maths","reasoning"],"core_areas":["reasoning"],"testers":["Epoch AI"],"tests":{"gpqa":{"value":0.24621212121212122,"norm":0.2462,"version":"deepseek-llm-67b-chat"},"math5":{"value":0.06391616314199396,"norm":0.0639,"version":"deepseek-llm-67b-chat"},"aime":{"value":0.008333333333333333,"norm":0.0083,"version":"deepseek-llm-67b-chat"}},"variant":"DeepSeek LLM 67B","match":"exact"}},{"id":"qwen-72b","name":"Qwen-72B","lab":"Alibaba","date":"2023-11-30","type":"open-weights","major":true,"open":true,"release":{"title":"Qwen-72B","source":"https://github.com/QwenLM/Qwen","verified":true,"confidence":"high"}},{"id":"gemini-1-0-nano","name":"Gemini 1.0 Nano","lab":"Google","date":"2023-12-06","type":"flagship","major":true,"open":false,"release":{"title":"Gemini 1.0 (Pro/Nano; Ultra announced)","source":"https://blog.google/innovation-and-ai/technology/ai/google-gemini-ai/","verified":true,"confidence":"high"}},{"id":"gemini-1-0-pro","name":"Gemini 1.0 Pro","lab":"Google","date":"2023-12-06","type":"flagship","major":true,"open":false,"release":{"title":"Gemini 1.0 (Pro/Nano; Ultra announced)","source":"https://blog.google/innovation-and-ai/technology/ai/google-gemini-ai/","verified":true,"confidence":"high"},"index":{"score":92.9,"lo":75,"hi":110.8,"se":5.43,"solve":11.1,"status":"rated","areas":["agents","maths","reasoning"],"core_areas":["reasoning"],"testers":["Epoch AI","Redwood Research","Stanford CRFM"],"tests":{"gpqa":{"value":0.33964646464646464,"norm":0.3396,"version":"gemini-1.0-pro-001"},"mmlu":{"value":0.7,"norm":0.7,"version":"gemini-1.0-pro-001"},"x:DTBench":{"value":0.4586,"norm":0.4586,"version":"gemini-1.0-pro-001"},"aime":{"value":0.011111111111111112,"norm":0.0111,"version":"gemini-1.0-pro-001"},"math5":{"value":0.11244335347432025,"norm":0.1124,"version":"gemini-1.0-pro-001"}},"variant":"Gemini 1.0 Pro","match":"exact"}},{"id":"mixtral-8x7b","name":"Mixtral 8x7B","lab":"Mistral","date":"2023-12-08","type":"open-weights","major":true,"open":true,"release":{"title":"Mixtral 8x7B","source":"https://mistral.ai/news/mixtral-of-experts/","verified":false,"confidence":"high"},"index":{"score":91.3,"lo":72.7,"hi":110,"se":6.28,"solve":10.8,"status":"rated","areas":["agents","maths","reasoning"],"core_areas":["reasoning"],"testers":["Epoch AI","Redwood Research"],"tests":{"gpqa":{"value":0.3058712121212121,"norm":0.3059,"version":"Mixtral-8x7B-Instruct-v0.1"},"math5":{"value":0.09950906344410876,"norm":0.0995,"version":"open-mixtral-8x7b"},"x:DTBench":{"value":0.4961,"norm":0.4961,"version":"open-mixtral-8x7b"}},"variant":"Mixtral 8x7B","match":"exact"}},{"id":"mistral-medium","name":"Mistral Medium","lab":"Mistral","date":"2023-12-11","type":"flagship","major":false,"open":false,"index":{"score":104.5,"lo":77.6,"hi":131.3,"se":13.33,"solve":13.8,"status":"insufficient","areas":["agents"],"core_areas":[],"testers":["Redwood Research"],"tests":{"x:DTBench":{"value":0.533,"norm":0.533,"version":"mistral-medium-2312"}},"variant":"Mistral Medium","match":"self"},"date_from":"epoch"},{"id":"phi-2","name":"Phi-2","lab":"Microsoft","date":"2023-12-12","type":"open-weights","major":true,"open":true,"release":{"title":"Phi-2","source":"https://huggingface.co/microsoft/phi-2","verified":true,"confidence":"high"},"index":{"score":83.4,"lo":61.7,"hi":105.2,"se":9.31,"solve":9.4,"status":"insufficient","areas":["reasoning"],"core_areas":[],"testers":["Stanford CRFM"],"tests":{"mmlu":{"value":0.584,"norm":0.584,"version":"phi-2"}},"variant":"Phi-2","match":"exact"}},{"id":"glm-4","name":"GLM-4","lab":"Zhipu","date":"2024-01-16","type":"flagship","major":true,"open":false,"release":{"title":"GLM-4","source":null,"verified":false,"confidence":"high"}},{"id":"gpt-3-5-turbo-jan-2024","name":"GPT-3.5 Turbo (Jan 2024)","lab":"OpenAI","date":"2024-01-25","type":"flagship","major":false,"open":false,"index":{"score":90.3,"lo":71.6,"hi":109,"se":6.34,"solve":10.6,"status":"rated","areas":["agents","coding","maths","reasoning"],"core_areas":["maths","reasoning"],"testers":["Epoch AI","Havard T. Ihle","Redwood Research","Stanford CRFM"],"tests":{"x:WeirdML":{"value":0.0348,"norm":0.0348,"version":"gpt-3.5-turbo-0125"},"mmlu":{"value":0.673,"norm":0.673,"version":"gpt-3.5-turbo-0125"},"x:LMCA":{"value":9.65,"norm":0.0965,"version":"gpt-3.5-turbo-0125"},"aime":{"value":0.022222222222222223,"norm":0.0222,"version":"gpt-3.5-turbo-0125"},"x:Mystery Game Puzzles":{"value":0.03,"norm":0.03,"version":"gpt-3.5-turbo-0125"},"x:DTBench":{"value":0.4853,"norm":0.4853,"version":"gpt-3.5-turbo-0125"},"gpqa":{"value":0.27178030303030304,"norm":0.2718,"version":"gpt-3.5-turbo-0125"},"math5":{"value":0.1163141993957704,"norm":0.1163,"version":"gpt-3.5-turbo-0125"},"fm":{"value":0,"norm":0,"version":"gpt-3.5-turbo-0125"}},"variant":"GPT-3.5 Turbo (Jan 2024)","match":"self"},"date_from":"epoch"},{"id":"qwen1-5","name":"Qwen1.5","lab":"Alibaba","date":"2024-02-04","type":"open-weights","major":true,"open":true,"release":{"title":"Qwen1.5","source":"https://qwenlm.github.io/blog/qwen1.5/","verified":true,"confidence":"high"},"index":{"score":98.7,"lo":78.4,"hi":119.1,"se":8.02,"solve":12.3,"status":"insufficient","areas":["reasoning"],"core_areas":["reasoning"],"testers":["Epoch AI","Stanford CRFM"],"tests":{"mmlu":{"value":0.744,"norm":0.744,"version":"qwen1.5-32B"},"gpqa":{"value":0.307449494949495,"norm":0.3074,"version":"qwen1.5-32b-chat"}},"variant":"Qwen1.5-32B","match":"family"}},{"id":"qwen1-5-14b","name":"Qwen1.5-14B","lab":"Alibaba","date":"2024-02-04","type":"open-weights","major":false,"open":true,"index":{"score":95,"lo":72.8,"hi":117.2,"se":9.69,"solve":11.5,"status":"insufficient","areas":["reasoning"],"core_areas":[],"testers":["Stanford CRFM"],"tests":{"mmlu":{"value":0.686,"norm":0.686,"version":"qwen1.5-14B"}},"variant":"Qwen1.5-14B","match":"self"},"date_from":"epoch"},{"id":"qwen1-5-72b","name":"Qwen1.5-72B","lab":"Alibaba","date":"2024-02-04","type":"open-weights","major":false,"open":true,"index":{"score":94,"lo":61.6,"hi":126.5,"se":17.32,"solve":11.3,"status":"insufficient","areas":["reasoning"],"core_areas":["reasoning"],"testers":["Epoch AI"],"tests":{"gpqa":{"value":0.2881944444444444,"norm":0.2882,"version":"qwen1.5-72b-chat"}},"variant":"Qwen1.5-72B","match":"self"},"date_from":"epoch"},{"id":"qwen1-5-7b","name":"Qwen1.5-7B","lab":"Alibaba","date":"2024-02-04","type":"open-weights","major":false,"open":true,"index":{"score":88.1,"lo":66.3,"hi":109.9,"se":9.3,"solve":10.2,"status":"insufficient","areas":["reasoning"],"core_areas":[],"testers":["Stanford CRFM"],"tests":{"mmlu":{"value":0.626,"norm":0.626,"version":"Qwen1.5-7B"}},"variant":"Qwen1.5-7B","match":"self"},"date_from":"epoch"},{"id":"gemini-1-0-ultra","name":"Gemini 1.0 Ultra","lab":"Google","date":"2024-02-08","type":"flagship","major":true,"open":false,"release":{"title":"Gemini 1.0 Ultra (Gemini Advanced)","source":"https://blog.google/innovation-and-ai/products/google-gemini-next-generation-model-february-2024/","verified":false,"confidence":"high"}},{"id":"gemini-1-5-pro","name":"Gemini 1.5 Pro","lab":"Google","date":"2024-02-15","type":"flagship","major":true,"open":false,"release":{"title":"Gemini 1.5 Pro","source":"https://blog.google/innovation-and-ai/products/google-gemini-next-generation-model-february-2024/","verified":true,"confidence":"high"},"index":{"score":114,"lo":86.7,"hi":141.2,"se":13.61,"solve":16.8,"status":"insufficient","areas":["reasoning"],"core_areas":[],"testers":["Stanford CRFM"],"tests":{"mmlu":{"value":0.827,"norm":0.827,"version":"gemini-1.5-pro-001-feb24"}},"variant":"Gemini 1.5 Pro (Feb 2024)","match":"exact"}},{"id":"gemma","name":"Gemma","lab":"Google","date":"2024-02-21","type":"open-weights","major":true,"open":true,"release":{"title":"Gemma (2B/7B)","source":"https://blog.google/innovation-and-ai/technology/developers-tools/gemma-open-models/","verified":true,"confidence":"high"},"index":{"score":92.1,"lo":70.1,"hi":114.1,"se":9.47,"solve":10.9,"status":"insufficient","areas":["reasoning"],"core_areas":[],"testers":["Stanford CRFM"],"tests":{"mmlu":{"value":0.661,"norm":0.661,"version":"gemma-7b"}},"variant":"Gemma 7B","match":"family"}},{"id":"mistral-large","name":"Mistral Large","lab":"Mistral","date":"2024-02-26","type":"flagship","major":true,"open":false,"release":{"title":"Mistral Large","source":"https://mistral.ai/news/mistral-large/","verified":true,"confidence":"high"},"index":{"score":100.3,"lo":83.8,"hi":116.8,"se":3.47,"solve":12.7,"status":"rated","areas":["agents","coding","maths","reasoning","work"],"core_areas":["reasoning"],"testers":["Epoch AI","Redwood Research","Stanford CRFM","Vals AI"],"tests":{"vals:gpqa":{"value":47.73,"norm":0.4773,"version":"Vals AI"},"x:DTBench":{"value":0.559,"norm":0.559,"version":"mistral-large-2402"},"vals:lcb":{"value":37.09,"norm":0.3709,"version":"Vals AI"},"vals:mmlu_pro":{"value":69.71,"norm":0.6971,"version":"Vals AI"},"aime":{"value":0.019444444444444445,"norm":0.0194,"version":"mistral-large-2402"},"math5":{"value":0.24461858006042297,"norm":0.2446,"version":"mistral-large-2402"},"vals:tax_eval_v2":{"value":63.78,"norm":0.6378,"version":"Vals AI"},"vals:mgsm":{"value":87.24,"norm":0.8724,"version":"Vals AI"},"vals:math500":{"value":74.4,"norm":0.744,"version":"Vals AI"},"gpqa":{"value":0.38762626262626265,"norm":0.3876,"version":"mistral-large-2402"},"mmlu":{"value":0.688,"norm":0.688,"version":"mistral-large-2402"}},"variant":"Mistral Large","match":"exact"},"api":{"or_id":"mistralai/mistral-large","in":2,"out":6,"context":128000,"hf":null,"created":"2024-02-26"}},{"id":"claude-3-opus","name":"Claude 3 Opus","lab":"Anthropic","date":"2024-03-04","type":"flagship","major":true,"open":false,"release":{"title":"Claude 3 (Opus/Sonnet; Haiku 03-13)","source":"https://www.anthropic.com/news/claude-3-family","verified":true,"confidence":"high"},"index":{"score":107.5,"lo":91.1,"hi":124,"se":3.39,"solve":14.7,"status":"rated","areas":["agents","coding","long","maths","reasoning"],"core_areas":["long","reasoning"],"testers":["Epoch AI","Havard T. Ihle","METR","Redwood Research","SimpleBench team","Stanford CRFM"],"tests":{"mmlu":{"value":0.846,"norm":0.846,"version":"claude-3-opus-20240229"},"x:SimpleQA Verified":{"value":0.126,"norm":0.126,"version":"claude-3-opus-20240229"},"metr":{"value":0.294753,"norm":0.1646,"version":"claude-3-opus-20240229"},"gpqa":{"value":0.4715909090909091,"norm":0.4716,"version":"claude-3-opus-20240229"},"x:DTBench":{"value":0.616,"norm":0.616,"version":"claude-3-opus-20240229"},"x:SimpleBench":{"value":0.235,"norm":0.235,"version":"claude-3-opus-20240229"},"x:LMCA":{"value":17,"norm":0.17,"version":"claude-3-opus-20240229"},"x:Chess Puzzles":{"value":0.04,"norm":0.04,"version":"claude-3-opus-20240229"},"aime":{"value":0.04722222222222222,"norm":0.0472,"version":"claude-3-opus-20240229"},"x:WeirdML":{"value":0.19219999999999998,"norm":0.1922,"version":"claude-3-opus-20240229"},"math5":{"value":0.37481117824773413,"norm":0.3748,"version":"claude-3-opus-20240229"}},"variant":"Claude 3 Opus","match":"exact"}},{"id":"claude-3-sonnet","name":"Claude 3 Sonnet","lab":"Anthropic","date":"2024-03-04","type":"flagship","major":true,"open":false,"release":{"title":"Claude 3 (Opus/Sonnet; Haiku 03-13)","source":"https://www.anthropic.com/news/claude-3-family","verified":true,"confidence":"high"},"index":{"score":98.3,"lo":81.4,"hi":115.1,"se":4.03,"solve":12.2,"status":"rated","areas":["agents","coding","maths","reasoning"],"core_areas":["reasoning"],"testers":["Epoch AI","Havard T. Ihle","Redwood Research","Stanford CRFM"],"tests":{"math5":{"value":0.18174093655589124,"norm":0.1817,"version":"claude-3-sonnet-20240229"},"gpqa":{"value":0.4059343434343434,"norm":0.4059,"version":"claude-3-sonnet-20240229"},"mmlu":{"value":0.759,"norm":0.759,"version":"claude-3-sonnet-20240229"},"x:DTBench":{"value":0.536,"norm":0.536,"version":"claude-3-sonnet-20240229"},"x:WeirdML":{"value":0.10160000000000001,"norm":0.1016,"version":"claude-3-sonnet-20240229"},"aime":{"value":0.025,"norm":0.025,"version":"claude-3-sonnet-20240229"}},"variant":"Claude 3 Sonnet","match":"exact"},"date_epoch":"2024-03-04","date_evidence":"https://www.anthropic.com/news/claude-3-family"},{"id":"claude-3-haiku","name":"Claude 3 Haiku","lab":"Anthropic","date":"2024-03-13","type":"flagship","major":false,"open":false,"index":{"score":95.4,"lo":78.4,"hi":112.5,"se":4.31,"solve":11.6,"status":"rated","areas":["agents","coding","maths","reasoning"],"core_areas":["reasoning"],"testers":["Epoch AI","Havard T. Ihle","Redwood Research","Stanford CRFM","Will Patrick"],"tests":{"aime":{"value":0.018055555555555554,"norm":0.0181,"version":"claude-3-haiku-20240307"},"gpqa":{"value":0.3630050505050505,"norm":0.363,"version":"claude-3-haiku-20240307"},"x:WeirdML":{"value":0.0984,"norm":0.0984,"version":"claude-3-haiku-20240307"},"x:DTBench":{"value":0.5013000000000001,"norm":0.5013,"version":"claude-3-haiku-20240307"},"x:LMCA":{"value":8.84,"norm":0.0884,"version":"claude-3-haiku-20240307"},"x:CadEval":{"value":0.12,"norm":0.12,"version":"claude-3-haiku-20240307"},"mmlu":{"value":0.738,"norm":0.738,"version":"claude-3-haiku-20240307"},"math5":{"value":0.1487915407854985,"norm":0.1488,"version":"claude-3-haiku-20240307"}},"variant":"Claude 3 Haiku","match":"self"},"date_epoch":"2024-03-07","date_evidence":"https://www.anthropic.com/news/claude-3-haiku","date_from":"epoch"},{"id":"grok-1-5","name":"Grok-1.5","lab":"xAI","date":"2024-03-28","type":"flagship","major":true,"open":false,"release":{"title":"Grok-1.5","source":"https://x.ai/news/grok-1.5","verified":true,"confidence":"high"}},{"id":"gpt-4-turbo-apr-2024","name":"GPT-4 Turbo (Apr 2024)","lab":"OpenAI","date":"2024-04-09","type":"flagship","major":false,"open":false,"index":{"score":108.3,"lo":91.9,"hi":124.7,"se":3.22,"solve":14.9,"status":"rated","areas":["coding","long","maths","reasoning"],"core_areas":["long","maths","reasoning"],"testers":["Epoch AI","Havard T. Ihle","METR","SimpleBench team","Stanford CRFM"],"tests":{"x:SimpleBench":{"value":0.251,"norm":0.251,"version":"gpt-4-turbo-2024-04-09"},"mmlu":{"value":0.813,"norm":0.813,"version":"gpt-4-turbo-2024-04-09"},"x:WeirdML":{"value":0.1801,"norm":0.1801,"version":"gpt-4-turbo-2024-04-09"},"fm":{"value":0.007017543859649123,"norm":0.007,"version":"gpt-4-turbo-2024-04-09"},"aime":{"value":0.06666666666666667,"norm":0.0667,"version":"gpt-4-turbo-2024-04-09"},"math5":{"value":0.46733383685800606,"norm":0.4673,"version":"gpt-4-turbo-2024-04-09"},"metr":{"value":0.367357,"norm":0.185,"version":"gpt-4-turbo-2024-04-09"},"gpqa":{"value":0.4659090909090909,"norm":0.4659,"version":"gpt-4-turbo-2024-04-09"}},"variant":"GPT-4 Turbo (Apr 2024)","match":"self"},"date_from":"epoch"},{"id":"mixtral-8x22b","name":"Mixtral 8x22B","lab":"Mistral","date":"2024-04-10","type":"open-weights","major":true,"open":true,"release":{"title":"Mixtral 8x22B","source":"https://mistral.ai/news/mixtral-8x22b/","verified":false,"confidence":"high"},"index":{"score":99.3,"lo":82.9,"hi":115.8,"se":3.34,"solve":12.5,"status":"rated","areas":["agents","coding","maths","reasoning"],"core_areas":["reasoning"],"testers":["Epoch AI","Havard T. Ihle","Redwood Research","Stanford CRFM"],"tests":{"math5":{"value":0.24244712990936557,"norm":0.2424,"version":"open-mixtral-8x22b"},"gpqa":{"value":0.34059343434343436,"norm":0.3406,"version":"open-mixtral-8x22b"},"mmlu":{"value":0.778,"norm":0.778,"version":"Mixtral-8x22B-v0.1"},"x:DTBench":{"value":0.5514,"norm":0.5514,"version":"Mixtral-8x22B-Instruct-v0.1"},"x:WeirdML":{"value":0.0317,"norm":0.0317,"version":"Mixtral-8x22B-Instruct-v0.1"}},"variant":"Mixtral 8x22B","match":"exact"},"api":{"or_id":"mistralai/mixtral-8x22b-instruct","in":2,"out":6,"context":65536,"hf":"mistralai/Mixtral-8x22B-Instruct-v0.1","created":"2024-04-17"}},{"id":"wizardlm-2-8x22b","name":"WizardLM-2 8x22B","lab":"Microsoft","date":"2024-04-15","type":"open-weights","major":false,"open":true,"index":{"score":101.4,"lo":84.9,"hi":117.8,"se":3.36,"solve":13,"status":"provisional","areas":["maths","reasoning"],"core_areas":["reasoning"],"testers":["Epoch AI"],"tests":{"math5":{"value":0.25736404833836857,"norm":0.2574,"version":"WizardLM-2-8x22B"},"gpqa":{"value":0.43434343434343436,"norm":0.4343,"version":"WizardLM-2-8x22B"}},"variant":"WizardLM-2 8x22B","match":"self"},"date_from":"epoch"},{"id":"llama-3","name":"Llama 3","lab":"Meta","date":"2024-04-18","type":"open-weights","major":true,"open":true,"release":{"title":"Llama 3","source":"https://ai.meta.com/blog/meta-llama-3/","verified":true,"confidence":"high"},"index":{"score":100.9,"lo":84.2,"hi":117.5,"se":3.69,"solve":12.8,"status":"rated","areas":["agents","maths","reasoning"],"core_areas":["reasoning"],"testers":["Epoch AI","Redwood Research","Stanford CRFM"],"tests":{"math5":{"value":0.225547583081571,"norm":0.2255,"version":"Meta-Llama-3-70B-Instruct"},"x:DTBench":{"value":0.5416,"norm":0.5416,"version":"Meta-Llama-3-70B-Instruct"},"aime":{"value":0.043055555555555555,"norm":0.0431,"version":"Meta-Llama-3-70B-Instruct"},"gpqa":{"value":0.40561868686868685,"norm":0.4056,"version":"Meta-Llama-3-70B-Instruct"},"mmlu":{"value":0.793,"norm":0.793,"version":"Meta-Llama-3-70B-Instruct"}},"variant":"Llama 3-70B","match":"family"}},{"id":"llama-3-8b","name":"Llama 3-8B","lab":"Meta","date":"2024-04-18","type":"open-weights","major":false,"open":true,"index":{"score":89.3,"lo":70.6,"hi":108,"se":6.37,"solve":10.4,"status":"rated","areas":["agents","maths","reasoning"],"core_areas":["reasoning"],"testers":["Epoch AI","Redwood Research","Stanford CRFM"],"tests":{"gpqa":{"value":0.26073232323232326,"norm":0.2607,"version":"Meta-Llama-3-8B-Instruct"},"x:DTBench":{"value":0.4393,"norm":0.4393,"version":"Meta-Llama-3-8B-Instruct"},"aime":{"value":0.019444444444444445,"norm":0.0194,"version":"Meta-Llama-3-8B-Instruct"},"mmlu":{"value":0.688,"norm":0.688,"version":"Meta-Llama-3-8B-Instruct"},"math5":{"value":0.0612726586102719,"norm":0.0613,"version":"Meta-Llama-3-8B-Instruct"}},"variant":"Llama 3-8B","match":"self"},"date_from":"epoch"},{"id":"phi-3","name":"Phi-3","lab":"Microsoft","date":"2024-04-23","type":"open-weights","major":true,"open":true,"release":{"title":"Phi-3","source":"https://huggingface.co/microsoft/Phi-3-mini-4k-instruct","verified":true,"confidence":"high"},"index":{"score":96.7,"lo":79.9,"hi":113.6,"se":4.01,"solve":11.9,"status":"provisional","areas":["maths","reasoning"],"core_areas":["reasoning"],"testers":["Epoch AI","Stanford CRFM"],"tests":{"mmlu":{"value":0.775,"norm":0.775,"version":"Phi-3-medium-128k-instruct"},"math5":{"value":0.17560422960725075,"norm":0.1756,"version":"Phi-3-medium-128k-instruct"},"gpqa":{"value":0.2758838383838384,"norm":0.2759,"version":"Phi-3-medium-128k-instruct"}},"variant":"phi-3-medium 14B","match":"family"}},{"id":"deepseek-v2","name":"DeepSeek-V2","lab":"DeepSeek","date":"2024-05-06","type":"open-weights","major":true,"open":true,"release":{"title":"DeepSeek-V2","source":"https://github.com/deepseek-ai/DeepSeek-V2","verified":true,"confidence":"high"}},{"id":"gpt-4o","name":"GPT-4o","lab":"OpenAI","date":"2024-05-13","type":"flagship","major":true,"open":false,"release":{"title":"GPT-4o","source":"https://openai.com/index/hello-gpt-4o/","verified":true,"confidence":"high"},"index":{"score":111.1,"lo":95,"hi":127.2,"se":2.68,"solve":15.8,"status":"rated","areas":["agents","long","maths","reasoning"],"core_areas":["long","reasoning"],"testers":["BALROG team","Epoch AI","METR","Redwood Research","Stanford CRFM"],"tests":{"x:LMCA":{"value":16.59,"norm":0.1659,"version":"gpt-4o-2024-05-13"},"x:Balrog":{"value":0.32299999999999995,"norm":0.323,"version":"gpt-4o-2024-05-13"},"metr":{"value":0.407636,"norm":0.1947,"version":"gpt-4o-2024-05-13"},"mmlu":{"value":0.842,"norm":0.842,"version":"gpt-4o-2024-05-13"},"math5":{"value":0.5104796072507553,"norm":0.5105,"version":"gpt-4o-2024-05-13"},"x:DTBench":{"value":0.6453,"norm":0.6453,"version":"gpt-4o-2024-05-13"},"aime":{"value":0.0625,"norm":0.0625,"version":"gpt-4o-2024-05-13"},"gpqa":{"value":0.4889520202020202,"norm":0.489,"version":"gpt-4o-2024-05-13"}},"variant":"GPT-4o (May 2024)","match":"exact"},"api":{"or_id":"openai/gpt-4o","in":2.5,"out":10,"context":128000,"hf":null,"created":"2024-05-13"}},{"id":"gemini-1-5-flash","name":"Gemini 1.5 Flash","lab":"Google","date":"2024-05-14","type":"small","major":true,"open":false,"release":{"title":"Gemini 1.5 Flash","source":"https://blog.google/technology/ai/google-gemini-update-flash-ai-assistant-io-2024/","verified":true,"confidence":"high"},"index":{"score":101.1,"lo":84.5,"hi":117.7,"se":3.65,"solve":12.9,"status":"rated","areas":["agents","maths","reasoning"],"core_areas":["reasoning"],"testers":["Epoch AI","Redwood Research","Stanford CRFM"],"tests":{"mmlu":{"value":0.779,"norm":0.779,"version":"gemini-1.5-flash-001"},"x:DTBench":{"value":0.5138,"norm":0.5138,"version":"gemini-1.5-flash-001"},"gpqa":{"value":0.40372474747474746,"norm":0.4037,"version":"gemini-1.5-flash-001"},"aime":{"value":0.03888888888888889,"norm":0.0389,"version":"gemini-1.5-flash-001"},"math5":{"value":0.2512273413897281,"norm":0.2512,"version":"gemini-1.5-flash-001"}},"variant":"Gemini 1.5 Flash (May 2024)","match":"exact"}},{"id":"gemini-1-5-pro-may-2024","name":"Gemini 1.5 Pro (May 2024)","lab":"Google","date":"2024-05-14","type":"flagship","major":false,"open":false,"index":{"score":106.8,"lo":90.5,"hi":123.1,"se":3.1,"solve":14.4,"status":"rated","areas":["agents","maths","reasoning"],"core_areas":["reasoning"],"testers":["Epoch AI","Redwood Research"],"tests":{"aime":{"value":0.06805555555555555,"norm":0.0681,"version":"gemini-1.5-pro-001"},"gpqa":{"value":0.4586489898989899,"norm":0.4586,"version":"gemini-1.5-pro-001"},"math5":{"value":0.4074773413897281,"norm":0.4075,"version":"gemini-1.5-pro-001"},"x:DTBench":{"value":0.5647,"norm":0.5647,"version":"gemini-1.5-pro-001"}},"variant":"Gemini 1.5 Pro (May 2024)","match":"self"},"date_from":"epoch"},{"id":"doubao-pro","name":"Doubao Pro","lab":"ByteDance","date":"2024-05-15","type":"flagship","major":true,"open":false,"release":{"title":"Doubao Pro","source":null,"verified":false,"confidence":"high"}},{"id":"mistral-7b-v0-3","name":"Mistral 7B v0.3","lab":"Mistral","date":"2024-05-22","type":"open-weights","major":false,"open":true,"index":{"score":81.6,"lo":61.2,"hi":102.1,"se":8.12,"solve":9.2,"status":"rated","areas":["agents","maths","reasoning"],"core_areas":["reasoning"],"testers":["Epoch AI","Redwood Research","Stanford CRFM"],"tests":{"x:DTBench":{"value":0.4254,"norm":0.4254,"version":"open-mistral-7b"},"aime":{"value":0.002777777777777778,"norm":0.0028,"version":"Mistral-7B-Instruct-v0.3"},"mmlu":{"value":0.599,"norm":0.599,"version":"Mistral-7B-Instruct-v0.3"},"gpqa":{"value":0.1518308080808081,"norm":0.1518,"version":"Mistral-7B-Instruct-v0.3"},"math5":{"value":0.0368202416918429,"norm":0.0368,"version":"open-mistral-7b"}},"variant":"Mistral 7B v0.3","match":"self"},"date_epoch":"2023-09-27","date_evidence":"https://docs.mistral.ai/models/mistral-7b-0-3","date_from":"epoch"},{"id":"codestral","name":"Codestral","lab":"Mistral","date":"2024-05-29","type":"open-weights","major":true,"open":true,"release":{"title":"Codestral","source":"https://mistral.ai/news/codestral/","verified":true,"confidence":"high"}},{"id":"qwen2","name":"Qwen2","lab":"Alibaba","date":"2024-06-07","type":"open-weights","major":true,"open":true,"release":{"title":"Qwen2","source":"https://qwenlm.github.io/blog/qwen2/","verified":true,"confidence":"high"},"index":{"score":105.8,"lo":89.7,"hi":121.9,"se":2.7,"solve":14.2,"status":"rated","areas":["coding","long","maths","reasoning"],"core_areas":["long","reasoning"],"testers":["Epoch AI","Havard T. Ihle","METR","Stanford CRFM"],"tests":{"mmlu":{"value":0.824,"norm":0.824,"version":"qwen2-72b-instruct"},"metr":{"value":0.298999,"norm":0.1659,"version":"qwen2-72b-instruct"},"gpqa":{"value":0.4078282828282828,"norm":0.4078,"version":"qwen2-72b-instruct"},"x:WeirdML":{"value":0.113,"norm":0.113,"version":"qwen2-72b-instruct"},"math5":{"value":0.39067220543806647,"norm":0.3907,"version":"qwen2-72b-instruct"}},"variant":"Qwen2-72B","match":"family"}},{"id":"nemotron-4-340b","name":"Nemotron-4 340B","lab":"NVIDIA","date":"2024-06-14","type":"open-weights","major":true,"open":true,"release":{"title":"Nemotron-4 340B","source":"https://blogs.nvidia.com/blog/nemotron-4-synthetic-data-generation-llm-training/","verified":true,"confidence":"high"}},{"id":"deepseek-coder-v2","name":"DeepSeek-Coder-V2","lab":"DeepSeek","date":"2024-06-17","type":"open-weights","major":true,"open":true,"release":{"title":"DeepSeek-Coder-V2","source":"https://api-docs.deepseek.com/updates/","verified":false,"confidence":"high"}},{"id":"claude-3-5-sonnet","name":"Claude 3.5 Sonnet (Jun 2024)","lab":"Anthropic","date":"2024-06-20","type":"flagship","major":true,"open":false,"release":{"title":"Claude 3.5 Sonnet","source":"https://www.anthropic.com/news/claude-3-5-sonnet","verified":true,"confidence":"high"},"index":{"score":113,"lo":96.7,"hi":129.4,"se":3.14,"solve":16.5,"status":"rated","areas":["agents","coding","long","maths","reasoning"],"core_areas":["long","reasoning"],"testers":["Chase Brower","Epoch AI","Havard T. Ihle","METR","Redwood Research","SimpleBench team","Stanford CRFM"],"tests":{"aime":{"value":0.06527777777777778,"norm":0.0653,"version":"claude-3-5-sonnet-20240620"},"mmlu":{"value":0.865,"norm":0.865,"version":"claude-3-5-sonnet-20240620"},"x:DTBench":{"value":0.6777,"norm":0.6777,"version":"claude-3-5-sonnet-20240620"},"x:FrontierMath-2025-02-28-Private":{"value":0.010344827586206896,"norm":0.0103,"version":"claude-3-5-sonnet-20240620"},"x:FrontierMath-Tier-4-2025-07-01-Private":{"value":0,"norm":0,"version":"claude-3-5-sonnet-20240620"},"x:SimpleBench":{"value":0.275,"norm":0.275,"version":"claude-3-5-sonnet-20240620"},"gpqa":{"value":0.5404040404040404,"norm":0.5404,"version":"claude-3-5-sonnet-20240620"},"x:VPCT":{"value":0.33,"norm":0.33,"version":"claude-3-5-sonnet-20240620"},"x:WeirdML":{"value":0.3097,"norm":0.3097,"version":"claude-3-5-sonnet-20240620"},"metr":{"value":0.401502,"norm":0.1933,"version":"claude-3-5-sonnet-20240620"},"math5":{"value":0.5168051359516617,"norm":0.5168,"version":"claude-3-5-sonnet-20240620"}},"variant":"Claude 3.5 Sonnet","match":"exact"}},{"id":"gemma-2-9b","name":"Gemma 2 9B","lab":"Google","date":"2024-06-24","type":"open-weights","major":false,"open":true,"index":{"score":97.1,"lo":79.8,"hi":114.3,"se":4.58,"solve":12,"status":"provisional","areas":["maths","reasoning"],"core_areas":["reasoning"],"testers":["Epoch AI","Stanford CRFM"],"tests":{"mmlu":{"value":0.721,"norm":0.721,"version":"gemma-2-9b-it"},"math5":{"value":0.2100641993957704,"norm":0.2101,"version":"gemma-2-9b-it"},"aime":{"value":0.005555555555555556,"norm":0.0056,"version":"gemma-2-9b-it"},"gpqa":{"value":0.2746212121212121,"norm":0.2746,"version":"gemma-2-9b-it"}},"variant":"Gemma 2 9B","match":"self"},"date_from":"epoch"},{"id":"gemma-2","name":"Gemma 2","lab":"Google","date":"2024-06-27","type":"open-weights","major":true,"open":true,"release":{"title":"Gemma 2","source":"https://blog.google/innovation-and-ai/technology/developers-tools/google-gemma-2/","verified":true,"confidence":"high"},"index":{"score":100.3,"lo":83.7,"hi":117,"se":3.73,"solve":12.7,"status":"rated","areas":["agents","maths","reasoning"],"core_areas":["reasoning"],"testers":["Epoch AI","Redwood Research","Stanford CRFM"],"tests":{"x:LMCA":{"value":7.1,"norm":0.071,"version":"gemma-2-27b"},"x:DTBench":{"value":0.48,"norm":0.48,"version":"gemma-2-27b"},"mmlu":{"value":0.757,"norm":0.757,"version":"gemma-2-27b-it"},"aime":{"value":0.013888888888888888,"norm":0.0139,"version":"gemma-2-27b-it"},"math5":{"value":0.2788897280966767,"norm":0.2789,"version":"gemma-2-27b-it"},"gpqa":{"value":0.3648989898989899,"norm":0.3649,"version":"gemma-2-27b-it"}},"variant":"Gemma 2 27B","match":"family"},"api":{"or_id":"google/gemma-2-27b-it","in":0.65,"out":0.65,"context":8192,"hf":"google/gemma-2-27b-it","created":"2024-07-13"}},{"id":"gpt-4o-mini","name":"GPT-4o mini","lab":"OpenAI","date":"2024-07-18","type":"small","major":true,"open":false,"release":{"title":"GPT-4o mini","source":"https://developers.openai.com/api/docs/models/gpt-4o-mini-tts","verified":false,"confidence":"high"},"index":{"score":103.6,"lo":87.1,"hi":120.1,"se":3.49,"solve":13.6,"status":"rated","areas":["agents","coding","maths","novel","reasoning","work"],"core_areas":["maths","novel","reasoning"],"testers":["ARC Prize","Aider","BALROG team","Chase Brower","Epoch AI","GeoBench team","Havard T. Ihle","Redwood Research","SimpleBench team","Stanford CRFM","Vals AI"],"tests":{"x:Chess Puzzles":{"value":0.03,"norm":0.03,"version":"gpt-4o-mini-2024-07-18"},"vals:gpqa":{"value":44.19,"norm":0.4419,"version":"Vals AI"},"vals:corp_fin_v2":{"value":45.46,"norm":0.4546,"version":"Vals AI"},"vals:mmlu_pro":{"value":62.73,"norm":0.6273,"version":"Vals AI"},"x:WeirdML":{"value":0.1176,"norm":0.1176,"version":"gpt-4o-mini-2024-07-18"},"aime":{"value":0.06944444444444445,"norm":0.0694,"version":"gpt-4o-mini-2024-07-18"},"x:Mystery Game Puzzles":{"value":0.12,"norm":0.12,"version":"gpt-4o-mini-2024-07-18"},"aider":{"value":3.6,"norm":0.036,"version":"gpt-4o-mini-2024-07-18"},"vals:tax_eval_v2":{"value":60.55,"norm":0.6055,"version":"Vals AI"},"fm":{"value":0.007017543859649123,"norm":0.007,"version":"gpt-4o-mini-2024-07-18"},"gpqa":{"value":0.37720959595959597,"norm":0.3772,"version":"gpt-4o-mini-2024-07-18"},"x:SimpleQA Verified":{"value":0.083,"norm":0.083,"version":"gpt-4o-mini-2024-07-18"},"vals:mmmu":{"value":56.56,"norm":0.5656,"version":"Vals AI"},"x:Balrog":{"value":0.174,"norm":0.174,"version":"gpt-4o-mini-2024-07-18"},"x:LMCA":{"value":10.37,"norm":0.1037,"version":"gpt-4o-mini-2024-07-18"},"vals:mortgage_tax":{"value":54.49,"norm":0.5449,"version":"Vals AI"},"x:SimpleBench":{"value":0.107,"norm":0.107,"version":"gpt-4o-mini-2024-07-18"},"vals:lcb":{"value":26.42,"norm":0.2642,"version":"Vals AI"},"arc:arc_agi_2":{"value":0,"norm":0,"version":"ARC Prize"},"x:DTBench":{"value":0.544,"norm":0.544,"version":"gpt-4o-mini-2024-07-18"},"vals:mgsm":{"value":86.18,"norm":0.8618,"version":"Vals AI"},"vals:math500":{"value":72.6,"norm":0.726,"version":"Vals AI"},"mmlu":{"value":0.767,"norm":0.767,"version":"gpt-4o-mini-2024-07-18"},"x:GeoBench":{"value":0.64,"norm":0.64,"version":"gpt-4o-mini-2024-07-18"},"x:VPCT":{"value":0.34,"norm":0.34,"version":"gpt-4o-mini-2024-07-18"},"math5":{"value":0.5263406344410876,"norm":0.5263,"version":"gpt-4o-mini-2024-07-18"}},"variant":"GPT-4o mini","match":"exact"},"api":{"or_id":"openai/gpt-4o-mini","in":0.15,"out":0.6,"context":128000,"hf":null,"created":"2024-07-18"}},{"id":"mistral-nemo","name":"Mistral NeMo","lab":"Mistral","date":"2024-07-18","type":"open-weights","major":false,"open":true,"index":{"score":92.1,"lo":74,"hi":110.3,"se":5.75,"solve":10.9,"status":"rated","areas":["agents","maths","reasoning"],"core_areas":["reasoning"],"testers":["BALROG team","Epoch AI","Redwood Research"],"tests":{"x:DTBench":{"value":0.48610000000000003,"norm":0.4861,"version":"Mistral-Nemo-Instruct-2407"},"gpqa":{"value":0.2989267676767677,"norm":0.2989,"version":"open-mistral-nemo-2407"},"x:Balrog":{"value":0.17600000000000002,"norm":0.176,"version":"Mistral-Nemo-Instruct-2407"},"math5":{"value":0.1082892749244713,"norm":0.1083,"version":"open-mistral-nemo-2407"}},"variant":"Mistral NeMo","match":"self"},"api":{"or_id":"mistralai/mistral-nemo","in":0.029,"out":0.03,"context":131072,"hf":"mistralai/Mistral-Nemo-Instruct-2407","created":"2024-07-19"},"date_from":"epoch"},{"id":"llama-3-1","name":"Llama 3.1","lab":"Meta","date":"2024-07-23","type":"open-weights","major":true,"open":true,"release":{"title":"Llama 3.1 (405B)","source":"https://ai.meta.com/blog/meta-llama-3-1/","verified":true,"confidence":"high"},"index":{"score":110.4,"lo":94.3,"hi":126.5,"se":2.68,"solve":15.6,"status":"rated","areas":["agents","coding","maths","reasoning"],"core_areas":["reasoning"],"testers":["Epoch AI","Havard T. Ihle","Redwood Research","SimpleBench team","Stanford CRFM"],"tests":{"aime":{"value":0.09722222222222222,"norm":0.0972,"version":"Llama-3.1-405B-Instruct"},"gpqa":{"value":0.5091540404040404,"norm":0.5092,"version":"Llama-3.1-405B-Instruct"},"x:WeirdML":{"value":0.2138,"norm":0.2138,"version":"Llama-3.1-405B-Instruct"},"math5":{"value":0.49773413897280966,"norm":0.4977,"version":"Llama-3.1-405B-Instruct"},"x:SimpleBench":{"value":0.23,"norm":0.23,"version":"Llama-3.1-405B-Instruct"},"mmlu":{"value":0.845,"norm":0.845,"version":"Llama-3.1-405B-Instruct"},"x:DTBench":{"value":0.6138,"norm":0.6138,"version":"Llama-3.1-405B-Instruct"}},"variant":"Llama 3.1-405B","match":"family"}},{"id":"llama-3-1-70b","name":"Llama 3.1-70B","lab":"Meta","date":"2024-07-23","type":"open-weights","major":false,"open":true,"index":{"score":105.5,"lo":89.3,"hi":121.8,"se":2.96,"solve":14.1,"status":"rated","areas":["agents","coding","maths","reasoning"],"core_areas":["reasoning"],"testers":["BALROG team","Epoch AI","Havard T. Ihle","Redwood Research","Stanford CRFM"],"tests":{"x:WeirdML":{"value":0.0897,"norm":0.0897,"version":"Llama-3.1-70B-Instruct"},"gpqa":{"value":0.44191919191919193,"norm":0.4419,"version":"Llama-3.1-70B-Instruct"},"x:Balrog":{"value":0.27899999999999997,"norm":0.279,"version":"Llama-3.1-70B-Instruct"},"math5":{"value":0.36678625377643503,"norm":0.3668,"version":"Llama-3.1-70B-Instruct"},"x:DTBench":{"value":0.6,"norm":0.6,"version":"Llama-3.1-70B-Instruct"},"mmlu":{"value":0.801,"norm":0.801,"version":"Llama-3.1-70B-Instruct"},"aime":{"value":0.03611111111111111,"norm":0.0361,"version":"Llama-3.1-70B-Instruct"},"x:LMCA":{"value":14.85,"norm":0.1485,"version":"Llama-3.1-70B-Instruct"}},"variant":"Llama 3.1-70B","match":"self"},"api":{"or_id":"meta-llama/llama-3.1-70b-instruct","in":0.72,"out":0.72,"context":131072,"hf":"meta-llama/Meta-Llama-3.1-70B-Instruct","created":"2024-07-23"},"date_from":"epoch"},{"id":"llama-3-1-8b","name":"Llama 3.1-8B","lab":"Meta","date":"2024-07-23","type":"open-weights","major":false,"open":true,"index":{"score":88.6,"lo":70.2,"hi":107,"se":6.01,"solve":10.3,"status":"rated","areas":["agents","coding","maths","reasoning"],"core_areas":["reasoning"],"testers":["BALROG team","Epoch AI","Havard T. Ihle","Redwood Research","Stanford CRFM"],"tests":{"x:WeirdML":{"value":0.0173,"norm":0.0173,"version":"Llama-3.1-8B-Instruct"},"x:DTBench":{"value":0.5093,"norm":0.5093,"version":"Llama-3.1-8B-Instruct"},"gpqa":{"value":0.26957070707070707,"norm":0.2696,"version":"Llama-3.1-8B-Instruct"},"x:LMCA":{"value":5.37,"norm":0.0537,"version":"Llama-3.1-8B-Instruct"},"aime":{"value":0.016666666666666666,"norm":0.0167,"version":"Llama-3.1-8B-Instruct"},"mmlu":{"value":0.561,"norm":0.561,"version":"Llama-3.1-8B-Instruct"},"x:Balrog":{"value":0.151,"norm":0.151,"version":"Llama-3.1-8B-Instruct"},"math5":{"value":0.22875755287009064,"norm":0.2288,"version":"Llama-3.1-8B-Instruct"}},"variant":"Llama 3.1-8B","match":"self"},"api":{"or_id":"meta-llama/llama-3.1-8b-instruct","in":0.02,"out":0.04,"context":131072,"hf":"meta-llama/Meta-Llama-3.1-8B-Instruct","created":"2024-07-23"},"date_from":"epoch"},{"id":"mistral-large-2","name":"Mistral Large 2","lab":"Mistral","date":"2024-07-24","type":"flagship","major":true,"open":true,"release":{"title":"Mistral Large 2","source":"https://mistral.ai/news/mistral-large-2407/","verified":true,"confidence":"high"},"index":{"score":108.6,"lo":92.4,"hi":124.8,"se":2.85,"solve":15,"status":"rated","areas":["agents","maths","reasoning"],"core_areas":["reasoning"],"testers":["Epoch AI","Redwood Research","SimpleBench team","Stanford CRFM"],"tests":{"math5":{"value":0.4481684290030212,"norm":0.4482,"version":"mistral-large-2407"},"gpqa":{"value":0.49021464646464646,"norm":0.4902,"version":"mistral-large-2407"},"mmlu":{"value":0.8,"norm":0.8,"version":"mistral-large-2407"},"x:DTBench":{"value":0.6121,"norm":0.6121,"version":"mistral-large-2407"},"aime":{"value":0.08472222222222223,"norm":0.0847,"version":"mistral-large-2407"},"x:SimpleBench":{"value":0.225,"norm":0.225,"version":"mistral-large-2407"}},"variant":"Mistral Large 2 (Jul 2024)","match":"exact"}},{"id":"gpt-4o-aug-2024","name":"GPT-4o (Aug 2024)","lab":"OpenAI","date":"2024-08-06","type":"flagship","major":false,"open":false,"index":{"score":111.7,"lo":95.4,"hi":128,"se":3.07,"solve":16,"status":"rated","areas":["coding","maths","reasoning"],"core_areas":["maths","reasoning"],"testers":["Aider","Epoch AI","SimpleBench team","Stanford CRFM","Will Patrick"],"tests":{"x:FrontierMath-2025-02-28-Private":{"value":0.0034482758620689655,"norm":0.0034,"version":"gpt-4o-2024-08-06"},"gpqa":{"value":0.49210858585858586,"norm":0.4921,"version":"gpt-4o-2024-08-06"},"mmlu":{"value":0.843,"norm":0.843,"version":"gpt-4o-2024-08-06"},"x:SimpleQA Verified":{"value":0.26,"norm":0.26,"version":"gpt-4o-2024-08-06"},"fm":{"value":0.0035087719298245615,"norm":0.0035,"version":"gpt-4o-2024-08-06"},"aime":{"value":0.06388888888888888,"norm":0.0639,"version":"gpt-4o-2024-08-06"},"math5":{"value":0.5327605740181269,"norm":0.5328,"version":"gpt-4o-2024-08-06"},"aider":{"value":23.1,"norm":0.231,"version":"gpt-4o-2024-08-06"},"x:Chess Puzzles":{"value":0.09,"norm":0.09,"version":"gpt-4o-2024-08-06"},"x:SimpleBench":{"value":0.178,"norm":0.178,"version":"gpt-4o-2024-08-06"},"x:CadEval":{"value":0.26,"norm":0.26,"version":"gpt-4o-2024-08-06"}},"variant":"GPT-4o (Aug 2024)","match":"self"},"api":{"or_id":"openai/gpt-4o-2024-08-06","in":2.5,"out":10,"context":128000,"hf":null,"created":"2024-08-06"},"date_from":"epoch"},{"id":"grok-2","name":"Grok-2","lab":"xAI","date":"2024-08-13","type":"flagship","major":true,"open":false,"release":{"title":"Grok-2","source":"https://x.ai/news/grok-2","verified":true,"confidence":"high"}},{"id":"deepseek-v2-5-sep-2024","name":"DeepSeek-V2.5 (Sep 2024)","lab":"DeepSeek","date":"2024-09-06","type":"open-weights","major":false,"open":true,"index":{"score":112,"lo":93.7,"hi":130.3,"se":5.89,"solve":16.1,"status":"insufficient","areas":["coding"],"core_areas":[],"testers":["Aider"],"tests":{"aider":{"value":17.8,"norm":0.178,"version":"DeepSeek-V2.5"}},"variant":"DeepSeek-V2.5 (Sep 2024)","match":"self"},"date_from":"epoch"},{"id":"pixtral-12b","name":"Pixtral 12B","lab":"Mistral","date":"2024-09-11","type":"open-weights","major":true,"open":true,"release":{"title":"Pixtral 12B","source":"https://mistral.ai/news/pixtral-12b/","verified":false,"confidence":"high"},"index":{"score":70.2,"lo":33.7,"hi":106.6,"se":20.06,"solve":7.7,"status":"insufficient","areas":["reasoning"],"core_areas":[],"testers":["GeoBench team"],"tests":{"x:GeoBench":{"value":0.37,"norm":0.37,"version":"Pixtral-12B-2409"}},"variant":"Pixtral 12B","match":"exact"}},{"id":"o1-mini","name":"o1-mini","lab":"OpenAI","date":"2024-09-12","type":"reasoning","major":true,"open":false,"release":{"title":"o1-preview / o1-mini","source":"https://openai.com/index/introducing-openai-o1-preview/","verified":true,"confidence":"high"},"index":{"score":123.1,"lo":107.2,"hi":138.9,"se":2.11,"solve":20.5,"status":"rated","areas":["coding","maths","novel","reasoning"],"core_areas":["novel","reasoning"],"testers":["ARC Prize","Aider","Epoch AI","Havard T. Ihle","SimpleBench team"],"tests":{"aider":{"value":32.9,"norm":0.329,"version":"o1-mini-2024-09-12_unknown"},"arc:arc_agi_2":{"value":0.8,"norm":0.008,"version":"ARC Prize"},"x:WeirdML":{"value":0.3632,"norm":0.3632,"version":"o1-mini-2024-09-12_medium"},"gpqa":{"value":0.6237373737373737,"norm":0.6237,"version":"o1-mini-2024-09-12_high"},"x:SimpleBench":{"value":0.181,"norm":0.181,"version":"o1-mini-2024-09-12_medium"},"x:FrontierMath-2025-02-28-Private":{"value":0.017241379310344827,"norm":0.0172,"version":"o1-mini-2024-09-12_medium"},"math5":{"value":0.8918051359516617,"norm":0.8918,"version":"o1-mini-2024-09-12_high"},"arc:arc_agi_1":{"value":14,"norm":0.14,"version":"ARC Prize"},"aime":{"value":0.46944444444444444,"norm":0.4694,"version":"o1-mini-2024-09-12_high"}},"variant":"o1-mini","match":"exact"}},{"id":"o1-preview","name":"o1-preview","lab":"OpenAI","date":"2024-09-12","type":"reasoning","major":true,"open":false,"release":{"title":"o1-preview / o1-mini","source":"https://openai.com/index/introducing-openai-o1-preview/","verified":true,"confidence":"high"},"index":{"score":121.8,"lo":105.8,"hi":137.8,"se":2.47,"solve":20,"status":"rated","areas":["coding","long","maths","reasoning"],"core_areas":["long","reasoning"],"testers":["Epoch AI","Havard T. Ihle","METR","SimpleBench team"],"tests":{"metr":{"value":0.451441,"norm":0.2041,"version":"o1-preview-2024-09-12"},"aime":{"value":0.3111111111111111,"norm":0.3111,"version":"o1-preview-2024-09-12"},"x:SimpleBench":{"value":0.417,"norm":0.417,"version":"o1-preview-2024-09-12"},"x:WeirdML":{"value":0.4756,"norm":0.4756,"version":"o1-preview-2024-09-12"},"math5":{"value":0.8164652567975831,"norm":0.8165,"version":"o1-preview-2024-09-12"},"gpqa":{"value":0.5031565656565656,"norm":0.5032,"version":"o1-preview-2024-09-12"}},"variant":"o1-preview","match":"exact"}},{"id":"mistral-small-v24-09","name":"Mistral Small v24.09","lab":"Mistral","date":"2024-09-17","type":"open-weights","major":false,"open":true,"index":{"score":98.9,"lo":68.8,"hi":129,"se":15.69,"solve":12.4,"status":"insufficient","areas":["agents"],"core_areas":[],"testers":["Redwood Research"],"tests":{"x:DTBench":{"value":0.4953,"norm":0.4953,"version":"mistral-small-2409"}},"variant":"Mistral Small v24.09","match":"self"},"date_from":"epoch"},{"id":"qwen2-5-32b","name":"Qwen2.5-32B","lab":"Alibaba","date":"2024-09-17","type":"open-weights","major":false,"open":true,"index":{"score":110.8,"lo":94.6,"hi":127,"se":2.91,"solve":15.7,"status":"provisional","areas":["maths","reasoning"],"core_areas":["reasoning"],"testers":["Epoch AI"],"tests":{"aime":{"value":0.07361111111111111,"norm":0.0736,"version":"qwen2.5-32b-instruct"},"math5":{"value":0.5607061933534743,"norm":0.5607,"version":"qwen2.5-32b-instruct"},"gpqa":{"value":0.46085858585858586,"norm":0.4609,"version":"qwen2.5-32b-instruct"}},"variant":"Qwen2.5-32B","match":"self"},"date_from":"epoch"},{"id":"qwen2-5","name":"Qwen2.5","lab":"Alibaba","date":"2024-09-19","type":"open-weights","major":true,"open":true,"release":{"title":"Qwen2.5","source":"https://qwenlm.github.io/blog/qwen2.5/","verified":true,"confidence":"high"},"index":{"score":112.2,"lo":96.1,"hi":128.2,"se":2.55,"solve":16.2,"status":"rated","areas":["agents","coding","long","maths","reasoning"],"core_areas":["long","reasoning"],"testers":["BALROG team","Epoch AI","GeoBench team","Havard T. Ihle","METR","Redwood Research","Stanford CRFM"],"tests":{"x:DTBench":{"value":0.6293,"norm":0.6293,"version":"qwen2.5-72b-instruct"},"mmlu":{"value":0.834,"norm":0.834,"version":"qwen2.5-72b-instruct"},"math5":{"value":0.631703172205438,"norm":0.6317,"version":"qwen2.5-72b-instruct"},"gpqa":{"value":0.4914772727272727,"norm":0.4915,"version":"qwen2.5-72b-instruct"},"x:LMCA":{"value":13.42,"norm":0.1342,"version":"qwen2.5-72b-instruct"},"metr":{"value":0.357771,"norm":0.1826,"version":"qwen2.5-72b-instruct"},"aime":{"value":0.08055555555555556,"norm":0.0806,"version":"qwen2.5-72b-instruct"},"x:GeoBench":{"value":0.62,"norm":0.62,"version":"Qwen2.5-VL-72B-Instruct"},"x:Balrog":{"value":0.162,"norm":0.162,"version":"qwen2.5-72b-instruct"},"x:WeirdML":{"value":0.1597,"norm":0.1597,"version":"qwen2.5-72b-instruct"}},"variant":"Qwen2.5-72B","match":"family"},"api":{"or_id":"qwen/qwen-2.5-72b-instruct","in":0.36,"out":0.4,"context":32768,"hf":"Qwen/Qwen2.5-72B-Instruct","created":"2024-09-19"}},{"id":"qwen2-5-7b","name":"Qwen2.5-7B","lab":"Alibaba","date":"2024-09-19","type":"open-weights","major":false,"open":true,"index":{"score":95.5,"lo":76.5,"hi":114.5,"se":6.69,"solve":11.6,"status":"rated","areas":["agents","maths","reasoning"],"core_areas":["reasoning"],"testers":["BALROG team","Epoch AI","Redwood Research","Stanford CRFM"],"tests":{"aime":{"value":0.025,"norm":0.025,"version":"qwen2.5-7b-instruct"},"x:Balrog":{"value":0.078,"norm":0.078,"version":"qwen2.5-7b-instruct"},"gpqa":{"value":0.3547979797979798,"norm":0.3548,"version":"qwen2.5-7b-instruct"},"mmlu":{"value":0.729,"norm":0.729,"version":"qwen2.5-7b-instruct"},"x:LMCA":{"value":6.42,"norm":0.0642,"version":"qwen2.5-7b-instruct"},"x:DTBench":{"value":0.4773,"norm":0.4773,"version":"qwen2.5-7b-instruct"}},"variant":"Qwen2.5-7B","match":"self"},"date_from":"epoch"},{"id":"gemini-1-5-flash-sep-2024","name":"Gemini 1.5 Flash (Sep 2024)","lab":"Google","date":"2024-09-24","type":"flagship","major":false,"open":false,"index":{"score":112.5,"lo":96.3,"hi":128.7,"se":2.87,"solve":16.3,"status":"rated","areas":["agents","coding","maths","reasoning"],"core_areas":["reasoning"],"testers":["BALROG team","Epoch AI","GeoBench team","Havard T. Ihle","Redwood Research","Stanford CRFM"],"tests":{"x:Balrog":{"value":0.146,"norm":0.146,"version":"gemini-1.5-flash-002"},"x:WeirdML":{"value":0.2487,"norm":0.2487,"version":"gemini-1.5-flash-002"},"gpqa":{"value":0.47316919191919193,"norm":0.4732,"version":"gemini-1.5-flash-002"},"x:DTBench":{"value":0.5379,"norm":0.5379,"version":"gemini-1.5-flash-002"},"mmlu":{"value":0.739,"norm":0.739,"version":"gemini-1.5-flash-002"},"x:GeoBench":{"value":0.76,"norm":0.76,"version":"gemini-1.5-flash-002"},"math5":{"value":0.6186744712990937,"norm":0.6187,"version":"gemini-1.5-flash-002"},"aime":{"value":0.1625,"norm":0.1625,"version":"gemini-1.5-flash-002"},"x:FrontierMath-2025-02-28-Private":{"value":0,"norm":0,"version":"gemini-1.5-flash-002"}},"variant":"Gemini 1.5 Flash (Sep 2024)","match":"self"},"date_from":"epoch"},{"id":"gemini-1-5-pro-sept-2024","name":"Gemini 1.5 Pro (Sept 2024)","lab":"Google","date":"2024-09-24","type":"flagship","major":false,"open":false,"index":{"score":116.5,"lo":100.6,"hi":132.5,"se":2.31,"solve":17.8,"status":"rated","areas":["agents","coding","maths","reasoning"],"core_areas":["reasoning"],"testers":["BALROG team","Epoch AI","Havard T. Ihle","Redwood Research","Scale AI","SimpleBench team","Stanford CRFM","Will Patrick"],"tests":{"x:Balrog":{"value":0.21,"norm":0.21,"version":"gemini-1.5-pro-002"},"aime":{"value":0.23055555555555557,"norm":0.2306,"version":"gemini-1.5-pro-002"},"x:SimpleBench":{"value":0.271,"norm":0.271,"version":"gemini-1.5-pro-002"},"gpqa":{"value":0.5722853535353535,"norm":0.5723,"version":"gemini-1.5-pro-002"},"x:CadEval":{"value":0.34,"norm":0.34,"version":"gemini-1.5-pro-002"},"math5":{"value":0.7039274924471299,"norm":0.7039,"version":"gemini-1.5-pro-002"},"mmlu":{"value":0.869,"norm":0.869,"version":"gemini-1.5-pro-002"},"x:WeirdML":{"value":0.222,"norm":0.222,"version":"gemini-1.5-pro-002"},"x:DTBench":{"value":0.5898,"norm":0.5898,"version":"gemini-1.5-pro-002"},"hle":{"value":0.046,"norm":0.046,"version":"gemini-1.5-pro-002"}},"variant":"Gemini 1.5 Pro (Sept 2024)","match":"self"},"date_from":"epoch"},{"id":"llama-3-2-11b","name":"Llama 3.2 11B","lab":"Meta","date":"2024-09-24","type":"open-weights","major":false,"open":true,"index":{"score":83.2,"lo":61.9,"hi":104.5,"se":8.89,"solve":9.4,"status":"provisional","areas":["agents","reasoning"],"core_areas":[],"testers":["BALROG team","Stanford CRFM"],"tests":{"x:Balrog":{"value":0.168,"norm":0.168,"version":"Llama-3.2-11B-Vision-Instruct"},"mmlu":{"value":0.565,"norm":0.565,"version":"Llama-3.2-11B-Vision-Instruct"}},"variant":"Llama 3.2 11B","match":"self"},"date_from":"epoch"},{"id":"llama-3-2-1b","name":"Llama 3.2 1B","lab":"Meta","date":"2024-09-24","type":"open-weights","major":false,"open":true,"index":{"score":81.2,"lo":38.6,"hi":123.7,"se":24.08,"solve":9.1,"status":"rated","areas":["agents","maths","reasoning"],"core_areas":["reasoning"],"testers":["BALROG team","Epoch AI"],"tests":{"gpqa":{"value":0.23926767676767677,"norm":0.2393,"version":"Llama-3.2-1B-Instruct"},"x:Balrog":{"value":0.066,"norm":0.066,"version":"Llama-3.2-1B-Instruct"},"aime":{"value":0.005555555555555556,"norm":0.0056,"version":"Llama-3.2-1B-Instruct"}},"variant":"Llama 3.2 1B","match":"self"},"api":{"or_id":"meta-llama/llama-3.2-1b-instruct","in":0.027,"out":0.201,"context":60000,"hf":"meta-llama/Llama-3.2-1B-Instruct","created":"2024-09-25"},"date_from":"epoch"},{"id":"llama-3-2-3b","name":"Llama 3.2 3B","lab":"Meta","date":"2024-09-24","type":"open-weights","major":false,"open":true,"index":{"score":93.2,"lo":50.1,"hi":136.4,"se":24.47,"solve":11.1,"status":"insufficient","areas":["agents"],"core_areas":[],"testers":["BALROG team"],"tests":{"x:Balrog":{"value":0.10099999999999999,"norm":0.101,"version":"Llama-3.2-3B-Instruct"}},"variant":"Llama 3.2 3B","match":"self"},"api":{"or_id":"meta-llama/llama-3.2-3b-instruct","in":0.0509,"out":0.335,"context":131072,"hf":"meta-llama/Llama-3.2-3B-Instruct","created":"2024-09-25"},"date_from":"epoch"},{"id":"llama-3-2","name":"Llama 3.2","lab":"Meta","date":"2024-09-25","type":"open-weights","major":true,"open":true,"release":{"title":"Llama 3.2 (1B/3B text + 11B/90B vision)","source":"https://ai.meta.com/blog/llama-3-2-connect-2024-Vision-edge-mobile-devices/","verified":true,"confidence":"high"},"index":{"score":105.9,"lo":89.6,"hi":122.2,"se":3.08,"solve":14.2,"status":"rated","areas":["agents","maths","reasoning"],"core_areas":["reasoning"],"testers":["BALROG team","Epoch AI","GeoBench team","Stanford CRFM"],"tests":{"gpqa":{"value":0.41035353535353536,"norm":0.4104,"version":"Llama-3.2-90B-Vision-Instruct"},"x:Balrog":{"value":0.273,"norm":0.273,"version":"Llama-3.2-90B-Vision-Instruct"},"math5":{"value":0.39435422960725075,"norm":0.3944,"version":"Llama-3.2-90B-Vision-Instruct"},"aime":{"value":0.02638888888888889,"norm":0.0264,"version":"Llama-3.2-90B-Vision-Instruct"},"mmlu":{"value":0.803,"norm":0.803,"version":"Llama-3.2-90B-Vision-Instruct"},"x:GeoBench":{"value":0.52,"norm":0.52,"version":"Llama-3.2-90B-Vision-Instruct"}},"variant":"Llama 3.2 90B","match":"family"}},{"id":"gemini-1-5-flash-8b","name":"Gemini 1.5 Flash 8B","lab":"Google","date":"2024-10-03","type":"flagship","major":false,"open":false,"index":{"score":98.5,"lo":77.2,"hi":119.9,"se":8.89,"solve":12.3,"status":"rated","areas":["agents","maths","reasoning"],"core_areas":["reasoning"],"testers":["Epoch AI","Redwood Research"],"tests":{"aime":{"value":0.04583333333333333,"norm":0.0458,"version":"gemini-1.5-flash-8b-001"},"gpqa":{"value":0.32954545454545453,"norm":0.3295,"version":"gemini-1.5-flash-8b-001"},"x:DTBench":{"value":0.5004,"norm":0.5004,"version":"gemini-1.5-flash-8b-001"}},"variant":"Gemini 1.5 Flash 8B","match":"self"},"date_from":"epoch"},{"id":"llama-3-1-nemotron-70b","name":"Llama-3.1-Nemotron-70B","lab":"NVIDIA","date":"2024-10-15","type":"open-weights","major":true,"open":true,"release":{"title":"Llama-3.1-Nemotron-70B","source":null,"verified":false,"confidence":"high"}},{"id":"ministral-3b","name":"Ministral 3B","lab":"Mistral","date":"2024-10-16","type":"small","major":true,"open":false,"release":{"title":"Ministral 3B/8B","source":"https://mistral.ai/news/ministraux/","verified":true,"confidence":"high"},"index":{"score":93,"lo":75.1,"hi":110.9,"se":5.45,"solve":11.1,"status":"rated","areas":["agents","maths","reasoning"],"core_areas":["reasoning"],"testers":["Epoch AI","Redwood Research"],"tests":{"gpqa":{"value":0.25252525252525254,"norm":0.2525,"version":"ministral-3b-2410"},"math5":{"value":0.14444864048338368,"norm":0.1444,"version":"ministral-3b-2410"},"x:DTBench":{"value":0.5173,"norm":0.5173,"version":"ministral-3b-2410"},"x:LMCA":{"value":5.48,"norm":0.0548,"version":"ministral-3b-2410"}},"variant":"Ministral 3B","match":"exact"}},{"id":"ministral-8b","name":"Ministral 8B","lab":"Mistral","date":"2024-10-16","type":"small","major":true,"open":true,"release":{"title":"Ministral 3B/8B","source":"https://mistral.ai/news/ministraux/","verified":true,"confidence":"high"},"index":{"score":92.9,"lo":75,"hi":110.9,"se":5.55,"solve":11.1,"status":"rated","areas":["agents","maths","reasoning"],"core_areas":["reasoning"],"testers":["Epoch AI","Redwood Research"],"tests":{"gpqa":{"value":0.27146464646464646,"norm":0.2715,"version":"ministral-8b-2410"},"x:DTBench":{"value":0.4569,"norm":0.4569,"version":"ministral-8b-2410"},"math5":{"value":0.14935800604229607,"norm":0.1494,"version":"ministral-8b-2410"}},"variant":"Ministral 8B","match":"exact"}},{"id":"claude-3-5-sonnet-2024-10","name":"Claude 3.5 Sonnet","lab":"Anthropic","date":"2024-10-22","type":"flagship","major":true,"open":false,"release":{"title":"Claude 3.5 Sonnet (new) + 3.5 Haiku (avail. 11-04)","source":"https://www.anthropic.com/news/3-5-models-and-computer-use","verified":true,"confidence":"high"},"index":{"score":122.4,"lo":106.5,"hi":138.3,"se":2.13,"solve":20.2,"status":"rated","areas":["agents","coding","long","maths","reasoning"],"core_areas":["long","reasoning"],"testers":["Aider","BALROG team","Chase Brower","Epoch AI","GSO team","GeoBench team","Havard T. Ihle","METR","Redwood Research","Scale AI","SimpleBench team","Stanford CRFM","Will Patrick"],"tests":{"aider":{"value":51.6,"norm":0.516,"version":"claude-3-5-sonnet-20241022"},"gpqa":{"value":0.553030303030303,"norm":0.553,"version":"claude-3-5-sonnet-20241022"},"mmlu":{"value":0.873,"norm":0.873,"version":"claude-3-5-sonnet-20241022"},"x:WeirdML":{"value":0.3997,"norm":0.3997,"version":"claude-3-5-sonnet-20241022"},"math5":{"value":0.5694864048338368,"norm":0.5695,"version":"claude-3-5-sonnet-20241022"},"x:DTBench":{"value":0.6784,"norm":0.6784,"version":"claude-3-5-sonnet-20241022"},"x:CadEval":{"value":0.48,"norm":0.48,"version":"claude-3-5-sonnet-20241022"},"x:GSO-Bench":{"value":0.046,"norm":0.046,"version":"claude-3-5-sonnet-20241022"},"x:VPCT":{"value":0.33,"norm":0.33,"version":"claude-3-5-sonnet-20241022"},"hle":{"value":0.0408,"norm":0.0408,"version":"claude-3-5-sonnet-20241022"},"aime":{"value":0.08472222222222223,"norm":0.0847,"version":"claude-3-5-sonnet-20241022"},"x:SimpleBench":{"value":0.414,"norm":0.414,"version":"claude-3-5-sonnet-20241022"},"metr":{"value":0.452356,"norm":0.2043,"version":"claude-3-5-sonnet-20241022"},"x:FrontierMath-Tier-4-2025-07-01-Private":{"value":0,"norm":0,"version":"claude-3-5-sonnet-20241022"},"x:FrontierMath-2025-02-28-Private":{"value":0.020689655172413793,"norm":0.0207,"version":"claude-3-5-sonnet-20241022"},"x:GeoBench":{"value":0.62,"norm":0.62,"version":"claude-3-5-sonnet-20241022"},"x:Balrog":{"value":0.326,"norm":0.326,"version":"claude-3-5-sonnet-20241022"}},"variant":"Claude 3.5 Sonnet (October 2024)","match":"exact"}},{"id":"qwen-turbo","name":"Qwen-Turbo","lab":"Alibaba","date":"2024-11-01","type":"flagship","major":false,"open":false,"index":{"score":110.3,"lo":94.1,"hi":126.5,"se":2.94,"solve":15.5,"status":"provisional","areas":["maths","reasoning"],"core_areas":["reasoning"],"testers":["Epoch AI"],"tests":{"gpqa":{"value":0.41792929292929293,"norm":0.4179,"version":"qwen-turbo-2024-11-01"},"math5":{"value":0.5623111782477341,"norm":0.5623,"version":"qwen-turbo-2024-11-01"},"aime":{"value":0.06111111111111111,"norm":0.0611,"version":"qwen-turbo-2024-11-01"}},"variant":"Qwen-Turbo","match":"self"},"date_from":"epoch"},{"id":"claude-3-5-haiku","name":"Claude 3.5 Haiku","lab":"Anthropic","date":"2024-11-04","type":"flagship","major":true,"open":false,"release":{"title":"Claude 3.5 Sonnet (new) + 3.5 Haiku (avail. 11-04)","source":"https://www.anthropic.com/news/3-5-models-and-computer-use","verified":true,"confidence":"high"},"index":{"score":108.9,"lo":92.8,"hi":125,"se":2.55,"solve":15.1,"status":"rated","areas":["agents","coding","maths","reasoning","work"],"core_areas":["reasoning"],"testers":["Aider","BALROG team","Epoch AI","GeoBench team","Havard T. Ihle","Redwood Research","Stanford CRFM","Vals AI","Will Patrick"],"tests":{"x:DTBench":{"value":0.5666,"norm":0.5666,"version":"claude-3-5-haiku-20241022"},"aime":{"value":0.043055555555555555,"norm":0.0431,"version":"claude-3-5-haiku-20241022"},"vals:math500":{"value":64.2,"norm":0.642,"version":"Vals AI"},"vals:corp_fin_v2":{"value":50.82,"norm":0.5082,"version":"Vals AI"},"vals:legal_bench":{"value":70.33,"norm":0.7033,"version":"Vals AI"},"x:WeirdML":{"value":0.3073,"norm":0.3073,"version":"claude-3-5-haiku-20241022"},"vals:tax_eval_v2":{"value":57.36,"norm":0.5736,"version":"Vals AI"},"vals:mgsm":{"value":84.62,"norm":0.8462,"version":"Vals AI"},"x:CadEval":{"value":0.32,"norm":0.32,"version":"claude-3-5-haiku-20241022"},"aider":{"value":28,"norm":0.28,"version":"claude-3-5-haiku-20241022"},"mmlu":{"value":0.743,"norm":0.743,"version":"claude-3-5-haiku-20241022"},"x:Balrog":{"value":0.193,"norm":0.193,"version":"claude-3-5-haiku-20241022"},"vals:gpqa":{"value":37.88,"norm":0.3788,"version":"Vals AI"},"x:FrontierMath-2025-02-28-Private":{"value":0.0034482758620689655,"norm":0.0034,"version":"claude-3-5-haiku-20241022"},"x:GeoBench":{"value":0.34,"norm":0.34,"version":"claude-3-5-haiku-20241022"},"math5":{"value":0.46355740181268884,"norm":0.4636,"version":"claude-3-5-haiku-20241022"},"gpqa":{"value":0.3813131313131313,"norm":0.3813,"version":"claude-3-5-haiku-20241022"},"vals:lcb":{"value":41.92,"norm":0.4192,"version":"Vals AI"},"vals:mmlu_pro":{"value":64.12,"norm":0.6412,"version":"Vals AI"}},"variant":"Claude 3.5 Haiku","match":"exact"},"date_epoch":"2024-10-22","date_evidence":"https://www.aboutamazon.com/news/aws/amazon-bedrock-anthropic-ai-claude-3-5-sonnet (announced 22 Oct, available 4 Nov)"},{"id":"hunyuan-large","name":"Hunyuan-Large","lab":"Tencent","date":"2024-11-05","type":"open-weights","major":true,"open":true,"release":{"title":"Hunyuan-Large","source":null,"verified":false,"confidence":"high"}},{"id":"qwen2-5-coder-32b","name":"Qwen2.5-Coder-32B","lab":"Alibaba","date":"2024-11-12","type":"open-weights","major":true,"open":true,"release":{"title":"Qwen2.5-Coder-32B","source":"https://qwenlm.github.io/blog/qwen2.5-coder-family/","verified":true,"confidence":"high"},"index":{"score":111,"lo":92.4,"hi":129.6,"se":6.25,"solve":15.8,"status":"insufficient","areas":["coding"],"core_areas":[],"testers":["Aider"],"tests":{"aider":{"value":16.4,"norm":0.164,"version":"Qwen2.5-Coder-32B-Instruct"}},"variant":"Qwen2.5-Coder-32B-Instruct","match":"family"},"api":{"or_id":"qwen/qwen-2.5-coder-32b-instruct","in":0.66,"out":1,"context":32768,"hf":"Qwen/Qwen2.5-Coder-32B-Instruct","created":"2024-11-12"}},{"id":"mistral-large-2-nov-2024","name":"Mistral Large 2 (Nov 2024)","lab":"Mistral","date":"2024-11-18","type":"open-weights","major":false,"open":true,"index":{"score":110.2,"lo":93.8,"hi":126.6,"se":3.26,"solve":15.5,"status":"rated","areas":["agents","maths","reasoning"],"core_areas":["reasoning"],"testers":["Epoch AI","Redwood Research"],"tests":{"gpqa":{"value":0.5132575757575758,"norm":0.5133,"version":"mistral-large-2411"},"math5":{"value":0.5028323262839879,"norm":0.5028,"version":"mistral-large-2411"},"x:DTBench":{"value":0.6082,"norm":0.6082,"version":"mistral-large-2411"},"x:FrontierMath-2025-02-28-Private":{"value":0.0034482758620689655,"norm":0.0034,"version":"mistral-large-2411"},"aime":{"value":0.07777777777777778,"norm":0.0778,"version":"mistral-large-2411"}},"variant":"Mistral Large 2 (Nov 2024)","match":"self"},"date_from":"epoch"},{"id":"deepseek-r1-lite-preview","name":"DeepSeek-R1-Lite-Preview","lab":"DeepSeek","date":"2024-11-20","type":"reasoning","major":true,"open":false,"release":{"title":"DeepSeek-R1-Lite-Preview","source":"https://www.deepseek.com/en/news/r1-lite-preview/","verified":true,"confidence":"high"}},{"id":"gpt-4o-nov-2024","name":"GPT-4o (Nov 2024)","lab":"OpenAI","date":"2024-11-20","type":"flagship","major":false,"open":false,"index":{"score":110.5,"lo":94.2,"hi":126.8,"se":3.05,"solve":15.6,"status":"rated","areas":["coding","maths","reasoning"],"core_areas":["coding","reasoning"],"testers":["Aider","Chase Brower","Epoch AI","GSO team","GeoBench team","Havard T. Ihle","Scale AI"],"tests":{"swe":{"value":0.30991735537190085,"norm":0.3099,"version":"gpt-4o-2024-11-20"},"x:GSO-Bench":{"value":0,"norm":0,"version":"gpt-4o-2024-11-20"},"x:GeoBench":{"value":0.71,"norm":0.71,"version":"gpt-4o-2024-11-20"},"x:WeirdML":{"value":0.25120000000000003,"norm":0.2512,"version":"gpt-4o-2024-11-20"},"x:FrontierMath-2025-02-28-Private":{"value":0.0034482758620689655,"norm":0.0034,"version":"gpt-4o-2024-11-20"},"hle":{"value":0.027200000000000002,"norm":0.0272,"version":"gpt-4o-2024-11-20"},"x:VPCT":{"value":0.4,"norm":0.4,"version":"gpt-4o-2024-11-20"},"aider":{"value":18.2,"norm":0.182,"version":"gpt-4o-2024-11-20"},"gpqa":{"value":0.4788510101010101,"norm":0.4789,"version":"gpt-4o-2024-11-20"},"math5":{"value":0.49773413897280966,"norm":0.4977,"version":"gpt-4o-2024-11-20"},"aime":{"value":0.0625,"norm":0.0625,"version":"gpt-4o-2024-11-20"}},"variant":"GPT-4o (Nov 2024)","match":"self"},"api":{"or_id":"openai/gpt-4o-2024-11-20","in":2.5,"out":10,"context":128000,"hf":null,"created":"2024-11-20"},"date_from":"epoch"},{"id":"qwq-32b-preview","name":"QwQ-32B-Preview","lab":"Alibaba","date":"2024-11-28","type":"reasoning","major":true,"open":true,"release":{"title":"QwQ-32B-Preview","source":"https://qwen.ai/blog?id=qwq-32b-preview","verified":false,"confidence":"high"},"open_evidence":"https://huggingface.co/Qwen/QwQ-32B-Preview"},{"id":"amazon-nova","name":"Amazon Nova","lab":"Amazon","date":"2024-12-03","type":"flagship","major":true,"open":false,"release":{"title":"Amazon Nova","source":null,"verified":false,"confidence":"high"}},{"id":"amazon-nova-lite","name":"Amazon Nova Lite","lab":"Amazon","date":"2024-12-03","type":"flagship","major":false,"open":false,"index":{"score":104.6,"lo":80.9,"hi":128.4,"se":10.92,"solve":13.8,"status":"insufficient","areas":["reasoning"],"core_areas":["reasoning"],"testers":["Scale AI","Stanford CRFM"],"tests":{"mmlu":{"value":0.77,"norm":0.77,"version":"amazon.nova-lite-v1:0"},"hle":{"value":0.0364,"norm":0.0364,"version":"amazon.nova-lite-v1:0"}},"variant":"Amazon Nova Lite","match":"self"},"date_from":"epoch"},{"id":"amazon-nova-micro","name":"Amazon Nova Micro","lab":"Amazon","date":"2024-12-03","type":"flagship","major":false,"open":false,"index":{"score":97.6,"lo":75.1,"hi":120.2,"se":9.96,"solve":12.1,"status":"insufficient","areas":["reasoning"],"core_areas":[],"testers":["Stanford CRFM"],"tests":{"mmlu":{"value":0.708,"norm":0.708,"version":"amazon.nova-micro-v1:0"}},"variant":"Amazon Nova Micro","match":"self"},"date_from":"epoch"},{"id":"amazon-nova-pro","name":"Amazon Nova Pro","lab":"Amazon","date":"2024-12-03","type":"flagship","major":false,"open":false,"index":{"score":111.2,"lo":85.7,"hi":136.7,"se":12.33,"solve":15.8,"status":"insufficient","areas":["reasoning"],"core_areas":["reasoning"],"testers":["Scale AI","Stanford CRFM"],"tests":{"hle":{"value":0.044000000000000004,"norm":0.044,"version":"amazon.nova-pro-v1:0"},"mmlu":{"value":0.82,"norm":0.82,"version":"amazon.nova-pro-v1:0"}},"variant":"Amazon Nova Pro","match":"self"},"date_from":"epoch"},{"id":"o1","name":"o1","lab":"OpenAI","date":"2024-12-05","type":"reasoning","major":true,"open":false,"release":{"title":"o1 (full) + o1 pro","source":null,"verified":false,"confidence":"high"},"index":{"score":135.6,"lo":119.7,"hi":151.5,"se":2.17,"solve":27.4,"status":"rated","areas":["agents","coding","long","maths","reasoning"],"core_areas":["long","maths","reasoning"],"testers":["Aider","Chase Brower","Epoch AI","Fiction.live","GeoBench team","Havard T. Ihle","METR","Redwood Research","Scale AI","SimpleBench team","Will Patrick"],"tests":{"metr":{"value":0.510512,"norm":0.2155,"version":"o1-2024-12-17_medium"},"x:VPCT":{"value":0.37,"norm":0.37,"version":"o1-2024-12-17_medium"},"math5":{"value":0.947129909365559,"norm":0.9471,"version":"o1-2024-12-17_high"},"fm":{"value":0.14736842105263157,"norm":0.1474,"version":"o1-2024-12-17_high"},"gpqa":{"value":0.7676767676767676,"norm":0.7677,"version":"o1-2024-12-17_high"},"x:FrontierMath-2025-02-28-Private":{"value":0.09310344827586207,"norm":0.0931,"version":"o1-2024-12-17_high"},"x:GeoBench":{"value":0.8,"norm":0.8,"version":"o1-2024-12-17_medium"},"x:CadEval":{"value":0.56,"norm":0.56,"version":"o1-2024-12-17_medium"},"x:SimpleQA Verified":{"value":0.411,"norm":0.411,"version":"o1-2024-12-17_high"},"aider":{"value":61.7,"norm":0.617,"version":"o1-2024-12-17_high"},"x:DTBench":{"value":0.7467,"norm":0.7467,"version":"o1-2024-12-17_high"},"x:SimpleBench":{"value":0.401,"norm":0.401,"version":"o1-2024-12-17_high"},"hle":{"value":0.0796,"norm":0.0796,"version":"o1-2024-12-17_unknown"},"x:WeirdML":{"value":0.4608,"norm":0.4608,"version":"o1-2024-12-17_high"},"x:Fiction.LiveBench":{"value":0.833,"norm":0.833,"version":"o1-2024-12-17_medium"},"aime":{"value":0.7333333333333333,"norm":0.7333,"version":"o1-2024-12-17_medium"},"x:Chess Puzzles":{"value":0.11,"norm":0.11,"version":"o1-2024-12-17_low"},"x:LMCA":{"value":22.27,"norm":0.2227,"version":"o1-2024-12-17_high"}},"variant":"o1","match":"exact"},"modes":[{"label":"Pro","name":"o1-pro","index":{"score":118.5,"lo":54.4,"hi":182.5,"se":37.77,"solve":18.5,"status":"insufficient","areas":["reasoning"],"core_areas":["reasoning"],"testers":["Scale AI"],"tests":{"hle":{"value":0.0812,"norm":0.0812,"version":"o1-pro-2025-03-19"}}}}],"aliases":["o1-pro"]},{"id":"llama-3-3-70b","name":"Llama 3.3 70B","lab":"Meta","date":"2024-12-06","type":"open-weights","major":true,"open":true,"release":{"title":"Llama 3.3 70B","source":"https://about.fb.com/ltam/news/2024/12/el-futuro-de-la-ia-construido-con-llama/","verified":false,"confidence":"high"},"index":{"score":107.6,"lo":91.6,"hi":123.7,"se":2.51,"solve":14.7,"status":"rated","areas":["agents","coding","maths","reasoning"],"core_areas":["reasoning"],"testers":["BALROG team","Epoch AI","Fiction.live","Havard T. Ihle","Redwood Research","SimpleBench team","Stanford CRFM"],"tests":{"x:DTBench":{"value":0.5947,"norm":0.5947,"version":"Llama-3.3-70B-Instruct"},"gpqa":{"value":0.4744318181818182,"norm":0.4744,"version":"Llama-3.3-70B-Instruct"},"x:WeirdML":{"value":0.1444,"norm":0.1444,"version":"Llama-3.3-70B-Instruct"},"x:LMCA":{"value":17.52,"norm":0.1752,"version":"Llama-3.3-70B-Instruct"},"x:Fiction.LiveBench":{"value":0.333,"norm":0.333,"version":"Llama-3.3-70B-Instruct"},"math5":{"value":0.41597432024169184,"norm":0.416,"version":"Llama-3.3-70B-Instruct"},"x:SimpleBench":{"value":0.199,"norm":0.199,"version":"Llama-3.3-70B-Instruct"},"x:Balrog":{"value":0.23,"norm":0.23,"version":"Llama-3.3-70B-Instruct"},"aime":{"value":0.05138888888888889,"norm":0.0514,"version":"Llama-3.3-70B-Instruct"},"mmlu":{"value":0.791,"norm":0.791,"version":"Llama-3.3-70B-Instruct"}},"variant":"Llama 3.3 70B","match":"exact"},"api":{"or_id":"meta-llama/llama-3.3-70b-instruct","in":0.71,"out":0.71,"context":131072,"hf":"meta-llama/Llama-3.3-70B-Instruct","created":"2024-12-06"}},{"id":"gemini-2-0-flash","name":"Gemini 2.0 Flash (Dec 2024)","lab":"Google","date":"2024-12-11","type":"flagship","major":true,"open":false,"release":{"title":"Gemini 2.0 Flash (experimental)","source":"https://blog.google/innovation-and-ai/models-and-research/google-deepmind/google-gemini-ai-update-december-2024/","verified":true,"confidence":"high"},"index":{"score":122.4,"lo":105.8,"hi":139,"se":3.6,"solve":20.2,"status":"provisional","areas":["coding","reasoning"],"core_areas":[],"testers":["Aider","SimpleBench team","Stanford CRFM"],"tests":{"x:SimpleBench":{"value":0.311,"norm":0.311,"version":"gemini-exp-1206"},"mmlu":{"value":0.797,"norm":0.797,"version":"gemini-2.0-flash-exp"},"aider":{"value":38.2,"norm":0.382,"version":"gemini-exp-1206"}},"variant":"Gemini 2.0 Flash (Dec 2024)","match":"exact"}},{"id":"grok-2-dec-2024","name":"Grok-2 (Dec 2024)","lab":"xAI","date":"2024-12-12","type":"flagship","major":false,"open":false,"index":{"score":113.1,"lo":96.9,"hi":129.3,"se":2.8,"solve":16.5,"status":"rated","areas":["agents","coding","maths","reasoning","work"],"core_areas":["reasoning"],"testers":["Epoch AI","Havard T. Ihle","Redwood Research","SimpleBench team","Vals AI"],"tests":{"vals:gpqa":{"value":50.76,"norm":0.5076,"version":"Vals AI"},"vals:tax_eval_v2":{"value":67.05,"norm":0.6705,"version":"Vals AI"},"vals:corp_fin_v2":{"value":51.13,"norm":0.5113,"version":"Vals AI"},"x:FrontierMath-2025-02-28-Private":{"value":0.006896551724137931,"norm":0.0069,"version":"grok-2-1212"},"vals:lcb":{"value":38.68,"norm":0.3868,"version":"Vals AI"},"gpqa":{"value":0.5378787878787878,"norm":0.5379,"version":"grok-2-1212"},"aime":{"value":0.11527777777777778,"norm":0.1153,"version":"grok-2-1212"},"x:SimpleBench":{"value":0.227,"norm":0.227,"version":"grok-2-1212"},"vals:mmlu_pro":{"value":75.47,"norm":0.7547,"version":"Vals AI"},"x:WeirdML":{"value":0.2224,"norm":0.2224,"version":"grok-2-1212"},"math5":{"value":0.6351963746223565,"norm":0.6352,"version":"grok-2-1212"},"x:DTBench":{"value":0.6520999999999999,"norm":0.6521,"version":"grok-2-1212"},"vals:mgsm":{"value":86.15,"norm":0.8615,"version":"Vals AI"},"vals:math500":{"value":78.4,"norm":0.784,"version":"Vals AI"}},"variant":"Grok-2 (Dec 2024)","match":"self"},"date_from":"epoch"},{"id":"phi-4","name":"Phi-4","lab":"Microsoft","date":"2024-12-12","type":"open-weights","major":true,"open":true,"release":{"title":"Phi-4","source":null,"verified":false,"confidence":"high"},"index":{"score":113.9,"lo":97.7,"hi":130,"se":2.75,"solve":16.8,"status":"rated","areas":["agents","maths","reasoning"],"core_areas":["reasoning"],"testers":["BALROG team","Epoch AI"],"tests":{"aime":{"value":0.1375,"norm":0.1375,"version":"phi-4"},"gpqa":{"value":0.5606060606060606,"norm":0.5606,"version":"phi-4"},"x:Balrog":{"value":0.11599999999999999,"norm":0.116,"version":"phi-4"},"math5":{"value":0.6493580060422961,"norm":0.6494,"version":"phi-4"}},"variant":"Phi-4","match":"exact"},"api":{"or_id":"microsoft/phi-4","in":0.07,"out":0.14,"context":16384,"hf":"microsoft/phi-4","created":"2025-01-10"}},{"id":"gemini-2-0-flash-thinking","name":"Gemini 2.0 Flash Thinking","lab":"Google","date":"2024-12-19","type":"reasoning","major":true,"open":false,"release":{"title":"Gemini 2.0 Flash Thinking (exp)","source":"https://ai.google.dev/gemini-api/docs/changelog","verified":true,"confidence":"high"}},{"id":"deepseek-v3","name":"DeepSeek-V3","lab":"DeepSeek","date":"2024-12-26","type":"open-weights","major":true,"open":true,"release":{"title":"DeepSeek-V3","source":"https://api-docs.deepseek.com/news/news1226/","verified":false,"confidence":"high"},"index":{"score":119.1,"lo":103.1,"hi":135.1,"se":2.41,"solve":18.8,"status":"rated","areas":["coding","long","maths","reasoning"],"core_areas":["long","reasoning"],"testers":["Aider","Epoch AI","METR","SimpleBench team","Stanford CRFM"],"tests":{"mmlu":{"value":0.872,"norm":0.872,"version":"DeepSeek-V3"},"math5":{"value":0.6485083081570997,"norm":0.6485,"version":"DeepSeek-V3"},"aider":{"value":48.4,"norm":0.484,"version":"DeepSeek-V3"},"x:FrontierMath-2025-02-28-Private":{"value":0.017241379310344827,"norm":0.0172,"version":"DeepSeek-V3"},"gpqa":{"value":0.5653409090909091,"norm":0.5653,"version":"DeepSeek-V3"},"aime":{"value":0.15833333333333333,"norm":0.1583,"version":"DeepSeek-V3"},"x:SimpleBench":{"value":0.189,"norm":0.189,"version":"DeepSeek-V3"},"metr":{"value":0.473627,"norm":0.2086,"version":"DeepSeek-V3"}},"variant":"DeepSeek-V3","match":"exact"},"api":{"or_id":"deepseek/deepseek-chat","in":0.2574,"out":1.0287,"context":163840,"hf":"deepseek-ai/DeepSeek-V3","created":"2024-12-26"}},{"id":"minimax-01","name":"MiniMax-01","lab":"MiniMax","date":"2025-01-15","type":"open-weights","major":true,"open":true,"release":{"title":"MiniMax-01","source":"https://huggingface.co/MiniMaxAI/MiniMax-Text-01","verified":false,"confidence":"high"}},{"id":"deepseek-r1","name":"DeepSeek-R1","lab":"DeepSeek","date":"2025-01-20","type":"reasoning","major":true,"open":true,"release":{"title":"DeepSeek-R1","source":"https://api-docs.deepseek.com/news/news250120/","verified":false,"confidence":"high"},"index":{"score":129.9,"lo":114,"hi":145.7,"se":1.97,"solve":24,"status":"rated","areas":["agents","coding","long","maths","reasoning"],"core_areas":["long","reasoning"],"testers":["Aider","BALROG team","Epoch AI","Fiction.live","Havard T. Ihle","METR","SimpleBench team"],"tests":{"x:SimpleBench":{"value":0.309,"norm":0.309,"version":"DeepSeek-R1"},"x:Balrog":{"value":0.349,"norm":0.349,"version":"DeepSeek-R1"},"aider":{"value":56.9,"norm":0.569,"version":"DeepSeek-R1"},"aime":{"value":0.5333333333333333,"norm":0.5333,"version":"DeepSeek-R1"},"x:WeirdML":{"value":0.3649,"norm":0.3649,"version":"DeepSeek-R1"},"math5":{"value":0.9305135951661632,"norm":0.9305,"version":"DeepSeek-R1"},"metr":{"value":0.519251,"norm":0.2171,"version":"DeepSeek-R1"},"x:Fiction.LiveBench":{"value":0.694,"norm":0.694,"version":"DeepSeek-R1"},"gpqa":{"value":0.7171717171717171,"norm":0.7172,"version":"DeepSeek-R1"}},"variant":"DeepSeek-R1","match":"exact"}},{"id":"deepseek-r1-distill-llama-70b","name":"DeepSeek-R1-Distill-Llama-70B","lab":"DeepSeek","date":"2025-01-20","type":"open-weights","major":false,"open":true,"index":{"score":125.4,"lo":109.3,"hi":141.5,"se":2.66,"solve":21.6,"status":"provisional","areas":["maths","reasoning"],"core_areas":["reasoning"],"testers":["Epoch AI"],"tests":{"gpqa":{"value":0.5574494949494949,"norm":0.5574,"version":"DeepSeek-R1-Distill-Llama-70B"},"aime":{"value":0.5138888888888888,"norm":0.5139,"version":"DeepSeek-R1-Distill-Llama-70B"},"math5":{"value":0.8989803625377644,"norm":0.899,"version":"DeepSeek-R1-Distill-Llama-70B"}},"variant":"DeepSeek-R1-Distill-Llama-70B","match":"self"},"date_from":"epoch"},{"id":"deepseek-r1-distill-qwen-1-5b","name":"DeepSeek-R1-Distill-Qwen-1.5B","lab":"DeepSeek","date":"2025-01-20","type":"open-weights","major":false,"open":true,"index":{"score":110.7,"lo":93.6,"hi":127.7,"se":4.29,"solve":15.7,"status":"provisional","areas":["maths","reasoning"],"core_areas":["reasoning"],"testers":["Epoch AI"],"tests":{"aime":{"value":0.21388888888888888,"norm":0.2139,"version":"DeepSeek-R1-Distill-Qwen-1.5B"},"gpqa":{"value":0.33585858585858586,"norm":0.3359,"version":"DeepSeek-R1-Distill-Qwen-1.5B"}},"variant":"DeepSeek-R1-Distill-Qwen-1.5B","match":"self"},"date_from":"epoch"},{"id":"deepseek-r1-distill-qwen-14b","name":"DeepSeek-R1-Distill-Qwen-14B","lab":"DeepSeek","date":"2025-01-20","type":"open-weights","major":false,"open":true,"index":{"score":123.8,"lo":107.7,"hi":139.9,"se":2.63,"solve":20.9,"status":"provisional","areas":["maths","reasoning"],"core_areas":["reasoning"],"testers":["Epoch AI"],"tests":{"aime":{"value":0.5055555555555555,"norm":0.5056,"version":"DeepSeek-R1-Distill-Qwen-14B"},"gpqa":{"value":0.44696969696969696,"norm":0.447,"version":"DeepSeek-R1-Distill-Qwen-14B"},"math5":{"value":0.8712235649546828,"norm":0.8712,"version":"DeepSeek-R1-Distill-Qwen-14B"}},"variant":"DeepSeek-R1-Distill-Qwen-14B","match":"self"},"date_from":"epoch"},{"id":"deepseek-r1-distill-qwen-32b","name":"DeepSeek-R1-Distill-Qwen-32B","lab":"DeepSeek","date":"2025-01-20","type":"open-weights","major":false,"open":true,"index":{"score":126.5,"lo":110.5,"hi":142.4,"se":2.34,"solve":22.2,"status":"rated","areas":["agents","maths","reasoning"],"core_areas":["reasoning"],"testers":["BALROG team","Epoch AI"],"tests":{"gpqa":{"value":0.6414141414141414,"norm":0.6414,"version":"DeepSeek-R1-Distill-Qwen-32B"},"x:Balrog":{"value":0.195,"norm":0.195,"version":"DeepSeek-R1-Distill-Qwen-32B"},"aime":{"value":0.5555555555555556,"norm":0.5556,"version":"DeepSeek-R1-Distill-Qwen-32B"}},"variant":"DeepSeek-R1-Distill-Qwen-32B","match":"self"},"date_from":"epoch"},{"id":"kimi-k1-5","name":"Kimi k1.5","lab":"Moonshot","date":"2025-01-20","type":"reasoning","major":true,"open":false,"release":{"title":"Kimi k1.5","source":"https://www.kimi.com/en/blog/","verified":true,"confidence":"high"}},{"id":"gemini-2-0-flash-thinking-jan-2025","name":"Gemini 2.0 Flash Thinking (Jan 2025)","lab":"Google","date":"2025-01-21","type":"flagship","major":false,"open":false,"index":{"score":122.9,"lo":107.1,"hi":138.7,"se":1.84,"solve":20.5,"status":"rated","areas":["coding","maths","reasoning"],"core_areas":["reasoning"],"testers":["Aider","Epoch AI","Fiction.live","GeoBench team","Scale AI","SimpleBench team"],"tests":{"x:SimpleBench":{"value":0.307,"norm":0.307,"version":"gemini-2.0-flash-thinking-exp-01-21"},"gpqa":{"value":0.5707070707070707,"norm":0.5707,"version":"gemini-2.0-flash-thinking-exp-01-21"},"x:GeoBench":{"value":0.7000000000000001,"norm":0.7,"version":"gemini-2.0-flash-thinking-exp-01-21"},"aime":{"value":0.5777777777777777,"norm":0.5778,"version":"gemini-2.0-flash-thinking-exp-01-21"},"aider":{"value":18.2,"norm":0.182,"version":"gemini-2.0-flash-thinking-exp-01-21"},"hle":{"value":0.06559999999999999,"norm":0.0656,"version":"gemini-2.0-flash-thinking-exp-01-21"},"x:Fiction.LiveBench":{"value":0.528,"norm":0.528,"version":"gemini-2.0-flash-thinking-exp-01-21"}},"variant":"Gemini 2.0 Flash Thinking (Jan 2025)","match":"self"},"date_from":"epoch"},{"id":"doubao-1-5-pro","name":"Doubao 1.5 Pro","lab":"ByteDance","date":"2025-01-22","type":"flagship","major":true,"open":false,"release":{"title":"Doubao 1.5 Pro","source":null,"verified":false,"confidence":"high"}},{"id":"qwen-plus-jan-2025","name":"Qwen Plus (Jan 2025)","lab":"Alibaba","date":"2025-01-25","type":"flagship","major":false,"open":false,"index":{"score":114.4,"lo":98.3,"hi":130.6,"se":2.77,"solve":17,"status":"provisional","areas":["maths","reasoning"],"core_areas":["reasoning"],"testers":["Epoch AI"],"tests":{"math5":{"value":0.6527567975830816,"norm":0.6528,"version":"qwen-plus-2025-01-25"},"aime":{"value":0.17777777777777778,"norm":0.1778,"version":"qwen-plus-2025-01-25"},"gpqa":{"value":0.4810606060606061,"norm":0.4811,"version":"qwen-plus-2025-01-25"}},"variant":"Qwen Plus (Jan 2025)","match":"self"},"api":{"or_id":"qwen/qwen-plus","in":0.26,"out":0.78,"context":1000000,"hf":null,"created":"2025-02-01"},"date_from":"epoch"},{"id":"qwen2-5-vl","name":"Qwen2.5-VL","lab":"Alibaba","date":"2025-01-26","type":"open-weights","major":true,"open":true,"release":{"title":"Qwen2.5-VL","source":"https://qwenlm.github.io/blog/qwen2.5-vl/","verified":true,"confidence":"high"},"api":{"or_id":"qwen/qwen2.5-vl-72b-instruct","in":0.8,"out":1,"context":128000,"hf":"Qwen/Qwen2.5-VL-72B-Instruct","created":"2025-02-01"}},{"id":"qwen2-5-max","name":"Qwen2.5-Max","lab":"Alibaba","date":"2025-01-28","type":"flagship","major":true,"open":false,"release":{"title":"Qwen2.5-Max","source":"https://qwenlm.github.io/blog/qwen2.5-max/","verified":true,"confidence":"high"},"index":{"score":118.4,"lo":102.4,"hi":134.3,"se":2.22,"solve":18.5,"status":"rated","areas":["coding","maths","reasoning"],"core_areas":["reasoning"],"testers":["Aider","Epoch AI","Fiction.live"],"tests":{"gpqa":{"value":0.5612373737373737,"norm":0.5612,"version":"qwen-max-2025-01-25"},"math5":{"value":0.6718277945619335,"norm":0.6718,"version":"qwen-max-2025-01-25"},"aider":{"value":21.8,"norm":0.218,"version":"qwen-max-2025-01-25"},"x:Fiction.LiveBench":{"value":0.667,"norm":0.667,"version":"qwen-max-2025-01-25"},"x:FrontierMath-2025-02-28-Private":{"value":0.010344827586206896,"norm":0.0103,"version":"qwen-max-2025-01-25"},"aime":{"value":0.16111111111111112,"norm":0.1611,"version":"qwen-max-2025-01-25"}},"variant":"Qwen2.5-Max","match":"exact"}},{"id":"gpt-4o-jan-2025","name":"GPT-4o (Jan 2025)","lab":"OpenAI","date":"2025-01-29","type":"flagship","major":false,"open":false,"index":{"score":123,"lo":106.8,"hi":139.3,"se":2.97,"solve":20.5,"status":"provisional","areas":["coding","reasoning"],"core_areas":[],"testers":["Aider","Fiction.live"],"tests":{"aider":{"value":27.1,"norm":0.271,"version":"chatgpt-4o-01-29-2025"},"x:Fiction.LiveBench":{"value":0.667,"norm":0.667,"version":"chatgpt-4o-01-29-2025"}},"variant":"GPT-4o (Jan 2025)","match":"self"},"date_from":"epoch"},{"id":"mistral-small-3","name":"Mistral Small 3","lab":"Mistral","date":"2025-01-30","type":"open-weights","major":true,"open":true,"release":{"title":"Mistral Small 3","source":"https://mistral.ai/news/mistral-small-3/","verified":true,"confidence":"high"},"index":{"score":108,"lo":91.7,"hi":124.3,"se":3.09,"solve":14.8,"status":"provisional","areas":["maths","reasoning"],"core_areas":["reasoning"],"testers":["Epoch AI"],"tests":{"aime":{"value":0.06666666666666667,"norm":0.0667,"version":"mistral-small-3-2501"},"gpqa":{"value":0.47285353535353536,"norm":0.4729,"version":"mistral-small-3-2501"},"math5":{"value":0.4481684290030212,"norm":0.4482,"version":"mistral-small-2501"}},"variant":"Mistral Small 3","match":"exact"},"api":{"or_id":"mistralai/mistral-small-24b-instruct-2501","in":0.05,"out":0.08,"context":32768,"hf":"mistralai/Mistral-Small-24B-Instruct-2501","created":"2025-01-30"}},{"id":"o3-mini-2025-01","name":"o3-mini","lab":"OpenAI","date":"2025-01-31","type":"reasoning","major":true,"open":false,"release":{"title":"o3-mini","source":"https://openai.com/index/openai-o3-mini/","verified":true,"confidence":"high"},"index":{"score":131.2,"lo":115.4,"hi":147,"se":1.82,"solve":24.7,"status":"rated","areas":["agents","coding","maths","novel","reasoning","work"],"core_areas":["maths","novel","reasoning"],"testers":["ARC Prize","Aider","Epoch AI","Fiction.live","GSO team","Havard T. Ihle","Redwood Research","SimpleBench team","Vals AI","Will Patrick"],"tests":{"x:LMCA":{"value":18.97,"norm":0.1897,"version":"o3-mini-2025-01-31_high"},"arc:arc_agi_1":{"value":34.5,"norm":0.345,"version":"ARC Prize"},"x:DTBench":{"value":0.688,"norm":0.688,"version":"o3-mini-2025-01-31_high"},"vals:mgsm":{"value":91.35,"norm":0.9135,"version":"Vals AI"},"x:FrontierMath-Tier-4-2025-07-01-Private":{"value":0.041666666666666664,"norm":0.0417,"version":"o3-mini-2025-01-31_high"},"vals:math500":{"value":91.8,"norm":0.918,"version":"Vals AI"},"vals:corp_fin_v2":{"value":45.3,"norm":0.453,"version":"Vals AI"},"vals:gpqa":{"value":75.5,"norm":0.755,"version":"Vals AI"},"math5":{"value":0.9648791540785498,"norm":0.9649,"version":"o3-mini-2025-01-31_high"},"arc:arc_agi_2":{"value":3,"norm":0.03,"version":"ARC Prize"},"x:Fiction.LiveBench":{"value":0.5,"norm":0.5,"version":"o3-mini-2025-01-31_medium"},"aider":{"value":60.4,"norm":0.604,"version":"o3-mini-2025-01-31_high"},"gpqa":{"value":0.7702020202020202,"norm":0.7702,"version":"o3-mini-2025-01-31_high"},"x:SimpleBench":{"value":0.228,"norm":0.228,"version":"o3-mini-2025-01-31_high"},"x:GSO-Bench":{"value":0.013000000000000001,"norm":0.013,"version":"o3-mini-2025-01-31_low"},"vals:tax_eval_v2":{"value":69.42,"norm":0.6942,"version":"Vals AI"},"x:SimpleQA Verified":{"value":0.153,"norm":0.153,"version":"o3-mini-2025-01-31_high"},"fm":{"value":0.18596491228070175,"norm":0.186,"version":"o3-mini-2025-01-31_high"},"x:CadEval":{"value":0.54,"norm":0.54,"version":"o3-mini-2025-01-31_medium"},"vals:lcb":{"value":71.48,"norm":0.7148,"version":"Vals AI"},"x:Chess Puzzles":{"value":0.15,"norm":0.15,"version":"o3-mini-2025-01-31_high"},"vals:mmlu_pro":{"value":78.69,"norm":0.7869,"version":"Vals AI"},"x:WeirdML":{"value":0.4375,"norm":0.4375,"version":"o3-mini-2025-01-31_high"},"aime":{"value":0.7694444444444445,"norm":0.7694,"version":"o3-mini-2025-01-31_high"},"x:Mystery Game Puzzles":{"value":0.07,"norm":0.07,"version":"o3-mini-2025-01-31_high"},"x:FrontierMath-2025-02-28-Private":{"value":0.12413793103448276,"norm":0.1241,"version":"o3-mini-2025-01-31_high"},"vals:legal_bench":{"value":71.54,"norm":0.7154,"version":"Vals AI"},"x:FrontierMath-Tier-4-v2-Private":{"value":0,"norm":0,"version":"o3-mini-2025-01-31_high"}},"variant":"o3-mini","match":"exact"},"events":[{"date":"2024-12-20","title":"o3 / o3-mini (announced; preview only)","source":"https://en.wikipedia.org/wiki/OpenAI_o3"}],"aliases":["o3-mini"]},{"id":"gemini-2-0-flash-2025-02","name":"Gemini 2.0 Flash","lab":"Google","date":"2025-02-05","type":"flagship","major":true,"open":false,"release":{"title":"Gemini 2.0 Pro (exp) + 2.0 Flash GA","source":"https://blog.google/innovation-and-ai/models-and-research/google-deepmind/gemini-model-updates-february-2025/","verified":true,"confidence":"high"},"index":{"score":121.7,"lo":105.8,"hi":137.7,"se":2.24,"solve":19.9,"status":"rated","areas":["agents","coding","maths","reasoning"],"core_areas":["reasoning"],"testers":["Epoch AI","Fiction.live","GeoBench team","Havard T. Ihle","Redwood Research","Will Patrick"],"tests":{"x:GeoBench":{"value":0.77,"norm":0.77,"version":"gemini-2.0-flash-001"},"x:WeirdML":{"value":0.2577,"norm":0.2577,"version":"gemini-2.0-flash-001"},"math5":{"value":0.8216578549848943,"norm":0.8217,"version":"gemini-2.0-flash-001"},"aime":{"value":0.3111111111111111,"norm":0.3111,"version":"gemini-2.0-flash-001"},"x:FrontierMath-2025-02-28-Private":{"value":0.017241379310344827,"norm":0.0172,"version":"gemini-2.0-flash-001"},"x:CadEval":{"value":0.3,"norm":0.3,"version":"gemini-2.0-flash-001"},"x:Fiction.LiveBench":{"value":0.611,"norm":0.611,"version":"gemini-2.0-flash-001"},"gpqa":{"value":0.6414141414141414,"norm":0.6414,"version":"gemini-2.0-flash-001"},"x:DTBench":{"value":0.632,"norm":0.632,"version":"gemini-2.0-flash-001"}},"variant":"Gemini 2.0 Flash (Feb 2025)","match":"exact"}},{"id":"gemini-2-0-flash-lite","name":"Gemini 2.0 Flash-Lite","lab":"Google","date":"2025-02-05","type":"flagship","major":false,"open":false,"index":{"score":103.4,"lo":76,"hi":130.8,"se":13.73,"solve":13.5,"status":"insufficient","areas":["agents"],"core_areas":[],"testers":["Redwood Research"],"tests":{"x:DTBench":{"value":0.5253,"norm":0.5253,"version":"gemini-2.0-flash-lite"}},"variant":"Gemini 2.0 Flash-Lite","match":"self"},"date_from":"epoch"},{"id":"gemini-2-0-pro","name":"Gemini 2.0 Pro","lab":"Google","date":"2025-02-05","type":"flagship","major":true,"open":false,"release":{"title":"Gemini 2.0 Pro (exp) + 2.0 Flash GA","source":"https://blog.google/innovation-and-ai/models-and-research/google-deepmind/gemini-model-updates-february-2025/","verified":true,"confidence":"high"},"index":{"score":122.3,"lo":106.4,"hi":138.3,"se":2.25,"solve":20.2,"status":"rated","areas":["coding","maths","reasoning","work"],"core_areas":["reasoning"],"testers":["Aider","Epoch AI","Fiction.live","GeoBench team","Vals AI"],"tests":{"aider":{"value":35.6,"norm":0.356,"version":"gemini-2.0-pro-exp-02-05"},"vals:tax_eval_v2":{"value":68.15,"norm":0.6815,"version":"Vals AI"},"x:Fiction.LiveBench":{"value":0.417,"norm":0.417,"version":"gemini-2.0-pro-exp-02-05"},"math5":{"value":0.8345921450151057,"norm":0.8346,"version":"gemini-2.0-pro-exp-02-05"},"vals:corp_fin_v2":{"value":43.44,"norm":0.4344,"version":"Vals AI"},"x:GeoBench":{"value":0.7755,"norm":0.7755,"version":"gemini-2.0-pro-exp-02-05"},"gpqa":{"value":0.6565656565656566,"norm":0.6566,"version":"gemini-2.0-pro-exp-02-05"}},"variant":"Gemini 2.0 Pro","match":"exact"}},{"id":"grok-3","name":"Grok 3","lab":"xAI","date":"2025-02-17","type":"flagship","major":true,"open":false,"release":{"title":"Grok 3 (+ Grok 3 mini)","source":"https://x.ai/news/grok-3","verified":true,"confidence":"high"}},{"id":"claude-3-7-sonnet","name":"Claude 3.7 Sonnet","lab":"Anthropic","date":"2025-02-24","type":"reasoning","major":true,"open":false,"release":{"title":"Claude 3.7 Sonnet","source":"https://www.anthropic.com/news/claude-3-7-sonnet","verified":true,"confidence":"high"},"index":{"score":135.2,"lo":119.3,"hi":151.1,"se":2.23,"solve":27.1,"status":"rated","areas":["agents","coding","long","maths","reasoning","work"],"core_areas":["coding","long","reasoning"],"testers":["Aider","Chase Brower","Epoch AI","Fiction.live","FutureSearch","GSO team","GeoBench team","METR","Scale AI","SimpleBench team","Vals AI","Will Patrick"],"tests":{"vals:tax_eval_v2":{"value":74.04,"norm":0.7404,"version":"Vals AI"},"x:VPCT":{"value":0.39,"norm":0.39,"version":"claude-3-7-sonnet-20250219"},"x:Fiction.LiveBench":{"value":0.833,"norm":0.833,"version":"claude-3-7-sonnet-20250219_8K"},"aider":{"value":64.9,"norm":0.649,"version":"claude-3-7-sonnet-20250219_32K"},"x:FrontierMath-2025-02-28-Private":{"value":0.041379310344827586,"norm":0.0414,"version":"claude-3-7-sonnet-20250219_16K"},"vals:legal_bench":{"value":82.52,"norm":0.8252,"version":"Vals AI"},"vals:lcb":{"value":60.44,"norm":0.6044,"version":"Vals AI"},"vals:mortgage_tax":{"value":68.68,"norm":0.6868,"version":"Vals AI"},"metr":{"value":0.599681,"norm":0.2305,"version":"claude-3-7-sonnet-20250219_16K"},"vals:mmmu":{"value":75.1,"norm":0.751,"version":"Vals AI"},"vals:math500":{"value":91.6,"norm":0.916,"version":"Vals AI"},"gpqa":{"value":0.7973484848484849,"norm":0.7973,"version":"claude-3-7-sonnet-20250219_64K"},"vals:gpqa":{"value":75.25,"norm":0.7525,"version":"Vals AI"},"x:GeoBench":{"value":0.68,"norm":0.68,"version":"claude-3-7-sonnet-20250219"},"hle":{"value":0.0804,"norm":0.0804,"version":"claude-3-7-sonnet-20250219_unknown"},"x:DeepResearch Bench":{"value":0.436,"norm":0.436,"version":"claude-3-7-sonnet-20250219_2K"},"swe":{"value":0.609504132231405,"norm":0.6095,"version":"claude-3-7-sonnet-20250219"},"vals:mmlu_pro":{"value":82.73,"norm":0.8273,"version":"Vals AI"},"x:GSO-Bench":{"value":0.038,"norm":0.038,"version":"claude-3-7-sonnet-20250219_unknown"},"vals:mgsm":{"value":92.98,"norm":0.9298,"version":"Vals AI"},"aime":{"value":0.5777777777777777,"norm":0.5778,"version":"claude-3-7-sonnet-20250219_64K"},"vals:corp_fin_v2":{"value":60.41,"norm":0.6041,"version":"Vals AI"},"math5":{"value":0.9116314199395771,"norm":0.9116,"version":"claude-3-7-sonnet-20250219_64K"},"x:SimpleBench":{"value":0.464,"norm":0.464,"version":"claude-3-7-sonnet-20250219_12K"},"x:CadEval":{"value":0.54,"norm":0.54,"version":"claude-3-7-sonnet-20250219"}},"variant":"Claude 3.7 Sonnet","match":"exact"}},{"id":"gpt-4-5","name":"GPT-4.5","lab":"OpenAI","date":"2025-02-27","type":"flagship","major":true,"open":false,"release":{"title":"GPT-4.5","source":"https://en.wikipedia.org/wiki/GPT-4.5","verified":false,"confidence":"high"},"index":{"score":124.7,"lo":108.9,"hi":140.5,"se":1.78,"solve":21.3,"status":"rated","areas":["coding","maths","novel","reasoning"],"core_areas":["novel","reasoning"],"testers":["ARC Prize","Aider","Chase Brower","Epoch AI","Fiction.live","Havard T. Ihle","Scale AI","SimpleBench team"],"tests":{"aider":{"value":44.9,"norm":0.449,"version":"gpt-4.5-preview-2025-02-27"},"arc:arc_agi_1":{"value":10.3,"norm":0.103,"version":"ARC Prize"},"x:Fiction.LiveBench":{"value":0.639,"norm":0.639,"version":"gpt-4.5-preview-2025-02-27"},"x:SimpleBench":{"value":0.345,"norm":0.345,"version":"gpt-4.5-preview-2025-02-27"},"x:WeirdML":{"value":0.3937,"norm":0.3937,"version":"gpt-4.5-preview-2025-02-27"},"aime":{"value":0.37777777777777777,"norm":0.3778,"version":"gpt-4.5-preview-2025-02-27"},"x:VPCT":{"value":0.45,"norm":0.45,"version":"gpt-4.5-preview-2025-02-27"},"hle":{"value":0.054400000000000004,"norm":0.0544,"version":"gpt-4.5-preview-2025-02-27"},"arc:arc_agi_2":{"value":0.8,"norm":0.008,"version":"ARC Prize"},"math5":{"value":0.7862537764350453,"norm":0.7863,"version":"gpt-4.5-preview-2025-02-27"},"gpqa":{"value":0.6868686868686869,"norm":0.6869,"version":"gpt-4.5-preview-2025-02-27"}},"variant":"GPT-4.5","match":"exact"}},{"id":"qwq-32b","name":"QwQ-32B","lab":"Alibaba","date":"2025-03-06","type":"reasoning","major":true,"open":true,"release":{"title":"QwQ-32B","source":"https://qwenlm.github.io/blog/qwq-32b/","verified":true,"confidence":"high"},"index":{"score":126.6,"lo":110.8,"hi":142.4,"se":1.84,"solve":22.2,"status":"rated","areas":["coding","maths","reasoning"],"core_areas":["reasoning"],"testers":["Aider","Epoch AI","Fiction.live"],"tests":{"x:Fiction.LiveBench":{"value":0.833,"norm":0.833,"version":"chutes/QwQ-32B"},"aime":{"value":0.5916666666666667,"norm":0.5917,"version":"QwQ-32B"},"gpqa":{"value":0.6534090909090909,"norm":0.6534,"version":"QwQ-32B"},"aider":{"value":20.9,"norm":0.209,"version":"QwQ-32B"}},"variant":"QwQ-32B","match":"exact"}},{"id":"gemma-3","name":"Gemma 3","lab":"Google","date":"2025-03-12","type":"open-weights","major":true,"open":true,"release":{"title":"Gemma 3","source":"https://blog.google/innovation-and-ai/technology/developers-tools/gemma-3/","verified":true,"confidence":"high"},"index":{"score":112.7,"lo":96.8,"hi":128.6,"se":2.2,"solve":16.4,"status":"rated","areas":["agents","coding","maths","reasoning"],"core_areas":["reasoning"],"testers":["Aider","Epoch AI","Fiction.live","GeoBench team","Redwood Research"],"tests":{"math5":{"value":0.740370090634441,"norm":0.7404,"version":"gemma-3-27b-it"},"x:Fiction.LiveBench":{"value":0.333,"norm":0.333,"version":"chutes/Gemma-3-27b-It"},"aime":{"value":0.225,"norm":0.225,"version":"gemma-3-27b-it"},"x:DTBench":{"value":0.5253,"norm":0.5253,"version":"gemma-3-27b-it"},"gpqa":{"value":0.4772727272727273,"norm":0.4773,"version":"gemma-3-27b-it"},"x:LMCA":{"value":12.28,"norm":0.1228,"version":"gemma-3-27b-it"},"aider":{"value":4.9,"norm":0.049,"version":"gemma-3-27b-it"},"x:GeoBench":{"value":0.52,"norm":0.52,"version":"gemma-3-27b-it"}},"variant":"Gemma 3 27B","match":"family"},"api":{"or_id":"google/gemma-3-27b-it","in":0.08,"out":0.45,"context":131072,"hf":"google/gemma-3-27b-it","created":"2025-03-12"}},{"id":"gemma-3-12b","name":"Gemma 3 12B","lab":"Google","date":"2025-03-12","type":"open-weights","major":false,"open":true,"index":{"score":106.5,"lo":88.9,"hi":124.2,"se":5.14,"solve":14.4,"status":"rated","areas":["agents","maths","reasoning"],"core_areas":["reasoning"],"testers":["Epoch AI","Redwood Research"],"tests":{"gpqa":{"value":0.39457070707070707,"norm":0.3946,"version":"gemma-3-12b-it"},"x:LMCA":{"value":4.46,"norm":0.0446,"version":"gemma-3-12b-it"},"x:DTBench":{"value":0.488,"norm":0.488,"version":"gemma-3-12b-it"},"aime":{"value":0.16666666666666666,"norm":0.1667,"version":"gemma-3-12b-it"}},"variant":"Gemma 3 12B","match":"self"},"api":{"or_id":"google/gemma-3-12b-it","in":0.05,"out":0.15,"context":131072,"hf":"google/gemma-3-12b-it","created":"2025-03-13"},"date_from":"epoch"},{"id":"gemma-3-1b","name":"Gemma 3 1B","lab":"Google","date":"2025-03-12","type":"open-weights","major":false,"open":true,"index":{"score":82.9,"lo":25.4,"hi":140.5,"se":33.68,"solve":9.4,"status":"provisional","areas":["maths","reasoning"],"core_areas":["reasoning"],"testers":["Epoch AI"],"tests":{"gpqa":{"value":0.1994949494949495,"norm":0.1995,"version":"gemma-3-1b-it"},"aime":{"value":0.011111111111111112,"norm":0.0111,"version":"gemma-3-1b-it"}},"variant":"Gemma 3 1B","match":"self"},"date_from":"epoch"},{"id":"gemma-3-4b","name":"Gemma 3 4B","lab":"Google","date":"2025-03-12","type":"open-weights","major":false,"open":true,"index":{"score":89.3,"lo":61.7,"hi":117,"se":13.92,"solve":10.4,"status":"rated","areas":["agents","maths","reasoning"],"core_areas":["reasoning"],"testers":["Epoch AI","Redwood Research"],"tests":{"x:LMCA":{"value":2.78,"norm":0.0278,"version":"gemma-3-4b-it"},"aime":{"value":0.075,"norm":0.075,"version":"gemma-3-4b-it"},"gpqa":{"value":0.23232323232323232,"norm":0.2323,"version":"gemma-3-4b-it"},"x:DTBench":{"value":0.5093,"norm":0.5093,"version":"gemma-3-4b-it"}},"variant":"Gemma 3 4B","match":"self"},"api":{"or_id":"google/gemma-3-4b-it","in":0.05,"out":0.1,"context":131072,"hf":"google/gemma-3-4b-it","created":"2025-03-14"},"date_from":"epoch"},{"id":"ernie-4-5","name":"ERNIE 4.5","lab":"Baidu","date":"2025-03-16","type":"flagship","major":true,"open":false,"release":{"title":"ERNIE 4.5","source":null,"verified":false,"confidence":"high"}},{"id":"ernie-x1","name":"ERNIE X1","lab":"Baidu","date":"2025-03-16","type":"reasoning","major":true,"open":false,"release":{"title":"ERNIE X1","source":null,"verified":false,"confidence":"high"}},{"id":"mistral-small-3-1","name":"Mistral Small 3.1","lab":"Mistral","date":"2025-03-17","type":"open-weights","major":false,"open":true,"index":{"score":107.9,"lo":91.7,"hi":124.1,"se":2.87,"solve":14.8,"status":"rated","areas":["agents","coding","maths","reasoning","work"],"core_areas":["reasoning"],"testers":["Epoch AI","Redwood Research","Vals AI"],"tests":{"vals:tax_eval_v2":{"value":58.3,"norm":0.583,"version":"Vals AI"},"vals:mgsm":{"value":84.22,"norm":0.8422,"version":"Vals AI"},"vals:mmmu":{"value":60.08,"norm":0.6008,"version":"Vals AI"},"x:DTBench":{"value":0.5864,"norm":0.5864,"version":"mistral-small-2503"},"vals:gpqa":{"value":44.19,"norm":0.4419,"version":"Vals AI"},"aime":{"value":0.058333333333333334,"norm":0.0583,"version":"mistral-small-2503"},"vals:mortgage_tax":{"value":61.21,"norm":0.6121,"version":"Vals AI"},"vals:mmlu_pro":{"value":66.02,"norm":0.6602,"version":"Vals AI"},"vals:math500":{"value":68.4,"norm":0.684,"version":"Vals AI"},"vals:corp_fin_v2":{"value":44.17,"norm":0.4417,"version":"Vals AI"},"gpqa":{"value":0.47474747474747475,"norm":0.4747,"version":"mistral-small-2503"},"vals:legal_bench":{"value":69.16,"norm":0.6916,"version":"Vals AI"},"vals:lcb":{"value":31.81,"norm":0.3181,"version":"Vals AI"},"math5":{"value":0.4677114803625378,"norm":0.4677,"version":"mistral-small-2503"}},"variant":"Mistral Small 3.1","match":"self"},"api":{"or_id":"mistralai/mistral-small-3.1-24b-instruct","in":0.351,"out":0.555,"context":128000,"hf":"mistralai/Mistral-Small-3.1-24B-Instruct-2503","created":"2025-03-17"},"date_from":"epoch"},{"id":"hunyuan-t1","name":"Hunyuan-T1","lab":"Tencent","date":"2025-03-21","type":"reasoning","major":true,"open":false,"release":{"title":"Hunyuan-T1","source":null,"verified":false,"confidence":"high"}},{"id":"deepseek-v3-mar-2025","name":"DeepSeek-V3 (Mar 2025)","lab":"DeepSeek","date":"2025-03-24","type":"open-weights","major":false,"open":true,"index":{"score":124.1,"lo":108.3,"hi":139.8,"se":1.84,"solve":21,"status":"rated","areas":["agents","coding","long","maths","reasoning"],"core_areas":["long","reasoning"],"testers":["Aider","Epoch AI","Fiction.live","Havard T. Ihle","METR","Redwood Research","SimpleBench team"],"tests":{"x:Fiction.LiveBench":{"value":0.5,"norm":0.5,"version":"chutes/DeepSeek-V3-0324"},"x:SimpleBench":{"value":0.272,"norm":0.272,"version":"DeepSeek-V3-0324"},"math5":{"value":0.75547583081571,"norm":0.7555,"version":"DeepSeek-V3-0324"},"x:DTBench":{"value":0.648,"norm":0.648,"version":"DeepSeek-V3-0324"},"gpqa":{"value":0.6761363636363636,"norm":0.6761,"version":"DeepSeek-V3-0324"},"aime":{"value":0.37777777777777777,"norm":0.3778,"version":"DeepSeek-V3-0324"},"metr":{"value":0.495813,"norm":0.2128,"version":"DeepSeek-V3-0324"},"x:WeirdML":{"value":0.3608,"norm":0.3608,"version":"DeepSeek-V3-0324"},"x:LMCA":{"value":15.47,"norm":0.1547,"version":"DeepSeek-V3-0324"},"aider":{"value":55.1,"norm":0.551,"version":"DeepSeek-V3-0324"}},"variant":"DeepSeek-V3 (Mar 2025)","match":"self"},"api":{"or_id":"deepseek/deepseek-chat-v3-0324","in":0.29,"out":1.14,"context":163840,"hf":"deepseek-ai/DeepSeek-V3-0324","created":"2025-03-24"},"date_from":"epoch"},{"id":"gemini-2-5-pro","name":"Gemini 2.5 Pro","lab":"Google","date":"2025-03-25","type":"reasoning","major":true,"open":false,"release":{"title":"Gemini 2.5 Pro","source":"https://blog.google/innovation-and-ai/models-and-research/google-deepmind/gemini-model-thinking-updates-march-2025/","verified":true,"confidence":"high"},"index":{"score":141.2,"lo":125.1,"hi":157.3,"se":2.62,"solve":31.2,"status":"rated","areas":["agents","coding","maths","reasoning"],"core_areas":["reasoning"],"testers":["Aider","BALROG team","Chase Brower","Epoch AI","Fiction.live","GeoBench team","Scale AI","SimpleBench team","Will Patrick"],"tests":{"hle":{"value":0.1816,"norm":0.1816,"version":"gemini-2.5-pro-exp-03-25"},"x:Fiction.LiveBench":{"value":0.667,"norm":0.667,"version":"gemini-2.5-pro-preview-03-25"},"x:VPCT":{"value":0.48,"norm":0.48,"version":"gemini-2.5-pro-preview-03-25"},"x:GeoBench":{"value":0.78,"norm":0.78,"version":"gemini-2.5-pro-exp-03-25"},"x:CadEval":{"value":0.64,"norm":0.64,"version":"gemini-2.5-pro-preview-03-25"},"x:Balrog":{"value":0.433,"norm":0.433,"version":"gemini-2.5-pro-exp-03-25"},"gpqa":{"value":0.8383838383838383,"norm":0.8384,"version":"gemini-2.5-pro-exp-03-25"},"math5":{"value":0.9556268882175226,"norm":0.9556,"version":"gemini-2.5-pro-preview-03-25"},"x:SimpleBench":{"value":0.516,"norm":0.516,"version":"gemini-2.5-pro-preview-03-25"},"aider":{"value":72.9,"norm":0.729,"version":"gemini-2.5-pro-preview-03-25"}},"variant":"Gemini 2.5 Pro (Mar 2025)","match":"exact"}},{"id":"gpt-4o-mar-2025","name":"GPT-4o (Mar 2025)","lab":"OpenAI","date":"2025-03-27","type":"flagship","major":false,"open":false,"index":{"score":126.6,"lo":110,"hi":143.2,"se":3.55,"solve":22.2,"status":"insufficient","areas":["coding"],"core_areas":[],"testers":["Aider"],"tests":{"aider":{"value":45.3,"norm":0.453,"version":"chatgpt-4o-03-27-2025"}},"variant":"GPT-4o (Mar 2025)","match":"self"},"date_from":"epoch"},{"id":"llama-4-maverick","name":"Llama 4 Maverick","lab":"Meta","date":"2025-04-05","type":"open-weights","major":true,"open":true,"release":{"title":"Llama 4 (Scout/Maverick)","source":"https://about.fb.com/ltam/news/2025/04/la-coleccion-de-modelos-llama-4-el-inicio-de-una-nueva-era-de-innovacion-multimodal-nativa-para-inteligencia-artificial/","verified":true,"confidence":"high"},"index":{"score":116,"lo":100.2,"hi":131.9,"se":2.08,"solve":17.6,"status":"rated","areas":["agents","coding","maths","novel","reasoning","work"],"core_areas":["novel","reasoning"],"testers":["ARC Prize","Aider","Epoch AI","Fiction.live","GeoBench team","Havard T. Ihle","Redwood Research","Scale AI","SimpleBench team","Vals AI"],"tests":{"arc:arc_agi_1":{"value":4.4,"norm":0.044,"version":"ARC Prize"},"aime":{"value":0.20555555555555555,"norm":0.2056,"version":"Llama-4-Maverick-17B-128E-Instruct-FP8"},"x:SimpleBench":{"value":0.277,"norm":0.277,"version":"Llama-4-Maverick-17B-128E-Instruct"},"vals:mmlu_pro":{"value":79.42,"norm":0.7942,"version":"Vals AI"},"arc:arc_agi_2":{"value":0,"norm":0,"version":"ARC Prize"},"x:WeirdML":{"value":0.2447,"norm":0.2447,"version":"Llama-4-Maverick-17B-128E-Instruct"},"math5":{"value":0.7301737160120846,"norm":0.7302,"version":"Llama-4-Maverick-17B-128E-Instruct-FP8"},"vals:medscribe":{"value":54.22,"norm":0.5422,"version":"Vals AI"},"x:Fiction.LiveBench":{"value":0.462,"norm":0.462,"version":"chutes/Llama-4-Maverick-17B-128E-Instruct"},"vals:legal_bench":{"value":77.81,"norm":0.7781,"version":"Vals AI"},"vals:lcb":{"value":47.25,"norm":0.4725,"version":"Vals AI"},"vals:medcode":{"value":36.51,"norm":0.3651,"version":"Vals AI"},"vals:mortgage_tax":{"value":58.51,"norm":0.5851,"version":"Vals AI"},"x:FrontierMath-2025-02-28-Private":{"value":0.006896551724137931,"norm":0.0069,"version":"Llama-4-Maverick-17B-128E-Instruct-FP8"},"vals:math500":{"value":85.2,"norm":0.852,"version":"Vals AI"},"x:LMCA":{"value":15.87,"norm":0.1587,"version":"Llama-4-Maverick-17B-128E-Instruct"},"hle":{"value":0.056799999999999996,"norm":0.0568,"version":"Llama-4-Maverick-17B-128E-Instruct"},"x:DTBench":{"value":0.6187,"norm":0.6187,"version":"Llama-4-Maverick-17B-128E-Instruct"},"vals:corp_fin_v2":{"value":49.73,"norm":0.4973,"version":"Vals AI"},"vals:gpqa":{"value":69.44,"norm":0.6944,"version":"Vals AI"},"x:GeoBench":{"value":0.52,"norm":0.52,"version":"Llama-4-Maverick-17B-128E-Instruct"},"aider":{"value":15.6,"norm":0.156,"version":"Llama-4-Maverick-17B-128E-Instruct"},"gpqa":{"value":0.6698232323232324,"norm":0.6698,"version":"Llama-4-Maverick-17B-128E-Instruct-FP8"},"vals:mmmu":{"value":71.69,"norm":0.7169,"version":"Vals AI"},"vals:tax_eval_v2":{"value":66.56,"norm":0.6656,"version":"Vals AI"},"vals:mgsm":{"value":92.44,"norm":0.9244,"version":"Vals AI"}},"variant":"Llama 4 Maverick","match":"exact"},"api":{"or_id":"meta-llama/llama-4-maverick","in":0.35,"out":1,"context":524288,"hf":"meta-llama/Llama-4-Maverick-17B-128E-Instruct","created":"2025-04-05"}},{"id":"llama-4-scout","name":"Llama 4 Scout","lab":"Meta","date":"2025-04-05","type":"open-weights","major":true,"open":true,"release":{"title":"Llama 4 (Scout/Maverick)","source":"https://about.fb.com/ltam/news/2025/04/la-coleccion-de-modelos-llama-4-el-inicio-de-una-nueva-era-de-innovacion-multimodal-nativa-para-inteligencia-artificial/","verified":true,"confidence":"high"},"index":{"score":112,"lo":95.9,"hi":128.1,"se":2.65,"solve":16.1,"status":"rated","areas":["agents","maths","novel","reasoning"],"core_areas":["novel","reasoning"],"testers":["ARC Prize","Epoch AI","Fiction.live","Redwood Research"],"tests":{"x:FrontierMath-2025-02-28-Private":{"value":0,"norm":0,"version":"Llama-4-Scout-17B-16E-Instruct"},"gpqa":{"value":0.5183080808080808,"norm":0.5183,"version":"Llama-4-Scout-17B-16E-Instruct"},"arc:arc_agi_1":{"value":0.5,"norm":0.005,"version":"ARC Prize"},"aime":{"value":0.07777777777777778,"norm":0.0778,"version":"Llama-4-Scout-17B-16E-Instruct"},"arc:arc_agi_2":{"value":0,"norm":0,"version":"ARC Prize"},"x:Fiction.LiveBench":{"value":0.36,"norm":0.36,"version":"chutes/Llama-4-Scout-17B-16E Instruct"},"math5":{"value":0.6227341389728097,"norm":0.6227,"version":"Llama-4-Scout-17B-16E-Instruct"},"x:LMCA":{"value":11.96,"norm":0.1196,"version":"Llama-4-Scout-17B-16E-Instruct"},"x:DTBench":{"value":0.5787,"norm":0.5787,"version":"Llama-4-Scout-17B-16E-Instruct"}},"variant":"Llama 4 Scout","match":"exact"},"api":{"or_id":"meta-llama/llama-4-scout","in":0.1,"out":0.3,"context":1310720,"hf":"meta-llama/Llama-4-Scout-17B-16E-Instruct","created":"2025-04-05"}},{"id":"qwq-plus","name":"QWQ-Plus","lab":"Alibaba","date":"2025-04-08","type":"flagship","major":false,"open":false,"index":{"score":127.4,"lo":109.3,"hi":145.6,"se":5.75,"solve":22.7,"status":"insufficient","areas":["reasoning"],"core_areas":["reasoning"],"testers":["Epoch AI"],"tests":{"gpqa":{"value":0.6540404040404041,"norm":0.654,"version":"qwq-plus"}},"variant":"QWQ-Plus","match":"self"},"date_from":"epoch"},{"id":"grok-3-mini","name":"Grok-3 mini","lab":"xAI","date":"2025-04-09","type":"flagship","major":false,"open":false,"index":{"score":130.6,"lo":114.8,"hi":146.4,"se":1.85,"solve":24.4,"status":"rated","areas":["coding","maths","novel","reasoning","work"],"core_areas":["novel","reasoning"],"testers":["ARC Prize","Aider","Epoch AI","Fiction.live","Havard T. Ihle","Vals AI"],"tests":{"aime":{"value":0.7777777777777778,"norm":0.7778,"version":"grok-3-mini-beta_high"},"x:Fiction.LiveBench":{"value":0.667,"norm":0.667,"version":"grok-3-mini-beta_medium"},"vals:mmlu_pro":{"value":81.37,"norm":0.8137,"version":"Vals AI"},"arc:arc_agi_2":{"value":0.4,"norm":0.004,"version":"ARC Prize"},"arc:arc_agi_1":{"value":16.5,"norm":0.165,"version":"ARC Prize"},"vals:mgsm":{"value":90.44,"norm":0.9044,"version":"Vals AI"},"x:FrontierMath-2025-02-28-Private":{"value":0.05862068965517241,"norm":0.0586,"version":"grok-3-mini-beta_high"},"vals:tax_eval_v2":{"value":72.98,"norm":0.7298,"version":"Vals AI"},"vals:math500":{"value":94.2,"norm":0.942,"version":"Vals AI"},"vals:lcb":{"value":76.22,"norm":0.7622,"version":"Vals AI"},"vals:gpqa":{"value":79.29,"norm":0.7929,"version":"Vals AI"},"x:WeirdML":{"value":0.4258,"norm":0.4258,"version":"grok-3-mini-beta_high"},"math5":{"value":0.9093655589123867,"norm":0.9094,"version":"grok-3-mini-beta_low"},"vals:corp_fin_v2":{"value":61.11,"norm":0.6111,"version":"Vals AI"},"aider":{"value":49.3,"norm":0.493,"version":"grok-3-mini-beta_high"},"gpqa":{"value":0.7626262626262627,"norm":0.7626,"version":"grok-3-mini-beta_low"},"vals:legal_bench":{"value":83.14,"norm":0.8314,"version":"Vals AI"}},"variant":"Grok-3 mini","match":"self"},"date_from":"epoch"},{"id":"gpt-4-1","name":"GPT-4.1","lab":"OpenAI","date":"2025-04-14","type":"flagship","major":true,"open":false,"release":{"title":"GPT-4.1","source":"https://en.wikipedia.org/wiki/GPT-4.1","verified":false,"confidence":"high"},"index":{"score":125.9,"lo":110.1,"hi":141.7,"se":1.88,"solve":21.9,"status":"rated","areas":["agents","coding","maths","novel","reasoning","work"],"core_areas":["coding","maths","novel","reasoning"],"testers":["ARC Prize","Aider","Epoch AI","Fiction.live","GeoBench team","Havard T. Ihle","Redwood Research","Scale AI","SimpleBench team","Vals AI","Will Patrick"],"tests":{"vals:lcb":{"value":54.67,"norm":0.5467,"version":"Vals AI"},"vals:mmmu":{"value":72.39,"norm":0.7239,"version":"Vals AI"},"math5":{"value":0.8300604229607251,"norm":0.8301,"version":"gpt-4.1-2025-04-14"},"vals:tax_eval_v2":{"value":75.06,"norm":0.7506,"version":"Vals AI"},"x:DTBench":{"value":0.6827,"norm":0.6827,"version":"gpt-4.1-2025-04-14"},"vals:mgsm":{"value":87.67,"norm":0.8767,"version":"Vals AI"},"x:FrontierMath-2025-02-28-Private":{"value":0.05517241379310345,"norm":0.0552,"version":"gpt-4.1-2025-04-14"},"x:CadEval":{"value":0.42,"norm":0.42,"version":"gpt-4.1-2025-04-14"},"x:WeirdML":{"value":0.3904,"norm":0.3904,"version":"gpt-4.1-2025-04-14"},"vals:mortgage_tax":{"value":65.94,"norm":0.6594,"version":"Vals AI"},"hle":{"value":0.054000000000000006,"norm":0.054,"version":"gpt-4.1-2025-04-14"},"x:Fiction.LiveBench":{"value":0.639,"norm":0.639,"version":"gpt-4.1-2025-04-14"},"vals:gpqa":{"value":65.4,"norm":0.654,"version":"Vals AI"},"x:SimpleQA Verified":{"value":0.311,"norm":0.311,"version":"gpt-4.1-2025-04-14"},"x:LMCA":{"value":25.61,"norm":0.2561,"version":"gpt-4.1-2025-04-14"},"arc:arc_agi_1":{"value":5.5,"norm":0.055,"version":"ARC Prize"},"fm":{"value":0.05964912280701754,"norm":0.0596,"version":"gpt-4.1-2025-04-14"},"vals:corp_fin_v2":{"value":63.05,"norm":0.6305,"version":"Vals AI"},"aime":{"value":0.38333333333333336,"norm":0.3833,"version":"gpt-4.1-2025-04-14"},"x:SimpleBench":{"value":0.27,"norm":0.27,"version":"gpt-4.1-2025-04-14"},"gpqa":{"value":0.6691919191919192,"norm":0.6692,"version":"gpt-4.1-2025-04-14"},"vals:legal_bench":{"value":83.1,"norm":0.831,"version":"Vals AI"},"swe":{"value":0.48541666666666666,"norm":0.4854,"version":"gpt-4.1-2025-04-14"},"x:FrontierMath-Tier-4-2025-07-01-Private":{"value":0,"norm":0,"version":"gpt-4.1-2025-04-14"},"x:Chess Puzzles":{"value":0.11,"norm":0.11,"version":"gpt-4.1-2025-04-14"},"x:GeoBench":{"value":0.72,"norm":0.72,"version":"gpt-4.1-2025-04-14"},"vals:math500":{"value":87.2,"norm":0.872,"version":"Vals AI"},"aider":{"value":52.4,"norm":0.524,"version":"gpt-4.1-2025-04-14"},"vals:mmlu_pro":{"value":80.5,"norm":0.805,"version":"Vals AI"},"arc:arc_agi_2":{"value":0.4,"norm":0.004,"version":"ARC Prize"}},"variant":"GPT-4.1","match":"exact"},"api":{"or_id":"openai/gpt-4.1","in":2,"out":8,"context":1047576,"hf":null,"created":"2025-04-14"}},{"id":"gpt-4-1-mini","name":"GPT-4.1 mini","lab":"OpenAI","date":"2025-04-14","type":"flagship","major":false,"open":false,"index":{"score":119.7,"lo":103.8,"hi":135.6,"se":2.09,"solve":19,"status":"rated","areas":["agents","coding","maths","novel","reasoning","work"],"core_areas":["maths","novel","reasoning"],"testers":["ARC Prize","Aider","Epoch AI","Fiction.live","Havard T. Ihle","Redwood Research","Vals AI","Will Patrick"],"tests":{"vals:mgsm":{"value":87.78,"norm":0.8778,"version":"Vals AI"},"vals:gpqa":{"value":67.93,"norm":0.6793,"version":"Vals AI"},"vals:legal_bench":{"value":78.04,"norm":0.7804,"version":"Vals AI"},"vals:mmmu":{"value":70.54,"norm":0.7054,"version":"Vals AI"},"vals:corp_fin_v2":{"value":57.93,"norm":0.5793,"version":"Vals AI"},"x:CadEval":{"value":0.16,"norm":0.16,"version":"gpt-4.1-mini-2025-04-14"},"gpqa":{"value":0.6584595959595959,"norm":0.6585,"version":"gpt-4.1-mini-2025-04-14"},"vals:mmlu_pro":{"value":77.22,"norm":0.7722,"version":"Vals AI"},"fm":{"value":0.06666666666666667,"norm":0.0667,"version":"gpt-4.1-mini-2025-04-14"},"aider":{"value":32.4,"norm":0.324,"version":"gpt-4.1-mini-2025-04-14"},"x:SimpleQA Verified":{"value":0.127,"norm":0.127,"version":"gpt-4.1-mini-2025-04-14"},"x:WeirdML":{"value":0.3761,"norm":0.3761,"version":"gpt-4.1-mini-2025-04-14"},"arc:arc_agi_2":{"value":0,"norm":0,"version":"ARC Prize"},"x:LMCA":{"value":21.12,"norm":0.2112,"version":"gpt-4.1-mini-2025-04-14"},"arc:arc_agi_1":{"value":3.5,"norm":0.035,"version":"ARC Prize"},"vals:mortgage_tax":{"value":65.5,"norm":0.655,"version":"Vals AI"},"x:Fiction.LiveBench":{"value":0.444,"norm":0.444,"version":"gpt-4.1-mini-2025-04-14"},"aime":{"value":0.44722222222222224,"norm":0.4472,"version":"gpt-4.1-mini-2025-04-14"},"x:Mystery Game Puzzles":{"value":0.07,"norm":0.07,"version":"gpt-4.1-mini-2025-04-14"},"x:FrontierMath-2025-02-28-Private":{"value":0.04482758620689655,"norm":0.0448,"version":"gpt-4.1-mini-2025-04-14"},"vals:lcb":{"value":58.16,"norm":0.5816,"version":"Vals AI"},"math5":{"value":0.8729229607250756,"norm":0.8729,"version":"gpt-4.1-mini-2025-04-14"},"vals:tax_eval_v2":{"value":71.91,"norm":0.7191,"version":"Vals AI"},"x:DTBench":{"value":0.688,"norm":0.688,"version":"gpt-4.1-mini-2025-04-14"},"x:Chess Puzzles":{"value":0.01,"norm":0.01,"version":"gpt-4.1-mini-2025-04-14"},"vals:math500":{"value":88,"norm":0.88,"version":"Vals AI"}},"variant":"GPT-4.1 mini","match":"self"},"api":{"or_id":"openai/gpt-4.1-mini","in":0.4,"out":1.6,"context":1047576,"hf":null,"created":"2025-04-14"},"date_from":"epoch"},{"id":"gpt-4-1-nano","name":"GPT-4.1 nano","lab":"OpenAI","date":"2025-04-14","type":"flagship","major":false,"open":false,"index":{"score":109.1,"lo":93,"hi":125.2,"se":2.69,"solve":15.2,"status":"rated","areas":["agents","coding","maths","novel","reasoning","work"],"core_areas":["novel","reasoning"],"testers":["ARC Prize","Aider","Epoch AI","Fiction.live","Havard T. Ihle","Redwood Research","Vals AI"],"tests":{"x:SimpleQA Verified":{"value":0.06,"norm":0.06,"version":"gpt-4.1-nano-2025-04-14"},"vals:mmmu":{"value":55.05,"norm":0.5505,"version":"Vals AI"},"x:LMCA":{"value":5.46,"norm":0.0546,"version":"gpt-4.1-nano-2025-04-14"},"arc:arc_agi_1":{"value":0,"norm":0,"version":"ARC Prize"},"x:Fiction.LiveBench":{"value":0.25,"norm":0.25,"version":"gpt-4.1-nano-2025-04-14"},"aider":{"value":8.9,"norm":0.089,"version":"gpt-4.1-nano-2025-04-14"},"vals:mmlu_pro":{"value":63.48,"norm":0.6348,"version":"Vals AI"},"vals:mortgage_tax":{"value":52.82,"norm":0.5282,"version":"Vals AI"},"vals:mgsm":{"value":69.27,"norm":0.6927,"version":"Vals AI"},"x:FrontierMath-2025-02-28-Private":{"value":0.010344827586206896,"norm":0.0103,"version":"gpt-4.1-nano-2025-04-14"},"gpqa":{"value":0.48926767676767674,"norm":0.4893,"version":"gpt-4.1-nano-2025-04-14"},"vals:legal_bench":{"value":61.06,"norm":0.6106,"version":"Vals AI"},"arc:arc_agi_2":{"value":0,"norm":0,"version":"ARC Prize"},"x:Chess Puzzles":{"value":0,"norm":0,"version":"gpt-4.1-nano-2025-04-14"},"vals:math500":{"value":80.2,"norm":0.802,"version":"Vals AI"},"vals:corp_fin_v2":{"value":42.08,"norm":0.4208,"version":"Vals AI"},"vals:lcb":{"value":42.72,"norm":0.4272,"version":"Vals AI"},"x:DTBench":{"value":0.5253,"norm":0.5253,"version":"gpt-4.1-nano-2025-04-14"},"x:WeirdML":{"value":0.1898,"norm":0.1898,"version":"gpt-4.1-nano-2025-04-14"},"math5":{"value":0.6999622356495468,"norm":0.7,"version":"gpt-4.1-nano-2025-04-14"},"vals:tax_eval_v2":{"value":60.75,"norm":0.6075,"version":"Vals AI"},"vals:gpqa":{"value":50.76,"norm":0.5076,"version":"Vals AI"},"aime":{"value":0.28888888888888886,"norm":0.2889,"version":"gpt-4.1-nano-2025-04-14"}},"variant":"GPT-4.1 nano","match":"self"},"api":{"or_id":"openai/gpt-4.1-nano","in":0.1,"out":0.4,"context":1047576,"hf":null,"created":"2025-04-14"},"date_from":"epoch"},{"id":"o3-2025-04","name":"o3","lab":"OpenAI","date":"2025-04-16","type":"reasoning","major":true,"open":false,"release":{"title":"o3 + o4-mini","source":"https://openai.com/index/introducing-o3-and-o4-mini/","verified":true,"confidence":"high"},"index":{"score":147.1,"lo":131.3,"hi":162.9,"se":1.74,"solve":36.1,"status":"rated","areas":["agents","coding","long","maths","novel","reasoning","work"],"core_areas":["coding","long","maths","novel","reasoning"],"testers":["ARC Prize","Aider","Chase Brower","Epoch AI","Fiction.live","FutureSearch","GSO team","GeoBench team","Havard T. Ihle","METR","Redwood Research","Scale AI","SimpleBench team","Tencent Hunyuan / Fudan","Vals AI","Will Patrick"],"tests":{"x:DeepResearch Bench":{"value":0.452,"norm":0.452,"version":"o3-2025-04-16_medium"},"x:FrontierMath-2025-02-28-Private":{"value":0.18685121107266436,"norm":0.1869,"version":"o3-2025-04-16_high"},"aider":{"value":81.3,"norm":0.813,"version":"o3-2025-04-16_high"},"x:GeoBench":{"value":0.74,"norm":0.74,"version":"o3-2025-04-16_medium"},"vals:mmmu":{"value":80.42,"norm":0.8042,"version":"Vals AI"},"vals:tax_eval_v2":{"value":74.57,"norm":0.7457,"version":"Vals AI"},"vals:medcode":{"value":47.29,"norm":0.4729,"version":"Vals AI"},"swe":{"value":0.6231884057971014,"norm":0.6232,"version":"o3-2025-04-16_medium"},"x:FrontierMath-Tier-4-2025-07-01-Private":{"value":0.020833333333333332,"norm":0.0208,"version":"o3-2025-04-16_high"},"x:CadEval":{"value":0.74,"norm":0.74,"version":"o3-2025-04-16_medium"},"x:Mystery Game Puzzles":{"value":0.29,"norm":0.29,"version":"o3-2025-04-16_high"},"vals:lcb":{"value":83.91,"norm":0.8391,"version":"Vals AI"},"x:VPCT":{"value":0.52,"norm":0.52,"version":"o3-2025-04-16_medium"},"x:Fiction.LiveBench":{"value":0.889,"norm":0.889,"version":"o3-2025-04-16_medium"},"x:DTBench":{"value":0.848,"norm":0.848,"version":"o3-2025-04-16_high"},"vals:math500":{"value":94.6,"norm":0.946,"version":"Vals AI"},"x:SimpleBench":{"value":0.531,"norm":0.531,"version":"o3-2025-04-16_high"},"x:GSO-Bench":{"value":0.0882,"norm":0.0882,"version":"o3-2025-04-16_high"},"arc:arc_agi_2":{"value":6.5,"norm":0.065,"version":"ARC Prize"},"fm":{"value":0.3333333333333333,"norm":0.3333,"version":"o3-2025-04-16_high"},"vals:mgsm":{"value":91.75,"norm":0.9175,"version":"Vals AI"},"x:Chess Puzzles":{"value":0.37,"norm":0.37,"version":"o3-2025-04-16_medium"},"vals:gpqa":{"value":84.09,"norm":0.8409,"version":"Vals AI"},"vals:corp_fin_v2":{"value":59.71,"norm":0.5971,"version":"Vals AI"},"vals:mmlu_pro":{"value":85.59,"norm":0.8559,"version":"Vals AI"},"metr":{"value":0.654376,"norm":0.2386,"version":"o3-2025-04-16_medium"},"x:CL-bench":{"value":0.17800000000000002,"norm":0.178,"version":"o3-2025-04-16_high"},"arc:arc_agi_1":{"value":75.7,"norm":0.757,"version":"ARC Prize"},"aime":{"value":0.8444444444444444,"norm":0.8444,"version":"o3-2025-04-16_medium"},"gpqa":{"value":0.8181818181818182,"norm":0.8182,"version":"o3-2025-04-16_high"},"vals:legal_bench":{"value":83.76,"norm":0.8376,"version":"Vals AI"},"x:SimpleQA Verified":{"value":0.494,"norm":0.494,"version":"o3-2025-04-16_high"},"x:WeirdML":{"value":0.5242,"norm":0.5242,"version":"o3-2025-04-16_high"},"math5":{"value":0.9777190332326284,"norm":0.9777,"version":"o3-2025-04-16_high"},"x:LMCA":{"value":39.7,"norm":0.397,"version":"o3-2025-04-16_high"},"vals:mortgage_tax":{"value":65.7,"norm":0.657,"version":"Vals AI"},"hle":{"value":0.20320000000000002,"norm":0.2032,"version":"o3-2025-04-16_high"},"vals:medscribe":{"value":76.65,"norm":0.7665,"version":"Vals AI"}},"variant":"o3","match":"exact"},"api":{"or_id":"openai/o3","in":2,"out":8,"context":200000,"hf":null,"created":"2025-04-16"},"events":[{"date":"2024-12-20","title":"o3 / o3-mini (announced; preview only)","source":"https://en.wikipedia.org/wiki/OpenAI_o3"}],"aliases":["o3","o3-pro"],"modes":[{"label":"Pro","name":"o3-pro","index":{"score":145.2,"lo":129.3,"hi":161.1,"se":2.14,"solve":34.4,"status":"rated","areas":["agents","coding","novel","reasoning"],"core_areas":["novel"],"testers":["ARC Prize","Aider","Fiction.live","Havard T. Ihle","Redwood Research"],"tests":{"aider":{"value":84.9,"norm":0.849,"version":"o3-pro-2025-06-10_high"},"x:DTBench":{"value":0.8693000000000001,"norm":0.8693,"version":"o3-pro-2025-06-10_high"},"x:WeirdML":{"value":0.5821000000000001,"norm":0.5821,"version":"o3-pro-2025-06-10_high"},"x:Fiction.LiveBench":{"value":0.972,"norm":0.972,"version":"o3-pro-2025-06-10_medium"},"arc:arc_agi_2":{"value":4.9,"norm":0.049,"version":"ARC Prize"},"x:LMCA":{"value":38.48,"norm":0.3848,"version":"o3-pro-2025-06-10_high"},"arc:arc_agi_1":{"value":59.3,"norm":0.593,"version":"ARC Prize"}}},"api":{"or_id":"openai/o3-pro","in":20,"out":80,"context":200000,"hf":null,"created":"2025-06-11"}}]},{"id":"o4-mini","name":"o4-mini","lab":"OpenAI","date":"2025-04-16","type":"reasoning","major":true,"open":false,"release":{"title":"o3 + o4-mini","source":"https://openai.com/index/introducing-o3-and-o4-mini/","verified":true,"confidence":"high"},"index":{"score":141.4,"lo":125.7,"hi":157.2,"se":1.76,"solve":31.4,"status":"rated","areas":["agents","coding","long","maths","novel","reasoning","work"],"core_areas":["long","maths","novel","reasoning"],"testers":["ARC Prize","Aider","Chase Brower","Epoch AI","Fiction.live","GSO team","GeoBench team","Havard T. Ihle","METR","Redwood Research","Scale AI","SimpleBench team","Vals AI","Will Patrick"],"tests":{"vals:corp_fin_v2":{"value":58.97,"norm":0.5897,"version":"Vals AI"},"vals:mmlu_pro":{"value":80.56,"norm":0.8056,"version":"Vals AI"},"vals:lcb":{"value":82.21,"norm":0.8221,"version":"Vals AI"},"x:FrontierMath-Tier-4-2025-07-01-Private":{"value":0.0625,"norm":0.0625,"version":"o4-mini-2025-04-16_high"},"vals:math500":{"value":94.2,"norm":0.942,"version":"Vals AI"},"x:CadEval":{"value":0.62,"norm":0.62,"version":"o4-mini-2025-04-16_medium"},"x:GeoBench":{"value":0.64,"norm":0.64,"version":"o4-mini-2025-04-16_medium"},"x:VPCT":{"value":0.575,"norm":0.575,"version":"o4-mini-2025-04-16_medium"},"vals:mmmu":{"value":79.67,"norm":0.7967,"version":"Vals AI"},"math5":{"value":0.978285498489426,"norm":0.9783,"version":"o4-mini-2025-04-16_high"},"vals:medcode":{"value":33.79,"norm":0.3379,"version":"Vals AI"},"vals:mortgage_tax":{"value":64.83,"norm":0.6483,"version":"Vals AI"},"hle":{"value":0.1808,"norm":0.1808,"version":"o4-mini-2025-04-16_high"},"aime":{"value":0.8166666666666667,"norm":0.8167,"version":"o4-mini-2025-04-16_high"},"aider":{"value":72,"norm":0.72,"version":"o4-mini-2025-04-16_high"},"x:WeirdML":{"value":0.5256000000000001,"norm":0.5256,"version":"o4-mini-2025-04-16_high"},"arc:arc_agi_2":{"value":6.1,"norm":0.061,"version":"ARC Prize"},"vals:medscribe":{"value":69.14,"norm":0.6914,"version":"Vals AI"},"x:Fiction.LiveBench":{"value":0.778,"norm":0.778,"version":"o4-mini-2025-04-16_medium"},"fm":{"value":0.36140350877192984,"norm":0.3614,"version":"o4-mini-2025-04-16_high"},"x:DTBench":{"value":0.7759999999999999,"norm":0.776,"version":"o4-mini-2025-04-16_high"},"vals:gpqa":{"value":74.5,"norm":0.745,"version":"Vals AI"},"x:GSO-Bench":{"value":0.036000000000000004,"norm":0.036,"version":"o4-mini-2025-04-16_high"},"x:LMCA":{"value":26.5,"norm":0.265,"version":"o4-mini-2025-04-16_high"},"x:SimpleBench":{"value":0.387,"norm":0.387,"version":"o4-mini-2025-04-16_high"},"x:Mystery Game Puzzles":{"value":0.05,"norm":0.05,"version":"o4-mini-2025-04-16_high"},"gpqa":{"value":0.7960858585858586,"norm":0.7961,"version":"o4-mini-2025-04-16_high"},"vals:legal_bench":{"value":79.19,"norm":0.7919,"version":"Vals AI"},"x:SimpleQA Verified":{"value":0.196,"norm":0.196,"version":"o4-mini-2025-04-16_low"},"metr":{"value":0.639269,"norm":0.2364,"version":"o4-mini-2025-04-16_medium"},"vals:tax_eval_v2":{"value":74.78,"norm":0.7478,"version":"Vals AI"},"arc:arc_agi_1":{"value":58.7,"norm":0.587,"version":"ARC Prize"},"vals:mgsm":{"value":93.42,"norm":0.9342,"version":"Vals AI"},"x:FrontierMath-2025-02-28-Private":{"value":0.2482758620689655,"norm":0.2483,"version":"o4-mini-2025-04-16_high"},"x:FrontierMath-Tier-4-v2-Private":{"value":0.04878048780487805,"norm":0.0488,"version":"o4-mini-2025-04-16_high"},"x:Chess Puzzles":{"value":0.22,"norm":0.22,"version":"o4-mini-2025-04-16_medium"}},"variant":"o4-mini","match":"exact"}},{"id":"gemini-2-5-flash","name":"Gemini 2.5 Flash","lab":"Google","date":"2025-04-17","type":"small","major":true,"open":false,"release":{"title":"Gemini 2.5 Flash","source":"https://blog.google/products-and-platforms/products/gemini/gemini-2-5-flash-preview/","verified":true,"confidence":"high"},"index":{"score":129.5,"lo":113.6,"hi":145.3,"se":1.93,"solve":23.7,"status":"rated","areas":["coding","maths","reasoning"],"core_areas":["reasoning"],"testers":["Aider","Chase Brower","Epoch AI","Fiction.live","GeoBench team","Havard T. Ihle","Scale AI"],"tests":{"aider":{"value":47.1,"norm":0.471,"version":"gemini-2.5-flash-preview-04-17"},"x:WeirdML":{"value":0.40950000000000003,"norm":0.4095,"version":"gemini-2.5-flash-preview-04-17 (16K thinking)"},"aime":{"value":0.7305555555555555,"norm":0.7306,"version":"gemini-2.5-flash-preview-04-17"},"hle":{"value":0.1208,"norm":0.1208,"version":"gemini-2.5-flash-preview-04-17"},"x:GeoBench":{"value":0.73,"norm":0.73,"version":"gemini-2.5-flash-preview-04-17"},"x:VPCT":{"value":0.38,"norm":0.38,"version":"gemini-2.5-flash-preview-04-17"},"x:Fiction.LiveBench":{"value":0.472,"norm":0.472,"version":"gemini-2.5-flash-preview-04-17"}},"variant":"Gemini 2.5 Flash (Apr 2025)","match":"exact"}},{"id":"qwen-plus-apr-2025","name":"Qwen Plus (Apr 2025)","lab":"Alibaba","date":"2025-04-28","type":"flagship","major":false,"open":false,"index":{"score":131.7,"lo":112.2,"hi":151.2,"se":7.23,"solve":25,"status":"provisional","areas":["agents","maths"],"core_areas":[],"testers":["Epoch AI","Redwood Research"],"tests":{"x:FrontierMath-2025-02-28-Private":{"value":0.017241379310344827,"norm":0.0172,"version":"qwen-plus-2025-04-28"},"x:DTBench":{"value":0.8107,"norm":0.8107,"version":"qwen-plus-2025-04-28"},"x:LMCA":{"value":23.96,"norm":0.2396,"version":"qwen-plus-2025-04-28"}},"variant":"Qwen Plus (Apr 2025)","match":"self"},"date_from":"epoch"},{"id":"qwen3-14b","name":"Qwen3-14B","lab":"Alibaba","date":"2025-04-28","type":"open-weights","major":false,"open":true,"index":{"score":128.9,"lo":113,"hi":144.8,"se":2.15,"solve":23.4,"status":"rated","areas":["agents","maths","reasoning"],"core_areas":["reasoning"],"testers":["Epoch AI","Fiction.live","Redwood Research"],"tests":{"x:Fiction.LiveBench":{"value":0.625,"norm":0.625,"version":"chutes/Qwen3-14B"},"x:DTBench":{"value":0.64,"norm":0.64,"version":"qwen3-14b"},"aime":{"value":0.6638888888888889,"norm":0.6639,"version":"qwen3-14b"},"gpqa":{"value":0.6376262626262627,"norm":0.6376,"version":"qwen3-14b"},"x:LMCA":{"value":18.19,"norm":0.1819,"version":"qwen3-14b"}},"variant":"Qwen3-14B","match":"self"},"api":{"or_id":"qwen/qwen3-14b","in":0.12,"out":0.24,"context":40960,"hf":"Qwen/Qwen3-14B","created":"2025-04-28"},"date_from":"epoch"},{"id":"qwen3-235b-a22b","name":"Qwen3-235B-A22B","lab":"Alibaba","date":"2025-04-28","type":"open-weights","major":false,"open":true,"index":{"score":130,"lo":114,"hi":145.9,"se":2.25,"solve":24,"status":"rated","areas":["agents","coding","maths","reasoning"],"core_areas":["reasoning"],"testers":["Aider","Epoch AI","Fiction.live","Havard T. Ihle","Redwood Research","SimpleBench team"],"tests":{"aider":{"value":59.6,"norm":0.596,"version":"Qwen3-235B-A22B"},"x:DTBench":{"value":0.7573000000000001,"norm":0.7573,"version":"Qwen3-235B-A22B"},"x:Fiction.LiveBench":{"value":0.677,"norm":0.677,"version":"chutes/Qwen3-235B-A22B"},"gpqa":{"value":0.7070707070707071,"norm":0.7071,"version":"qwen3-235b-a22b"},"x:WeirdML":{"value":0.3728,"norm":0.3728,"version":"Qwen3-235B-A22B"},"math5":{"value":0.6885699899295065,"norm":0.6886,"version":"qwen3-235b-a22b"},"x:LMCA":{"value":25,"norm":0.25,"version":"Qwen3-235B-A22B"},"x:SimpleBench":{"value":0.31,"norm":0.31,"version":"qwen3-235b-a22b"}},"variant":"Qwen3-235B-A22B","match":"self"},"date_from":"epoch"},{"id":"qwen3-30b-a3b","name":"Qwen3-30B-A3B","lab":"Alibaba","date":"2025-04-28","type":"open-weights","major":false,"open":true,"index":{"score":125.1,"lo":109.2,"hi":140.9,"se":2.01,"solve":21.5,"status":"rated","areas":["agents","coding","maths","reasoning"],"core_areas":["reasoning"],"testers":["Epoch AI","Fiction.live","Havard T. Ihle","Redwood Research"],"tests":{"x:Fiction.LiveBench":{"value":0.406,"norm":0.406,"version":"chutes/Qwen3-30B-A3B"},"gpqa":{"value":0.6167929292929293,"norm":0.6168,"version":"Qwen3-30B-A3B"},"x:LMCA":{"value":15.82,"norm":0.1582,"version":"Qwen3-30B-A3B"},"x:DTBench":{"value":0.6027,"norm":0.6027,"version":"Qwen3-30B-A3B"},"x:WeirdML":{"value":0.2975,"norm":0.2975,"version":"Qwen3-30B-A3B"},"aime":{"value":0.6277777777777778,"norm":0.6278,"version":"Qwen3-30B-A3B"}},"variant":"Qwen3-30B-A3B","match":"self"},"api":{"or_id":"qwen/qwen3-30b-a3b","in":0.12,"out":0.5,"context":40960,"hf":"Qwen/Qwen3-30B-A3B","created":"2025-04-29"},"date_from":"epoch"},{"id":"qwen3-32b","name":"Qwen3-32B","lab":"Alibaba","date":"2025-04-28","type":"open-weights","major":false,"open":true,"index":{"score":128.9,"lo":113.1,"hi":144.7,"se":1.83,"solve":23.4,"status":"rated","areas":["agents","coding","maths","reasoning"],"core_areas":["reasoning"],"testers":["Aider","Epoch AI","Fiction.live","Redwood Research"],"tests":{"aime":{"value":0.6694444444444444,"norm":0.6694,"version":"Qwen3-32B"},"x:LMCA":{"value":17.28,"norm":0.1728,"version":"Qwen3-32B"},"x:Fiction.LiveBench":{"value":0.742,"norm":0.742,"version":"chutes/Qwen3-32B"},"aider":{"value":40,"norm":0.4,"version":"Qwen3-32B"},"x:DTBench":{"value":0.6747,"norm":0.6747,"version":"Qwen3-32B"},"gpqa":{"value":0.6571969696969697,"norm":0.6572,"version":"Qwen3-32B"}},"variant":"Qwen3-32B","match":"self"},"api":{"or_id":"qwen/qwen3-32b","in":0.08,"out":0.28,"context":131072,"hf":"Qwen/Qwen3-32B","created":"2025-04-28"},"date_from":"epoch"},{"id":"qwen3-8b","name":"Qwen3-8B","lab":"Alibaba","date":"2025-04-28","type":"open-weights","major":false,"open":true,"index":{"score":125.5,"lo":109.6,"hi":141.3,"se":2.06,"solve":21.7,"status":"rated","areas":["agents","maths","reasoning"],"core_areas":["reasoning"],"testers":["Epoch AI","Fiction.live","Redwood Research"],"tests":{"aime":{"value":0.5611111111111111,"norm":0.5611,"version":"qwen3-8b"},"gpqa":{"value":0.5675505050505051,"norm":0.5676,"version":"qwen3-8b"},"x:LMCA":{"value":8.82,"norm":0.0882,"version":"qwen3-8b"},"x:DTBench":{"value":0.5972999999999999,"norm":0.5973,"version":"qwen3-8b"},"x:Fiction.LiveBench":{"value":0.621,"norm":0.621,"version":"chutes/Qwen3-8B"}},"variant":"Qwen3-8B","match":"self"},"date_from":"epoch"},{"id":"qwen3","name":"Qwen3","lab":"Alibaba","date":"2025-04-29","type":"open-weights","major":true,"open":true,"release":{"title":"Qwen3","source":"https://qwenlm.github.io/blog/qwen3/","verified":true,"confidence":"high"},"index":{"score":132.7,"lo":109.8,"hi":155.5,"se":10.21,"solve":25.6,"status":"provisional","areas":["coding","maths","reasoning","work"],"core_areas":[],"testers":["Vals AI"],"tests":{"vals:gpqa":{"value":70.2,"norm":0.702,"version":"Vals AI"},"vals:lcb":{"value":70.62,"norm":0.7062,"version":"Vals AI"},"vals:tax_eval_v2":{"value":70.65,"norm":0.7065,"version":"Vals AI"},"vals:mgsm":{"value":92.47,"norm":0.9247,"version":"Vals AI"},"vals:math500":{"value":94.6,"norm":0.946,"version":"Vals AI"},"vals:legal_bench":{"value":80.18,"norm":0.8018,"version":"Vals AI"},"vals:mmlu_pro":{"value":81.25,"norm":0.8125,"version":"Vals AI"}},"variant":"Qwen3","match":"exact"},"api":{"or_id":"qwen/qwen3-30b-a3b","in":0.12,"out":0.5,"context":40960,"hf":"Qwen/Qwen3-30B-A3B","created":"2025-04-29"}},{"id":"qwen3-1-7b","name":"Qwen3-1.7B","lab":"Alibaba","date":"2025-04-29","type":"open-weights","major":false,"open":true,"index":{"score":105.3,"lo":86.5,"hi":124,"se":6.4,"solve":14,"status":"provisional","areas":["maths","reasoning"],"core_areas":["reasoning"],"testers":["Epoch AI"],"tests":{"aime":{"value":0.08055555555555556,"norm":0.0806,"version":"qwen3-1.7b_none"},"gpqa":{"value":0.380050505050505,"norm":0.3801,"version":"qwen3-1.7b"}},"variant":"Qwen3-1.7B","match":"self"},"date_from":"epoch"},{"id":"qwen3-4b","name":"Qwen3-4B","lab":"Alibaba","date":"2025-04-29","type":"open-weights","major":false,"open":true,"index":{"score":118,"lo":99.5,"hi":136.5,"se":6.14,"solve":18.3,"status":"insufficient","areas":["reasoning"],"core_areas":["reasoning"],"testers":["Epoch AI"],"tests":{"gpqa":{"value":0.5233585858585859,"norm":0.5234,"version":"qwen3-4b"}},"variant":"Qwen3-4B","match":"self"},"date_from":"epoch"},{"id":"mimo-7b","name":"MiMo-7B","lab":"Xiaomi","date":"2025-04-30","type":"open-weights","major":true,"open":true,"release":{"title":"MiMo-7B","source":"https://huggingface.co/XiaomiMiMo/MiMo-7B-RL","verified":true,"confidence":"high"}},{"id":"nova-premier","name":"Nova Premier","lab":"Amazon","date":"2025-04-30","type":"flagship","major":true,"open":false,"release":{"title":"Nova Premier","source":"https://aws.amazon.com/ru/blogs/aws/amazon-nova-premier-our-most-capable-model-for-complex-tasks-and-teacher-for-model-distillation/","verified":true,"confidence":"high"}},{"id":"phi-4-reasoning","name":"Phi-4-reasoning","lab":"Microsoft","date":"2025-04-30","type":"open-weights","major":true,"open":true,"release":{"title":"Phi-4-reasoning","source":"https://huggingface.co/microsoft/Phi-4-reasoning","verified":true,"confidence":"high"}},{"id":"gemini-2-5-pro-may-2025","name":"Gemini 2.5 Pro (May 2025)","lab":"Google","date":"2025-05-06","type":"flagship","major":false,"open":false,"index":{"score":137.2,"lo":121,"hi":153.4,"se":2.82,"solve":28.4,"status":"rated","areas":["coding","maths","reasoning"],"core_areas":["reasoning"],"testers":["Aider","Chase Brower","Epoch AI","Fiction.live","GeoBench team","Scale AI"],"tests":{"hle":{"value":0.17800000000000002,"norm":0.178,"version":"gemini-2.5-pro-preview-05-06"},"x:VPCT":{"value":0.405,"norm":0.405,"version":"gemini-2.5-pro-preview-05-06"},"x:Fiction.LiveBench":{"value":0.667,"norm":0.667,"version":"gemini-2.5-pro-preview-05-06"},"aider":{"value":76.9,"norm":0.769,"version":"gemini-2.5-pro-preview-05-06"},"x:GeoBench":{"value":0.86,"norm":0.86,"version":"gemini-2.5-pro-preview-05-06"},"math5":{"value":0.9590256797583081,"norm":0.959,"version":"gemini-2.5-pro-preview-05-06"},"gpqa":{"value":0.6666666666666666,"norm":0.6667,"version":"gemini-2.5-pro-preview-05-06"}},"variant":"Gemini 2.5 Pro (May 2025)","match":"self"},"date_from":"epoch"},{"id":"mistral-medium-3","name":"Mistral Medium 3","lab":"Mistral","date":"2025-05-07","type":"flagship","major":true,"open":false,"release":{"title":"Mistral Medium 3","source":"https://mistral.ai/news/mistral-medium-3/","verified":true,"confidence":"high"},"index":{"score":120.1,"lo":103.9,"hi":136.3,"se":2.89,"solve":19.2,"status":"rated","areas":["agents","maths","reasoning"],"core_areas":["reasoning"],"testers":["Epoch AI","Redwood Research","Scale AI"],"tests":{"x:FrontierMath-2025-02-28-Private":{"value":0.0034602076124567475,"norm":0.0035,"version":"mistral-medium-2505"},"aime":{"value":0.32222222222222224,"norm":0.3222,"version":"mistral-medium-2505"},"math5":{"value":0.8162764350453172,"norm":0.8163,"version":"mistral-medium-2505"},"hle":{"value":0.0452,"norm":0.0452,"version":"mistral-medium-2505"},"gpqa":{"value":0.5953282828282829,"norm":0.5953,"version":"mistral-medium-2505"},"x:DTBench":{"value":0.6227,"norm":0.6227,"version":"mistral-medium-2505"}},"variant":"Mistral Medium 3","match":"exact"},"api":{"or_id":"mistralai/mistral-medium-3","in":0.44,"out":2.2,"context":131072,"hf":null,"created":"2025-05-07"}},{"id":"codex-1","name":"codex-1","lab":"OpenAI","date":"2025-05-16","type":"reasoning","major":true,"open":false,"release":{"title":"codex-1 (Codex research preview)","source":"https://openai.com/index/introducing-codex/","verified":true,"confidence":"high"}},{"id":"gemini-2-5-flash-may-2025","name":"Gemini 2.5 Flash (May 2025)","lab":"Google","date":"2025-05-20","type":"flagship","major":false,"open":false,"index":{"score":133.1,"lo":117.3,"hi":149,"se":2.13,"solve":25.8,"status":"rated","areas":["coding","maths","reasoning"],"core_areas":["reasoning"],"testers":["Aider","Epoch AI","Fiction.live","GeoBench team","Havard T. Ihle","Scale AI"],"tests":{"hle":{"value":0.10960000000000002,"norm":0.1096,"version":"gemini-2.5-flash-preview-05-20"},"aime":{"value":0.7083333333333334,"norm":0.7083,"version":"gemini-2.5-flash-preview-05-20"},"x:Fiction.LiveBench":{"value":0.778,"norm":0.778,"version":"gemini-2.5-flash-preview-05-20"},"aider":{"value":55.1,"norm":0.551,"version":"gemini-2.5-flash-preview-05-20_23K"},"x:WeirdML":{"value":0.4095,"norm":0.4095,"version":"gemini-2.5-flash-preview-05-20_16K"},"x:GeoBench":{"value":0.76,"norm":0.76,"version":"gemini-2.5-flash-preview-05-20"}},"variant":"Gemini 2.5 Flash (May 2025)","match":"self"},"date_from":"epoch"},{"id":"gemma-3n","name":"Gemma 3n","lab":"Google","date":"2025-05-20","type":"small","major":true,"open":true,"release":{"title":"Gemma 3n (preview; full 06-26)","source":null,"verified":false,"confidence":"high"},"open_evidence":"https://huggingface.co/google/gemma-3n-E4B-it"},{"id":"devstral","name":"Devstral","lab":"Mistral","date":"2025-05-21","type":"open-weights","major":true,"open":true,"release":{"title":"Devstral","source":"https://mistral.ai/news/devstral/","verified":true,"confidence":"high"}},{"id":"claude-opus-4","name":"Claude Opus 4","lab":"Anthropic","date":"2025-05-22","type":"flagship","major":true,"open":false,"release":{"title":"Claude Opus 4 + Sonnet 4","source":"https://www.anthropic.com/news/claude-4","verified":true,"confidence":"high"},"index":{"score":135.7,"lo":119.9,"hi":151.4,"se":1.81,"solve":27.4,"status":"rated","areas":["agents","coding","long","maths","novel","reasoning","work"],"core_areas":["coding","long","novel","reasoning"],"testers":["ARC Prize","Aider","Chase Brower","Epoch AI","Fiction.live","FutureSearch","GSO team","GeoBench team","Havard T. Ihle","METR","Redwood Research","Scale AI","SimpleBench team","Vals AI"],"tests":{"vals:mgsm":{"value":93.78,"norm":0.9378,"version":"Vals AI"},"aime":{"value":0.6444444444444445,"norm":0.6444,"version":"claude-opus-4-20250514_27K"},"x:GeoBench":{"value":0.49,"norm":0.49,"version":"claude-opus-4-20250514_32K"},"x:DTBench":{"value":0.816,"norm":0.816,"version":"claude-opus-4-20250514_unknown"},"gpqa":{"value":0.7626262626262627,"norm":0.7626,"version":"claude-opus-4-20250514_16K"},"vals:mmlu_pro":{"value":86.17,"norm":0.8617,"version":"Vals AI"},"vals:legal_bench":{"value":83.07,"norm":0.8307,"version":"Vals AI"},"x:GSO-Bench":{"value":0.069,"norm":0.069,"version":"claude-opus-4-20250514"},"vals:lcb":{"value":70.19,"norm":0.7019,"version":"Vals AI"},"x:FrontierMath-Tier-4-2025-07-01-Private":{"value":0.041666666666666664,"norm":0.0417,"version":"claude-opus-4-20250514_27K"},"x:Fiction.LiveBench":{"value":0.611,"norm":0.611,"version":"claude-opus-4-20250514"},"arc:arc_agi_2":{"value":8.6,"norm":0.086,"version":"ARC Prize"},"metr":{"value":0.639292,"norm":0.2364,"version":"claude-opus-4-20250514_16K"},"x:WeirdML":{"value":0.4372,"norm":0.4372,"version":"claude-opus-4-20250514_16K"},"math5":{"value":0.850453172205438,"norm":0.8505,"version":"claude-opus-4-20250514"},"vals:tax_eval_v2":{"value":71.91,"norm":0.7191,"version":"Vals AI"},"x:LMCA":{"value":37.43,"norm":0.3743,"version":"claude-opus-4-20250514_unknown"},"arc:arc_agi_1":{"value":35.7,"norm":0.357,"version":"ARC Prize"},"vals:gpqa":{"value":71.72,"norm":0.7172,"version":"Vals AI"},"x:SimpleBench":{"value":0.588,"norm":0.588,"version":"claude-opus-4-20250514_unknown"},"x:DeepResearch Bench":{"value":0.46799999999999997,"norm":0.468,"version":"claude-opus-4-20250514_2K"},"vals:mortgage_tax":{"value":58.59,"norm":0.5859,"version":"Vals AI"},"swe":{"value":0.7066115702479339,"norm":0.7066,"version":"claude-opus-4-20250514"},"aider":{"value":72,"norm":0.72,"version":"claude-opus-4-20250514_32K"},"x:FrontierMath-2025-02-28-Private":{"value":0.04482758620689655,"norm":0.0448,"version":"claude-opus-4-20250514"},"vals:math500":{"value":90.4,"norm":0.904,"version":"Vals AI"},"x:VPCT":{"value":0.38,"norm":0.38,"version":"claude-opus-4-20250514_16K"},"vals:mmmu":{"value":73.31,"norm":0.7331,"version":"Vals AI"},"hle":{"value":0.1072,"norm":0.1072,"version":"claude-opus-4-20250514_unknown"}},"variant":"Claude Opus 4","match":"exact"}},{"id":"claude-sonnet-4","name":"Claude Sonnet 4","lab":"Anthropic","date":"2025-05-22","type":"flagship","major":true,"open":false,"release":{"title":"Claude Opus 4 + Sonnet 4","source":"https://www.anthropic.com/news/claude-4","verified":true,"confidence":"high"},"index":{"score":133,"lo":117.2,"hi":148.8,"se":1.78,"solve":25.8,"status":"rated","areas":["agents","coding","long","maths","novel","reasoning","work"],"core_areas":["long","novel","reasoning"],"testers":["ARC Prize","Aider","Chase Brower","Epoch AI","Fiction.live","FutureSearch","GSO team","GeoBench team","Havard T. Ihle","METR","Redwood Research","Scale AI","SimpleBench team","Vals AI"],"tests":{"vals:tax_eval_v2":{"value":72,"norm":0.72,"version":"Vals AI"},"arc:arc_agi_1":{"value":40,"norm":0.4,"version":"ARC Prize"},"vals:medcode":{"value":34.96,"norm":0.3496,"version":"Vals AI"},"vals:medscribe":{"value":72.41,"norm":0.7241,"version":"Vals AI"},"x:SimpleBench":{"value":0.455,"norm":0.455,"version":"claude-sonnet-4-20250514_unknown"},"metr":{"value":0.619725,"norm":0.2335,"version":"claude-sonnet-4-20250514_16K"},"x:WeirdML":{"value":0.4611,"norm":0.4611,"version":"claude-sonnet-4-20250514_16K"},"arc:arc_agi_2":{"value":5.9,"norm":0.059,"version":"ARC Prize"},"scale:swe_bench_pro":{"value":42.7,"norm":0.427,"version":"Scale AI"},"vals:mortgage_tax":{"value":62.47,"norm":0.6247,"version":"Vals AI"},"x:Fiction.LiveBench":{"value":0.469,"norm":0.469,"version":"claude-sonnet-4-20250514"},"vals:corp_fin_v2":{"value":61.23,"norm":0.6123,"version":"Vals AI"},"gpqa":{"value":0.7918781725888325,"norm":0.7919,"version":"claude-sonnet-4-20250514_59K"},"vals:lcb":{"value":62.39,"norm":0.6239,"version":"Vals AI"},"x:VPCT":{"value":0.34,"norm":0.34,"version":"claude-sonnet-4-20250514_32K"},"math5":{"value":0.8436555891238671,"norm":0.8437,"version":"claude-sonnet-4-20250514"},"x:LMCA":{"value":29,"norm":0.29,"version":"claude-sonnet-4-20250514_unknown"},"x:FrontierMath-Tier-4-2025-07-01-Private":{"value":0,"norm":0,"version":"claude-sonnet-4-20250514"},"x:DTBench":{"value":0.7706999999999999,"norm":0.7707,"version":"claude-sonnet-4-20250514_unknown"},"x:FrontierMath-2025-02-28-Private":{"value":0.041379310344827586,"norm":0.0414,"version":"claude-sonnet-4-20250514"},"vals:legal_bench":{"value":82.95,"norm":0.8295,"version":"Vals AI"},"x:GeoBench":{"value":0.37,"norm":0.37,"version":"claude-sonnet-4-20250514"},"aime":{"value":0.7111111111111111,"norm":0.7111,"version":"claude-sonnet-4-20250514_32K"},"aider":{"value":61.3,"norm":0.613,"version":"claude-sonnet-4-20250514_32K"},"vals:mmlu_pro":{"value":83.86,"norm":0.8386,"version":"Vals AI"},"hle":{"value":0.0776,"norm":0.0776,"version":"claude-sonnet-4-20250514_unknown"},"vals:mgsm":{"value":93.02,"norm":0.9302,"version":"Vals AI"},"vals:gpqa":{"value":75,"norm":0.75,"version":"Vals AI"},"vals:math500":{"value":93.8,"norm":0.938,"version":"Vals AI"},"x:GSO-Bench":{"value":0.049,"norm":0.049,"version":"claude-sonnet-4-20250514_unknown"},"vals:mmmu":{"value":74.93,"norm":0.7493,"version":"Vals AI"},"x:DeepResearch Bench":{"value":0.466,"norm":0.466,"version":"claude-sonnet-4-20250514_2K"}},"variant":"Claude Sonnet 4","match":"exact"}},{"id":"deepseek-r1-0528","name":"DeepSeek-R1-0528","lab":"DeepSeek","date":"2025-05-28","type":"reasoning","major":true,"open":true,"release":{"title":"DeepSeek-R1-0528","source":"https://api-docs.deepseek.com/news/news250528/","verified":false,"confidence":"high"},"index":{"score":135.4,"lo":119.5,"hi":151.4,"se":2.3,"solve":27.2,"status":"rated","areas":["agents","coding","long","maths","reasoning"],"core_areas":["long","reasoning"],"testers":["Aider","Epoch AI","Fiction.live","FutureSearch","Havard T. Ihle","METR","SimpleBench team"],"tests":{"x:WeirdML":{"value":0.41630000000000006,"norm":0.4163,"version":"DeepSeek-R1-0528"},"math5":{"value":0.9663897280966768,"norm":0.9664,"version":"DeepSeek-R1-0528"},"x:DeepResearch Bench":{"value":0.351,"norm":0.351,"version":"DeepSeek-R1-0528"},"aider":{"value":71.4,"norm":0.714,"version":"DeepSeek-R1-0528"},"aime":{"value":0.6638888888888889,"norm":0.6639,"version":"DeepSeek-R1-0528"},"x:SimpleBench":{"value":0.408,"norm":0.408,"version":"DeepSeek-R1-0528"},"x:Fiction.LiveBench":{"value":0.75,"norm":0.75,"version":"chutes/DeepSeek-R1-0528"},"gpqa":{"value":0.7632575757575758,"norm":0.7633,"version":"DeepSeek-R1-0528"},"metr":{"value":0.537826,"norm":0.2204,"version":"DeepSeek-R1-0528"}},"variant":"DeepSeek-R1 (May 2025)","match":"exact"},"open_evidence":"https://huggingface.co/deepseek-ai/DeepSeek-R1-0528"},{"id":"gemini-2-5-pro-jun-2025","name":"Gemini 2.5 Pro (Jun 2025)","lab":"Google","date":"2025-06-05","type":"flagship","major":false,"open":false,"index":{"score":144.9,"lo":128.9,"hi":160.9,"se":2.34,"solve":34.2,"status":"rated","areas":["agents","coding","long","maths","reasoning"],"core_areas":["coding","long","maths","reasoning"],"testers":["Aider","Chase Brower","Epoch AI","Fiction.live","FutureSearch","GSO team","GeoBench team","Havard T. Ihle","METR","Redwood Research","Scale AI","SimpleBench team","Terminal-Bench team"],"tests":{"x:DeepResearch Bench":{"value":0.428,"norm":0.428,"version":"gemini-2.5-pro-preview-06-05"},"x:FrontierMath-2025-02-28-Private":{"value":0.1413793103448276,"norm":0.1414,"version":"gemini-2.5-pro"},"x:SimpleBench":{"value":0.624,"norm":0.624,"version":"gemini-2.5-pro-preview-06-05"},"x:LMCA":{"value":34.77,"norm":0.3477,"version":"gemini-2.5-pro"},"hle":{"value":0.2164,"norm":0.2164,"version":"gemini-2.5-pro-preview-06-05"},"tb":{"value":0.32640449438200003,"norm":0.3264,"version":"gemini-2.5-pro"},"gpqa":{"value":0.8529040404040404,"norm":0.8529,"version":"gemini-2.5-pro"},"x:WeirdML":{"value":0.5403,"norm":0.5403,"version":"gemini-2.5-pro_16K"},"x:DTBench":{"value":0.8240000000000001,"norm":0.824,"version":"gemini-2.5-pro"},"aime":{"value":0.8472222222222222,"norm":0.8472,"version":"gemini-2.5-pro"},"x:FrontierMath-Tier-4-2025-07-01-Private":{"value":0.041666666666666664,"norm":0.0417,"version":"gemini-2.5-pro"},"aider":{"value":83.1,"norm":0.831,"version":"gemini-2.5-pro-preview-06-05_32K"},"metr":{"value":0.5544,"norm":0.2232,"version":"gemini-2.5-pro-preview-06-05"},"x:Fiction.LiveBench":{"value":0.917,"norm":0.917,"version":"gemini-2.5-pro-preview-06-05"},"x:FrontierMath-Tier-4-v2-Private":{"value":0,"norm":0,"version":"gemini-2.5-pro"},"x:VPCT":{"value":0.464,"norm":0.464,"version":"gemini-2.5-pro-preview-06-05"},"swe":{"value":0.5755693581780539,"norm":0.5756,"version":"gemini-2.5-pro"},"fm":{"value":0.24561403508771928,"norm":0.2456,"version":"gemini-2.5-pro"},"x:GeoBench":{"value":0.81,"norm":0.81,"version":"gemini-2.5-pro-preview-06-05"},"x:GSO-Bench":{"value":0.0392,"norm":0.0392,"version":"gemini-2.5-pro-preview-06-05"}},"variant":"Gemini 2.5 Pro (Jun 2025)","match":"self"},"api":{"or_id":"google/gemini-2.5-pro","in":1.25,"out":10,"context":1048576,"hf":null,"created":"2025-06-17"},"date_from":"epoch"},{"id":"magistral-medium","name":"Magistral Medium","lab":"Mistral","date":"2025-06-10","type":"reasoning","major":true,"open":false,"release":{"title":"Magistral (Small/Medium)","source":"https://mistral.ai/news/magistral/","verified":true,"confidence":"high"},"index":{"score":116,"lo":91.5,"hi":140.5,"se":11.53,"solve":17.5,"status":"insufficient","areas":["novel"],"core_areas":["novel"],"testers":["ARC Prize"],"tests":{"arc:arc_agi_2":{"value":0,"norm":0,"version":"ARC Prize"},"arc:arc_agi_1":{"value":6.1,"norm":0.061,"version":"ARC Prize"}},"variant":"Magistral Medium","match":"exact"}},{"id":"magistral-small","name":"Magistral Small","lab":"Mistral","date":"2025-06-10","type":"reasoning","major":true,"open":true,"release":{"title":"Magistral (Small/Medium)","source":"https://mistral.ai/news/magistral/","verified":true,"confidence":"high"},"index":{"score":114.6,"lo":87.9,"hi":141.2,"se":13.17,"solve":17,"status":"insufficient","areas":["novel"],"core_areas":["novel"],"testers":["ARC Prize"],"tests":{"arc:arc_agi_1":{"value":5,"norm":0.05,"version":"ARC Prize"},"arc:arc_agi_2":{"value":0,"norm":0,"version":"ARC Prize"}},"variant":"Magistral Small","match":"exact"}},{"id":"magistral-small-1-0","name":"Magistral Small 1.0","lab":"Mistral","date":"2025-06-10","type":"open-weights","major":false,"open":true,"index":{"score":118.7,"lo":102.6,"hi":134.8,"se":2.7,"solve":18.6,"status":"provisional","areas":["maths","reasoning"],"core_areas":["reasoning"],"testers":["Epoch AI"],"tests":{"gpqa":{"value":0.5606060606060606,"norm":0.5606,"version":"magistral-small-2506"},"aime":{"value":0.3,"norm":0.3,"version":"magistral-small-2506"}},"variant":"Magistral Small 1.0","match":"self"},"date_from":"epoch"},{"id":"seed-1-6","name":"Seed 1.6","lab":"ByteDance","date":"2025-06-11","type":"reasoning","major":true,"open":false,"release":{"title":"Seed 1.6","source":null,"verified":false,"confidence":"high"},"index":{"score":131.8,"lo":115.1,"hi":148.4,"se":3.71,"solve":25,"status":"provisional","areas":["maths","reasoning"],"core_areas":["reasoning"],"testers":["Epoch AI"],"tests":{"gpqa":{"value":0.7758838383838383,"norm":0.7759,"version":"seed-1.6"},"x:SimpleQA Verified":{"value":0.174,"norm":0.174,"version":"seed-1.6"},"x:Chess Puzzles":{"value":0.05,"norm":0.05,"version":"seed-1.6"},"aime":{"value":0.6944444444444444,"norm":0.6944,"version":"seed-1.6"}},"variant":"Doubao-Seed-1.6","match":"family"}},{"id":"minimax-m1","name":"MiniMax-M1","lab":"MiniMax","date":"2025-06-16","type":"open-weights","major":true,"open":true,"release":{"title":"MiniMax-M1","source":"https://huggingface.co/collections/MiniMaxAI/minimax-m1-68502ad9634ec0eeac8cf094","verified":true,"confidence":"high"},"index":{"score":133.4,"lo":115.3,"hi":151.5,"se":5.66,"solve":26,"status":"insufficient","areas":["reasoning"],"core_areas":[],"testers":["Fiction.live"],"tests":{"x:Fiction.LiveBench":{"value":0.694,"norm":0.694,"version":"MiniMax-M1-80k"}},"variant":"MiniMax-M1-80k","match":"family"}},{"id":"gemini-2-5-flash-jun-2025","name":"Gemini 2.5 Flash (Jun 2025)","lab":"Google","date":"2025-06-17","type":"flagship","major":false,"open":false,"index":{"score":130.7,"lo":113.4,"hi":148,"se":4.72,"solve":24.4,"status":"rated","areas":["agents","coding","maths","reasoning"],"core_areas":[],"testers":["BALROG team","Epoch AI","Redwood Research","SimpleBench team","Terminal-Bench team"],"tests":{"x:Balrog":{"value":0.335,"norm":0.335,"version":"gemini-2.5-flash"},"x:LMCA":{"value":27.49,"norm":0.2749,"version":"gemini-2.5-flash"},"x:SimpleBench":{"value":0.41200000000000003,"norm":0.412,"version":"gemini-2.5-flash"},"tb":{"value":0.16853932584300002,"norm":0.1685,"version":"gemini-2.5-flash"},"x:FrontierMath-Tier-4-2025-07-01-Private":{"value":0.041666666666666664,"norm":0.0417,"version":"gemini-2.5-flash"},"x:DTBench":{"value":0.7653,"norm":0.7653,"version":"gemini-2.5-flash"},"x:FrontierMath-2025-02-28-Private":{"value":0.04844290657439446,"norm":0.0484,"version":"gemini-2.5-flash"}},"variant":"Gemini 2.5 Flash (Jun 2025)","match":"self"},"api":{"or_id":"google/gemini-2.5-flash","in":0.3,"out":2.5,"context":1048576,"hf":null,"created":"2025-06-17"},"date_from":"epoch"},{"id":"gemini-2-5-flash-lite","name":"Gemini 2.5 Flash-Lite","lab":"Google","date":"2025-06-17","type":"small","major":true,"open":false,"release":{"title":"Gemini 2.5 Flash-Lite (preview; 2.5 Pro/Flash GA)","source":"https://developers.googleblog.com/en/gemini-2-5-thinking-model-updates/","verified":true,"confidence":"high"},"index":{"score":120.3,"lo":103.5,"hi":137,"se":3.85,"solve":19.3,"status":"rated","areas":["agents","coding","reasoning"],"core_areas":[],"testers":["Fiction.live","Havard T. Ihle","Redwood Research"],"tests":{"x:WeirdML":{"value":0.3522,"norm":0.3522,"version":"gemini-2.5-flash-lite-preview-06-17-thinking"},"x:DTBench":{"value":0.6283,"norm":0.6283,"version":"gemini-2.5-flash-lite-preview-06-17-thinking"},"x:LMCA":{"value":18.07,"norm":0.1807,"version":"gemini-2.5-flash-lite-preview-06-17-thinking"},"x:Fiction.LiveBench":{"value":0.472,"norm":0.472,"version":"gemini-2.5-flash-lite-preview-06-17-thinking"}},"variant":"Gemini 2.5 Flash-Lite (Jun 2025)","match":"exact"},"api":{"or_id":"google/gemini-2.5-flash-lite","in":0.1,"out":0.4,"context":1048576,"hf":null,"created":"2025-07-22"}},{"id":"mistral-small-3-2","name":"Mistral Small 3.2","lab":"Mistral","date":"2025-06-20","type":"open-weights","major":false,"open":true,"index":{"score":117.3,"lo":101.2,"hi":133.5,"se":2.76,"solve":18.1,"status":"rated","areas":["agents","maths","reasoning"],"core_areas":["reasoning"],"testers":["Epoch AI","Redwood Research"],"tests":{"aime":{"value":0.30277777777777776,"norm":0.3028,"version":"mistral-small-3.2-2506"},"x:DTBench":{"value":0.5987,"norm":0.5987,"version":"mistral-small-2506"},"gpqa":{"value":0.49053030303030304,"norm":0.4905,"version":"mistral-small-3.2-2506"}},"variant":"Mistral Small 3.2","match":"self"},"api":{"or_id":"mistralai/mistral-small-3.2-24b-instruct","in":0.075,"out":0.2,"context":256000,"hf":"mistralai/Mistral-Small-3.2-24B-Instruct-2506","created":"2025-06-20"},"date_from":"epoch"},{"id":"hunyuan-a13b","name":"Hunyuan-A13B","lab":"Tencent","date":"2025-06-27","type":"open-weights","major":true,"open":true,"release":{"title":"Hunyuan-A13B","source":"https://huggingface.co/tencent/Hunyuan-A13B-Instruct","verified":true,"confidence":"high"}},{"id":"grok-4","name":"Grok 4","lab":"xAI","date":"2025-07-09","type":"flagship","major":true,"open":false,"release":{"title":"Grok 4 (+ Heavy)","source":"https://x.ai/news/grok-4","verified":true,"confidence":"high"},"index":{"score":148.1,"lo":132.3,"hi":164,"se":1.93,"solve":37.1,"status":"rated","areas":["agents","coding","long","maths","novel","reasoning","work"],"core_areas":["long","novel"],"testers":["ARC Prize","Aider","BALROG team","Epoch AI","Fiction.live","GeoBench team","Havard T. Ihle","METR","SimpleBench team","Terminal-Bench team","Vals AI"],"tests":{"tb":{"value":0.231460674157,"norm":0.2315,"version":"grok-4-0709"},"vals:gpqa":{"value":88.13,"norm":0.8813,"version":"Vals AI"},"x:GeoBench":{"value":0.45,"norm":0.45,"version":"grok-4-0709"},"vals:lcb":{"value":83.25,"norm":0.8325,"version":"Vals AI"},"metr":{"value":0.665803,"norm":0.2402,"version":"grok-4-0709"},"x:WeirdML":{"value":0.4573,"norm":0.4573,"version":"grok-4-0709"},"x:FrontierMath-Tier-4-2025-07-01-Private":{"value":0.020833333333333332,"norm":0.0208,"version":"grok-4-0709"},"x:Fiction.LiveBench":{"value":0.944,"norm":0.944,"version":"grok-4-0709"},"vals:math500":{"value":96.2,"norm":0.962,"version":"Vals AI"},"aider":{"value":79.6,"norm":0.796,"version":"grok-4-0709_high"},"vals:swebench":{"value":57.8,"norm":0.578,"version":"Vals AI"},"vals:mortgage_tax":{"value":44.48,"norm":0.4448,"version":"Vals AI"},"x:FrontierMath-2025-02-28-Private":{"value":0.19655172413793104,"norm":0.1966,"version":"grok-4-0709"},"arc:arc_agi_2":{"value":29.4,"norm":0.294,"version":"ARC Prize"},"x:Balrog":{"value":0.436,"norm":0.436,"version":"grok-4-0709"},"vals:mmmu":{"value":76.27,"norm":0.7627,"version":"Vals AI"},"vals:mgsm":{"value":90.91,"norm":0.9091,"version":"Vals AI"},"x:SimpleBench":{"value":0.605,"norm":0.605,"version":"grok-4-0709"},"vals:mmlu_pro":{"value":85.3,"norm":0.853,"version":"Vals AI"},"vals:legal_bench":{"value":83.19,"norm":0.8319,"version":"Vals AI"},"arc:arc_agi_1":{"value":79.6,"norm":0.796,"version":"ARC Prize"},"vals:medscribe":{"value":78.15,"norm":0.7815,"version":"Vals AI"},"vals:corp_fin_v2":{"value":66.04,"norm":0.6604,"version":"Vals AI"},"vals:tax_eval_v2":{"value":65.09,"norm":0.6509,"version":"Vals AI"},"vals:medcode":{"value":38.08,"norm":0.3808,"version":"Vals AI"}},"variant":"Grok 4","match":"exact"}},{"id":"kimi-k2","name":"Kimi K2","lab":"Moonshot","date":"2025-07-11","type":"open-weights","major":true,"open":true,"release":{"title":"Kimi K2","source":"https://www.kimi.com/en/blog/kimi-k2","verified":false,"confidence":"high"},"index":{"score":130.9,"lo":114.9,"hi":147,"se":2.58,"solve":24.5,"status":"provisional","areas":["coding","reasoning"],"core_areas":[],"testers":["Aider","Fiction.live","GSO team","Havard T. Ihle","SimpleBench team","Terminal-Bench team"],"tests":{"x:GSO-Bench":{"value":0.049,"norm":0.049,"version":"Kimi-K2-Instruct"},"aider":{"value":59.1,"norm":0.591,"version":"Kimi-K2-Instruct"},"tb":{"value":0.278,"norm":0.278,"version":"Kimi-K2-Instruct"},"x:SimpleBench":{"value":0.263,"norm":0.263,"version":"Kimi-K2-Instruct"},"x:WeirdML":{"value":0.3936,"norm":0.3936,"version":"kimi-k2-0711-preview"},"x:Fiction.LiveBench":{"value":0.611,"norm":0.611,"version":"Kimi-K2-Instruct"}},"variant":"Kimi K2 (Jul 2025)","match":"exact"},"api":{"or_id":"moonshotai/kimi-k2","in":0.57,"out":2.3,"context":131072,"hf":"moonshotai/Kimi-K2-Instruct","created":"2025-07-11"}},{"id":"glm-4-5-air","name":"GLM-4.5-Air","lab":"Zhipu","date":"2025-07-20","type":"open-weights","major":false,"open":true,"index":{"score":120.2,"lo":64.1,"hi":176.2,"se":32.76,"solve":19.2,"status":"provisional","areas":["coding","reasoning"],"core_areas":["reasoning"],"testers":["GSO team","Scale AI"],"tests":{"hle":{"value":0.0812,"norm":0.0812,"version":"GLM-4.5-Air"},"x:GSO-Bench":{"value":0.0294,"norm":0.0294,"version":"GLM-4.5-Air"}},"variant":"GLM-4.5-Air","match":"self"},"api":{"or_id":"z-ai/glm-4.5-air","in":0.13,"out":0.85,"context":131072,"hf":"zai-org/GLM-4.5-Air","created":"2025-07-25"},"date_from":"epoch"},{"id":"qwen3-coder","name":"Qwen3-Coder","lab":"Alibaba","date":"2025-07-22","type":"open-weights","major":true,"open":true,"release":{"title":"Qwen3-Coder","source":"https://qwenlm.github.io/blog/qwen3-coder/","verified":true,"confidence":"high"},"index":{"score":132.8,"lo":114.7,"hi":151,"se":5.68,"solve":25.7,"status":"insufficient","areas":["coding"],"core_areas":[],"testers":["GSO team","Havard T. Ihle","Terminal-Bench team"],"tests":{"x:GSO-Bench":{"value":0.049,"norm":0.049,"version":"Qwen3-Coder-480B-A35B-Instruct"},"x:WeirdML":{"value":0.4117,"norm":0.4117,"version":"Qwen3-Coder-480B-A35B-Instruct"},"tb":{"value":0.239,"norm":0.239,"version":"Qwen3-Coder-480B-A35B-Instruct"}},"variant":"Qwen3-Coder-480B-A35B","match":"family"},"api":{"or_id":"qwen/qwen3-coder","in":0.3,"out":1,"context":262144,"hf":"Qwen/Qwen3-Coder-480B-A35B-Instruct","created":"2025-07-23"}},{"id":"qwen3-235b-a22b-instruct-jul-2025","name":"Qwen3-235B-A22B-Instruct (Jul 2025)","lab":"Alibaba","date":"2025-07-25","type":"open-weights","major":false,"open":true,"index":{"score":130.2,"lo":114.1,"hi":146.3,"se":2.62,"solve":24.2,"status":"rated","areas":["agents","coding","reasoning"],"core_areas":[],"testers":["Aider","Fiction.live","Havard T. Ihle","Redwood Research"],"tests":{"x:LMCA":{"value":23.52,"norm":0.2352,"version":"Qwen3-235B-A22B-Instruct-2507"},"aider":{"value":59.6,"norm":0.596,"version":"Qwen3-235B-A22B-Instruct-2507"},"x:Fiction.LiveBench":{"value":0.529,"norm":0.529,"version":"parasail-qwen3-235b-a22b-instruct-2507"},"x:DTBench":{"value":0.784,"norm":0.784,"version":"Qwen3-235B-A22B-Instruct-2507"},"x:WeirdML":{"value":0.387,"norm":0.387,"version":"Qwen3-235B-A22B-Instruct-2507"}},"variant":"Qwen3-235B-A22B-Instruct (Jul 2025)","match":"self"},"api":{"or_id":"qwen/qwen3-235b-a22b-2507","in":0.14,"out":0.8,"context":262144,"hf":"Qwen/Qwen3-235B-A22B-Instruct-2507","created":"2025-07-21"},"date_from":"epoch"},{"id":"qwen3-235b-a22b-thinking-jul-2025","name":"Qwen3-235B-A22B-Thinking (Jul 2025)","lab":"Alibaba","date":"2025-07-25","type":"open-weights","major":false,"open":true,"index":{"score":137.8,"lo":121.3,"hi":154.3,"se":3.45,"solve":28.8,"status":"rated","areas":["agents","coding","maths","reasoning"],"core_areas":["reasoning"],"testers":["Epoch AI","Fiction.live","Havard T. Ihle","Redwood Research"],"tests":{"gpqa":{"value":0.8005050505050505,"norm":0.8005,"version":"Qwen3-235B-A22B-Thinking-2507"},"x:DTBench":{"value":0.8027,"norm":0.8027,"version":"Qwen3-235B-A22B-Thinking-2507"},"x:LMCA":{"value":29.31,"norm":0.2931,"version":"Qwen3-235B-A22B-Thinking-2507"},"x:FrontierMath-2025-02-28-Private":{"value":0.08480565371024736,"norm":0.0848,"version":"Qwen/Qwen3-235B-A22B-Thinking-2507"},"aime":{"value":0.8666666666666667,"norm":0.8667,"version":"Qwen3-235B-A22B-Thinking-2507"},"x:SimpleQA Verified":{"value":0.4044044044044044,"norm":0.4044,"version":"Qwen3-235B-A22B-Thinking-2507"},"x:WeirdML":{"value":0.4104,"norm":0.4104,"version":"Qwen3-235B-A22B-Thinking-2507"},"x:FrontierMath-Tier-4-2025-07-01-Private":{"value":0,"norm":0,"version":"Qwen/Qwen3-235B-A22B-Thinking-2507"},"x:Mystery Game Puzzles":{"value":0.09,"norm":0.09,"version":"Qwen3-235B-A22B-Thinking-2507"},"x:Fiction.LiveBench":{"value":0.75,"norm":0.75,"version":"chutes/Qwen3-235B-A22B-Thinking-2507"}},"variant":"Qwen3-235B-A22B-Thinking (Jul 2025)","match":"self"},"api":{"or_id":"qwen/qwen3-235b-a22b-thinking-2507","in":0.45,"out":3.5,"context":128000,"hf":"Qwen/Qwen3-235B-A22B-Thinking-2507","created":"2025-07-25"},"date_from":"epoch"},{"id":"step-3","name":"Step 3","lab":"StepFun","date":"2025-07-25","type":"open-weights","major":true,"open":true,"release":{"title":"Step 3","source":null,"verified":false,"confidence":"high"}},{"id":"glm-4-5","name":"GLM-4.5","lab":"Zhipu","date":"2025-07-28","type":"open-weights","major":true,"open":true,"release":{"title":"GLM-4.5","source":"https://z.ai/blog/glm-4.5","verified":true,"confidence":"high"},"index":{"score":131.7,"lo":113.2,"hi":150.3,"se":6.19,"solve":25,"status":"rated","areas":["coding","maths","reasoning","work"],"core_areas":["reasoning"],"testers":["Havard T. Ihle","Scale AI","Vals AI"],"tests":{"vals:mgsm":{"value":90.84,"norm":0.9084,"version":"Vals AI"},"vals:corp_fin_v2":{"value":60.96,"norm":0.6096,"version":"Vals AI"},"vals:gpqa":{"value":72.22,"norm":0.7222,"version":"Vals AI"},"x:WeirdML":{"value":0.4058,"norm":0.4058,"version":"glm-4.5_thinking"},"vals:tax_eval_v2":{"value":72.4,"norm":0.724,"version":"Vals AI"},"vals:legal_bench":{"value":75.63,"norm":0.7563,"version":"Vals AI"},"vals:mmlu_pro":{"value":81.22,"norm":0.8122,"version":"Vals AI"},"hle":{"value":0.08320000000000001,"norm":0.0832,"version":"glm-4.5"},"vals:math500":{"value":94,"norm":0.94,"version":"Vals AI"},"vals:lcb":{"value":67.45,"norm":0.6745,"version":"Vals AI"}},"variant":"GLM-4.5","match":"exact"},"api":{"or_id":"z-ai/glm-4.5","in":0.6,"out":2.2,"context":131072,"hf":"zai-org/GLM-4.5","created":"2025-07-25"}},{"id":"qwen3-30b-a3b-instruct-jul-2025","name":"Qwen3-30B-A3B-Instruct (Jul 2025)","lab":"Alibaba","date":"2025-07-29","type":"open-weights","major":false,"open":true,"index":{"score":127.3,"lo":111.4,"hi":143.3,"se":2.3,"solve":22.6,"status":"rated","areas":["agents","maths","reasoning"],"core_areas":["reasoning"],"testers":["Epoch AI","Redwood Research"],"tests":{"x:DTBench":{"value":0.672,"norm":0.672,"version":"qwen3-30b-a3b-instruct-2507"},"aime":{"value":0.6222222222222222,"norm":0.6222,"version":"qwen3-30b-a3b-instruct-2507"},"x:LMCA":{"value":22.37,"norm":0.2237,"version":"qwen3-30b-a3b-instruct-2507"},"gpqa":{"value":0.5561868686868687,"norm":0.5562,"version":"qwen3-30b-a3b-instruct-2507"}},"variant":"Qwen3-30B-A3B-Instruct (Jul 2025)","match":"self"},"api":{"or_id":"qwen/qwen3-30b-a3b-instruct-2507","in":0.1,"out":0.3,"context":262144,"hf":"Qwen/Qwen3-30B-A3B-Instruct-2507","created":"2025-07-29"},"date_from":"epoch"},{"id":"qwen3-30b-a3b-thinking-jul-2025","name":"Qwen3-30B-A3B-Thinking (Jul 2025)","lab":"Alibaba","date":"2025-07-30","type":"open-weights","major":false,"open":true,"index":{"score":131.1,"lo":115,"hi":147.1,"se":2.51,"solve":24.6,"status":"rated","areas":["agents","maths","reasoning"],"core_areas":["reasoning"],"testers":["Epoch AI","Redwood Research"],"tests":{"x:DTBench":{"value":0.6933,"norm":0.6933,"version":"qwen3-30b-a3b-thinking-2507"},"x:LMCA":{"value":19.53,"norm":0.1953,"version":"qwen3-30b-a3b-thinking-2507"},"aime":{"value":0.7027777777777777,"norm":0.7028,"version":"qwen3-30b-a3b-thinking-2507"},"gpqa":{"value":0.7007575757575758,"norm":0.7008,"version":"qwen3-30b-a3b-thinking-2507"}},"variant":"Qwen3-30B-A3B-Thinking (Jul 2025)","match":"self"},"date_from":"epoch"},{"id":"claude-opus-4-1","name":"Claude Opus 4.1","lab":"Anthropic","date":"2025-08-05","type":"flagship","major":true,"open":false,"release":{"title":"Claude Opus 4.1","source":"https://www.anthropic.com/news/claude-opus-4-1","verified":true,"confidence":"high"},"index":{"score":139.8,"lo":123.6,"hi":156.1,"se":2.97,"solve":30.2,"status":"rated","areas":["agents","coding","long","maths","reasoning","work"],"core_areas":["coding","long","maths","reasoning"],"testers":["Chase Brower","Epoch AI","FutureSearch","Havard T. Ihle","METR","Redwood Research","Scale AI","SimpleBench team","Terminal-Bench team","Vals AI"],"tests":{"x:SimpleBench":{"value":0.6,"norm":0.6,"version":"claude-opus-4-1-20250805_unknown"},"vals:mmlu_pro":{"value":87.92,"norm":0.8792,"version":"Vals AI"},"x:EBR-bench":{"value":0.079365,"norm":0.0794,"version":"claude-opus-4-1-20250805"},"x:DeepResearch Bench":{"value":0.483,"norm":0.483,"version":"claude-opus-4-1-20250805"},"vals:tax_eval_v2":{"value":73.67,"norm":0.7367,"version":"Vals AI"},"swe":{"value":0.7334710743801653,"norm":0.7335,"version":"claude-opus-4-1-20250805"},"x:FrontierMath-Tier-4-2025-07-01-Private":{"value":0.041666666666666664,"norm":0.0417,"version":"claude-opus-4-1-20250805_27K"},"vals:mgsm":{"value":94.44,"norm":0.9444,"version":"Vals AI"},"x:LMCA":{"value":37.08,"norm":0.3708,"version":"claude-opus-4-1-20250805_unknown"},"vals:medscribe":{"value":73.9,"norm":0.739,"version":"Vals AI"},"vals:gpqa":{"value":76.26,"norm":0.7626,"version":"Vals AI"},"x:WeirdML":{"value":0.4586,"norm":0.4586,"version":"claude-opus-4-1-20250805_16K"},"x:VPCT":{"value":0.35,"norm":0.35,"version":"claude-opus-4-1-20250805"},"vals:mortgage_tax":{"value":61.09,"norm":0.6109,"version":"Vals AI"},"x:DTBench":{"value":0.8,"norm":0.8,"version":"claude-opus-4-1-20250805_unknown"},"x:FrontierMath-2025-02-28-Private":{"value":0.07241379310344828,"norm":0.0724,"version":"claude-opus-4-1-20250805_27K"},"aime":{"value":0.6888888888888889,"norm":0.6889,"version":"claude-opus-4-1-20250805_27K"},"vals:legal_bench":{"value":83.46,"norm":0.8346,"version":"Vals AI"},"hle":{"value":0.1152,"norm":0.1152,"version":"claude-opus-4-1-20250805_unknown"},"fm":{"value":0.12631578947368421,"norm":0.1263,"version":"claude-opus-4-1-20250805_32K"},"tb":{"value":0.38,"norm":0.38,"version":"claude-opus-4-1-20250805"},"x:FrontierMath-Tier-4-v2-Private":{"value":0.024390243902439025,"norm":0.0244,"version":"claude-opus-4-1-20250805_32K"},"vals:math500":{"value":95.4,"norm":0.954,"version":"Vals AI"},"x:Mystery Game Puzzles":{"value":0.21,"norm":0.21,"version":"claude-opus-4-1-20250805_24K"},"gpqa":{"value":0.7727272727272727,"norm":0.7727,"version":"claude-opus-4-1-20250805_16K"},"vals:lcb":{"value":66.46,"norm":0.6646,"version":"Vals AI"},"metr":{"value":0.668069,"norm":0.2405,"version":"claude-opus-4-1-20250805_16K"},"vals:mmmu":{"value":77.51,"norm":0.7751,"version":"Vals AI"},"vals:medcode":{"value":47.23,"norm":0.4723,"version":"Vals AI"}},"variant":"Claude Opus 4.1","match":"exact"},"api":{"or_id":"anthropic/claude-opus-4.1","in":15,"out":75,"context":200000,"hf":null,"created":"2025-08-05"}},{"id":"gpt-oss-120b","name":"gpt-oss-120b","lab":"OpenAI","date":"2025-08-05","type":"open-weights","major":true,"open":true,"release":{"title":"gpt-oss-120b/20b","source":"https://openai.com/index/gpt-oss-model-card/","verified":true,"confidence":"high"},"index":{"score":129.1,"lo":113.3,"hi":144.9,"se":1.79,"solve":23.5,"status":"rated","areas":["agents","coding","long","maths","reasoning","work"],"core_areas":["agents","long","reasoning"],"testers":["Aider","Epoch AI","Fiction.live","Havard T. Ihle","METR","Mercor","Redwood Research","Scale AI","SimpleBench team","Terminal-Bench team","Vals AI","Yann Henon"],"tests":{"x:WeirdML":{"value":0.4817,"norm":0.4817,"version":"openai/gpt-oss-120b_high"},"x:Fiction.LiveBench":{"value":0.444,"norm":0.444,"version":"chutes/gpt-oss-120b_high"},"vals:swebench":{"value":33.6,"norm":0.336,"version":"Vals AI"},"apex":{"value":0.044000000000000004,"norm":0.044,"version":"gpt-oss-120b"},"metr":{"value":0.56634,"norm":0.2252,"version":"gpt-oss-120b"},"aime":{"value":0.8888888888888888,"norm":0.8889,"version":"openai/gpt-oss-120b_high"},"vals:corp_fin_v2":{"value":58.24,"norm":0.5824,"version":"Vals AI"},"x:SimpleBench":{"value":0.22100000000000003,"norm":0.221,"version":"gpt-oss-120b"},"x:Surface Evolver Bench":{"value":0.25,"norm":0.25,"version":"gpt-oss-120b"},"x:DTBench":{"value":0.7626999999999999,"norm":0.7627,"version":"gpt-oss-120b_high"},"vals:math500":{"value":94.8,"norm":0.948,"version":"Vals AI"},"aider":{"value":41.8,"norm":0.418,"version":"openai/gpt-oss-120b_high"},"vals:gpqa":{"value":78.54,"norm":0.7854,"version":"Vals AI"},"vals:mmlu_pro":{"value":79.17,"norm":0.7917,"version":"Vals AI"},"vals:lcb":{"value":83.23,"norm":0.8323,"version":"Vals AI"},"x:LMCA":{"value":22.14,"norm":0.2214,"version":"gpt-oss-120b_high"},"tb":{"value":0.187,"norm":0.187,"version":"gpt-oss-120b"},"x:Mystery Game Puzzles":{"value":0.02,"norm":0.02,"version":"gpt-oss-120b_medium"},"gpqa":{"value":0.7575757575757576,"norm":0.7576,"version":"openai/gpt-oss-120b_high"},"vals:legal_bench":{"value":75.94,"norm":0.7594,"version":"Vals AI"},"scale:swe_bench_pro":{"value":16.2,"norm":0.162,"version":"Scale AI"},"vals:mgsm":{"value":92.04,"norm":0.9204,"version":"Vals AI"},"vals:tax_eval_v2":{"value":71.59,"norm":0.7159,"version":"Vals AI"}},"variant":"gpt-oss-120b","match":"exact"},"api":{"or_id":"openai/gpt-oss-120b","in":0.15,"out":0.6,"context":131072,"hf":"openai/gpt-oss-120b","created":"2025-08-05"}},{"id":"gpt-oss-20b","name":"gpt-oss-20b","lab":"OpenAI","date":"2025-08-05","type":"open-weights","major":true,"open":true,"release":{"title":"gpt-oss-120b/20b","source":"https://openai.com/index/gpt-oss-model-card/","verified":true,"confidence":"high"},"index":{"score":127.8,"lo":111.9,"hi":143.7,"se":2.12,"solve":22.8,"status":"rated","areas":["agents","coding","maths","reasoning","work"],"core_areas":["reasoning"],"testers":["Epoch AI","Havard T. Ihle","Redwood Research","Terminal-Bench team","Vals AI"],"tests":{"vals:tax_eval_v2":{"value":63.7,"norm":0.637,"version":"Vals AI"},"vals:gpqa":{"value":68.94,"norm":0.6894,"version":"Vals AI"},"vals:lcb":{"value":80.39,"norm":0.8039,"version":"Vals AI"},"x:LMCA":{"value":14.51,"norm":0.1451,"version":"gpt-oss-20b_high"},"gpqa":{"value":0.6079545454545454,"norm":0.608,"version":"gpt-oss-20b_medium"},"tb":{"value":0.031,"norm":0.031,"version":"gpt-oss-20b"},"vals:mmlu_pro":{"value":71.64,"norm":0.7164,"version":"Vals AI"},"vals:legal_bench":{"value":70.85,"norm":0.7085,"version":"Vals AI"},"x:DTBench":{"value":0.68,"norm":0.68,"version":"gpt-oss-20b_high"},"vals:mgsm":{"value":89.02,"norm":0.8902,"version":"Vals AI"},"aime":{"value":0.6527777777777778,"norm":0.6528,"version":"gpt-oss-20b_medium"},"x:WeirdML":{"value":0.4093,"norm":0.4093,"version":"gpt-oss-20b_high"},"vals:math500":{"value":94.2,"norm":0.942,"version":"Vals AI"},"vals:corp_fin_v2":{"value":53.15,"norm":0.5315,"version":"Vals AI"}},"variant":"gpt-oss-20b","match":"exact"},"api":{"or_id":"openai/gpt-oss-20b","in":0.018,"out":0.09,"context":131072,"hf":"openai/gpt-oss-20b","created":"2025-08-05"}},{"id":"gpt-5","name":"GPT-5","lab":"OpenAI","date":"2025-08-07","type":"flagship","major":true,"open":false,"release":{"title":"GPT-5","source":"https://openai.com/index/introducing-gpt-5/","verified":true,"confidence":"high"},"index":{"score":148.5,"lo":132.7,"hi":164.2,"se":1.66,"solve":37.3,"status":"rated","areas":["agents","coding","long","maths","novel","reasoning","work"],"core_areas":["coding","long","maths","novel","reasoning"],"testers":["ARC Prize","Aider","BALROG team","Chase Brower","Epoch AI","Fiction.live","FutureSearch","GSO team","GeoBench team","Havard T. Ihle","METR","Redwood Research","Scale AI","SimpleBench team","Terminal-Bench team","Vals AI"],"tests":{"vals:gpqa":{"value":85.61,"norm":0.8561,"version":"Vals AI"},"x:GeoBench":{"value":0.81,"norm":0.81,"version":"gpt-5-2025-08-07_medium"},"x:SimpleBench":{"value":0.5670000000000001,"norm":0.567,"version":"gpt-5-2025-08-07_high"},"metr":{"value":0.696053,"norm":0.2443,"version":"gpt-5-2025-08-07_medium"},"vals:tax_eval_v2":{"value":73.39,"norm":0.7339,"version":"Vals AI"},"x:LMCA":{"value":39.99,"norm":0.3999,"version":"gpt-5-2025-08-07_high"},"hle":{"value":0.2532,"norm":0.2532,"version":"gpt-5-2025-08-07_unknown"},"vals:medscribe":{"value":83.65,"norm":0.8365,"version":"Vals AI"},"vals:corp_fin_v2":{"value":61.07,"norm":0.6107,"version":"Vals AI"},"x:FrontierMath-Tier-4-v2-Private":{"value":0.21951219512195122,"norm":0.2195,"version":"gpt-5-2025-08-07_high"},"aime":{"value":0.9138888888888889,"norm":0.9139,"version":"gpt-5-2025-08-07_high"},"vals:math500":{"value":96,"norm":0.96,"version":"Vals AI"},"scale:swe_bench_pro":{"value":41.78,"norm":0.4178,"version":"Scale AI"},"x:EBR-bench":{"value":0.126984,"norm":0.127,"version":"gpt-5-2025-08-07_high"},"vals:vibe-code":{"value":20.09,"norm":0.2009,"version":"Vals AI"},"x:VPCT":{"value":0.66,"norm":0.66,"version":"gpt-5-2025-08-07_high"},"math5":{"value":0.9813066465256798,"norm":0.9813,"version":"gpt-5-2025-08-07_high"},"fm":{"value":0.5543859649122806,"norm":0.5544,"version":"gpt-5-2025-08-07_high"},"vals:mgsm":{"value":92.84,"norm":0.9284,"version":"Vals AI"},"arc:arc_agi_2":{"value":9.9,"norm":0.099,"version":"ARC Prize"},"vals:legal_bench":{"value":86.02,"norm":0.8602,"version":"Vals AI"},"vals:mmmu":{"value":81.5,"norm":0.815,"version":"Vals AI"},"vals:medcode":{"value":49.63,"norm":0.4963,"version":"Vals AI"},"vals:mortgage_tax":{"value":65.45,"norm":0.6545,"version":"Vals AI"},"x:MirrorCode":{"value":0.044444444444444446,"norm":0.0444,"version":"gpt-5-2025-08-07_high"},"x:Fiction.LiveBench":{"value":0.972,"norm":0.972,"version":"gpt-5-2025-08-07_medium"},"tb":{"value":0.352,"norm":0.352,"version":"gpt-5-2025-08-07_medium"},"x:Chess Puzzles":{"value":0.36,"norm":0.36,"version":"gpt-5-2025-08-07_high"},"aider":{"value":88,"norm":0.88,"version":"gpt-5-2025-08-07_high"},"vals:swebench":{"value":69,"norm":0.69,"version":"Vals AI"},"x:Balrog":{"value":0.32799999999999996,"norm":0.328,"version":"gpt-5-2025-08-07_minimal"},"arc:arc_agi_1":{"value":65.7,"norm":0.657,"version":"ARC Prize"},"swe":{"value":0.7355371900826446,"norm":0.7355,"version":"gpt-5-2025-08-07_high"},"x:DTBench":{"value":0.9067000000000001,"norm":0.9067,"version":"gpt-5-2025-08-07_high"},"gpqa":{"value":0.8617424242424242,"norm":0.8617,"version":"gpt-5-2025-08-07_high"},"x:FrontierMath-2025-02-28-Private":{"value":0.32413793103448274,"norm":0.3241,"version":"gpt-5-2025-08-07_high"},"x:Mystery Game Puzzles":{"value":0.23,"norm":0.23,"version":"gpt-5-2025-08-07_high"},"vals:mmlu_pro":{"value":86.54,"norm":0.8654,"version":"Vals AI"},"x:GSO-Bench":{"value":0.06860000000000001,"norm":0.0686,"version":"gpt-5-2025-08-07_high"},"vals:lcb":{"value":85.91,"norm":0.8591,"version":"Vals AI"},"x:SimpleQA Verified":{"value":0.501,"norm":0.501,"version":"gpt-5-2025-08-07_high"},"x:WeirdML":{"value":0.6070000000000001,"norm":0.607,"version":"gpt-5-2025-08-07_high"},"x:DeepResearch Bench":{"value":0.49600000000000005,"norm":0.496,"version":"gpt-5-2025-08-07_low"},"x:FrontierMath-Tier-4-2025-07-01-Private":{"value":0.125,"norm":0.125,"version":"gpt-5-2025-08-07_high"}},"variant":"GPT-5","match":"exact"},"api":{"or_id":"openai/gpt-5","in":1.25,"out":10,"context":400000,"hf":null,"created":"2025-08-07"},"modes":[{"label":"Pro","name":"GPT-5 Pro","index":{"score":150.9,"lo":135.1,"hi":166.8,"se":2.05,"solve":39.7,"status":"rated","areas":["coding","maths","novel","reasoning"],"core_areas":["maths","novel","reasoning"],"testers":["ARC Prize","Epoch AI","Havard T. Ihle","Scale AI","SimpleBench team"],"tests":{"x:SimpleBench":{"value":0.616,"norm":0.616,"version":"gpt-5-pro-2025-10-06_unknown"},"hle":{"value":0.3164,"norm":0.3164,"version":"gpt-5-pro-2025-10-06_unknown"},"x:FrontierMath-Tier-4-v2-Private":{"value":0.1951219512195122,"norm":0.1951,"version":"gpt-5-pro-2025-10-06_high"},"x:Chess Puzzles":{"value":0.48,"norm":0.48,"version":"gpt-5-pro-2025-10-06_high"},"fm":{"value":0.5578947368421052,"norm":0.5579,"version":"gpt-5-pro-2025-10-06_high"},"x:WeirdML":{"value":0.6039,"norm":0.6039,"version":"gpt-5-pro-2025-10-06_high"},"x:FrontierMath-Tier-4-2025-07-01-Private":{"value":0.146,"norm":0.146,"version":"gpt-5-pro-2025-10-06_high"},"arc:arc_agi_2":{"value":18.3,"norm":0.183,"version":"ARC Prize"},"arc:arc_agi_1":{"value":70.2,"norm":0.702,"version":"ARC Prize"}}},"api":{"or_id":"openai/gpt-5-pro","in":15,"out":120,"context":400000,"hf":null,"created":"2025-10-06"}}],"aliases":["gpt-5-pro"]},{"id":"gpt-5-mini","name":"GPT-5 mini","lab":"OpenAI","date":"2025-08-07","type":"flagship","major":false,"open":false,"index":{"score":140,"lo":124.2,"hi":155.8,"se":1.98,"solve":30.4,"status":"rated","areas":["agents","coding","maths","novel","reasoning","work"],"core_areas":["coding","maths","novel","reasoning"],"testers":["ARC Prize","Chase Brower","Epoch AI","Fiction.live","Havard T. Ihle","Redwood Research","Scale AI","Terminal-Bench team","Vals AI"],"tests":{"vals:lcb":{"value":86.6,"norm":0.866,"version":"Vals AI"},"x:WeirdML":{"value":0.5267000000000001,"norm":0.5267,"version":"gpt-5-mini-2025-08-07_high"},"arc:arc_agi_1":{"value":54.3,"norm":0.543,"version":"ARC Prize"},"aime":{"value":0.8666666666666667,"norm":0.8667,"version":"gpt-5-mini-2025-08-07_high"},"x:Mystery Game Puzzles":{"value":0.1,"norm":0.1,"version":"gpt-5-mini-2025-08-07_minimal"},"vals:tax_eval_v2":{"value":75.22,"norm":0.7522,"version":"Vals AI"},"x:LMCA":{"value":34.2,"norm":0.342,"version":"gpt-5-mini-2025-08-07_high"},"vals:mortgage_tax":{"value":66.89,"norm":0.6689,"version":"Vals AI"},"x:FrontierMath-Tier-4-2025-07-01-Private":{"value":0.0625,"norm":0.0625,"version":"gpt-5-mini-2025-08-07_high"},"tb":{"value":0.24044943820200002,"norm":0.2404,"version":"gpt-5-mini-2025-08-07_unknown"},"x:FrontierMath-Tier-4-v2-Private":{"value":0.12195121951219512,"norm":0.122,"version":"gpt-5-mini-2025-08-07_high"},"arc:arc_agi_2":{"value":4.4,"norm":0.044,"version":"ARC Prize"},"vals:vibe-code":{"value":14.17,"norm":0.1417,"version":"Vals AI"},"x:VPCT":{"value":0.402,"norm":0.402,"version":"gpt-5-mini-2025-08-07_medium"},"vals:mmmu":{"value":78.91,"norm":0.7891,"version":"Vals AI"},"x:DTBench":{"value":0.8053,"norm":0.8053,"version":"gpt-5-mini-2025-08-07_high"},"vals:medcode":{"value":43.04,"norm":0.4304,"version":"Vals AI"},"vals:medscribe":{"value":80.58,"norm":0.8058,"version":"Vals AI"},"fm":{"value":0.4666666666666667,"norm":0.4667,"version":"gpt-5-mini-2025-08-07_high"},"gpqa":{"value":0.75,"norm":0.75,"version":"gpt-5-mini-2025-08-07_high"},"x:Chess Puzzles":{"value":0.24,"norm":0.24,"version":"gpt-5-mini-2025-08-07_high"},"vals:gpqa":{"value":80.3,"norm":0.803,"version":"Vals AI"},"vals:math500":{"value":94.8,"norm":0.948,"version":"Vals AI"},"x:SimpleQA Verified":{"value":0.216,"norm":0.216,"version":"gpt-5-mini-2025-08-07_high"},"x:Fiction.LiveBench":{"value":0.694,"norm":0.694,"version":"gpt-5-mini-2025-08-07_medium"},"vals:corp_fin_v2":{"value":60.18,"norm":0.6018,"version":"Vals AI"},"vals:mmlu_pro":{"value":82.23,"norm":0.8223,"version":"Vals AI"},"vals:swebench":{"value":60.8,"norm":0.608,"version":"Vals AI"},"vals:legal_bench":{"value":81.77,"norm":0.8177,"version":"Vals AI"},"math5":{"value":0.9784743202416919,"norm":0.9785,"version":"gpt-5-mini-2025-08-07_high"},"hle":{"value":0.19440000000000002,"norm":0.1944,"version":"gpt-5-mini-2025-08-07_unknown"},"swe":{"value":0.6466942148760331,"norm":0.6467,"version":"gpt-5-mini-2025-08-07_medium"},"vals:mgsm":{"value":92.58,"norm":0.9258,"version":"Vals AI"},"x:FrontierMath-2025-02-28-Private":{"value":0.27241379310344827,"norm":0.2724,"version":"gpt-5-mini-2025-08-07_high"}},"variant":"GPT-5 mini","match":"self"},"api":{"or_id":"openai/gpt-5-mini","in":0.25,"out":2,"context":400000,"hf":null,"created":"2025-08-07"},"date_from":"epoch"},{"id":"gpt-5-nano","name":"GPT-5 nano","lab":"OpenAI","date":"2025-08-07","type":"flagship","major":false,"open":false,"index":{"score":124.7,"lo":108.7,"hi":140.7,"se":2.44,"solve":21.3,"status":"rated","areas":["agents","coding","maths","novel","reasoning","work"],"core_areas":["maths","novel","reasoning"],"testers":["ARC Prize","Chase Brower","Epoch AI","Fiction.live","Havard T. Ihle","Redwood Research","Terminal-Bench team","Vals AI"],"tests":{"vals:medscribe":{"value":72.86,"norm":0.7286,"version":"Vals AI"},"fm":{"value":0.2,"norm":0.2,"version":"gpt-5-nano-2025-08-07_high"},"x:FrontierMath-2025-02-28-Private":{"value":0.08275862068965517,"norm":0.0828,"version":"gpt-5-nano-2025-08-07_high"},"x:Chess Puzzles":{"value":0.22,"norm":0.22,"version":"gpt-5-nano-2025-08-07_high"},"x:SimpleQA Verified":{"value":0.117,"norm":0.117,"version":"gpt-5-nano-2025-08-07_high"},"x:VPCT":{"value":0.372,"norm":0.372,"version":"gpt-5-nano-2025-08-07_high"},"vals:mmlu_pro":{"value":76.07,"norm":0.7607,"version":"Vals AI"},"vals:medcode":{"value":30.44,"norm":0.3044,"version":"Vals AI"},"vals:mortgage_tax":{"value":53.62,"norm":0.5362,"version":"Vals AI"},"vals:gpqa":{"value":63.38,"norm":0.6338,"version":"Vals AI"},"aime":{"value":0.8111111111111111,"norm":0.8111,"version":"gpt-5-nano-2025-08-07_high"},"arc:arc_agi_2":{"value":2.6,"norm":0.026,"version":"ARC Prize"},"vals:legal_bench":{"value":50.13,"norm":0.5013,"version":"Vals AI"},"math5":{"value":0.952416918429003,"norm":0.9524,"version":"gpt-5-nano-2025-08-07_medium"},"vals:tax_eval_v2":{"value":67.38,"norm":0.6738,"version":"Vals AI"},"x:LMCA":{"value":7.93,"norm":0.0793,"version":"gpt-5-nano-2025-08-07_high"},"x:FrontierMath-Tier-4-2025-07-01-Private":{"value":0.020833333333333332,"norm":0.0208,"version":"gpt-5-nano-2025-08-07_medium"},"x:FrontierMath-Tier-4-v2-Private":{"value":0.024390243902439025,"norm":0.0244,"version":"gpt-5-nano-2025-08-07_high"},"vals:math500":{"value":93.8,"norm":0.938,"version":"Vals AI"},"x:DTBench":{"value":0.6267,"norm":0.6267,"version":"gpt-5-nano-2025-08-07_high"},"vals:mgsm":{"value":89.31,"norm":0.8931,"version":"Vals AI"},"arc:arc_agi_1":{"value":20.7,"norm":0.207,"version":"ARC Prize"},"x:Fiction.LiveBench":{"value":0.444,"norm":0.444,"version":"gpt-5-nano-2025-08-07_medium"},"tb":{"value":0.079,"norm":0.079,"version":"gpt-5-nano-2025-08-07_medium"},"gpqa":{"value":0.6944444444444444,"norm":0.6944,"version":"gpt-5-nano-2025-08-07_high"},"x:Mystery Game Puzzles":{"value":0.09,"norm":0.09,"version":"gpt-5-nano-2025-08-07_low"},"vals:lcb":{"value":70.22,"norm":0.7022,"version":"Vals AI"},"x:WeirdML":{"value":0.38060000000000005,"norm":0.3806,"version":"gpt-5-nano-2025-08-07_high"},"vals:mmmu":{"value":70.94,"norm":0.7094,"version":"Vals AI"}},"variant":"GPT-5 nano","match":"self"},"api":{"or_id":"openai/gpt-5-nano","in":0.05,"out":0.4,"context":400000,"hf":null,"created":"2025-08-07"},"date_from":"epoch"},{"id":"mistral-medium-3-1","name":"Mistral Medium 3.1","lab":"Mistral","date":"2025-08-12","type":"flagship","major":false,"open":false,"index":{"score":120.2,"lo":102.2,"hi":138.1,"se":5.49,"solve":19.2,"status":"rated","areas":["agents","coding","maths","reasoning","work"],"core_areas":[],"testers":["Havard T. Ihle","Redwood Research","Vals AI"],"tests":{"vals:legal_bench":{"value":61.98,"norm":0.6198,"version":"Vals AI"},"vals:tax_eval_v2":{"value":70.32,"norm":0.7032,"version":"Vals AI"},"vals:lcb":{"value":44.85,"norm":0.4485,"version":"Vals AI"},"vals:mmlu_pro":{"value":75.29,"norm":0.7529,"version":"Vals AI"},"x:WeirdML":{"value":0.33130000000000004,"norm":0.3313,"version":"mistral-medium-2508"},"vals:mmmu":{"value":62.97,"norm":0.6297,"version":"Vals AI"},"x:LMCA":{"value":19.05,"norm":0.1905,"version":"mistral-medium-2508"},"x:DTBench":{"value":0.6747,"norm":0.6747,"version":"mistral-medium-2508"},"vals:corp_fin_v2":{"value":50.74,"norm":0.5074,"version":"Vals AI"},"vals:math500":{"value":87,"norm":0.87,"version":"Vals AI"},"vals:mgsm":{"value":90.87,"norm":0.9087,"version":"Vals AI"},"vals:mortgage_tax":{"value":36.45,"norm":0.3645,"version":"Vals AI"}},"variant":"Mistral Medium 3.1","match":"self"},"api":{"or_id":"mistralai/mistral-medium-3.1","in":0.44,"out":2.2,"context":131072,"hf":null,"created":"2025-08-13"},"date_from":"epoch"},{"id":"deepseek-v3-1","name":"DeepSeek-V3.1","lab":"DeepSeek","date":"2025-08-21","type":"open-weights","major":true,"open":true,"release":{"title":"DeepSeek-V3.1 (hybrid reasoning)","source":"https://api-docs.deepseek.com/news/news250821/","verified":false,"confidence":"high"},"index":{"score":128,"lo":111.4,"hi":144.6,"se":3.6,"solve":22.9,"status":"rated","areas":["agents","coding","reasoning","work"],"core_areas":[],"testers":["Fiction.live","Havard T. Ihle","Redwood Research","SimpleBench team","Vals AI"],"tests":{"x:Fiction.LiveBench":{"value":0.528,"norm":0.528,"version":"DeepSeek-V3.1"},"x:DTBench":{"value":0.8267,"norm":0.8267,"version":"DeepSeek-V3.1_thinking"},"x:WeirdML":{"value":0.3836,"norm":0.3836,"version":"DeepSeek-V3.1_thinking"},"x:SimpleBench":{"value":0.4,"norm":0.4,"version":"DeepSeek-V3.1"},"x:LMCA":{"value":24.29,"norm":0.2429,"version":"DeepSeek-V3.1_thinking"},"vals:corp_fin_v2":{"value":51.48,"norm":0.5148,"version":"Vals AI"}},"variant":"DeepSeek-V3.1","match":"exact"},"api":{"or_id":"deepseek/deepseek-chat-v3.1","in":0.25,"out":0.95,"context":163840,"hf":"deepseek-ai/DeepSeek-V3.1","created":"2025-08-21"}},{"id":"grok-2-5","name":"Grok 2.5","lab":"xAI","date":"2025-08-23","type":"open-weights","major":true,"open":true,"release":{"title":"Grok 2.5 open weights","source":"https://techcrunch.com/2025/08/24/elon-musk-says-xai-has-open-sourced-grok-2-5/","verified":false,"confidence":"high"}},{"id":"grok-code-fast-1","name":"Grok Code Fast 1","lab":"xAI","date":"2025-08-28","type":"small","major":true,"open":false,"release":{"title":"Grok Code Fast 1","source":"https://x.ai/news/grok-code-fast-1","verified":true,"confidence":"high"}},{"id":"mai-1-preview","name":"MAI-1-preview","lab":"Microsoft","date":"2025-08-28","type":"flagship","major":true,"open":false,"release":{"title":"MAI-1-preview","source":null,"verified":false,"confidence":"high"}},{"id":"longcat-flash","name":"LongCat-Flash","lab":"Meituan","date":"2025-09-01","type":"open-weights","major":true,"open":true,"release":{"title":"LongCat-Flash","source":"https://huggingface.co/meituan-longcat/LongCat-Flash-Chat","verified":false,"confidence":"high"}},{"id":"kimi-k2-sep-2025","name":"Kimi K2 (Sep 2025)","lab":"Moonshot","date":"2025-09-05","type":"open-weights","major":false,"open":true,"index":{"score":132,"lo":115.8,"hi":148.2,"se":2.82,"solve":25.2,"status":"provisional","areas":["coding","reasoning"],"core_areas":[],"testers":["Aider","Fiction.live","Havard T. Ihle"],"tests":{"x:WeirdML":{"value":0.3668,"norm":0.3668,"version":"Kimi-K2-Instruct-0905"},"aider":{"value":59.1,"norm":0.591,"version":"moonshotai/kimi-k2-0905"},"x:Fiction.LiveBench":{"value":0.667,"norm":0.667,"version":"fireworks/Kimi-K2-Instruct-0905"}},"variant":"Kimi K2 (Sep 2025)","match":"self"},"api":{"or_id":"moonshotai/kimi-k2-0905","in":0.6,"out":2.5,"context":262144,"hf":"moonshotai/Kimi-K2-Instruct-0905","created":"2025-09-04"},"date_from":"epoch"},{"id":"qwen3-max","name":"Qwen3-Max","lab":"Alibaba","date":"2025-09-05","type":"flagship","major":true,"open":false,"release":{"title":"Qwen3-Max (preview; official 09-24)","source":null,"verified":false,"confidence":"high"},"index":{"score":134.9,"lo":118.5,"hi":151.2,"se":3.16,"solve":26.9,"status":"rated","areas":["agents","coding","maths","reasoning","work"],"core_areas":["maths","reasoning"],"testers":["Epoch AI","Fiction.live","Redwood Research","Tencent Hunyuan / Fudan","Vals AI"],"tests":{"vals:medscribe":{"value":72.71,"norm":0.7271,"version":"Vals AI"},"math5":{"value":0.9712990936555891,"norm":0.9713,"version":"qwen3-max-2025-09-23"},"x:LMCA":{"value":28.27,"norm":0.2827,"version":"qwen3-max-2025-09-23"},"x:DTBench":{"value":0.8212999999999999,"norm":0.8213,"version":"qwen3-max-2025-09-23"},"gpqa":{"value":0.726010101010101,"norm":0.726,"version":"qwen3-max-2025-09-23"},"vals:gpqa":{"value":84.85,"norm":0.8485,"version":"Vals AI"},"aime":{"value":0.7333333333333333,"norm":0.7333,"version":"qwen3-max-2025-09-23"},"vals:legal_bench":{"value":81.86,"norm":0.8186,"version":"Vals AI"},"vals:vibe-code":{"value":3.51,"norm":0.0351,"version":"Vals AI"},"vals:medcode":{"value":31.37,"norm":0.3137,"version":"Vals AI"},"x:Fiction.LiveBench":{"value":0.667,"norm":0.667,"version":"qwen3-max-2025-09-23"},"vals:lcb":{"value":78.22,"norm":0.7822,"version":"Vals AI"},"x:CL-bench":{"value":0.145,"norm":0.145,"version":"qwen3-max-2025-09-23"},"fm":{"value":0.18947368421052632,"norm":0.1895,"version":"qwen3-max-2025-09-23"},"vals:mgsm":{"value":92.15,"norm":0.9215,"version":"Vals AI"},"vals:corp_fin_v2":{"value":68.03,"norm":0.6803,"version":"Vals AI"},"x:Mystery Game Puzzles":{"value":0.05,"norm":0.05,"version":"qwen3-max-2025-09-23"},"vals:mmlu_pro":{"value":84.98,"norm":0.8498,"version":"Vals AI"},"x:SimpleQA Verified":{"value":0.4874874874874875,"norm":0.4875,"version":"qwen3-max-2025-09-23"},"vals:tax_eval_v2":{"value":73.96,"norm":0.7396,"version":"Vals AI"}},"variant":"Qwen3-Max","match":"exact"}},{"id":"qwen3-next-80b-a3b","name":"Qwen3-Next-80B-A3B","lab":"Alibaba","date":"2025-09-11","type":"open-weights","major":true,"open":true,"release":{"title":"Qwen3-Next-80B-A3B","source":null,"verified":false,"confidence":"high"},"api":{"or_id":"qwen/qwen3-next-80b-a3b-instruct","in":0.1,"out":1.1,"context":262144,"hf":"Qwen/Qwen3-Next-80B-A3B-Instruct","created":"2025-09-11"}},{"id":"gpt-5-codex","name":"GPT-5-Codex","lab":"OpenAI","date":"2025-09-15","type":"reasoning","major":true,"open":false,"release":{"title":"GPT-5-Codex","source":"https://openai.com/index/introducing-upgrades-to-codex/","verified":true,"confidence":"high"},"index":{"score":148.4,"lo":131.1,"hi":165.6,"se":4.61,"solve":37.3,"status":"insufficient","areas":["coding"],"core_areas":[],"testers":["Havard T. Ihle","Terminal-Bench team","Vals AI"],"tests":{"x:WeirdML":{"value":0.5453,"norm":0.5453,"version":"gpt-5-codex_high"},"tb":{"value":0.434269662921,"norm":0.4343,"version":"gpt-5-codex"},"vals:lcb":{"value":84.72,"norm":0.8472,"version":"Vals AI"}},"variant":"GPT-5-Codex","match":"exact"}},{"id":"magistral-small-1-2","name":"Magistral Small 1.2","lab":"Mistral","date":"2025-09-18","type":"open-weights","major":false,"open":true,"index":{"score":116.6,"lo":100.5,"hi":132.8,"se":2.73,"solve":17.8,"status":"rated","areas":["agents","coding","maths","reasoning","work"],"core_areas":["reasoning"],"testers":["Epoch AI","Redwood Research","Vals AI"],"tests":{"gpqa":{"value":0.476010101010101,"norm":0.476,"version":"magistral-small-2509"},"vals:mmlu_pro":{"value":62.13,"norm":0.6213,"version":"Vals AI"},"x:DTBench":{"value":0.6131000000000001,"norm":0.6131,"version":"magistral-small-2509"},"vals:gpqa":{"value":58.33,"norm":0.5833,"version":"Vals AI"},"vals:legal_bench":{"value":40.02,"norm":0.4002,"version":"Vals AI"},"vals:lcb":{"value":72.13,"norm":0.7213,"version":"Vals AI"},"vals:tax_eval_v2":{"value":60.3,"norm":0.603,"version":"Vals AI"},"aime":{"value":0.28055555555555556,"norm":0.2806,"version":"magistral-small-2509"},"vals:mortgage_tax":{"value":62.12,"norm":0.6212,"version":"Vals AI"},"vals:mgsm":{"value":86.25,"norm":0.8625,"version":"Vals AI"},"vals:mmmu":{"value":65.2,"norm":0.652,"version":"Vals AI"},"vals:corp_fin_v2":{"value":44.02,"norm":0.4402,"version":"Vals AI"}},"variant":"Magistral Small 1.2","match":"self"},"date_from":"epoch"},{"id":"grok-4-fast","name":"Grok 4 Fast","lab":"xAI","date":"2025-09-19","type":"small","major":true,"open":false,"release":{"title":"Grok 4 Fast","source":"https://x.ai/news/grok-4-fast","verified":true,"confidence":"high"},"index":{"score":142.5,"lo":125.4,"hi":159.7,"se":4.42,"solve":32.3,"status":"rated","areas":["agents","coding","maths","reasoning","work"],"core_areas":[],"testers":["Fiction.live","Havard T. Ihle","Redwood Research","Vals AI"],"tests":{"vals:vibe-code":{"value":0,"norm":0,"version":"Vals AI"},"vals:mmmu":{"value":72.78,"norm":0.7278,"version":"Vals AI"},"vals:medcode":{"value":37.38,"norm":0.3738,"version":"Vals AI"},"vals:swebench":{"value":45.4,"norm":0.454,"version":"Vals AI"},"x:WeirdML":{"value":0.4286,"norm":0.4286,"version":"grok-4-fast"},"vals:tax_eval_v2":{"value":75.7,"norm":0.757,"version":"Vals AI"},"vals:corp_fin_v2":{"value":66.9,"norm":0.669,"version":"Vals AI"},"vals:lcb":{"value":78.97,"norm":0.7897,"version":"Vals AI"},"vals:mortgage_tax":{"value":42.09,"norm":0.4209,"version":"Vals AI"},"vals:medscribe":{"value":81.63,"norm":0.8163,"version":"Vals AI"},"x:Fiction.LiveBench":{"value":0.944,"norm":0.944,"version":"grok-4-fast"},"x:DTBench":{"value":0.8267,"norm":0.8267,"version":"grok-4-fast"},"vals:mmlu_pro":{"value":79.7,"norm":0.797,"version":"Vals AI"},"vals:mgsm":{"value":90.87,"norm":0.9087,"version":"Vals AI"},"vals:gpqa":{"value":85.35,"norm":0.8535,"version":"Vals AI"},"vals:legal_bench":{"value":80.6,"norm":0.806,"version":"Vals AI"}},"variant":"Grok 4 Fast","match":"exact"}},{"id":"deepseek-v3-1-terminus","name":"DeepSeek-V3.1-Terminus","lab":"DeepSeek","date":"2025-09-22","type":"open-weights","major":false,"open":true,"index":{"score":138.2,"lo":117.7,"hi":158.8,"se":8.2,"solve":29.1,"status":"insufficient","areas":["agents"],"core_areas":[],"testers":["Redwood Research"],"tests":{"x:LMCA":{"value":28.58,"norm":0.2858,"version":"DeepSeek-V3.1-Terminus"},"x:DTBench":{"value":0.8133,"norm":0.8133,"version":"DeepSeek-V3.1-Terminus"}},"variant":"DeepSeek-V3.1-Terminus","match":"self"},"api":{"or_id":"deepseek/deepseek-v3.1-terminus","in":0.27,"out":1,"context":163840,"hf":"deepseek-ai/DeepSeek-V3.1-Terminus","created":"2025-09-22"},"date_from":"epoch"},{"id":"gemini-2-5-flash-sep-2025","name":"Gemini 2.5 Flash (Sep 2025)","lab":"Google","date":"2025-09-25","type":"flagship","major":false,"open":false,"index":{"score":131.6,"lo":113.7,"hi":149.6,"se":5.53,"solve":25,"status":"provisional","areas":["coding","reasoning"],"core_areas":[],"testers":["Chase Brower","Havard T. Ihle","Terminal-Bench team"],"tests":{"x:VPCT":{"value":0.462,"norm":0.462,"version":"gemini-2.5-flash-preview-09-2025"},"tb":{"value":0.169,"norm":0.169,"version":"gemini-2.5-flash-preview-09-2025"},"x:WeirdML":{"value":0.4191,"norm":0.4191,"version":"gemini-2.5-flash-preview-09-2025_16k"}},"variant":"Gemini 2.5 Flash (Sep 2025)","match":"self"},"date_from":"epoch"},{"id":"gemini-2-5-flash-lite-sep-2025","name":"Gemini 2.5 Flash-Lite (Sep 2025)","lab":"Google","date":"2025-09-25","type":"flagship","major":false,"open":false,"index":{"score":116.3,"lo":-3.3,"hi":236,"se":72.11,"solve":17.7,"status":"insufficient","areas":["reasoning"],"core_areas":[],"testers":["Chase Brower"],"tests":{"x:VPCT":{"value":0.3,"norm":0.3,"version":"gemini-2.5-flash-lite-preview-09-2025"}},"variant":"Gemini 2.5 Flash-Lite (Sep 2025)","match":"self"},"date_from":"epoch"},{"id":"claude-sonnet-4-5","name":"Claude Sonnet 4.5","lab":"Anthropic","date":"2025-09-29","type":"flagship","major":true,"open":false,"release":{"title":"Claude Sonnet 4.5","source":"https://www.anthropic.com/news/claude-sonnet-4-5","verified":true,"confidence":"high"},"index":{"score":144.3,"lo":128.5,"hi":160.1,"se":1.85,"solve":33.7,"status":"rated","areas":["agents","coding","long","maths","novel","reasoning","work"],"core_areas":["coding","long","maths","novel","reasoning"],"testers":["ARC Prize","Chase Brower","Epoch AI","FutureSearch","GSO team","GeoBench team","Havard T. Ihle","METR","Redwood Research","Scale AI","SimpleBench team","Terminal-Bench team","Vals AI"],"tests":{"x:Chess Puzzles":{"value":0.1,"norm":0.1,"version":"claude-sonnet-4-5-20250929_32K"},"x:FrontierMath-Tier-4-2025-07-01-Private":{"value":0.041666666666666664,"norm":0.0417,"version":"claude-sonnet-4-5-20250929_32K"},"vals:vibe-code":{"value":22.62,"norm":0.2262,"version":"Vals AI"},"metr":{"value":0.673842,"norm":0.2413,"version":"claude-sonnet-4-5-20250929_16K"},"vals:tax_eval_v2":{"value":73.3,"norm":0.733,"version":"Vals AI"},"x:LMCA":{"value":38.76,"norm":0.3876,"version":"claude-sonnet-4-5-20250929_unknown"},"vals:mortgage_tax":{"value":63.99,"norm":0.6399,"version":"Vals AI"},"hle":{"value":0.13720000000000002,"norm":0.1372,"version":"claude-sonnet-4-5-20250929_unknown"},"x:FrontierMath-Tier-4-v2-Private":{"value":0.024390243902439025,"norm":0.0244,"version":"claude-sonnet-4-5-20250929_32K"},"vals:gpqa":{"value":81.63,"norm":0.8163,"version":"Vals AI"},"aime":{"value":0.7777777777777778,"norm":0.7778,"version":"claude-sonnet-4-5-20250929_32K"},"x:SimpleBench":{"value":0.543,"norm":0.543,"version":"claude-sonnet-4-5-20250929_unknown"},"vals:swebench":{"value":70,"norm":0.7,"version":"Vals AI"},"x:FrontierMath-2025-02-28-Private":{"value":0.1522491349480969,"norm":0.1522,"version":"claude-sonnet-4-5-20250929_32K"},"vals:lcb":{"value":73,"norm":0.73,"version":"Vals AI"},"x:EBR-bench":{"value":0.023809523809523808,"norm":0.0238,"version":"claude-sonnet-4-5-20250929"},"x:GeoBench":{"value":0.6,"norm":0.6,"version":"claude-sonnet-4-5-20250929_32K"},"x:VPCT":{"value":0.398,"norm":0.398,"version":"claude-sonnet-4-5-20250929_32K"},"tb":{"value":0.42808988764,"norm":0.4281,"version":"claude-sonnet-4-5-20250929_unknown"},"vals:corp_fin_v2":{"value":61.97,"norm":0.6197,"version":"Vals AI"},"x:Mystery Game Puzzles":{"value":0.17,"norm":0.17,"version":"claude-sonnet-4-5-20250929_48K"},"gpqa":{"value":0.8232323232323232,"norm":0.8232,"version":"claude-sonnet-4-5-20250929_59K"},"x:GSO-Bench":{"value":0.1471,"norm":0.1471,"version":"claude-sonnet-4-5-20250929_unknown"},"x:SimpleQA Verified":{"value":0.307,"norm":0.307,"version":"claude-sonnet-4-5-20250929_59K"},"vals:mmlu_pro":{"value":87.36,"norm":0.8736,"version":"Vals AI"},"vals:medscribe":{"value":84.51,"norm":0.8451,"version":"Vals AI"},"x:DTBench":{"value":0.8320000000000001,"norm":0.832,"version":"claude-sonnet-4-5-20250929_unknown"},"vals:medcode":{"value":44.13,"norm":0.4413,"version":"Vals AI"},"vals:legal_bench":{"value":84.08,"norm":0.8408,"version":"Vals AI"},"x:WeirdML":{"value":0.4771,"norm":0.4771,"version":"claude-sonnet-4-5-20250929_16K"},"vals:mmmu":{"value":79.31,"norm":0.7931,"version":"Vals AI"},"math5":{"value":0.9773413897280967,"norm":0.9773,"version":"claude-sonnet-4-5-20250929_32K"},"scale:swe_bench_pro":{"value":43.6,"norm":0.436,"version":"Scale AI"},"fm":{"value":0.23859649122807017,"norm":0.2386,"version":"claude-sonnet-4-5-20250929_32K"},"vals:mgsm":{"value":94.33,"norm":0.9433,"version":"Vals AI"},"x:DeepResearch Bench":{"value":0.526,"norm":0.526,"version":"claude-sonnet-4-5-20250929_2K"},"swe":{"value":0.7128099173553719,"norm":0.7128,"version":"claude-sonnet-4-5-20250929"},"arc:arc_agi_2":{"value":13.6,"norm":0.136,"version":"ARC Prize"},"arc:arc_agi_1":{"value":63.7,"norm":0.637,"version":"ARC Prize"}},"variant":"Claude Sonnet 4.5","match":"exact"},"api":{"or_id":"anthropic/claude-sonnet-4.5","in":3,"out":15,"context":1000000,"hf":null,"created":"2025-09-29"}},{"id":"deepseek-v3-2-exp","name":"DeepSeek-V3.2-Exp","lab":"DeepSeek","date":"2025-09-29","type":"open-weights","major":true,"open":true,"release":{"title":"DeepSeek-V3.2-Exp (sparse attention)","source":"https://api-docs.deepseek.com/news/news250929/","verified":false,"confidence":"high"},"index":{"score":140.3,"lo":124,"hi":156.7,"se":3.16,"solve":30.6,"status":"rated","areas":["agents","coding","reasoning"],"core_areas":[],"testers":["Aider","Fiction.live","Havard T. Ihle","Redwood Research","Tencent Hunyuan / Fudan"],"tests":{"x:Fiction.LiveBench":{"value":0.833,"norm":0.833,"version":"DeepSeek-V3.2-Exp_high"},"x:WeirdML":{"value":0.3948,"norm":0.3948,"version":"DeepSeek-V3.2-Exp_thinking"},"aider":{"value":74.2,"norm":0.742,"version":"DeepSeek-V3.2-Exp_thinking"},"x:DTBench":{"value":0.8773000000000001,"norm":0.8773,"version":"DeepSeek-V3.2-Exp_thinking"},"x:CL-bench":{"value":0.132,"norm":0.132,"version":"DeepSeek-V3.2-Exp_thinking"},"x:LMCA":{"value":29.11,"norm":0.2911,"version":"DeepSeek-V3.2-Exp_thinking"}},"variant":"DeepSeek-V3.2-Exp","match":"exact"},"api":{"or_id":"deepseek/deepseek-v3.2-exp","in":0.27,"out":0.41,"context":163840,"hf":"deepseek-ai/DeepSeek-V3.2-Exp","created":"2025-09-29"}},{"id":"glm-4-6","name":"GLM-4.6","lab":"Zhipu","date":"2025-09-30","type":"open-weights","major":true,"open":true,"release":{"title":"GLM-4.6","source":"https://huggingface.co/zai-org/GLM-4.6-FP8","verified":true,"confidence":"high"},"index":{"score":126.7,"lo":108.4,"hi":144.9,"se":5.85,"solve":22.3,"status":"rated","areas":["coding","maths","reasoning","work"],"core_areas":[],"testers":["Epoch AI","Scale AI","Terminal-Bench team","Vals AI"],"tests":{"x:FrontierMath-2025-02-28-Private":{"value":0.03819444444444445,"norm":0.0382,"version":"zai-org/GLM-4.6"},"vals:legal_bench":{"value":79.61,"norm":0.7961,"version":"Vals AI"},"vals:tax_eval_v2":{"value":66.23,"norm":0.6623,"version":"Vals AI"},"vals:corp_fin_v2":{"value":56.84,"norm":0.5684,"version":"Vals AI"},"x:FrontierMath-Tier-4-2025-07-01-Private":{"value":0.02127659574468085,"norm":0.0213,"version":"zai-org/GLM-4.6"},"vals:gpqa":{"value":74.5,"norm":0.745,"version":"Vals AI"},"scale:swe_bench_pro":{"value":9.67,"norm":0.0967,"version":"Scale AI"},"vals:mgsm":{"value":89.75,"norm":0.8975,"version":"Vals AI"},"tb":{"value":0.24531835206,"norm":0.2453,"version":"glm-4.6"},"vals:mmlu_pro":{"value":82.2,"norm":0.822,"version":"Vals AI"},"vals:lcb":{"value":81.04,"norm":0.8104,"version":"Vals AI"},"vals:vibe-code":{"value":3.09,"norm":0.0309,"version":"Vals AI"}},"variant":"GLM-4.6","match":"exact"},"api":{"or_id":"z-ai/glm-4.6","in":0.5,"out":2,"context":204800,"hf":"zai-org/GLM-4.6","created":"2025-09-30"}},{"id":"ling-1t","name":"Ling-1T","lab":"Ant Group","date":"2025-10-09","type":"open-weights","major":true,"open":true,"release":{"title":"Ling-1T","source":"https://huggingface.co/inclusionAI/Ling-1T","verified":false,"confidence":"high"}},{"id":"ring-1t","name":"Ring-1T","lab":"Ant Group","date":"2025-10-14","type":"open-weights","major":true,"open":true,"release":{"title":"Ring-1T","source":"https://huggingface.co/inclusionAI/Ring-1T","verified":false,"confidence":"high"}},{"id":"claude-haiku-4-5","name":"Claude Haiku 4.5","lab":"Anthropic","date":"2025-10-15","type":"small","major":true,"open":false,"release":{"title":"Claude Haiku 4.5","source":"https://www.anthropic.com/news/claude-haiku-4-5","verified":true,"confidence":"high"},"index":{"score":136.6,"lo":120.7,"hi":152.5,"se":2.15,"solve":28,"status":"rated","areas":["agents","coding","maths","novel","reasoning","work"],"core_areas":["novel","reasoning","work"],"testers":["ARC Prize","BALROG team","Epoch AI","FutureSearch","Havard T. Ihle","Redwood Research","Scale AI","Terminal-Bench team","Vals AI"],"tests":{"gpqa":{"value":0.7121212121212122,"norm":0.7121,"version":"claude-haiku-4-5-20251001_32K"},"vals:public-benefits-bench":{"value":54.26,"norm":0.5426,"version":"Vals AI"},"vals:programbench":{"value":0,"norm":0,"version":"Vals AI"},"x:WeirdML":{"value":0.454,"norm":0.454,"version":"claude-haiku-4-5-20251001"},"scale:swe_bench_pro":{"value":39.45,"norm":0.3945,"version":"Scale AI"},"vals:mortgage_tax":{"value":62.16,"norm":0.6216,"version":"Vals AI"},"vals:emb":{"value":22.66,"norm":0.2266,"version":"Vals AI"},"vals:mgsm":{"value":92.15,"norm":0.9215,"version":"Vals AI"},"vals:gpqa":{"value":72.22,"norm":0.7222,"version":"Vals AI"},"aime":{"value":0.6666666666666666,"norm":0.6667,"version":"claude-haiku-4-5-20251001_32K"},"vals:code-migration":{"value":7.55,"norm":0.0755,"version":"Vals AI"},"x:SimpleQA Verified":{"value":0.132,"norm":0.132,"version":"claude-haiku-4-5-20251001"},"vals:tax_eval_v2":{"value":67.54,"norm":0.6754,"version":"Vals AI"},"x:LMCA":{"value":30.94,"norm":0.3094,"version":"claude-haiku-4-5-20251001_unknown"},"tb":{"value":0.283146067416,"norm":0.2831,"version":"claude-haiku-4-5-20251001_unknown"},"vals:swebench":{"value":66.6,"norm":0.666,"version":"Vals AI"},"vals:lcb":{"value":41.17,"norm":0.4117,"version":"Vals AI"},"arc:arc_agi_2":{"value":4,"norm":0.04,"version":"ARC Prize"},"arc:arc_agi_1":{"value":47.7,"norm":0.477,"version":"ARC Prize"},"x:DTBench":{"value":0.736,"norm":0.736,"version":"claude-haiku-4-5-20251001_unknown"},"x:DeepResearch Bench":{"value":0.455,"norm":0.455,"version":"claude-haiku-4-5-20251001_low"},"vals:corp_fin_v2":{"value":60.61,"norm":0.6061,"version":"Vals AI"},"x:FrontierMath-2025-02-28-Private":{"value":0.059027777777777776,"norm":0.059,"version":"claude-haiku-4-5-20251001_32K"},"vals:legal_bench":{"value":81.24,"norm":0.8124,"version":"Vals AI"},"vals:vibe-code":{"value":11.39,"norm":0.1139,"version":"Vals AI"},"math5":{"value":0.9636087945413192,"norm":0.9636,"version":"claude-haiku-4-5-20251001_32K"},"vals:mmlu_pro":{"value":78.72,"norm":0.7872,"version":"Vals AI"},"vals:medcode":{"value":32.68,"norm":0.3268,"version":"Vals AI"},"x:FrontierMath-Tier-4-2025-07-01-Private":{"value":0.020833333333333332,"norm":0.0208,"version":"claude-haiku-4-5-20251001_32K"},"vals:fabv2":{"value":31.01,"norm":0.3101,"version":"Vals AI"},"vals:tax_agent_bench":{"value":5.86,"norm":0.0586,"version":"Vals AI"},"vals:legal_research":{"value":10.58,"norm":0.1058,"version":"Vals AI"},"x:Balrog":{"value":0.312,"norm":0.312,"version":"claude-haiku-4-5-20251001_1K"},"vals:mmmu":{"value":46.07,"norm":0.4607,"version":"Vals AI"},"vals:medscribe":{"value":85.23,"norm":0.8523,"version":"Vals AI"},"x:Chess Puzzles":{"value":0.08,"norm":0.08,"version":"claude-haiku-4-5-20251001_59K"}},"variant":"Claude Haiku 4.5","match":"exact"},"api":{"or_id":"anthropic/claude-haiku-4.5","in":1,"out":5,"context":200000,"hf":null,"created":"2025-10-15"}},{"id":"minimax-m2","name":"MiniMax-M2","lab":"MiniMax","date":"2025-10-27","type":"open-weights","major":true,"open":true,"release":{"title":"MiniMax-M2","source":null,"verified":false,"confidence":"high"},"index":{"score":128.9,"lo":109.3,"hi":148.5,"se":7.31,"solve":23.4,"status":"provisional","areas":["coding","reasoning"],"core_areas":[],"testers":["SimpleBench team","Terminal-Bench team"],"tests":{"x:SimpleBench":{"value":0.25,"norm":0.25,"version":"MiniMax-M2"},"tb":{"value":0.300374531835,"norm":0.3004,"version":"MiniMax-M2"}},"variant":"MiniMax-M2","match":"exact"}},{"id":"kimi-linear-48b-a3b","name":"Kimi Linear 48B-A3B","lab":"Moonshot","date":"2025-10-30","type":"open-weights","major":true,"open":true,"release":{"title":"Kimi Linear 48B-A3B","source":"https://huggingface.co/moonshotai/Kimi-Linear-48B-A3B-Instruct","verified":false,"confidence":"med"}},{"id":"kimi-k2-thinking","name":"Kimi K2 Thinking","lab":"Moonshot","date":"2025-11-06","type":"reasoning","major":true,"open":true,"release":{"title":"Kimi K2 Thinking","source":"https://www.kimi.com/en/blog/kimi-k2-thinking","verified":false,"confidence":"high"},"index":{"score":139.3,"lo":122.7,"hi":155.8,"se":3.51,"solve":29.8,"status":"rated","areas":["coding","long","maths","reasoning"],"core_areas":["long","reasoning"],"testers":["Epoch AI","Havard T. Ihle","METR","SimpleBench team","Tencent Hunyuan / Fudan","Terminal-Bench team"],"tests":{"tb":{"value":0.35730337078700003,"norm":0.3573,"version":"kimi-k2-thinking"},"x:FrontierMath-2025-02-28-Private":{"value":0.21403508771929824,"norm":0.214,"version":"moonshotai/Kimi-K2-Thinking"},"metr":{"value":0.591943,"norm":0.2293,"version":"kimi-k2-thinking"},"aime":{"value":0.8305555555555556,"norm":0.8306,"version":"kimi-k2-thinking-turbo"},"gpqa":{"value":0.8421717171717171,"norm":0.8422,"version":"kimi-k2-thinking-turbo"},"x:FrontierMath-Tier-4-2025-07-01-Private":{"value":0,"norm":0,"version":"moonshotai/Kimi-K2-Thinking"},"x:CL-bench":{"value":0.176,"norm":0.176,"version":"moonshotai/Kimi-K2-Thinking"},"x:SimpleBench":{"value":0.396,"norm":0.396,"version":"kimi-k2-thinking"},"x:WeirdML":{"value":0.4279,"norm":0.4279,"version":"kimi-k2-thinking"}},"variant":"Kimi K2 Thinking","match":"exact"},"api":{"or_id":"moonshotai/kimi-k2-thinking","in":0.6,"out":2.5,"context":262144,"hf":"moonshotai/Kimi-K2-Thinking","created":"2025-11-06"}},{"id":"gpt-5-1","name":"GPT-5.1","lab":"OpenAI","date":"2025-11-12","type":"flagship","major":true,"open":false,"release":{"title":"GPT-5.1","source":"https://openai.com/index/gpt-5-system-card-addendum-gpt-5-1/","verified":true,"confidence":"high"},"index":{"score":150,"lo":134.2,"hi":165.7,"se":1.75,"solve":38.8,"status":"rated","areas":["agents","coding","maths","novel","reasoning","work"],"core_areas":["coding","novel","reasoning"],"testers":["ARC Prize","Chase Brower","Epoch AI","FutureSearch","GSO team","Havard T. Ihle","Redwood Research","Scale AI","SimpleBench team","Tencent Hunyuan / Fudan","Terminal-Bench team","Vals AI"],"tests":{"vals:legal_bench":{"value":85.68,"norm":0.8568,"version":"Vals AI"},"arc:arc_agi_2":{"value":17.6,"norm":0.176,"version":"ARC Prize"},"x:DeepResearch Bench":{"value":0.428,"norm":0.428,"version":"gpt-5.1-2025-11-13_low"},"gpqa":{"value":0.8762626262626263,"norm":0.8763,"version":"gpt-5.1-2025-11-13_high"},"x:DTBench":{"value":0.9013,"norm":0.9013,"version":"gpt-5.1-2025-11-13_high"},"tb":{"value":0.476,"norm":0.476,"version":"gpt-5.1-2025-11-13_medium"},"x:FrontierMath-2025-02-28-Private":{"value":0.3103448275862069,"norm":0.3103,"version":"gpt-5.1-2025-11-13_high"},"x:Chess Puzzles":{"value":0.42,"norm":0.42,"version":"gpt-5.1-2025-11-13_high"},"swe":{"value":0.6797520661157025,"norm":0.6798,"version":"gpt-5.1-2025-11-13_high"},"x:FrontierMath-Tier-4-2025-07-01-Private":{"value":0.125,"norm":0.125,"version":"gpt-5.1-2025-11-13_high"},"vals:corp_fin_v2":{"value":63.83,"norm":0.6383,"version":"Vals AI"},"x:WeirdML":{"value":0.6077,"norm":0.6077,"version":"gpt-5.1-2025-11-13_high"},"x:CL-bench":{"value":0.237,"norm":0.237,"version":"gpt-5.1-2025-11-13_high"},"vals:mortgage_tax":{"value":61.37,"norm":0.6137,"version":"Vals AI"},"vals:medscribe":{"value":88.09,"norm":0.8809,"version":"Vals AI"},"vals:mgsm":{"value":92.98,"norm":0.9298,"version":"Vals AI"},"vals:gpqa":{"value":86.62,"norm":0.8662,"version":"Vals AI"},"vals:swebench":{"value":69.8,"norm":0.698,"version":"Vals AI"},"vals:lcb":{"value":86.49,"norm":0.8649,"version":"Vals AI"},"vals:mmmu":{"value":83.18,"norm":0.8318,"version":"Vals AI"},"vals:tax_eval_v2":{"value":74.86,"norm":0.7486,"version":"Vals AI"},"hle":{"value":0.2368,"norm":0.2368,"version":"gpt-5.1-2025-11-13_unknown"},"aime":{"value":0.8861111111111111,"norm":0.8861,"version":"gpt-5.1-2025-11-13_high"},"x:SimpleBench":{"value":0.532,"norm":0.532,"version":"gpt-5.1-2025-11-13_high"},"x:Mystery Game Puzzles":{"value":0.19,"norm":0.19,"version":"gpt-5.1-2025-11-13_low"},"vals:mmlu_pro":{"value":86.38,"norm":0.8638,"version":"Vals AI"},"x:GSO-Bench":{"value":0.1373,"norm":0.1373,"version":"gpt-5.1-2025-11-13_unknown"},"vals:vibe-code":{"value":24.61,"norm":0.2461,"version":"Vals AI"},"x:SimpleQA Verified":{"value":0.48,"norm":0.48,"version":"gpt-5.1-2025-11-13_high"},"x:VPCT":{"value":0.587,"norm":0.587,"version":"gpt-5.1-2025-11-13_high"},"x:LMCA":{"value":43.89,"norm":0.4389,"version":"gpt-5.1-2025-11-13_high"},"arc:arc_agi_1":{"value":72.8,"norm":0.728,"version":"ARC Prize"},"vals:medcode":{"value":52.73,"norm":0.5273,"version":"Vals AI"}},"variant":"GPT-5.1","match":"exact"},"api":{"or_id":"openai/gpt-5.1","in":1.25,"out":10,"context":400000,"hf":null,"created":"2025-11-13"}},{"id":"gpt-5-1-codex","name":"GPT-5.1-Codex","lab":"OpenAI","date":"2025-11-12","type":"flagship","major":false,"open":false,"index":{"score":141.4,"lo":124.5,"hi":158.3,"se":4.09,"solve":31.4,"status":"insufficient","areas":["coding"],"core_areas":[],"testers":["Terminal-Bench team","Vals AI"],"tests":{"tb":{"value":0.369,"norm":0.369,"version":"gpt-5.1-codex"},"vals:vibe-code":{"value":13.11,"norm":0.1311,"version":"Vals AI"},"vals:lcb":{"value":85.55,"norm":0.8555,"version":"Vals AI"}},"variant":"GPT-5.1-Codex","match":"self"},"api":{"or_id":"openai/gpt-5.1-codex","in":1.25,"out":10,"context":400000,"hf":null,"created":"2025-11-13"},"date_from":"epoch"},{"id":"ernie-5-0","name":"ERNIE 5.0","lab":"Baidu","date":"2025-11-13","type":"flagship","major":true,"open":false,"release":{"title":"ERNIE 5.0","source":null,"verified":false,"confidence":"high"}},{"id":"grok-4-1","name":"Grok 4.1","lab":"xAI","date":"2025-11-17","type":"flagship","major":true,"open":false,"release":{"title":"Grok 4.1","source":"https://x.ai/news/grok-4-1","verified":true,"confidence":"high"}},{"id":"gemini-3-pro","name":"Gemini 3 Pro","lab":"Google","date":"2025-11-18","type":"flagship","major":true,"open":false,"release":{"title":"Gemini 3 Pro","source":"https://blog.google/products-and-platforms/products/gemini/gemini-3/","verified":true,"confidence":"high"},"index":{"score":160.3,"lo":144.6,"hi":176.1,"se":1.64,"solve":49.8,"status":"rated","areas":["agents","coding","long","maths","novel","reasoning","work"],"core_areas":["coding","long","novel","reasoning"],"testers":["ARC Prize","BALROG team","Chase Brower","Epoch AI","FutureSearch","GSO team","GeoBench team","Havard T. Ihle","METR","Scale AI","SimpleBench team","Tencent Hunyuan / Fudan","Terminal-Bench team","Vals AI"],"tests":{"vals:legal_bench":{"value":87.03,"norm":0.8703,"version":"Vals AI"},"x:Balrog":{"value":0.5810000000000001,"norm":0.581,"version":"gemini-3-pro-preview"},"x:WeirdML":{"value":0.6993,"norm":0.6993,"version":"gemini-3-pro-preview"},"arc:arc_agi_1":{"value":75,"norm":0.75,"version":"ARC Prize"},"hle":{"value":0.37520000000000003,"norm":0.3752,"version":"gemini-3-pro-preview"},"swe":{"value":0.7293388429752066,"norm":0.7293,"version":"gemini-3-pro-preview"},"x:GeoBench":{"value":0.84,"norm":0.84,"version":"gemini-3-pro-preview"},"scale:swe_bench_pro":{"value":43.3,"norm":0.433,"version":"Scale AI"},"metr":{"value":0.709822,"norm":0.2461,"version":"gemini-3-pro-preview"},"vals:medcode":{"value":52.2,"norm":0.522,"version":"Vals AI"},"vals:mortgage_tax":{"value":69.08,"norm":0.6908,"version":"Vals AI"},"x:FrontierMath-2025-02-28-Private":{"value":0.376,"norm":0.376,"version":"gemini-3-pro-preview"},"vals:lcb":{"value":86.41,"norm":0.8641,"version":"Vals AI"},"x:CL-bench":{"value":0.158,"norm":0.158,"version":"gemini-3-pro-preview"},"vals:mgsm":{"value":93.93,"norm":0.9393,"version":"Vals AI"},"vals:corp_fin_v2":{"value":63.67,"norm":0.6367,"version":"Vals AI"},"aime":{"value":0.9138888888888889,"norm":0.9139,"version":"gemini-3-pro-preview"},"vals:math500":{"value":96.4,"norm":0.964,"version":"Vals AI"},"vals:mmlu_pro":{"value":90.1,"norm":0.901,"version":"Vals AI"},"arc:arc_agi_2":{"value":54,"norm":0.54,"version":"ARC Prize"},"vals:swebench":{"value":76.4,"norm":0.764,"version":"Vals AI"},"x:GSO-Bench":{"value":0.1863,"norm":0.1863,"version":"gemini-3-pro-preview"},"vals:vibe-code":{"value":14.3,"norm":0.143,"version":"Vals AI"},"x:VPCT":{"value":0.91,"norm":0.91,"version":"gemini-3-pro-preview"},"vals:mmmu":{"value":87.51,"norm":0.8751,"version":"Vals AI"},"vals:medscribe":{"value":72.04,"norm":0.7204,"version":"Vals AI"},"x:FrontierMath-Tier-4-2025-07-01-Private":{"value":0.1875,"norm":0.1875,"version":"gemini-3-pro-preview"},"vals:gpqa":{"value":91.67,"norm":0.9167,"version":"Vals AI"},"x:DeepResearch Bench":{"value":0.46299999999999997,"norm":0.463,"version":"gemini-3-pro-preview_low"},"vals:tax_eval_v2":{"value":72.57,"norm":0.7257,"version":"Vals AI"},"tb":{"value":0.5685393258429999,"norm":0.5685,"version":"gemini-3-pro-preview"},"gpqa":{"value":0.9261363636363636,"norm":0.9261,"version":"gemini-3-pro-preview"},"x:SimpleBench":{"value":0.7640000000000001,"norm":0.764,"version":"gemini-3-pro-preview"}},"variant":"Gemini 3 Pro","match":"exact"}},{"id":"gpt-5-1-codex-max","name":"GPT-5.1-Codex-Max","lab":"OpenAI","date":"2025-11-19","type":"reasoning","major":true,"open":false,"release":{"title":"GPT-5.1-Codex-Max","source":"https://openai.com/index/gpt-5-1-codex-max/","verified":true,"confidence":"high"},"index":{"score":144.2,"lo":127,"hi":161.4,"se":4.55,"solve":33.6,"status":"provisional","areas":["coding","long"],"core_areas":["long"],"testers":["METR","Vals AI"],"tests":{"vals:lcb":{"value":83.56,"norm":0.8356,"version":"Vals AI"},"vals:vibe-code":{"value":22.17,"norm":0.2217,"version":"Vals AI"},"metr":{"value":0.708301,"norm":0.2459,"version":"gpt-5.1-codex-max"}},"variant":"GPT-5.1-Codex-Max","match":"exact"},"api":{"or_id":"openai/gpt-5.1-codex-max","in":1.25,"out":10,"context":400000,"hf":null,"created":"2025-12-04"}},{"id":"grok-4-1-fast","name":"Grok 4.1 Fast","lab":"xAI","date":"2025-11-19","type":"small","major":true,"open":false,"release":{"title":"Grok 4.1 Fast (+ Agent Tools API)","source":"https://x.ai/news/grok-4-1-fast","verified":true,"confidence":"high"},"index":{"score":144.4,"lo":126.2,"hi":162.7,"se":5.86,"solve":33.8,"status":"rated","areas":["agents","coding","maths","reasoning","work"],"core_areas":[],"testers":["Redwood Research","SimpleBench team","Vals AI"],"tests":{"vals:vibe-code":{"value":1.2,"norm":0.012,"version":"Vals AI"},"vals:mmlu_pro":{"value":84.18,"norm":0.8418,"version":"Vals AI"},"vals:corp_fin_v2":{"value":65.97,"norm":0.6597,"version":"Vals AI"},"vals:mortgage_tax":{"value":42.61,"norm":0.4261,"version":"Vals AI"},"vals:swebench":{"value":41.4,"norm":0.414,"version":"Vals AI"},"vals:legal_bench":{"value":82.45,"norm":0.8245,"version":"Vals AI"},"vals:lcb":{"value":80.64,"norm":0.8064,"version":"Vals AI"},"x:SimpleBench":{"value":0.56,"norm":0.56,"version":"grok-4-1-fast-reasoning"},"vals:tax_eval_v2":{"value":73.14,"norm":0.7314,"version":"Vals AI"},"x:DTBench":{"value":0.8773000000000001,"norm":0.8773,"version":"grok-4-1-fast-reasoning"},"vals:medcode":{"value":28.08,"norm":0.2808,"version":"Vals AI"},"vals:sage":{"value":74.47,"norm":0.7447,"version":"Vals AI"},"vals:public-benefits-bench":{"value":44.79,"norm":0.4479,"version":"Vals AI"},"vals:gpqa":{"value":84.34,"norm":0.8434,"version":"Vals AI"},"vals:mmmu":{"value":72.66,"norm":0.7266,"version":"Vals AI"},"vals:mgsm":{"value":89.53,"norm":0.8953,"version":"Vals AI"},"vals:medscribe":{"value":78.73,"norm":0.7873,"version":"Vals AI"}},"variant":"Grok 4.1 Fast","match":"exact"}},{"id":"claude-opus-4-5","name":"Claude Opus 4.5","lab":"Anthropic","date":"2025-11-24","type":"flagship","major":true,"open":false,"release":{"title":"Claude Opus 4.5","source":"https://www.anthropic.com/news/claude-opus-4-5","verified":true,"confidence":"high"},"index":{"score":151.8,"lo":135.8,"hi":167.7,"se":2.2,"solve":40.6,"status":"rated","areas":["agents","coding","long","maths","reasoning","work"],"core_areas":["coding","long","maths","reasoning"],"testers":["BALROG team","Chase Brower","Epoch AI","FutureSearch","GSO team","GeoBench team","Havard T. Ihle","LiveBench","METR","Redwood Research","Scale AI","SimpleBench team","Tencent Hunyuan / Fudan","Terminal-Bench team","Vals AI"],"tests":{"x:DeepResearch Bench":{"value":0.5479999999999999,"norm":0.548,"version":"claude-opus-4-5-20251101_high"},"x:LMCA":{"value":44.46,"norm":0.4446,"version":"claude-opus-4-5-20251101_high"},"aime":{"value":0.8611111111111112,"norm":0.8611,"version":"claude-opus-4-5-20251101_32K"},"livebench:data-analysis":{"value":74.44,"norm":0.7444,"version":"LiveBench"},"x:Furniture Assembly":{"value":0.2833333333333333,"norm":0.2833,"version":"claude-opus-4-5-20251101_64K"},"vals:corp_fin_v2":{"value":65.07,"norm":0.6507,"version":"Vals AI"},"x:GeoBench":{"value":0.75,"norm":0.75,"version":"claude-opus-4-5-20251101_32K"},"x:GSO-Bench":{"value":0.265,"norm":0.265,"version":"claude-opus-4-5-20251101"},"vals:tax_eval_v2":{"value":74.86,"norm":0.7486,"version":"Vals AI"},"vals:vibe-code":{"value":20.63,"norm":0.2063,"version":"Vals AI"},"x:SimpleQA Verified":{"value":0.457,"norm":0.457,"version":"claude-opus-4-5-20251101_32K"},"livebench:if":{"value":62.55,"norm":0.6255,"version":"LiveBench"},"vals:mmmu":{"value":82.95,"norm":0.8295,"version":"Vals AI"},"fm":{"value":0.34385964912280703,"norm":0.3439,"version":"claude-opus-4-5-20251101_32K"},"x:Mystery Game Puzzles":{"value":0.22,"norm":0.22,"version":"claude-opus-4-5-20251101_48K"},"vals:legal_bench":{"value":84.6,"norm":0.846,"version":"Vals AI"},"vals:lcb":{"value":83.67,"norm":0.8367,"version":"Vals AI"},"x:Chess Puzzles":{"value":0.14,"norm":0.14,"version":"claude-opus-4-5-20251101_59K"},"scale:swe_bench_pro":{"value":45.89,"norm":0.4589,"version":"Scale AI"},"x:EBR-bench":{"value":0.14285714285714285,"norm":0.1429,"version":"claude-opus-4-5-20251101_128K"},"x:VPCT":{"value":0.4,"norm":0.4,"version":"claude-opus-4-5-20251101_32K"},"x:DTBench":{"value":0.8987,"norm":0.8987,"version":"claude-opus-4-5-20251101_high"},"vals:mgsm":{"value":95.2,"norm":0.952,"version":"Vals AI"},"x:CL-bench":{"value":0.211,"norm":0.211,"version":"claude-opus-4-5-20251101_unknown"},"vals:medscribe":{"value":85.32,"norm":0.8532,"version":"Vals AI"},"livebench:mathematics":{"value":90.39,"norm":0.9039,"version":"LiveBench"},"tb":{"value":0.577528089888,"norm":0.5775,"version":"claude-opus-4-5-20251101_unknown"},"livebench:agentic-coding":{"value":39.7,"norm":0.397,"version":"LiveBench"},"livebench:language":{"value":81.26,"norm":0.8126,"version":"LiveBench"},"metr":{"value":0.74969,"norm":0.2512,"version":"claude-opus-4-5-20251101_16K"},"x:WeirdML":{"value":0.6374000000000001,"norm":0.6374,"version":"claude-opus-4-5-20251101_16K"},"vals:medcode":{"value":49.16,"norm":0.4916,"version":"Vals AI"},"vals:mortgage_tax":{"value":68.68,"norm":0.6868,"version":"Vals AI"},"x:FrontierMath-Tier-4-2025-07-01-Private":{"value":0.041666666666666664,"norm":0.0417,"version":"claude-opus-4-5-20251101_32K"},"livebench:coding":{"value":79.65,"norm":0.7965,"version":"LiveBench"},"gpqa":{"value":0.860479797979798,"norm":0.8605,"version":"claude-opus-4-5-20251101_32K"},"vals:gpqa":{"value":85.86,"norm":0.8586,"version":"Vals AI"},"hle":{"value":0.252,"norm":0.252,"version":"claude-opus-4-5-20251101_unknown"},"swe":{"value":0.7665289256198347,"norm":0.7665,"version":"claude-opus-4-5-20251101"},"x:FrontierMath-2025-02-28-Private":{"value":0.20689655172413793,"norm":0.2069,"version":"claude-opus-4-5-20251101_32K"},"x:FrontierMath-Tier-4-v2-Private":{"value":0.04878048780487805,"norm":0.0488,"version":"claude-opus-4-5-20251101_32K"},"x:Balrog":{"value":0.435,"norm":0.435,"version":"claude-opus-4-5-20251101"},"x:SimpleBench":{"value":0.62,"norm":0.62,"version":"claude-opus-4-5-20251101_unknown"},"vals:mmlu_pro":{"value":87.26,"norm":0.8726,"version":"Vals AI"},"vals:swebench":{"value":76.4,"norm":0.764,"version":"Vals AI"},"livebench:reasoning":{"value":80.09,"norm":0.8009,"version":"LiveBench"}},"variant":"Claude Opus 4.5","match":"exact"},"api":{"or_id":"anthropic/claude-opus-4.5","in":5,"out":25,"context":200000,"hf":null,"created":"2025-11-24"}},{"id":"deepseekmath-v2","name":"DeepSeekMath-V2","lab":"DeepSeek","date":"2025-11-27","type":"open-weights","major":true,"open":true,"release":{"title":"DeepSeekMath-V2","source":null,"verified":false,"confidence":"high"}},{"id":"deepseek-v3-2","name":"DeepSeek-V3.2","lab":"DeepSeek","date":"2025-12-01","type":"open-weights","major":true,"open":true,"release":{"title":"DeepSeek-V3.2 (+ Speciale)","source":"https://api-docs.deepseek.com/news/news251201/","verified":false,"confidence":"high"},"index":{"score":141.2,"lo":125,"hi":157.4,"se":2.92,"solve":31.2,"status":"rated","areas":["agents","coding","maths","reasoning"],"core_areas":["agents","reasoning"],"testers":["Aider","Epoch AI","Mercor","Redwood Research","Tencent Hunyuan / Fudan","Terminal-Bench team"],"tests":{"aider":{"value":74.2,"norm":0.742,"version":"deepseek-reasoner"},"x:LMCA":{"value":28.82,"norm":0.2882,"version":"deepseek/deepseek-v3.2"},"x:DTBench":{"value":0.856,"norm":0.856,"version":"deepseek/deepseek-v3.2"},"x:FrontierMath-2025-02-28-Private":{"value":0.221,"norm":0.221,"version":"fireworks/deepseek-v3p2"},"aime":{"value":0.878174603174603,"norm":0.8782,"version":"deepseek-reasoner"},"apex":{"value":0.21300000000000002,"norm":0.213,"version":"deepseek/deepseek-v3.2"},"x:FrontierMath-Tier-4-2025-07-01-Private":{"value":0.021,"norm":0.021,"version":"fireworks/deepseek-v3p2"},"tb":{"value":0.395505617978,"norm":0.3955,"version":"deepseek/deepseek-v3.2"},"gpqa":{"value":0.8342352092352093,"norm":0.8342,"version":"deepseek-reasoner"},"x:CL-bench":{"value":0.12400000000000001,"norm":0.124,"version":"deepseek/deepseek-v3.2"}},"variant":"DeepSeek-V3.2","match":"exact"},"api":{"or_id":"deepseek/deepseek-v3.2","in":0.26,"out":0.38,"context":163840,"hf":"deepseek-ai/DeepSeek-V3.2","created":"2025-12-01"}},{"id":"deepseek-v3-2-speciale","name":"DeepSeek-V3.2-Speciale","lab":"DeepSeek","date":"2025-12-01","type":"open-weights","major":false,"open":true,"index":{"score":143.4,"lo":125.1,"hi":161.8,"se":5.95,"solve":33,"status":"provisional","areas":["coding","reasoning"],"core_areas":[],"testers":["Havard T. Ihle","SimpleBench team"],"tests":{"x:SimpleBench":{"value":0.526,"norm":0.526,"version":"DeepSeek-V3.2-Speciale"},"x:WeirdML":{"value":0.4673,"norm":0.4673,"version":"DeepSeek-V3.2-Speciale"}},"variant":"DeepSeek-V3.2-Speciale","match":"self"},"date_from":"epoch"},{"id":"mistral-large-3","name":"Mistral Large 3","lab":"Mistral","date":"2025-12-02","type":"open-weights","major":true,"open":true,"release":{"title":"Mistral Large 3 (+ Ministral 3)","source":"https://mistral.ai/news/mistral-3/","verified":true,"confidence":"high"},"index":{"score":116.9,"lo":98.5,"hi":135.3,"se":6.03,"solve":17.9,"status":"rated","areas":["agents","coding","maths","reasoning","work"],"core_areas":[],"testers":["Redwood Research","SimpleBench team","Vals AI"],"tests":{"vals:mmlu_pro":{"value":79.82,"norm":0.7982,"version":"Vals AI"},"vals:tax_eval_v2":{"value":73.06,"norm":0.7306,"version":"Vals AI"},"vals:mgsm":{"value":85.42,"norm":0.8542,"version":"Vals AI"},"vals:mortgage_tax":{"value":52.11,"norm":0.5211,"version":"Vals AI"},"vals:gpqa":{"value":68.43,"norm":0.6843,"version":"Vals AI"},"x:SimpleBench":{"value":0.204,"norm":0.204,"version":"mistral-large-2512"},"vals:corp_fin_v2":{"value":61.03,"norm":0.6103,"version":"Vals AI"},"vals:legal_bench":{"value":79.14,"norm":0.7914,"version":"Vals AI"},"vals:lcb":{"value":55.34,"norm":0.5534,"version":"Vals AI"},"x:DTBench":{"value":0.6507,"norm":0.6507,"version":"mistral-large-2512"},"x:LMCA":{"value":16.7,"norm":0.167,"version":"mistral-large-2512"},"vals:swebench":{"value":41.4,"norm":0.414,"version":"Vals AI"},"vals:mmmu":{"value":66.19,"norm":0.6619,"version":"Vals AI"}},"variant":"Mistral Large 3","match":"exact"},"api":{"or_id":"mistralai/mistral-large-2512","in":0.5,"out":1.5,"context":262144,"hf":"mistralai/Mistral-Large-3-675B-Instruct-2512","created":"2025-12-01"}},{"id":"nova-2","name":"Nova 2","lab":"Amazon","date":"2025-12-02","type":"flagship","major":true,"open":false,"release":{"title":"Nova 2","source":"https://www.aboutamazon.com/news/aws/aws-re-invent-2025-ai-news-updates","verified":true,"confidence":"high"}},{"id":"devstral-2","name":"Devstral 2","lab":"Mistral","date":"2025-12-09","type":"open-weights","major":true,"open":true,"release":{"title":"Devstral 2 (+ Devstral Small 2)","source":"https://mistral.ai/news/devstral-2-vibe-cli/","verified":true,"confidence":"high"},"index":{"score":118.2,"lo":99.7,"hi":136.7,"se":6.17,"solve":18.4,"status":"insufficient","areas":["coding"],"core_areas":[],"testers":["Vals AI"],"tests":{"vals:lcb":{"value":51.84,"norm":0.5184,"version":"Vals AI"},"vals:swebench":{"value":62.8,"norm":0.628,"version":"Vals AI"}},"variant":"Devstral 2","match":"exact"},"api":{"or_id":"mistralai/devstral-2512","in":0.4,"out":2,"context":262144,"hf":"mistralai/Devstral-2-123B-Instruct-2512","created":"2025-12-09"}},{"id":"gpt-5-2","name":"GPT-5.2","lab":"OpenAI","date":"2025-12-11","type":"flagship","major":true,"open":false,"release":{"title":"GPT-5.2","source":"https://openai.com/zh-Hans-CN/index/introducing-gpt-5-2/","verified":false,"confidence":"high"},"index":{"score":161.8,"lo":146.1,"hi":177.6,"se":1.61,"solve":51.4,"status":"rated","areas":["agents","coding","long","maths","novel","reasoning","work"],"core_areas":["coding","long","maths","novel","reasoning"],"testers":["ARC Prize","Chase Brower","Epoch AI","FutureSearch","GSO team","Havard T. Ihle","METR","Redwood Research","Scale AI","SimpleBench team","Tencent Hunyuan / Fudan","Terminal-Bench team","Vals AI"],"tests":{"vals:swebench":{"value":75.8,"norm":0.758,"version":"Vals AI"},"x:GSO-Bench":{"value":0.2745,"norm":0.2745,"version":"gpt-5.2-2025-12-11_high"},"x:CL-bench":{"value":0.182,"norm":0.182,"version":"gpt-5.2-2025-12-11_unknown"},"arc:arc_agi_1":{"value":94.5,"norm":0.945,"version":"ARC Prize"},"hle":{"value":0.278,"norm":0.278,"version":"gpt-5.2-2025-12-11_unknown"},"swe":{"value":0.737603305785124,"norm":0.7376,"version":"gpt-5.2-2025-12-11_high"},"vals:medscribe":{"value":84.39,"norm":0.8439,"version":"Vals AI"},"tb":{"value":0.540449438202,"norm":0.5404,"version":"gpt-5.2-2025-12-11_unknown"},"gpqa":{"value":0.914,"norm":0.914,"version":"gpt-5.2-2025-12-11_xhigh"},"x:FrontierMath-2025-02-28-Private":{"value":0.407,"norm":0.407,"version":"gpt-5.2-2025-12-11_xhigh"},"x:Mystery Game Puzzles":{"value":0.23,"norm":0.23,"version":"gpt-5.2-2025-12-11_high"},"scale:swe_bench_pro":{"value":29.94,"norm":0.2994,"version":"Scale AI"},"vals:legal_bench":{"value":82.76,"norm":0.8276,"version":"Vals AI"},"x:EBR-bench":{"value":0.230159,"norm":0.2302,"version":"gpt-5.2-2025-12-11_xhigh"},"x:VPCT":{"value":0.84,"norm":0.84,"version":"gpt-5.2-2025-12-11_xhigh"},"vals:mmmu":{"value":86.67,"norm":0.8667,"version":"Vals AI"},"x:LMCA":{"value":43.91,"norm":0.4391,"version":"gpt-5.2-2025-12-11_xhigh"},"vals:medcode":{"value":49.75,"norm":0.4975,"version":"Vals AI"},"vals:mortgage_tax":{"value":67.13,"norm":0.6713,"version":"Vals AI"},"x:FrontierMath-Tier-4-2025-07-01-Private":{"value":0.188,"norm":0.188,"version":"gpt-5.2-2025-12-11_high"},"x:Furniture Assembly":{"value":0.38333333333333336,"norm":0.3833,"version":"gpt-5.2-2025-12-11_xhigh"},"fm":{"value":0.674,"norm":0.674,"version":"gpt-5.2-2025-12-11_xhigh"},"vals:gpqa":{"value":91.67,"norm":0.9167,"version":"Vals AI"},"vals:vibe-code":{"value":53.5,"norm":0.535,"version":"Vals AI"},"metr":{"value":0.752875,"norm":0.2516,"version":"gpt-5.2-2025-12-11_high"},"x:DeepResearch Bench":{"value":0.41100000000000003,"norm":0.411,"version":"gpt-5.2-2025-12-11_low"},"vals:tax_eval_v2":{"value":75.76,"norm":0.7576,"version":"Vals AI"},"vals:mgsm":{"value":94,"norm":0.94,"version":"Vals AI"},"vals:corp_fin_v2":{"value":65.89,"norm":0.6589,"version":"Vals AI"},"x:FrontierMath-Tier-4-v2-Private":{"value":0.317,"norm":0.317,"version":"gpt-5.2-2025-12-11_xhigh"},"x:Chess Puzzles":{"value":0.48,"norm":0.48,"version":"gpt-5.2-2025-12-11_xhigh"},"aime":{"value":0.9611111111111111,"norm":0.9611,"version":"gpt-5.2-2025-12-11_high"},"x:SimpleBench":{"value":0.458,"norm":0.458,"version":"gpt-5.2-2025-12-11_high"},"arc:arc_agi_2":{"value":72.9,"norm":0.729,"version":"ARC Prize"},"vals:lcb":{"value":85.36,"norm":0.8536,"version":"Vals AI"},"x:SimpleQA Verified":{"value":0.371,"norm":0.371,"version":"gpt-5.2-2025-12-11_xhigh"},"x:WeirdML":{"value":0.7219,"norm":0.7219,"version":"gpt-5.2-2025-12-11_xhigh"},"x:MirrorCode":{"value":0.32222222222222224,"norm":0.3222,"version":"gpt-5.2-2025-12-11_xhigh"},"vals:mmlu_pro":{"value":86.23,"norm":0.8623,"version":"Vals AI"},"x:DTBench":{"value":0.9093000000000001,"norm":0.9093,"version":"gpt-5.2-2025-12-11_xhigh"}},"variant":"GPT-5.2","match":"exact"},"api":{"or_id":"openai/gpt-5.2","in":1.75,"out":14,"context":400000,"hf":null,"created":"2025-12-10"},"modes":[{"label":"Pro","name":"GPT-5.2 Pro","index":{"score":163.3,"lo":147.4,"hi":179.1,"se":2.03,"solve":52.9,"status":"rated","areas":["maths","novel","reasoning"],"core_areas":["maths","novel"],"testers":["ARC Prize","Epoch AI","SimpleBench team"],"tests":{"fm":{"value":0.74,"norm":0.74,"version":"gpt-5.2-pro-2025-12-11_xhigh"},"arc:arc_agi_1":{"value":90.5,"norm":0.905,"version":"ARC Prize"},"x:SimpleBench":{"value":0.574,"norm":0.574,"version":"gpt-5.2-pro-2025-12-11"},"arc:arc_agi_2":{"value":54.2,"norm":0.542,"version":"ARC Prize"},"x:FrontierMath-Tier-4-v2-Private":{"value":0.46,"norm":0.46,"version":"gpt-5.2-pro-2025-12-11_xhigh"},"x:Chess Puzzles":{"value":0.6,"norm":0.6,"version":"gpt-5.2-pro-2025-12-11_xhigh"}}},"api":{"or_id":"openai/gpt-5.2-pro","in":21,"out":168,"context":400000,"hf":null,"created":"2025-12-10"}}],"aliases":["gpt-5-2-pro"]},{"id":"nemotron-3-nano","name":"Nemotron 3 Nano","lab":"NVIDIA","date":"2025-12-15","type":"open-weights","major":true,"open":true,"release":{"title":"Nemotron 3 Nano","source":"https://nvidianews.nvidia.com/news/nvidia-debuts-nemotron-3-family-of-open-models","verified":true,"confidence":"high"},"api":{"or_id":"nvidia/nemotron-3-nano-30b-a3b","in":0.06,"out":0.24,"context":262144,"hf":"nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16","created":"2025-12-14"}},{"id":"mimo-v2-flash","name":"MiMo-V2-Flash","lab":"Xiaomi","date":"2025-12-16","type":"open-weights","major":true,"open":true,"release":{"title":"MiMo-V2-Flash","source":"https://mimo.xiaomi.com/mimo-v2-flash","verified":true,"confidence":"high"},"index":{"score":117.4,"lo":96.4,"hi":138.5,"se":8.63,"solve":18.1,"status":"insufficient","areas":["reasoning"],"core_areas":[],"testers":["Vals AI"],"tests":{"vals:gpqa":{"value":59.34,"norm":0.5934,"version":"Vals AI"}},"variant":"MiMo-V2-Flash","match":"exact"}},{"id":"gemini-3-flash","name":"Gemini 3 Flash","lab":"Google","date":"2025-12-17","type":"small","major":true,"open":false,"release":{"title":"Gemini 3 Flash","source":"https://blog.google/products-and-platforms/products/gemini/gemini-3-flash/","verified":true,"confidence":"high"},"index":{"score":154.4,"lo":138.7,"hi":170.1,"se":1.61,"solve":43.3,"status":"rated","areas":["agents","coding","maths","novel","reasoning","work"],"core_areas":["coding","maths","novel","reasoning","work"],"testers":["ARC Prize","BALROG team","Chase Brower","Epoch AI","FutureSearch","GSO team","GeoBench team","Havard T. Ihle","Redwood Research","Scale AI","SimpleBench team","Terminal-Bench team","Vals AI"],"tests":{"tb":{"value":0.516853932584,"norm":0.5169,"version":"gemini-3-flash-preview"},"vals:corp_fin_v2":{"value":66.43,"norm":0.6643,"version":"Vals AI"},"vals:swebench":{"value":75,"norm":0.75,"version":"Vals AI"},"x:FrontierMath-2025-02-28-Private":{"value":0.356401384083045,"norm":0.3564,"version":"gemini-3-flash-preview"},"x:SimpleQA Verified":{"value":0.668,"norm":0.668,"version":"gemini-3-flash-preview_high"},"vals:mmmu":{"value":87.63,"norm":0.8763,"version":"Vals AI"},"arc:arc_agi_2":{"value":33.6,"norm":0.336,"version":"ARC Prize"},"arc:arc_agi_1":{"value":84.7,"norm":0.847,"version":"ARC Prize"},"x:FrontierMath-Tier-4-v2-Private":{"value":0.17073170731707318,"norm":0.1707,"version":"gemini-3-flash-preview"},"x:FrontierMath-Tier-4-2025-07-01-Private":{"value":0.041666666666666664,"norm":0.0417,"version":"gemini-3-flash-preview"},"vals:tax_agent_bench":{"value":11.84,"norm":0.1184,"version":"Vals AI"},"gpqa":{"value":0.8939393939393939,"norm":0.8939,"version":"gemini-3-flash-preview_high"},"vals:legal_research":{"value":18.27,"norm":0.1827,"version":"Vals AI"},"vals:lcb":{"value":85.59,"norm":0.8559,"version":"Vals AI"},"x:Chess Puzzles":{"value":0.4,"norm":0.4,"version":"gemini-3-flash-preview_high"},"x:DTBench":{"value":0.8906999999999999,"norm":0.8907,"version":"gemini-3-flash-preview_high"},"x:DeepResearch Bench":{"value":0.498,"norm":0.498,"version":"gemini-3-flash-preview_low"},"vals:medscribe":{"value":69.92,"norm":0.6992,"version":"Vals AI"},"vals:vibe-code":{"value":20.2,"norm":0.202,"version":"Vals AI"},"x:VPCT":{"value":0.726,"norm":0.726,"version":"gemini-3-flash-preview"},"vals:programbench":{"value":0,"norm":0,"version":"Vals AI"},"vals:mortgage_tax":{"value":68.72,"norm":0.6872,"version":"Vals AI"},"x:WeirdML":{"value":0.616,"norm":0.616,"version":"gemini-3-flash-preview"},"vals:mgsm":{"value":93.31,"norm":0.9331,"version":"Vals AI"},"vals:medcode":{"value":55.92,"norm":0.5592,"version":"Vals AI"},"swe":{"value":0.7541322314049587,"norm":0.7541,"version":"gemini-3-flash-preview"},"vals:emb":{"value":26.02,"norm":0.2602,"version":"Vals AI"},"vals:fabv2":{"value":42.55,"norm":0.4255,"version":"Vals AI"},"x:Mystery Game Puzzles":{"value":0.26,"norm":0.26,"version":"gemini-3-flash-preview_low"},"vals:legal_bench":{"value":86.86,"norm":0.8686,"version":"Vals AI"},"vals:gpqa":{"value":87.88,"norm":0.8788,"version":"Vals AI"},"scale:swe_bench_pro":{"value":34.63,"norm":0.3463,"version":"Scale AI"},"x:GSO-Bench":{"value":0.098,"norm":0.098,"version":"gemini-3-flash-preview"},"vals:tax_eval_v2":{"value":73.88,"norm":0.7388,"version":"Vals AI"},"x:LMCA":{"value":43.13,"norm":0.4313,"version":"gemini-3-flash-preview_high"},"x:GeoBench":{"value":0.88,"norm":0.88,"version":"gemini-3-flash-preview"},"x:Balrog":{"value":0.48100000000000004,"norm":0.481,"version":"gemini-3-flash-preview"},"x:SimpleBench":{"value":0.611,"norm":0.611,"version":"gemini-3-flash-preview"},"vals:mmlu_pro":{"value":88.59,"norm":0.8859,"version":"Vals AI"},"vals:code-migration":{"value":6.39,"norm":0.0639,"version":"Vals AI"},"aime":{"value":0.9555555555555556,"norm":0.9556,"version":"gemini-3-flash-preview_high"},"fm":{"value":0.512280701754386,"norm":0.5123,"version":"gemini-3-flash-preview"}},"variant":"Gemini 3 Flash","match":"exact"},"api":{"or_id":"google/gemini-3-flash-preview","in":0.5,"out":3,"context":1048576,"hf":null,"created":"2025-12-17"}},{"id":"gpt-5-2-codex","name":"GPT-5.2-Codex","lab":"OpenAI","date":"2025-12-18","type":"reasoning","major":true,"open":false,"release":{"title":"GPT-5.2-Codex","source":"https://openai.com/index/introducing-gpt-5-2-codex/","verified":true,"confidence":"med"},"index":{"score":152.2,"lo":135.4,"hi":169.1,"se":3.96,"solve":41.1,"status":"rated","areas":["coding","maths","reasoning"],"core_areas":[],"testers":["LiveBench","Scale AI","Vals AI"],"tests":{"livebench:reasoning":{"value":77.71,"norm":0.7771,"version":"LiveBench"},"livebench:mathematics":{"value":88.77,"norm":0.8877,"version":"LiveBench"},"livebench:agentic-coding":{"value":49.39,"norm":0.4939,"version":"LiveBench"},"livebench:language":{"value":73.68,"norm":0.7368,"version":"LiveBench"},"vals:swebench":{"value":72.4,"norm":0.724,"version":"Vals AI"},"vals:vibe-code":{"value":37.91,"norm":0.3791,"version":"Vals AI"},"livebench:data-analysis":{"value":78.2,"norm":0.782,"version":"LiveBench"},"scale:swe_bench_pro":{"value":41.04,"norm":0.4104,"version":"Scale AI"},"vals:lcb":{"value":87.99,"norm":0.8799,"version":"Vals AI"},"livebench:if":{"value":66.45,"norm":0.6645,"version":"LiveBench"},"livebench:coding":{"value":83.62,"norm":0.8362,"version":"LiveBench"}},"variant":"GPT-5.2-Codex","match":"exact"},"api":{"or_id":"openai/gpt-5.2-codex","in":1.75,"out":14,"context":400000,"hf":null,"created":"2026-01-14"}},{"id":"glm-4-7","name":"GLM-4.7","lab":"Zhipu","date":"2025-12-22","type":"open-weights","major":true,"open":true,"release":{"title":"GLM-4.7","source":"https://huggingface.co/zai-org/GLM-4.7","verified":true,"confidence":"high"},"index":{"score":139.8,"lo":123,"hi":156.6,"se":3.95,"solve":30.2,"status":"rated","areas":["coding","maths","reasoning","work"],"core_areas":["reasoning"],"testers":["Epoch AI","SimpleBench team","Tencent Hunyuan / Fudan","Terminal-Bench team","Vals AI"],"tests":{"vals:tax_eval_v2":{"value":68.77,"norm":0.6877,"version":"Vals AI"},"vals:mgsm":{"value":88.18,"norm":0.8818,"version":"Vals AI"},"x:CL-bench":{"value":0.159,"norm":0.159,"version":"zai-org/glm-4.7"},"vals:medcode":{"value":32.77,"norm":0.3277,"version":"Vals AI"},"vals:legal_bench":{"value":83.36,"norm":0.8336,"version":"Vals AI"},"x:SimpleBench":{"value":0.47700000000000004,"norm":0.477,"version":"zai-org/glm-4.7"},"vals:mmlu_pro":{"value":82.74,"norm":0.8274,"version":"Vals AI"},"vals:medscribe":{"value":68.63,"norm":0.6863,"version":"Vals AI"},"vals:corp_fin_v2":{"value":46.39,"norm":0.4639,"version":"Vals AI"},"gpqa":{"value":0.8333333333333334,"norm":0.8333,"version":"glm-4.7"},"vals:lcb":{"value":82.23,"norm":0.8223,"version":"Vals AI"},"x:FrontierMath-Tier-4-2025-07-01-Private":{"value":0,"norm":0,"version":"zai-org/GLM-4.7"},"x:FrontierMath-2025-02-28-Private":{"value":0.024390243902439025,"norm":0.0244,"version":"zai-org/GLM-4.7"},"vals:gpqa":{"value":80.05,"norm":0.8005,"version":"Vals AI"},"aime":{"value":0.8333333333333334,"norm":0.8333,"version":"glm-4.7"},"tb":{"value":0.333895131086,"norm":0.3339,"version":"glm-4.7"},"x:SimpleQA Verified":{"value":0.322,"norm":0.322,"version":"glm-4.7"},"vals:swebench":{"value":69.4,"norm":0.694,"version":"Vals AI"},"x:Chess Puzzles":{"value":0.21,"norm":0.21,"version":"glm-4.7"}},"variant":"GLM-4.7","match":"exact"},"api":{"or_id":"z-ai/glm-4.7","in":0.4,"out":1.75,"context":204800,"hf":"zai-org/GLM-4.7","created":"2025-12-22"}},{"id":"minimax-m2-1","name":"MiniMax-M2.1","lab":"MiniMax","date":"2025-12-23","type":"open-weights","major":true,"open":true,"release":{"title":"MiniMax-M2.1","source":"https://huggingface.co/MiniMaxAI/MiniMax-M2.1","verified":false,"confidence":"high"},"index":{"score":135.3,"lo":117.4,"hi":153.2,"se":5.48,"solve":27.2,"status":"rated","areas":["coding","maths","reasoning","work"],"core_areas":[],"testers":["SimpleBench team","Terminal-Bench team","Vals AI"],"tests":{"vals:mgsm":{"value":87.85,"norm":0.8785,"version":"Vals AI"},"vals:gpqa":{"value":78.54,"norm":0.7854,"version":"Vals AI"},"vals:legal_bench":{"value":75.45,"norm":0.7545,"version":"Vals AI"},"vals:tax_eval_v2":{"value":66.35,"norm":0.6635,"version":"Vals AI"},"x:SimpleBench":{"value":0.34700000000000003,"norm":0.347,"version":"MiniMax-M2.1"},"vals:medscribe":{"value":80.78,"norm":0.8078,"version":"Vals AI"},"tb":{"value":0.292134831461,"norm":0.2921,"version":"MiniMax-M2.1"},"vals:math500":{"value":89,"norm":0.89,"version":"Vals AI"},"vals:corp_fin_v2":{"value":58.9,"norm":0.589,"version":"Vals AI"},"vals:mmlu_pro":{"value":87.05,"norm":0.8705,"version":"Vals AI"},"vals:swebench":{"value":74.8,"norm":0.748,"version":"Vals AI"},"vals:lcb":{"value":81.76,"norm":0.8176,"version":"Vals AI"},"vals:medcode":{"value":34.08,"norm":0.3408,"version":"Vals AI"}},"variant":"MiniMax-M2.1","match":"exact"}},{"id":"qwen3-max-thinking","name":"Qwen3-Max-Thinking","lab":"Alibaba","date":"2026-01-26","type":"reasoning","major":true,"open":false,"release":{"title":"Qwen3-Max-Thinking","source":"https://qwen.ai/blog?id=qwen3-max-thinking&ref=testingcatalog.com","verified":false,"confidence":"high"}},{"id":"kimi-k2-5","name":"Kimi K2.5","lab":"Moonshot","date":"2026-01-27","type":"open-weights","major":true,"open":true,"release":{"title":"Kimi K2.5","source":"https://www.kimi.com/blog/kimi-k2-5?ref=upstract.com","verified":false,"confidence":"high"},"index":{"score":145.3,"lo":129.5,"hi":161.2,"se":2.01,"solve":34.6,"status":"rated","areas":["coding","maths","novel","reasoning","work"],"core_areas":["coding","novel","reasoning","work"],"testers":["ARC Prize","Epoch AI","Fiction.live","Havard T. Ihle","Scale AI","SimpleBench team","Tencent Hunyuan / Fudan","Terminal-Bench team","Vals AI"],"tests":{"vals:legal_research":{"value":15.87,"norm":0.1587,"version":"Vals AI"},"vals:fabv2":{"value":35.79,"norm":0.3579,"version":"Vals AI"},"x:WeirdML":{"value":0.456,"norm":0.456,"version":"kimi-k2.5"},"vals:tax_eval_v2":{"value":74.2,"norm":0.742,"version":"Vals AI"},"x:FrontierMath-Tier-4-2025-07-01-Private":{"value":0.042,"norm":0.042,"version":"fireworks/kimi-k2p5"},"vals:emb":{"value":28.47,"norm":0.2847,"version":"Vals AI"},"tb":{"value":0.43202247191,"norm":0.432,"version":"kimi-k2.5"},"x:FrontierMath-2025-02-28-Private":{"value":0.279,"norm":0.279,"version":"fireworks/kimi-k2p5"},"vals:gpqa":{"value":84.09,"norm":0.8409,"version":"Vals AI"},"aime":{"value":0.922,"norm":0.922,"version":"fireworks/kimi-k2p5"},"x:SimpleBench":{"value":0.46799999999999997,"norm":0.468,"version":"kimi-k2.5"},"vals:swebench":{"value":70,"norm":0.7,"version":"Vals AI"},"vals:vibe-code":{"value":17.54,"norm":0.1754,"version":"Vals AI"},"x:SimpleQA Verified":{"value":0.343,"norm":0.343,"version":"kimi-k2.5"},"vals:medscribe":{"value":76.44,"norm":0.7644,"version":"Vals AI"},"vals:corp_fin_v2":{"value":68.26,"norm":0.6826,"version":"Vals AI"},"vals:code-migration":{"value":6.96,"norm":0.0696,"version":"Vals AI"},"vals:lcb":{"value":83.87,"norm":0.8387,"version":"Vals AI"},"vals:mmmu":{"value":84.33,"norm":0.8433,"version":"Vals AI"},"swe":{"value":0.737603305785124,"norm":0.7376,"version":"kimi-k2.5"},"x:Fiction.LiveBench":{"value":0.861,"norm":0.861,"version":"kimi-k2.5"},"gpqa":{"value":0.876,"norm":0.876,"version":"fireworks/kimi-k2p5"},"vals:mmlu_pro":{"value":85.91,"norm":0.8591,"version":"Vals AI"},"arc:arc_agi_2":{"value":11.8,"norm":0.118,"version":"ARC Prize"},"x:CL-bench":{"value":0.193,"norm":0.193,"version":"kimi-k2.5"},"arc:arc_agi_1":{"value":65.3,"norm":0.653,"version":"ARC Prize"},"vals:medcode":{"value":39.32,"norm":0.3932,"version":"Vals AI"},"vals:mortgage_tax":{"value":66.53,"norm":0.6653,"version":"Vals AI"},"hle":{"value":0.24370000000000003,"norm":0.2437,"version":"kimi-k2.5"}},"variant":"Kimi K2.5","match":"exact"},"api":{"or_id":"moonshotai/kimi-k2.5","in":0.5,"out":2.5,"context":262144,"hf":"moonshotai/Kimi-K2.5","created":"2026-01-27"}},{"id":"step-3-5-flash","name":"Step 3.5 Flash","lab":"StepFun","date":"2026-02-02","type":"open-weights","major":true,"open":true,"release":{"title":"Step 3.5 Flash","source":"https://huggingface.co/stepfun-ai/Step-3.5-Flash","verified":true,"confidence":"high"}},{"id":"claude-opus-4-6","name":"Claude Opus 4.6","lab":"Anthropic","date":"2026-02-05","type":"flagship","major":true,"open":false,"release":{"title":"Claude Opus 4.6","source":"https://www.anthropic.com/news/claude-opus-4-6","verified":true,"confidence":"high"},"index":{"score":165,"lo":149.3,"hi":180.8,"se":1.71,"solve":54.7,"status":"rated","areas":["agents","coding","long","maths","novel","reasoning","work"],"core_areas":["agents","coding","long","maths","novel","reasoning"],"testers":["ARC Prize","Epoch AI","FutureSearch","GSO team","Havard T. Ihle","LiveBench","METR","Mercor","Redwood Research","Scale AI","SimpleBench team","Tencent Hunyuan / Fudan","Terminal-Bench team","Vals AI"],"tests":{"arc:arc_agi_2":{"value":69.2,"norm":0.692,"version":"ARC Prize"},"scale:swe_bench_pro":{"value":51.9,"norm":0.519,"version":"Scale AI"},"hle":{"value":0.3444,"norm":0.3444,"version":"claude-opus-4-6_max"},"apex":{"value":0.46299999999999997,"norm":0.463,"version":"claude-opus-4-6_max"},"metr":{"value":0.788644,"norm":0.2559,"version":"claude-opus-4-6_unknown"},"x:DeepResearch Bench":{"value":0.5529999999999999,"norm":0.553,"version":"claude-opus-4-6_high"},"livebench:data-analysis":{"value":69.89,"norm":0.6989,"version":"LiveBench"},"fm":{"value":0.6596491228070176,"norm":0.6596,"version":"claude-opus-4-6_max"},"vals:corp_fin_v2":{"value":67.02,"norm":0.6702,"version":"Vals AI"},"vals:lcb":{"value":84.68,"norm":0.8468,"version":"Vals AI"},"vals:mmmu":{"value":83.87,"norm":0.8387,"version":"Vals AI"},"x:GSO-Bench":{"value":0.412,"norm":0.412,"version":"claude-opus-4-6_high"},"arc:arc_agi_1":{"value":94,"norm":0.94,"version":"ARC Prize"},"livebench:reasoning":{"value":88.67,"norm":0.8867,"version":"LiveBench"},"x:DTBench":{"value":0.912,"norm":0.912,"version":"claude-opus-4-6_max"},"aime":{"value":0.9444444444444444,"norm":0.9444,"version":"claude-opus-4-6_64K"},"x:Furniture Assembly":{"value":0.2833333333333333,"norm":0.2833,"version":"claude-opus-4-6_max"},"tb":{"value":0.629213483146,"norm":0.6292,"version":"claude-opus-4-6_unknown"},"x:Mystery Game Puzzles":{"value":0.25,"norm":0.25,"version":"claude-opus-4-6_max"},"x:SimpleQA Verified":{"value":0.47,"norm":0.47,"version":"claude-opus-4-6_max"},"vals:mortgage_tax":{"value":68.52,"norm":0.6852,"version":"Vals AI"},"livebench:if":{"value":63.31,"norm":0.6331,"version":"LiveBench"},"x:CL-bench":{"value":0.207,"norm":0.207,"version":"claude-opus-4-6_unknown"},"vals:medscribe":{"value":86.74,"norm":0.8674,"version":"Vals AI"},"livebench:mathematics":{"value":89.32,"norm":0.8932,"version":"LiveBench"},"vals:legal_bench":{"value":85.3,"norm":0.853,"version":"Vals AI"},"x:EBR-bench":{"value":0.126984,"norm":0.127,"version":"claude-opus-4-6_max"},"vals:vibe-code":{"value":57.57,"norm":0.5757,"version":"Vals AI"},"vals:mmlu_pro":{"value":89.11,"norm":0.8911,"version":"Vals AI"},"vals:tax_eval_v2":{"value":75.96,"norm":0.7596,"version":"Vals AI"},"swe":{"value":0.7871900826446281,"norm":0.7872,"version":"claude-opus-4-6"},"livebench:agentic-coding":{"value":48.99,"norm":0.4899,"version":"LiveBench"},"gpqa":{"value":0.9053030303030303,"norm":0.9053,"version":"claude-opus-4-6_32K"},"livebench:language":{"value":83.27,"norm":0.8327,"version":"LiveBench"},"vals:swebench":{"value":78.2,"norm":0.782,"version":"Vals AI"},"x:FrontierMath-2025-02-28-Private":{"value":0.407,"norm":0.407,"version":"claude-opus-4-6_max"},"vals:gpqa":{"value":89.65,"norm":0.8965,"version":"Vals AI"},"x:SimpleBench":{"value":0.6759999999999999,"norm":0.676,"version":"claude-opus-4-6"},"x:LMCA":{"value":55.76,"norm":0.5576,"version":"claude-opus-4-6_max"},"x:WeirdML":{"value":0.7795000000000001,"norm":0.7795,"version":"claude-opus-4-6_high"},"livebench:coding":{"value":78.18,"norm":0.7818,"version":"LiveBench"},"vals:medcode":{"value":49.13,"norm":0.4913,"version":"Vals AI"},"x:FrontierMath-Tier-4-v2-Private":{"value":0.2682926829268293,"norm":0.2683,"version":"claude-opus-4-6_max"},"x:MirrorCode":{"value":0.3333333333333333,"norm":0.3333,"version":"claude-opus-4-6_max"},"x:FrontierMath-Tier-4-2025-07-01-Private":{"value":0.229,"norm":0.229,"version":"claude-opus-4-6_max"},"x:Chess Puzzles":{"value":0.21,"norm":0.21,"version":"claude-opus-4-6_max"}},"variant":"Claude Opus 4.6","match":"exact"},"api":{"or_id":"anthropic/claude-opus-4.6","in":5,"out":25,"context":1000000,"hf":null,"created":"2026-02-04"}},{"id":"gpt-5-3-codex","name":"GPT-5.3-Codex","lab":"OpenAI","date":"2026-02-05","type":"reasoning","major":true,"open":false,"release":{"title":"GPT-5.3-Codex","source":"https://openai.com/ja-JP/index/introducing-gpt-5-3-codex/","verified":false,"confidence":"high"},"index":{"score":165.4,"lo":149,"hi":181.9,"se":3.32,"solve":55.1,"status":"provisional","areas":["coding","long"],"core_areas":["coding","long"],"testers":["Epoch AI","Havard T. Ihle","METR","Terminal-Bench team","Vals AI"],"tests":{"vals:lcb":{"value":87.31,"norm":0.8731,"version":"Vals AI"},"swe":{"value":0.7479338842975206,"norm":0.7479,"version":"gpt-5.3-codex_high"},"metr":{"value":0.745439,"norm":0.2507,"version":"gpt-5.3-codex"},"vals:vibe-code":{"value":61.77,"norm":0.6177,"version":"Vals AI"},"vals:ioi":{"value":53.83,"norm":0.5383,"version":"Vals AI"},"tb":{"value":0.647191011236,"norm":0.6472,"version":"gpt-5.3-codex"},"vals:swebench":{"value":78,"norm":0.78,"version":"Vals AI"},"x:WeirdML":{"value":0.793,"norm":0.793,"version":"gpt-5.3-codex"}},"variant":"GPT-5.3 Codex","match":"exact"},"api":{"or_id":"openai/gpt-5.3-codex","in":1.75,"out":14,"context":400000,"hf":null,"created":"2026-02-24"}},{"id":"glm-5","name":"GLM-5","lab":"Zhipu","date":"2026-02-11","type":"open-weights","major":true,"open":true,"release":{"title":"GLM-5","source":"https://huggingface.co/zai-org/GLM-5","verified":false,"confidence":"high"},"index":{"score":141.5,"lo":125.6,"hi":157.3,"se":2.1,"solve":31.4,"status":"rated","areas":["coding","maths","novel","reasoning","work"],"core_areas":["coding","novel","reasoning"],"testers":["ARC Prize","Epoch AI","Havard T. Ihle","SimpleBench team","Tencent Hunyuan / Fudan","Terminal-Bench team","Vals AI"],"tests":{"x:CL-bench":{"value":0.187,"norm":0.187,"version":"glm-5"},"arc:arc_agi_1":{"value":44.7,"norm":0.447,"version":"ARC Prize"},"vals:gpqa":{"value":83.33,"norm":0.8333,"version":"Vals AI"},"vals:swebench":{"value":71.4,"norm":0.714,"version":"Vals AI"},"vals:vibe-code":{"value":23.36,"norm":0.2336,"version":"Vals AI"},"x:SimpleBench":{"value":0.532,"norm":0.532,"version":"glm-5"},"vals:mmlu_pro":{"value":86.03,"norm":0.8603,"version":"Vals AI"},"vals:legal_bench":{"value":84.06,"norm":0.8406,"version":"Vals AI"},"vals:lcb":{"value":81.87,"norm":0.8187,"version":"Vals AI"},"tb":{"value":0.524157303371,"norm":0.5242,"version":"glm-5"},"x:FrontierMath-2025-02-28-Private":{"value":0.16433566433566432,"norm":0.1643,"version":"glm-5"},"arc:arc_agi_2":{"value":4.9,"norm":0.049,"version":"ARC Prize"},"x:WeirdML":{"value":0.4817,"norm":0.4817,"version":"glm-5"},"vals:tax_eval_v2":{"value":70.03,"norm":0.7003,"version":"Vals AI"},"swe":{"value":0.7207792207792207,"norm":0.7208,"version":"glm-5"},"x:FrontierMath-Tier-4-2025-07-01-Private":{"value":0.021,"norm":0.021,"version":"glm-5"},"vals:corp_fin_v2":{"value":62.9,"norm":0.629,"version":"Vals AI"},"gpqa":{"value":0.8781725888324873,"norm":0.8782,"version":"glm-5"},"aime":{"value":0.8,"norm":0.8,"version":"glm-5"}},"variant":"GLM-5","match":"exact"},"api":{"or_id":"z-ai/glm-5","in":1,"out":3.2,"context":204800,"hf":"zai-org/GLM-5","created":"2026-02-11"}},{"id":"gpt-5-3-codex-spark","name":"GPT-5.3-Codex-Spark","lab":"OpenAI","date":"2026-02-12","type":"small","major":true,"open":false,"release":{"title":"GPT-5.3-Codex-Spark","source":"https://openai.com/index/introducing-gpt-5-3-codex-spark/","verified":true,"confidence":"high"}},{"id":"minimax-m2-5","name":"MiniMax-M2.5","lab":"MiniMax","date":"2026-02-12","type":"open-weights","major":true,"open":true,"release":{"title":"MiniMax-M2.5","source":"https://huggingface.co/MiniMaxAI/MiniMax-M2.5","verified":true,"confidence":"high"},"index":{"score":143.7,"lo":127.7,"hi":159.7,"se":2.34,"solve":33.2,"status":"rated","areas":["coding","novel","reasoning","work"],"core_areas":["novel"],"testers":["ARC Prize","Tencent Hunyuan / Fudan","Terminal-Bench team","Vals AI"],"tests":{"x:CL-bench":{"value":0.114,"norm":0.114,"version":"MiniMax-M2.5"},"arc:arc_agi_1":{"value":63.7,"norm":0.637,"version":"ARC Prize"},"vals:gpqa":{"value":82.07,"norm":0.8207,"version":"Vals AI"},"vals:tax_eval_v2":{"value":68.15,"norm":0.6815,"version":"Vals AI"},"vals:mmlu_pro":{"value":80.09,"norm":0.8009,"version":"Vals AI"},"vals:lcb":{"value":79.21,"norm":0.7921,"version":"Vals AI"},"vals:corp_fin_v2":{"value":59.6,"norm":0.596,"version":"Vals AI"},"arc:arc_agi_2":{"value":4.9,"norm":0.049,"version":"ARC Prize"},"vals:vibe-code":{"value":14.85,"norm":0.1485,"version":"Vals AI"},"tb":{"value":0.422471910112,"norm":0.4225,"version":"MiniMax-M2.5"},"vals:legal_bench":{"value":79.96,"norm":0.7996,"version":"Vals AI"},"vals:swebench":{"value":74.2,"norm":0.742,"version":"Vals AI"}},"variant":"MiniMax-M2.5","match":"exact"},"api":{"or_id":"minimax/minimax-m2.5","in":0.295,"out":1.2,"context":198000,"hf":"MiniMaxAI/MiniMax-M2.5","created":"2026-02-12"}},{"id":"doubao-seed-2-0-pro","name":"Doubao-Seed-2.0 Pro","lab":"ByteDance","date":"2026-02-14","type":"flagship","major":false,"open":false,"index":{"score":146.9,"lo":127.3,"hi":166.6,"se":7.33,"solve":36,"status":"provisional","areas":["maths","reasoning"],"core_areas":["reasoning"],"testers":["Epoch AI"],"tests":{"x:SimpleQA Verified":{"value":0.394,"norm":0.394,"version":"doubao-seed-2.0-pro-260215_high"},"gpqa":{"value":0.8819444444444444,"norm":0.8819,"version":"doubao-seed-2.0-pro-260215_high"},"x:Chess Puzzles":{"value":0.2,"norm":0.2,"version":"doubao-seed-2.0-pro-260215_high"},"aime":{"value":0.9388888888888889,"norm":0.9389,"version":"doubao-seed-2.0-pro-260215_high"}},"variant":"Doubao-Seed-2.0 Pro","match":"self"},"date_from":"epoch"},{"id":"seed-2-0","name":"Seed 2.0","lab":"ByteDance","date":"2026-02-14","type":"flagship","major":true,"open":false,"release":{"title":"Seed 2.0","source":"https://seed.bytedance.com/en/blog/seed2-0-%E6%AD%A3%E5%BC%8F%E5%8F%91%E5%B8%83","verified":true,"confidence":"high"}},{"id":"qwen-3-5-plus-hosted-397b-a17b","name":"Qwen 3.5 Plus (hosted 397B-A17B)","lab":"Alibaba","date":"2026-02-16","type":"flagship","major":false,"open":false,"index":{"score":140.9,"lo":123.8,"hi":158.1,"se":4.48,"solve":31,"status":"rated","areas":["agents","coding","maths","reasoning","work"],"core_areas":["reasoning"],"testers":["Epoch AI","Redwood Research","Tencent Hunyuan / Fudan","Vals AI"],"tests":{"x:Mystery Game Puzzles":{"value":0.17,"norm":0.17,"version":"qwen3.5-plus_none"},"gpqa":{"value":0.8484848484848485,"norm":0.8485,"version":"qwen3.5-plus"},"x:FrontierMath-2025-02-28-Private":{"value":0.2103448275862069,"norm":0.2103,"version":"qwen3.5-plus"},"vals:mmlu_pro":{"value":87.18,"norm":0.8718,"version":"Vals AI"},"x:CL-bench":{"value":0.198,"norm":0.198,"version":"qwen3.5-plus"},"vals:gpqa":{"value":87.37,"norm":0.8737,"version":"Vals AI"},"vals:legal_bench":{"value":85.1,"norm":0.851,"version":"Vals AI"},"vals:lcb":{"value":85.33,"norm":0.8533,"version":"Vals AI"},"vals:mmmu":{"value":22.77,"norm":0.2277,"version":"Vals AI"},"vals:vibe-code":{"value":15.74,"norm":0.1574,"version":"Vals AI"},"x:SimpleQA Verified":{"value":0.2535070140280561,"norm":0.2535,"version":"qwen3.5-plus"},"x:LMCA":{"value":36.36,"norm":0.3636,"version":"qwen3.5-plus"},"vals:mortgage_tax":{"value":60.77,"norm":0.6077,"version":"Vals AI"},"x:DTBench":{"value":0.8053,"norm":0.8053,"version":"qwen3.5-plus"},"x:Chess Puzzles":{"value":0.09,"norm":0.09,"version":"qwen3.5-plus"},"aime":{"value":0.8666666666666667,"norm":0.8667,"version":"qwen3.5-plus"},"x:FrontierMath-Tier-4-2025-07-01-Private":{"value":0.020833333333333332,"norm":0.0208,"version":"qwen3.5-plus"},"vals:corp_fin_v2":{"value":65.31,"norm":0.6531,"version":"Vals AI"},"vals:swebench":{"value":71.2,"norm":0.712,"version":"Vals AI"}},"variant":"Qwen 3.5 Plus (hosted 397B-A17B)","match":"self"},"date_from":"epoch"},{"id":"qwen3-5","name":"Qwen3.5","lab":"Alibaba","date":"2026-02-16","type":"open-weights","major":true,"open":true,"release":{"title":"Qwen3.5 (397B open + 3.5-Plus)","source":"https://qwenlm.github.io/qwen-code-docs/en/blog/updates/weekly-update-2026-03-03/","verified":false,"confidence":"high"},"index":{"score":143.7,"lo":126.9,"hi":160.6,"se":4,"solve":33.2,"status":"rated","areas":["agents","maths","reasoning","work"],"core_areas":["agents","maths","reasoning"],"testers":["Epoch AI","Mercor","Redwood Research","Vals AI"],"tests":{"apex":{"value":0.249,"norm":0.249,"version":"qwen3.5-397b-a17b"},"fm":{"value":0.312280701754386,"norm":0.3123,"version":"qwen3.5-397b-a17b_none"},"x:DTBench":{"value":0.8747,"norm":0.8747,"version":"qwen3.5-397b-a17b"},"x:Mystery Game Puzzles":{"value":0.18,"norm":0.18,"version":"qwen3.5-397b-a17b_none"},"gpqa":{"value":0.8636363636363636,"norm":0.8636,"version":"qwen3.5-397b-a17b_none"},"aime":{"value":0.8888888888888888,"norm":0.8889,"version":"qwen3.5-397b-a17b"},"x:LMCA":{"value":37.89,"norm":0.3789,"version":"qwen3.5-397b-a17b"},"vals:sage":{"value":85.53,"norm":0.8553,"version":"Vals AI"}},"variant":"Qwen3.5 397B-A17B","match":"family"},"api":{"or_id":"qwen/qwen3.5-397b-a17b","in":0.45,"out":3,"context":262144,"hf":"Qwen/Qwen3.5-397B-A17B","created":"2026-02-16"}},{"id":"claude-sonnet-4-6","name":"Claude Sonnet 4.6","lab":"Anthropic","date":"2026-02-17","type":"flagship","major":true,"open":false,"release":{"title":"Claude Sonnet 4.6","source":"https://www.anthropic.com/news/claude-sonnet-4-6","verified":true,"confidence":"high"},"index":{"score":158.7,"lo":142.9,"hi":174.5,"se":1.87,"solve":48,"status":"rated","areas":["agents","coding","maths","novel","reasoning","work"],"core_areas":["agents","coding","novel","reasoning","work"],"testers":["ARC Prize","Datacurve","Epoch AI","FutureSearch","GeoBench team","Havard T. Ihle","LiveBench","Mercor","Redwood Research","Vals AI"],"tests":{"vals:mortgage_tax":{"value":67.73,"norm":0.6773,"version":"Vals AI"},"vals:corp_fin_v2":{"value":65.31,"norm":0.6531,"version":"Vals AI"},"gpqa":{"value":0.8737373737373737,"norm":0.8737,"version":"claude-sonnet-4-6_32K"},"vals:legal_research":{"value":38.46,"norm":0.3846,"version":"Vals AI"},"vals:lcb":{"value":82.09,"norm":0.8209,"version":"Vals AI"},"x:Chess Puzzles":{"value":0.23,"norm":0.23,"version":"claude-sonnet-4-6_max"},"vals:mmlu_pro":{"value":87.34,"norm":0.8734,"version":"Vals AI"},"arc:arc_agi_2":{"value":60.4,"norm":0.604,"version":"ARC Prize"},"vals:swebench":{"value":77.4,"norm":0.774,"version":"Vals AI"},"vals:code-migration":{"value":39.89,"norm":0.3989,"version":"Vals AI"},"x:SimpleQA Verified":{"value":0.355,"norm":0.355,"version":"claude-sonnet-4-6_high"},"apex":{"value":0.43,"norm":0.43,"version":"claude-sonnet-4-6_high"},"x:DeepResearch Bench":{"value":0.549,"norm":0.549,"version":"claude-sonnet-4-6_high"},"vals:emb":{"value":60.15,"norm":0.6015,"version":"Vals AI"},"livebench:coding":{"value":79.27,"norm":0.7927,"version":"LiveBench"},"x:FrontierMath-2025-02-28-Private":{"value":0.324,"norm":0.324,"version":"claude-sonnet-4-6_16K"},"vals:vibe-code":{"value":51.48,"norm":0.5148,"version":"Vals AI"},"x:Mystery Game Puzzles":{"value":0.16,"norm":0.16,"version":"claude-sonnet-4-6_low"},"arc:arc_agi_1":{"value":86.5,"norm":0.865,"version":"ARC Prize"},"livebench:if":{"value":63.22,"norm":0.6322,"version":"LiveBench"},"x:WeirdML":{"value":0.6607,"norm":0.6607,"version":"claude-sonnet-4-6_medium"},"x:DTBench":{"value":0.8987,"norm":0.8987,"version":"claude-sonnet-4-6_max"},"x:LMCA":{"value":46.51,"norm":0.4651,"version":"claude-sonnet-4-6_max"},"aime":{"value":0.857936507936508,"norm":0.8579,"version":"claude-sonnet-4-6_32K"},"x:FrontierMath-Tier-4-2025-07-01-Private":{"value":0.083,"norm":0.083,"version":"claude-sonnet-4-6_16K"},"livebench:data-analysis":{"value":77.95,"norm":0.7795,"version":"LiveBench"},"vals:tax_agent_bench":{"value":25.84,"norm":0.2584,"version":"Vals AI"},"vals:legal_bench":{"value":82.12,"norm":0.8212,"version":"Vals AI"},"x:DeepSWE":{"value":0.29933481152993346,"norm":0.2993,"version":"claude-sonnet-4-6_high"},"vals:tax_eval_v2":{"value":77.11,"norm":0.7711,"version":"Vals AI"},"livebench:reasoning":{"value":84.77,"norm":0.8477,"version":"LiveBench"},"vals:mmmu":{"value":83.58,"norm":0.8358,"version":"Vals AI"},"swe":{"value":0.7520661157024794,"norm":0.7521,"version":"claude-sonnet-4-6"},"vals:fabv2":{"value":51.03,"norm":0.5103,"version":"Vals AI"},"livebench:mathematics":{"value":86.99,"norm":0.8699,"version":"LiveBench"},"livebench:agentic-coding":{"value":42.63,"norm":0.4263,"version":"LiveBench"},"livebench:language":{"value":76.1,"norm":0.761,"version":"LiveBench"},"vals:skillsbench":{"value":49.05,"norm":0.4905,"version":"Vals AI"},"vals:public-benefits-bench":{"value":62.45,"norm":0.6245,"version":"Vals AI"},"vals:gpqa":{"value":85.61,"norm":0.8561,"version":"Vals AI"},"x:GeoBench":{"value":0.73,"norm":0.73,"version":"claude-sonnet-4-6_32K"},"vals:programbench":{"value":0.5,"norm":0.005,"version":"Vals AI"}},"variant":"Claude Sonnet 4.6","match":"exact"},"api":{"or_id":"anthropic/claude-sonnet-4.6","in":3,"out":15,"context":1000000,"hf":null,"created":"2026-02-17"}},{"id":"grok-4-20","name":"Grok 4.20","lab":"xAI","date":"2026-02-17","type":"flagship","major":true,"open":false,"release":{"title":"Grok 4.20 (beta; GA/API 03-10)","source":"https://docs.x.ai/developers/models/grok-4.20","verified":false,"confidence":"high"},"index":{"score":156.2,"lo":140.3,"hi":172.2,"se":2.22,"solve":45.3,"status":"rated","areas":["agents","coding","maths","novel","reasoning","work"],"core_areas":["maths","novel","reasoning","work"],"testers":["ARC Prize","Epoch AI","Havard T. Ihle","Redwood Research","Tencent Hunyuan / Fudan","Vals AI"],"tests":{"x:Chess Puzzles":{"value":0.29,"norm":0.29,"version":"grok-4.20-0309-reasoning"},"vals:mmlu_pro":{"value":86.25,"norm":0.8625,"version":"Vals AI"},"vals:code-migration":{"value":0.31,"norm":0.0031,"version":"Vals AI"},"vals:lcb":{"value":84.27,"norm":0.8427,"version":"Vals AI"},"vals:mmmu":{"value":83.47,"norm":0.8347,"version":"Vals AI"},"vals:tax_eval_v2":{"value":74.12,"norm":0.7412,"version":"Vals AI"},"arc:arc_agi_1":{"value":89.5,"norm":0.895,"version":"ARC Prize"},"vals:mortgage_tax":{"value":45.35,"norm":0.4535,"version":"Vals AI"},"vals:medscribe":{"value":63.41,"norm":0.6341,"version":"Vals AI"},"fm":{"value":0.44912280701754387,"norm":0.4491,"version":"grok-4.20-0309-reasoning"},"vals:legal_research":{"value":13.94,"norm":0.1394,"version":"Vals AI"},"aime":{"value":0.9222222222222223,"norm":0.9222,"version":"grok-4.20-0309-reasoning"},"arc:arc_agi_2":{"value":65.1,"norm":0.651,"version":"ARC Prize"},"x:SimpleQA Verified":{"value":0.302,"norm":0.302,"version":"grok-4.20-0309-reasoning"},"x:WeirdML":{"value":0.5226,"norm":0.5226,"version":"grok-4-20"},"vals:fabv2":{"value":28.49,"norm":0.2849,"version":"Vals AI"},"arc:arc_agi_3":{"value":0.1,"norm":0.001,"version":"ARC Prize"},"gpqa":{"value":0.8933080808080808,"norm":0.8933,"version":"grok-4.20-0309-reasoning"},"vals:gpqa":{"value":88.64,"norm":0.8864,"version":"Vals AI"},"x:DTBench":{"value":0.9013,"norm":0.9013,"version":"grok-4-20"},"x:CL-bench":{"value":0.222,"norm":0.222,"version":"grok-4-20"},"x:LMCA":{"value":38.66,"norm":0.3866,"version":"grok-4-20"},"x:FrontierMath-Tier-4-v2-Private":{"value":0.17073170731707318,"norm":0.1707,"version":"grok-4.20-0309-reasoning"},"vals:swebench":{"value":72.2,"norm":0.722,"version":"Vals AI"},"vals:legal_bench":{"value":77.74,"norm":0.7774,"version":"Vals AI"},"vals:tax_agent_bench":{"value":6.95,"norm":0.0695,"version":"Vals AI"},"vals:vibe-code":{"value":4.06,"norm":0.0406,"version":"Vals AI"},"vals:medcode":{"value":32.16,"norm":0.3216,"version":"Vals AI"},"vals:emb":{"value":11.74,"norm":0.1174,"version":"Vals AI"},"vals:corp_fin_v2":{"value":63.67,"norm":0.6367,"version":"Vals AI"}},"variant":"Grok 4.20","match":"exact"},"api":{"or_id":"x-ai/grok-4.20","in":1.25,"out":2.5,"context":2000000,"hf":null,"created":"2026-03-31"}},{"id":"gemini-3-1-pro","name":"Gemini 3.1 Pro","lab":"Google","date":"2026-02-19","type":"flagship","major":true,"open":false,"release":{"title":"Gemini 3.1 Pro","source":"https://blog.google/innovation-and-ai/models-and-research/gemini-models/gemini-3-1-pro/","verified":true,"confidence":"high"},"index":{"score":161.4,"lo":145.6,"hi":177.1,"se":1.76,"solve":50.9,"status":"rated","areas":["agents","coding","long","maths","novel","reasoning","work"],"core_areas":["agents","coding","long","maths","novel","reasoning","work"],"testers":["ARC Prize","BALROG team","Datacurve","Epoch AI","FutureSearch","GSO team","GeoBench team","Havard T. Ihle","METR","Mercor","Redwood Research","Scale AI","SimpleBench team","Tencent Hunyuan / Fudan","Vals AI"],"tests":{"vals:legal_research":{"value":20.67,"norm":0.2067,"version":"Vals AI"},"x:FrontierMath-Tier-4-v2-Private":{"value":0.2682926829268293,"norm":0.2683,"version":"gemini-3.1-pro-preview"},"x:GeoBench":{"value":0.84,"norm":0.84,"version":"gemini-3.1-pro-preview"},"gpqa":{"value":0.9444444444444444,"norm":0.9444,"version":"gemini-3.1-pro-preview_high"},"vals:ioi":{"value":51.83,"norm":0.5183,"version":"Vals AI"},"vals:vibe-code":{"value":32.03,"norm":0.3203,"version":"Vals AI"},"vals:mysterymechanism":{"value":20.72,"norm":0.2072,"version":"Vals AI"},"x:Mystery Game Puzzles":{"value":0.34,"norm":0.34,"version":"gemini-3.1-pro-preview_high"},"x:DeepSWE":{"value":0.1172566371681416,"norm":0.1173,"version":"gemini-3.1-pro-preview"},"apex":{"value":0.35100000000000003,"norm":0.351,"version":"gemini-3.1-pro-preview"},"metr":{"value":0.77035,"norm":0.2537,"version":"gemini-3.1-pro-preview"},"vals:proof_bench":{"value":26,"norm":0.26,"version":"Vals AI"},"vals:vcb-1-100":{"value":6.69,"norm":0.0669,"version":"Vals AI"},"x:LMCA":{"value":53.82,"norm":0.5382,"version":"gemini-3.1-pro-preview_high"},"arc:arc_agi_1":{"value":98,"norm":0.98,"version":"ARC Prize"},"arc:arc_agi_3":{"value":0.4,"norm":0.004,"version":"ARC Prize"},"x:FrontierMath-2025-02-28-Private":{"value":0.369,"norm":0.369,"version":"gemini-3.1-pro-preview"},"vals:terminal-bench-4":{"value":2.52,"norm":0.0252,"version":"Vals AI"},"x:EBR-bench":{"value":0.14285714285714285,"norm":0.1429,"version":"gemini-3.1-pro-preview"},"x:SimpleQA Verified":{"value":0.735,"norm":0.735,"version":"gemini-3.1-pro-preview_high"},"x:Balrog":{"value":0.57,"norm":0.57,"version":"gemini-3.1-pro-preview"},"x:WeirdML":{"value":0.7206999999999999,"norm":0.7207,"version":"gemini-3.1-pro-preview"},"x:MirrorCode":{"value":0.08888888888888888,"norm":0.0889,"version":"gemini-3.1-pro-preview_high"},"fm":{"value":0.5964912280701754,"norm":0.5965,"version":"gemini-3.1-pro-preview"},"x:Chess Puzzles":{"value":0.51,"norm":0.51,"version":"gemini-3.1-pro-preview_high"},"vals:terminal-bench-science":{"value":1.43,"norm":0.0143,"version":"Vals AI"},"vals:tax_agent_bench":{"value":9.3,"norm":0.093,"version":"Vals AI"},"vals:mmlu_pro":{"value":90.99,"norm":0.9099,"version":"Vals AI"},"arc:arc_agi_2":{"value":77.1,"norm":0.771,"version":"ARC Prize"},"scale:swe_bench_pro":{"value":46.1,"norm":0.461,"version":"Scale AI"},"vals:swebench":{"value":78.8,"norm":0.788,"version":"Vals AI"},"x:DeepResearch Bench":{"value":0.478,"norm":0.478,"version":"gemini-3.1-pro-preview_high"},"vals:biomysterybench":{"value":60,"norm":0.6,"version":"Vals AI"},"vals:srebench":{"value":0,"norm":0,"version":"Vals AI"},"hle":{"value":0.4644,"norm":0.4644,"version":"gemini-3.1-pro-preview"},"swe":{"value":0.756198347107438,"norm":0.7562,"version":"gemini-3.1-pro-preview-customtools"},"vals:sage":{"value":83.18,"norm":0.8318,"version":"Vals AI"},"x:FrontierMath-Tier-4-2025-07-01-Private":{"value":0.167,"norm":0.167,"version":"gemini-3.1-pro-preview"},"x:DTBench":{"value":0.9707,"norm":0.9707,"version":"gemini-3.1-pro-preview_medium"},"vals:corp_fin_v2":{"value":64.49,"norm":0.6449,"version":"Vals AI"},"vals:gpqa":{"value":95.45,"norm":0.9545,"version":"Vals AI"},"aime":{"value":0.956,"norm":0.956,"version":"gemini-3.1-pro-preview"},"vals:fabv2":{"value":42.98,"norm":0.4298,"version":"Vals AI"},"x:SimpleBench":{"value":0.7959999999999999,"norm":0.796,"version":"gemini-3.1-pro-preview"},"vals:legal_bench":{"value":87.4,"norm":0.874,"version":"Vals AI"},"vals:public-benefits-bench":{"value":53.79,"norm":0.5379,"version":"Vals AI"},"x:GSO-Bench":{"value":0.2255,"norm":0.2255,"version":"gemini-3.1-pro-preview"},"vals:programbench":{"value":0,"norm":0,"version":"Vals AI"},"vals:code-migration":{"value":17.31,"norm":0.1731,"version":"Vals AI"},"vals:lcb":{"value":88.49,"norm":0.8849,"version":"Vals AI"},"vals:mmmu":{"value":88.21,"norm":0.8821,"version":"Vals AI"},"x:CL-bench":{"value":0.20800000000000002,"norm":0.208,"version":"gemini-3.1-pro-preview"},"vals:tax_eval_v2":{"value":72.88,"norm":0.7288,"version":"Vals AI"},"vals:medcode":{"value":59.06,"norm":0.5906,"version":"Vals AI"},"vals:mortgage_tax":{"value":69.4,"norm":0.694,"version":"Vals AI"},"vals:medscribe":{"value":76.11,"norm":0.7611,"version":"Vals AI"},"x:Furniture Assembly":{"value":0.26666666666666666,"norm":0.2667,"version":"gemini-3.1-pro-preview_high"},"vals:emb":{"value":52.62,"norm":0.5262,"version":"Vals AI"}},"variant":"Gemini 3.1 Pro","match":"exact"},"api":{"or_id":"google/gemini-3.1-pro-preview","in":2,"out":12,"context":1048576,"hf":null,"created":"2026-02-19"}},{"id":"qwen3-5-122b-a10b","name":"Qwen3.5-122B-A10B","lab":"Alibaba","date":"2026-02-24","type":"open-weights","major":false,"open":true,"index":{"score":143.5,"lo":123.4,"hi":163.5,"se":7.73,"solve":33,"status":"insufficient","areas":["agents"],"core_areas":[],"testers":["Epoch AI","Redwood Research"],"tests":{"x:Mystery Game Puzzles":{"value":0.17,"norm":0.17,"version":"qwen3.5-122b-a10b_none"},"x:LMCA":{"value":32.23,"norm":0.3223,"version":"qwen3.5-122b-a10b_none"},"x:DTBench":{"value":0.8427,"norm":0.8427,"version":"qwen3.5-122b-a10b_none"}},"variant":"Qwen3.5-122B-A10B","match":"self"},"api":{"or_id":"qwen/qwen3.5-122b-a10b","in":0.26,"out":2.08,"context":262144,"hf":"Qwen/Qwen3.5-122B-A10B","created":"2026-02-25"},"date_from":"epoch"},{"id":"qwen3-5-27b","name":"Qwen3.5-27B","lab":"Alibaba","date":"2026-02-24","type":"open-weights","major":false,"open":true,"index":{"score":137.4,"lo":119.3,"hi":155.5,"se":5.68,"solve":28.5,"status":"provisional","areas":["agents","coding"],"core_areas":[],"testers":["Havard T. Ihle","Redwood Research"],"tests":{"x:LMCA":{"value":33.98,"norm":0.3398,"version":"qwen3.5-27B"},"x:WeirdML":{"value":0.39530000000000004,"norm":0.3953,"version":"qwen3.5-27B"},"x:DTBench":{"value":0.8240000000000001,"norm":0.824,"version":"qwen3.5-27B"}},"variant":"Qwen3.5-27B","match":"self"},"api":{"or_id":"qwen/qwen3.5-27b","in":0.3,"out":2.4,"context":262144,"hf":"Qwen/Qwen3.5-27B","created":"2026-02-25"},"date_from":"epoch"},{"id":"qwen3-5-35b-a3b","name":"Qwen3.5-35B-A3B","lab":"Alibaba","date":"2026-02-24","type":"open-weights","major":false,"open":true,"index":{"score":134.7,"lo":118.4,"hi":150.9,"se":2.91,"solve":26.8,"status":"rated","areas":["agents","maths","reasoning"],"core_areas":["reasoning"],"testers":["Epoch AI","Redwood Research"],"tests":{"x:LMCA":{"value":29.47,"norm":0.2947,"version":"qwen3.5-35b-a3b"},"x:DTBench":{"value":0.8,"norm":0.8,"version":"qwen3.5-35b-a3b"},"gpqa":{"value":0.8345959595959596,"norm":0.8346,"version":"qwen3.5-35b-a3b"},"aime":{"value":0.7,"norm":0.7,"version":"qwen3.5-35b-a3b_none"}},"variant":"Qwen3.5-35B-A3B","match":"self"},"api":{"or_id":"qwen/qwen3.5-35b-a3b","in":0.14,"out":1,"context":262144,"hf":"Qwen/Qwen3.5-35B-A3B","created":"2026-02-25"},"date_from":"epoch"},{"id":"qwen3-5-4b","name":"Qwen3.5-4B","lab":"Alibaba","date":"2026-02-24","type":"open-weights","major":false,"open":true,"index":{"score":127.1,"lo":111,"hi":143.2,"se":2.63,"solve":22.5,"status":"insufficient","areas":["maths"],"core_areas":[],"testers":["Epoch AI"],"tests":{"aime":{"value":0.5583333333333333,"norm":0.5583,"version":"qwen3.5-4b_none"}},"variant":"Qwen3.5-4B","match":"self"},"date_from":"epoch"},{"id":"qwen3-5-9b","name":"Qwen3.5-9B","lab":"Alibaba","date":"2026-02-24","type":"open-weights","major":false,"open":true,"index":{"score":130.4,"lo":114.4,"hi":146.4,"se":2.46,"solve":24.2,"status":"rated","areas":["agents","maths","reasoning"],"core_areas":["reasoning"],"testers":["Epoch AI","Redwood Research"],"tests":{"x:LMCA":{"value":24.47,"norm":0.2447,"version":"qwen3.5-9B"},"aime":{"value":0.6166666666666667,"norm":0.6167,"version":"qwen3.5-9B_none"},"x:DTBench":{"value":0.7120000000000001,"norm":0.712,"version":"qwen3.5-9B"},"gpqa":{"value":0.7897727272727273,"norm":0.7898,"version":"qwen3.5-9B"}},"variant":"Qwen3.5-9B","match":"self"},"api":{"or_id":"qwen/qwen3.5-9b","in":0.156,"out":0.229,"context":262144,"hf":"Qwen/Qwen3.5-9B","created":"2026-03-10"},"date_from":"epoch"},{"id":"qwen-3-5-flash-hosted-35b-a3b","name":"Qwen 3.5 Flash (hosted 35B-A3B)","lab":"Alibaba","date":"2026-02-25","type":"flagship","major":false,"open":false,"index":{"score":138.3,"lo":121.2,"hi":155.4,"se":4.4,"solve":29.1,"status":"rated","areas":["agents","coding","maths","reasoning","work"],"core_areas":["maths","reasoning"],"testers":["Epoch AI","Redwood Research","Vals AI"],"tests":{"vals:gpqa":{"value":82.83,"norm":0.8283,"version":"Vals AI"},"gpqa":{"value":0.8232323232323232,"norm":0.8232,"version":"qwen3.5-flash"},"vals:mmlu_pro":{"value":84.06,"norm":0.8406,"version":"Vals AI"},"x:SimpleQA Verified":{"value":0.2032032032032032,"norm":0.2032,"version":"qwen3.5-flash"},"vals:mmmu":{"value":81.91,"norm":0.8191,"version":"Vals AI"},"x:LMCA":{"value":29.14,"norm":0.2914,"version":"qwen3.5-flash"},"vals:mortgage_tax":{"value":67.37,"norm":0.6737,"version":"Vals AI"},"x:FrontierMath-2025-02-28-Private":{"value":0.06206896551724138,"norm":0.0621,"version":"qwen3.5-flash"},"x:Mystery Game Puzzles":{"value":0.2,"norm":0.2,"version":"qwen3.5-flash"},"vals:medcode":{"value":33,"norm":0.33,"version":"Vals AI"},"vals:medscribe":{"value":70.62,"norm":0.7062,"version":"Vals AI"},"vals:sage":{"value":84.9,"norm":0.849,"version":"Vals AI"},"x:FrontierMath-Tier-4-2025-07-01-Private":{"value":0,"norm":0,"version":"qwen3.5-flash"},"vals:corp_fin_v2":{"value":63.56,"norm":0.6356,"version":"Vals AI"},"vals:swebench":{"value":64.4,"norm":0.644,"version":"Vals AI"},"vals:legal_bench":{"value":84.28,"norm":0.8428,"version":"Vals AI"},"vals:tax_eval_v2":{"value":72.16,"norm":0.7216,"version":"Vals AI"},"fm":{"value":0.1824561403508772,"norm":0.1825,"version":"qwen3.5-flash_none"},"x:DTBench":{"value":0.8293,"norm":0.8293,"version":"qwen3.5-flash"},"x:Chess Puzzles":{"value":0.14,"norm":0.14,"version":"qwen3.5-flash"},"aime":{"value":0.8444444444444444,"norm":0.8444,"version":"qwen3.5-flash"},"vals:lcb":{"value":83.28,"norm":0.8328,"version":"Vals AI"}},"variant":"Qwen 3.5 Flash (hosted 35B-A3B)","match":"self"},"date_from":"epoch"},{"id":"gemini-3-1-flash-lite","name":"Gemini 3.1 Flash-Lite","lab":"Google","date":"2026-03-03","type":"small","major":true,"open":false,"release":{"title":"Gemini 3.1 Flash-Lite","source":"https://blog.google/innovation-and-ai/models-and-research/gemini-models/gemini-3-1-flash-lite/","verified":true,"confidence":"high"},"index":{"score":138.5,"lo":122.2,"hi":154.8,"se":3.04,"solve":29.3,"status":"rated","areas":["agents","coding","maths","reasoning","work"],"core_areas":["maths","reasoning","work"],"testers":["Epoch AI","FutureSearch","GeoBench team","Havard T. Ihle","Redwood Research","Scale AI","Vals AI"],"tests":{"vals:mmlu_pro":{"value":86.24,"norm":0.8624,"version":"Vals AI"},"vals:programbench":{"value":0,"norm":0,"version":"Vals AI"},"gpqa":{"value":0.8181818181818182,"norm":0.8182,"version":"gemini-3.1-flash-lite_high"},"x:GeoBench":{"value":0.8300000000000001,"norm":0.83,"version":"gemini-3.1-flash-lite"},"vals:swebench":{"value":62.8,"norm":0.628,"version":"Vals AI"},"hle":{"value":0.0864,"norm":0.0864,"version":"gemini-3.1-flash-lite"},"x:WeirdML":{"value":0.5219,"norm":0.5219,"version":"gemini-3.1-flash-lite"},"vals:mmmu":{"value":82.49,"norm":0.8249,"version":"Vals AI"},"vals:tax_eval_v2":{"value":71.79,"norm":0.7179,"version":"Vals AI"},"x:LMCA":{"value":35.01,"norm":0.3501,"version":"gemini-3.1-flash-lite_high"},"vals:medscribe":{"value":63.9,"norm":0.639,"version":"Vals AI"},"aime":{"value":0.8,"norm":0.8,"version":"gemini-3.1-flash-lite_high"},"vals:emb":{"value":8.63,"norm":0.0863,"version":"Vals AI"},"vals:fabv2":{"value":29.99,"norm":0.2999,"version":"Vals AI"},"vals:legal_bench":{"value":83.76,"norm":0.8376,"version":"Vals AI"},"vals:legal_research":{"value":3.36,"norm":0.0336,"version":"Vals AI"},"vals:gpqa":{"value":81.06,"norm":0.8106,"version":"Vals AI"},"x:DeepResearch Bench":{"value":0.373,"norm":0.373,"version":"gemini-3.1-flash-lite_low"},"vals:medcode":{"value":47.6,"norm":0.476,"version":"Vals AI"},"vals:tax_agent_bench":{"value":3.11,"norm":0.0311,"version":"Vals AI"},"vals:code-migration":{"value":4.61,"norm":0.0461,"version":"Vals AI"},"vals:mortgage_tax":{"value":68.04,"norm":0.6804,"version":"Vals AI"},"fm":{"value":0.2771929824561403,"norm":0.2772,"version":"gemini-3.1-flash-lite_high"},"vals:corp_fin_v2":{"value":59.36,"norm":0.5936,"version":"Vals AI"},"vals:lcb":{"value":80.12,"norm":0.8012,"version":"Vals AI"},"vals:vibe-code":{"value":0,"norm":0,"version":"Vals AI"},"x:DTBench":{"value":0.768,"norm":0.768,"version":"gemini-3.1-flash-lite_high"}},"variant":"Gemini 3.1 Flash-Lite","match":"exact"},"api":{"or_id":"google/gemini-3.1-flash-lite","in":0.25,"out":1.5,"context":1048576,"hf":null,"created":"2026-05-07"}},{"id":"gpt-5-4","name":"GPT-5.4","lab":"OpenAI","date":"2026-03-05","type":"flagship","major":false,"open":false,"index":{"score":165.8,"lo":150,"hi":181.6,"se":1.86,"solve":55.5,"status":"rated","areas":["agents","coding","long","maths","novel","reasoning","work"],"core_areas":["agents","coding","long","maths","novel","reasoning"],"testers":["ARC Prize","Datacurve","Epoch AI","FutureSearch","GSO team","Havard T. Ihle","LiveBench","METR","Mercor","Redwood Research","Scale AI","Tencent Hunyuan / Fudan","Vals AI"],"tests":{"vals:lcb":{"value":84.14,"norm":0.8414,"version":"Vals AI"},"x:EBR-bench":{"value":0.253968,"norm":0.254,"version":"gpt-5.4-2026-03-05_xhigh"},"livebench:if":{"value":70.22,"norm":0.7022,"version":"LiveBench"},"x:DTBench":{"value":0.9440000000000001,"norm":0.944,"version":"gpt-5.4-2026-03-05_xhigh"},"x:CL-bench":{"value":0.279,"norm":0.279,"version":"gpt-5.4-2026-03-05_xhigh"},"vals:medcode":{"value":41.29,"norm":0.4129,"version":"Vals AI"},"hle":{"value":0.3624,"norm":0.3624,"version":"gpt-5.4-2026-03-05_xhigh"},"x:FrontierMath-Tier-4-2025-07-01-Private":{"value":0.271,"norm":0.271,"version":"gpt-5.4-2026-03-05_xhigh"},"livebench:data-analysis":{"value":79.31,"norm":0.7931,"version":"LiveBench"},"livebench:coding":{"value":77.54,"norm":0.7754,"version":"LiveBench"},"fm":{"value":0.7859649122807018,"norm":0.786,"version":"gpt-5.4-2026-03-05_xhigh"},"vals:mmlu_pro":{"value":87.48,"norm":0.8748,"version":"Vals AI"},"x:GSO-Bench":{"value":0.31370000000000003,"norm":0.3137,"version":"gpt-5.4-2026-03-05_xhigh"},"vals:code-migration":{"value":34.98,"norm":0.3498,"version":"Vals AI"},"vals:mmmu":{"value":87.51,"norm":0.8751,"version":"Vals AI"},"x:DeepResearch Bench":{"value":0.35100000000000003,"norm":0.351,"version":"gpt-5.4-2026-03-05_low"},"vals:tax_eval_v2":{"value":73.96,"norm":0.7396,"version":"Vals AI"},"x:LMCA":{"value":51.97,"norm":0.5197,"version":"gpt-5.4-2026-03-05_xhigh"},"vals:mortgage_tax":{"value":68.32,"norm":0.6832,"version":"Vals AI"},"livebench:language":{"value":82.63,"norm":0.8263,"version":"LiveBench"},"vals:skillsbench":{"value":51.71,"norm":0.5171,"version":"Vals AI"},"vals:programbench":{"value":0.5,"norm":0.005,"version":"Vals AI"},"livebench:reasoning":{"value":88.12,"norm":0.8812,"version":"LiveBench"},"vals:vibe-code":{"value":67.42,"norm":0.6742,"version":"Vals AI"},"metr":{"value":0.743443,"norm":0.2504,"version":"gpt-5.4-2026-03-05_xhigh"},"arc:arc_agi_1":{"value":93.7,"norm":0.937,"version":"ARC Prize"},"x:Furniture Assembly":{"value":0.375,"norm":0.375,"version":"gpt-5.4-2026-03-05_xhigh"},"vals:gpqa":{"value":91.67,"norm":0.9167,"version":"Vals AI"},"scale:swe_bench_pro":{"value":59.1,"norm":0.591,"version":"Scale AI"},"vals:legal_bench":{"value":86.04,"norm":0.8604,"version":"Vals AI"},"x:DeepSWE":{"value":0.5176991150442478,"norm":0.5177,"version":"gpt-5.4-2026-03-05_xhigh"},"x:SimpleQA Verified":{"value":0.451,"norm":0.451,"version":"gpt-5.4-2026-03-05_xhigh"},"apex":{"value":0.524,"norm":0.524,"version":"gpt-5.4-2026-03-05_unknown"},"x:WeirdML":{"value":0.777,"norm":0.777,"version":"gpt-5.4-2026-03-05_xhigh"},"swe":{"value":0.768595041322314,"norm":0.7686,"version":"gpt-5.4-2026-03-05_high"},"vals:medscribe":{"value":77.55,"norm":0.7755,"version":"Vals AI"},"vals:corp_fin_v2":{"value":65.27,"norm":0.6527,"version":"Vals AI"},"x:Chess Puzzles":{"value":0.48,"norm":0.48,"version":"gpt-5.4-2026-03-05_xhigh"},"x:Mystery Game Puzzles":{"value":0.37,"norm":0.37,"version":"gpt-5.4-2026-03-05_xhigh"},"arc:arc_agi_2":{"value":74,"norm":0.74,"version":"ARC Prize"},"x:MirrorCode":{"value":0.23333333333333334,"norm":0.2333,"version":"gpt-5.4-2026-03-05_xhigh"},"arc:arc_agi_3":{"value":0.2,"norm":0.002,"version":"ARC Prize"},"livebench:mathematics":{"value":94.15,"norm":0.9415,"version":"LiveBench"},"livebench:agentic-coding":{"value":53.84,"norm":0.5384,"version":"LiveBench"},"gpqa":{"value":0.933,"norm":0.933,"version":"gpt-5.4-2026-03-05_xhigh"},"x:FrontierMath-2025-02-28-Private":{"value":0.476,"norm":0.476,"version":"gpt-5.4-2026-03-05_xhigh"},"x:FrontierMath-Tier-4-v2-Private":{"value":0.49,"norm":0.49,"version":"gpt-5.4-2026-03-05_xhigh"},"aime":{"value":0.9777777777777777,"norm":0.9778,"version":"gpt-5.4-2026-03-05_high"},"vals:swebench":{"value":78.2,"norm":0.782,"version":"Vals AI"}},"variant":"GPT-5.4","match":"self"},"api":{"or_id":"openai/gpt-5.4","in":2.5,"out":15,"context":1050000,"hf":null,"created":"2026-03-05"},"modes":[{"label":"Pro","name":"GPT-5.4 Pro","index":{"score":171.1,"lo":155,"hi":187.1,"se":2.6,"solve":60.3,"status":"rated","areas":["coding","maths","novel","reasoning"],"core_areas":["maths","novel","reasoning"],"testers":["ARC Prize","Epoch AI","Havard T. Ihle","Scale AI","SimpleBench team"],"tests":{"x:SimpleBench":{"value":0.741,"norm":0.741,"version":"gpt-5.4-pro-2026-03-05_unknown"},"x:FrontierMath-Tier-4-v2-Private":{"value":0.5853658536585366,"norm":0.5854,"version":"gpt-5.4-pro-2026-03-05_xhigh"},"x:Chess Puzzles":{"value":0.59,"norm":0.59,"version":"gpt-5.4-pro-2026-03-05_xhigh"},"gpqa":{"value":0.946,"norm":0.946,"version":"gpt-5.4-pro-2026-03-05_xhigh"},"arc:arc_agi_2":{"value":83.3,"norm":0.833,"version":"ARC Prize"},"arc:arc_agi_1":{"value":94.5,"norm":0.945,"version":"ARC Prize"},"hle":{"value":0.44320000000000004,"norm":0.4432,"version":"gpt-5.4-pro-2026-03-05_unknown"},"x:SimpleQA Verified":{"value":0.463,"norm":0.463,"version":"gpt-5.4-pro-2026-03-05_xhigh"},"fm":{"value":0.8245614035087719,"norm":0.8246,"version":"gpt-5.4-pro-2026-03-05_xhigh"},"x:FrontierMath-2025-02-28-Private":{"value":0.5,"norm":0.5,"version":"gpt-5.4-pro-2026-03-05_xhigh"},"x:WeirdML":{"value":0.5744,"norm":0.5744,"version":"gpt-5.4-pro-2026-03-05_none"}}},"api":{"or_id":"openai/gpt-5.4-pro","in":30,"out":180,"context":1050000,"hf":null,"created":"2026-03-05"}}],"aliases":["gpt-5-4-pro"],"date_from":"epoch"},{"id":"gpt-5-4-thinking","name":"GPT-5.4 Thinking","lab":"OpenAI","date":"2026-03-05","type":"flagship","major":true,"open":false,"release":{"title":"GPT-5.4 (Thinking/Pro; mini/nano 03-17)","source":"https://openai.com/index/gpt-5-4-thinking-system-card/","verified":true,"confidence":"high"}},{"id":"nemotron-3-super","name":"Nemotron 3 Super","lab":"NVIDIA","date":"2026-03-11","type":"open-weights","major":true,"open":true,"release":{"title":"Nemotron 3 Super","source":"https://blogs.nvidia.com/blog/nemotron-3-super-agentic-ai/","verified":true,"confidence":"high"},"index":{"score":130.5,"lo":109.8,"hi":151.1,"se":8.28,"solve":24.3,"status":"insufficient","areas":["coding"],"core_areas":[],"testers":["Havard T. Ihle"],"tests":{"x:WeirdML":{"value":0.3801,"norm":0.3801,"version":"nemotron-3-super"}},"variant":"Nemotron 3 Super","match":"exact"},"api":{"or_id":"nvidia/nemotron-3-super-120b-a12b","in":0.075,"out":0.4,"context":262144,"hf":"nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8","created":"2026-03-11"}},{"id":"mistral-small-4","name":"Mistral Small 4","lab":"Mistral","date":"2026-03-16","type":"open-weights","major":true,"open":true,"release":{"title":"Mistral Small 4","source":"https://mistral.ai/news/mistral-small-4/","verified":true,"confidence":"high"},"index":{"score":119,"lo":99.7,"hi":138.4,"se":7.01,"solve":18.8,"status":"rated","areas":["agents","coding","work"],"core_areas":[],"testers":["Redwood Research","Vals AI"],"tests":{"x:DTBench":{"value":0.7093,"norm":0.7093,"version":"mistral-small-2603"},"vals:vibe-code":{"value":0,"norm":0,"version":"Vals AI"},"vals:sage":{"value":59.16,"norm":0.5916,"version":"Vals AI"},"x:LMCA":{"value":20.61,"norm":0.2061,"version":"mistral-small-2603"}},"variant":"Mistral Small 4","match":"exact"},"api":{"or_id":"mistralai/mistral-small-2603","in":0.15,"out":0.6,"context":262144,"hf":"mistralai/Mistral-Small-4-119B-2603","created":"2026-03-16"}},{"id":"gpt-5-4-mini","name":"GPT-5.4 Mini","lab":"OpenAI","date":"2026-03-17","type":"flagship","major":false,"open":false,"index":{"score":146.9,"lo":131.1,"hi":162.8,"se":2.14,"solve":36,"status":"rated","areas":["agents","coding","maths","novel","reasoning","work"],"core_areas":["coding","maths","novel","reasoning","work"],"testers":["ARC Prize","Epoch AI","FutureSearch","Havard T. Ihle","Redwood Research","Vals AI"],"tests":{"vals:vibe-code":{"value":47.97,"norm":0.4797,"version":"Vals AI"},"x:LMCA":{"value":40.79,"norm":0.4079,"version":"gpt-5.4-mini-2026-03-17_xhigh"},"vals:emb":{"value":45.43,"norm":0.4543,"version":"Vals AI"},"vals:corp_fin_v2":{"value":60.92,"norm":0.6092,"version":"Vals AI"},"vals:legal_research":{"value":12.5,"norm":0.125,"version":"Vals AI"},"vals:gpqa":{"value":83.08,"norm":0.8308,"version":"Vals AI"},"vals:mmmu":{"value":79.25,"norm":0.7925,"version":"Vals AI"},"x:DeepResearch Bench":{"value":0.363,"norm":0.363,"version":"gpt-5.4-mini-2026-03-17_low"},"vals:tax_eval_v2":{"value":71.22,"norm":0.7122,"version":"Vals AI"},"x:FrontierMath-Tier-4-2025-07-01-Private":{"value":0.0208,"norm":0.0208,"version":"gpt-5.4-mini-2026-03-17_high"},"x:FrontierMath-Tier-4-v2-Private":{"value":0.0975609756097561,"norm":0.0976,"version":"gpt-5.4-mini-2026-03-17_xhigh"},"x:Chess Puzzles":{"value":0.23,"norm":0.23,"version":"gpt-5.4-mini-2026-03-17_high"},"gpqa":{"value":0.8686868686868687,"norm":0.8687,"version":"gpt-5.4-mini-2026-03-17_xhigh"},"x:SimpleQA Verified":{"value":0.294,"norm":0.294,"version":"gpt-5.4-mini-2026-03-17_high"},"arc:arc_agi_1":{"value":63.7,"norm":0.637,"version":"ARC Prize"},"vals:mortgage_tax":{"value":63.51,"norm":0.6351,"version":"Vals AI"},"vals:tax_agent_bench":{"value":8.81,"norm":0.0881,"version":"Vals AI"},"x:Mystery Game Puzzles":{"value":0.11,"norm":0.11,"version":"gpt-5.4-mini-2026-03-17_none"},"vals:mmlu_pro":{"value":84.55,"norm":0.8455,"version":"Vals AI"},"vals:programbench":{"value":0,"norm":0,"version":"Vals AI"},"vals:lcb":{"value":81.46,"norm":0.8146,"version":"Vals AI"},"x:WeirdML":{"value":0.603,"norm":0.603,"version":"gpt-5.4-mini-2026-03-17_high"},"fm":{"value":0.512280701754386,"norm":0.5123,"version":"gpt-5.4-mini-2026-03-17_xhigh"},"x:DTBench":{"value":0.8,"norm":0.8,"version":"gpt-5.4-mini-2026-03-17_xhigh"},"x:FrontierMath-2025-02-28-Private":{"value":0.2828,"norm":0.2828,"version":"gpt-5.4-mini-2026-03-17_high"},"aime":{"value":0.8888888888888888,"norm":0.8889,"version":"gpt-5.4-mini-2026-03-17_xhigh"},"vals:fabv2":{"value":45.36,"norm":0.4536,"version":"Vals AI"},"arc:arc_agi_2":{"value":18.9,"norm":0.189,"version":"ARC Prize"},"vals:swebench":{"value":73,"norm":0.73,"version":"Vals AI"},"vals:terminal-bench-4":{"value":2.52,"norm":0.0252,"version":"Vals AI"},"vals:code-migration":{"value":12.94,"norm":0.1294,"version":"Vals AI"}},"variant":"GPT-5.4 Mini","match":"self"},"api":{"or_id":"openai/gpt-5.4-mini","in":0.75,"out":4.5,"context":400000,"hf":null,"created":"2026-03-17"},"date_from":"epoch"},{"id":"gpt-5-4-nano","name":"GPT-5.4 Nano","lab":"OpenAI","date":"2026-03-17","type":"flagship","major":false,"open":false,"index":{"score":141.3,"lo":125.4,"hi":157.2,"se":2.22,"solve":31.3,"status":"rated","areas":["agents","coding","maths","novel","reasoning","work"],"core_areas":["maths","novel","reasoning","work"],"testers":["ARC Prize","Epoch AI","Havard T. Ihle","Redwood Research","Vals AI"],"tests":{"vals:mmlu_pro":{"value":77.17,"norm":0.7717,"version":"Vals AI"},"x:LMCA":{"value":36.86,"norm":0.3686,"version":"gpt-5.4-nano-2026-03-17_xhigh"},"vals:mmmu":{"value":73.58,"norm":0.7358,"version":"Vals AI"},"vals:corp_fin_v2":{"value":61.19,"norm":0.6119,"version":"Vals AI"},"vals:tax_agent_bench":{"value":4.97,"norm":0.0497,"version":"Vals AI"},"gpqa":{"value":0.7847,"norm":0.7847,"version":"gpt-5.4-nano-2026-03-17_high"},"x:Chess Puzzles":{"value":0.45,"norm":0.45,"version":"gpt-5.4-nano-2026-03-17_xhigh"},"x:SimpleQA Verified":{"value":0.117,"norm":0.117,"version":"gpt-5.4-nano-2026-03-17_high"},"vals:legal_bench":{"value":77.92,"norm":0.7792,"version":"Vals AI"},"x:FrontierMath-Tier-4-v2-Private":{"value":0.12195121951219512,"norm":0.122,"version":"gpt-5.4-nano-2026-03-17_high"},"arc:arc_agi_1":{"value":51.5,"norm":0.515,"version":"ARC Prize"},"x:WeirdML":{"value":0.49229999999999996,"norm":0.4923,"version":"gpt-5.4-nano-2026-03-17_high"},"x:DTBench":{"value":0.8027,"norm":0.8027,"version":"gpt-5.4-nano-2026-03-17_xhigh"},"vals:medscribe":{"value":77.09,"norm":0.7709,"version":"Vals AI"},"aime":{"value":0.8778,"norm":0.8778,"version":"gpt-5.4-nano-2026-03-17_high"},"vals:emb":{"value":44.75,"norm":0.4475,"version":"Vals AI"},"fm":{"value":0.44912280701754387,"norm":0.4491,"version":"gpt-5.4-nano-2026-03-17_high"},"x:Mystery Game Puzzles":{"value":0.09,"norm":0.09,"version":"gpt-5.4-nano-2026-03-17_none"},"vals:tax_eval_v2":{"value":67.42,"norm":0.6742,"version":"Vals AI"},"vals:legal_research":{"value":6.25,"norm":0.0625,"version":"Vals AI"},"vals:gpqa":{"value":77.53,"norm":0.7753,"version":"Vals AI"},"vals:vibe-code":{"value":26.1,"norm":0.261,"version":"Vals AI"},"arc:arc_agi_2":{"value":5.7,"norm":0.057,"version":"ARC Prize"},"vals:code-migration":{"value":14.47,"norm":0.1447,"version":"Vals AI"},"vals:mortgage_tax":{"value":59.1,"norm":0.591,"version":"Vals AI"},"vals:medcode":{"value":41.03,"norm":0.4103,"version":"Vals AI"},"x:FrontierMath-Tier-4-2025-07-01-Private":{"value":0.0625,"norm":0.0625,"version":"gpt-5.4-nano-2026-03-17_high"},"vals:fabv2":{"value":38.22,"norm":0.3822,"version":"Vals AI"},"vals:swebench":{"value":69.8,"norm":0.698,"version":"Vals AI"},"x:FrontierMath-2025-02-28-Private":{"value":0.2586,"norm":0.2586,"version":"gpt-5.4-nano-2026-03-17_high"},"vals:lcb":{"value":84.01,"norm":0.8401,"version":"Vals AI"}},"variant":"GPT-5.4 Nano","match":"self"},"api":{"or_id":"openai/gpt-5.4-nano","in":0.2,"out":1.25,"context":400000,"hf":null,"created":"2026-03-17"},"date_from":"epoch"},{"id":"minimax-m2-7","name":"MiniMax-M2.7","lab":"MiniMax","date":"2026-03-18","type":"open-weights","major":true,"open":true,"release":{"title":"MiniMax-M2.7","source":"https://www.minimax.io/news/minimax-m27-en","verified":true,"confidence":"high"},"index":{"score":132.4,"lo":113.6,"hi":151.2,"se":6.44,"solve":25.4,"status":"rated","areas":["coding","reasoning","work"],"core_areas":["work"],"testers":["Havard T. Ihle","Vals AI"],"tests":{"vals:legal_bench":{"value":83.98,"norm":0.8398,"version":"Vals AI"},"vals:vibe-code":{"value":11.93,"norm":0.1193,"version":"Vals AI"},"vals:code-migration":{"value":8.69,"norm":0.0869,"version":"Vals AI"},"vals:legal_research":{"value":10.58,"norm":0.1058,"version":"Vals AI"},"vals:mmlu_pro":{"value":80.43,"norm":0.8043,"version":"Vals AI"},"vals:tax_eval_v2":{"value":66.56,"norm":0.6656,"version":"Vals AI"},"vals:tax_agent_bench":{"value":6.87,"norm":0.0687,"version":"Vals AI"},"vals:lcb":{"value":79.93,"norm":0.7993,"version":"Vals AI"},"vals:medscribe":{"value":79.87,"norm":0.7987,"version":"Vals AI"},"vals:gpqa":{"value":86.62,"norm":0.8662,"version":"Vals AI"},"vals:fabv2":{"value":27.89,"norm":0.2789,"version":"Vals AI"},"vals:swebench":{"value":73.8,"norm":0.738,"version":"Vals AI"},"vals:programbench":{"value":0,"norm":0,"version":"Vals AI"},"x:WeirdML":{"value":0.36950000000000005,"norm":0.3695,"version":"MiniMax-M2.7"},"vals:medcode":{"value":34.44,"norm":0.3444,"version":"Vals AI"},"vals:emb":{"value":28.45,"norm":0.2845,"version":"Vals AI"},"vals:corp_fin_v2":{"value":61.19,"norm":0.6119,"version":"Vals AI"}},"variant":"MiniMax-M2.7","match":"exact"},"api":{"or_id":"minimax/minimax-m2.7","in":0.21,"out":0.84,"context":196608,"hf":"MiniMaxAI/MiniMax-M2.7","created":"2026-03-18"}},{"id":"gemma-4","name":"Gemma 4","lab":"Google","date":"2026-04-02","type":"open-weights","major":true,"open":true,"release":{"title":"Gemma 4","source":"https://blog.google/innovation-and-ai/technology/developers-tools/gemma-4/","verified":true,"confidence":"high"},"index":{"score":137.2,"lo":121,"hi":153.5,"se":2.92,"solve":28.4,"status":"rated","areas":["agents","coding","maths","reasoning","work"],"core_areas":["reasoning"],"testers":["Epoch AI","Havard T. Ihle","Redwood Research","Vals AI","Yann Henon"],"tests":{"x:Surface Evolver Bench":{"value":0.30624999999999997,"norm":0.3062,"version":"gemma-4-31b-it"},"x:DTBench":{"value":0.8267,"norm":0.8267,"version":"gemma-4-31b-it"},"aime":{"value":0.7333333333333333,"norm":0.7333,"version":"gemma-4-31b-it_minimal"},"x:WeirdML":{"value":0.5226,"norm":0.5226,"version":"gemma-4-31b-it"},"vals:sage":{"value":82.78,"norm":0.8278,"version":"Vals AI"},"x:SimpleQA Verified":{"value":0.104,"norm":0.104,"version":"gemma-4-31b-it"},"x:LMCA":{"value":39.25,"norm":0.3925,"version":"gemma-4-31b-it"},"gpqa":{"value":0.7575757575757576,"norm":0.7576,"version":"gemma-4-31b-it_minimal"},"vals:mortgage_tax":{"value":61.37,"norm":0.6137,"version":"Vals AI"}},"variant":"Gemma 4 31B IT","match":"family"}},{"id":"gemma-4-26b-a4b","name":"Gemma 4 26B A4B","lab":"Google","date":"2026-04-02","type":"open-weights","major":false,"open":true,"index":{"score":135.1,"lo":119,"hi":151.3,"se":2.79,"solve":27.1,"status":"rated","areas":["agents","coding","maths","reasoning"],"core_areas":["reasoning"],"testers":["Epoch AI","Havard T. Ihle","Redwood Research"],"tests":{"aime":{"value":0.8222222222222222,"norm":0.8222,"version":"gemma-4-26b-a4b-it_minimal"},"x:WeirdML":{"value":0.3517,"norm":0.3517,"version":"gemma-4-26b-a4b"},"x:DTBench":{"value":0.7493000000000001,"norm":0.7493,"version":"gemma-4-26b-a4b"},"x:LMCA":{"value":29.68,"norm":0.2968,"version":"gemma-4-26b-a4b"},"gpqa":{"value":0.7323232323232324,"norm":0.7323,"version":"gemma-4-26b-a4b-it_minimal"}},"variant":"Gemma 4 26B A4B","match":"self"},"api":{"or_id":"google/gemma-4-26b-a4b-it","in":0.0765,"out":0.255,"context":262144,"hf":"google/gemma-4-26B-A4B-it","created":"2026-04-03"},"date_from":"epoch"},{"id":"qwen3-6-plus","name":"Qwen3.6-Plus","lab":"Alibaba","date":"2026-04-02","type":"flagship","major":true,"open":false,"release":{"title":"Qwen3.6-Plus","source":null,"verified":false,"confidence":"high"},"index":{"score":144.9,"lo":127.9,"hi":161.9,"se":4.27,"solve":34.2,"status":"rated","areas":["agents","coding","maths","reasoning","work"],"core_areas":["coding","maths","reasoning","work"],"testers":["Epoch AI","LiveBench","Redwood Research","Tencent Hunyuan / Fudan","Vals AI"],"tests":{"vals:mmlu_pro":{"value":87.67,"norm":0.8767,"version":"Vals AI"},"vals:swebench":{"value":73.4,"norm":0.734,"version":"Vals AI"},"aime":{"value":0.9333333333333333,"norm":0.9333,"version":"qwen3.6-plus"},"x:FrontierMath-Tier-4-2025-07-01-Private":{"value":0.08333333333333333,"norm":0.0833,"version":"qwen3.6-plus"},"vals:emb":{"value":32.87,"norm":0.3287,"version":"Vals AI"},"vals:tax_agent_bench":{"value":7.68,"norm":0.0768,"version":"Vals AI"},"x:FrontierMath-2025-02-28-Private":{"value":0.2620689655172414,"norm":0.2621,"version":"qwen3.6-plus"},"livebench:reasoning":{"value":75.83,"norm":0.7583,"version":"LiveBench"},"x:DTBench":{"value":0.8187000000000001,"norm":0.8187,"version":"qwen3.6-plus"},"x:CL-bench":{"value":0.203,"norm":0.203,"version":"qwen3.6-plus"},"x:LMCA":{"value":33.09,"norm":0.3309,"version":"qwen3.6-plus"},"livebench:data-analysis":{"value":69.91,"norm":0.6991,"version":"LiveBench"},"livebench:coding":{"value":78.18,"norm":0.7818,"version":"LiveBench"},"livebench:agentic-coding":{"value":41.36,"norm":0.4136,"version":"LiveBench"},"x:Mystery Game Puzzles":{"value":0.12,"norm":0.12,"version":"qwen3.6-plus_none"},"vals:legal_research":{"value":14.9,"norm":0.149,"version":"Vals AI"},"vals:lcb":{"value":85.95,"norm":0.8595,"version":"Vals AI"},"vals:gpqa":{"value":87.37,"norm":0.8737,"version":"Vals AI"},"vals:code-migration":{"value":11.1,"norm":0.111,"version":"Vals AI"},"livebench:if":{"value":58.34,"norm":0.5834,"version":"LiveBench"},"vals:mmmu":{"value":84.16,"norm":0.8416,"version":"Vals AI"},"vals:mortgage_tax":{"value":67.97,"norm":0.6797,"version":"Vals AI"},"vals:corp_fin_v2":{"value":61.93,"norm":0.6193,"version":"Vals AI"},"x:Chess Puzzles":{"value":0.16,"norm":0.16,"version":"qwen3.6-plus"},"vals:programbench":{"value":0,"norm":0,"version":"Vals AI"},"vals:vibe-code":{"value":25.56,"norm":0.2556,"version":"Vals AI"},"x:SimpleQA Verified":{"value":0.44144144144144143,"norm":0.4414,"version":"qwen3.6-plus"},"vals:medcode":{"value":36.89,"norm":0.3689,"version":"Vals AI"},"vals:fabv2":{"value":40.85,"norm":0.4085,"version":"Vals AI"},"fm":{"value":0.3824561403508772,"norm":0.3825,"version":"qwen3.6-plus"},"livebench:language":{"value":74.99,"norm":0.7499,"version":"LiveBench"},"vals:legal_bench":{"value":84.23,"norm":0.8423,"version":"Vals AI"},"vals:tax_eval_v2":{"value":74.73,"norm":0.7473,"version":"Vals AI"},"swe":{"value":0.5785123966942148,"norm":0.5785,"version":"qwen3.6-plus"},"vals:medscribe":{"value":76.96,"norm":0.7696,"version":"Vals AI"},"livebench:mathematics":{"value":83.72,"norm":0.8372,"version":"LiveBench"},"gpqa":{"value":0.8838383838383839,"norm":0.8838,"version":"qwen3.6-plus"}},"variant":"Qwen 3.6 Plus","match":"exact"},"api":{"or_id":"qwen/qwen3.6-plus","in":0.325,"out":1.95,"context":1000000,"hf":null,"created":"2026-04-02"}},{"id":"claude-mythos-preview","name":"Claude Mythos Preview","lab":"Anthropic","date":"2026-04-07","type":"flagship","major":true,"open":false,"release":{"title":"Claude Mythos Preview (restricted; Project Glasswing)","source":"https://www.anthropic.com/research/mythos-preview","verified":true,"confidence":"high"},"index":{"score":143.7,"lo":82.7,"hi":204.7,"se":35.85,"solve":33.2,"status":"insufficient","areas":["long"],"core_areas":["long"],"testers":["METR"],"tests":{"metr":{"value":0.85205,"norm":0.2631,"version":"claude-mythos-preview-early"}},"variant":"Claude Mythos Preview","match":"exact"}},{"id":"glm-5-1","name":"GLM-5.1","lab":"Zhipu","date":"2026-04-07","type":"open-weights","major":true,"open":true,"release":{"title":"GLM-5.1","source":"https://huggingface.co/zai-org/GLM-5.1","verified":false,"confidence":"high"},"index":{"score":150.6,"lo":134.1,"hi":167.1,"se":3.5,"solve":39.4,"status":"rated","areas":["agents","coding","maths","reasoning","work"],"core_areas":["agents","coding","maths","reasoning","work"],"testers":["Epoch AI","Havard T. Ihle","Mercor","SimpleBench team","Vals AI"],"tests":{"vals:corp_fin_v2":{"value":64.45,"norm":0.6445,"version":"Vals AI"},"vals:programbench":{"value":0,"norm":0,"version":"Vals AI"},"vals:vibe-code":{"value":31.46,"norm":0.3146,"version":"Vals AI"},"x:SimpleQA Verified":{"value":0.34,"norm":0.34,"version":"glm-5.1"},"x:WeirdML":{"value":0.5710000000000001,"norm":0.571,"version":"glm-5.1"},"swe":{"value":0.7417,"norm":0.7417,"version":"glm-5.1"},"x:FrontierMath-Tier-4-2025-07-01-Private":{"value":0.125,"norm":0.125,"version":"glm-5.1"},"vals:legal_bench":{"value":84.39,"norm":0.8439,"version":"Vals AI"},"vals:mmlu_pro":{"value":86.9,"norm":0.869,"version":"Vals AI"},"vals:medcode":{"value":41.6,"norm":0.416,"version":"Vals AI"},"aime":{"value":0.9333333333333333,"norm":0.9333,"version":"glm-5.1"},"vals:fabv2":{"value":44.79,"norm":0.4479,"version":"Vals AI"},"x:SimpleBench":{"value":0.551,"norm":0.551,"version":"glm-5.1"},"gpqa":{"value":0.898989898989899,"norm":0.899,"version":"glm-5.1"},"vals:swebench":{"value":76.4,"norm":0.764,"version":"Vals AI"},"x:FrontierMath-2025-02-28-Private":{"value":0.33448275862068966,"norm":0.3345,"version":"glm-5.1"},"vals:public-benefits-bench":{"value":61.84,"norm":0.6184,"version":"Vals AI"},"apex":{"value":0.409,"norm":0.409,"version":"glm-5.1"},"vals:tax_eval_v2":{"value":71.19,"norm":0.7119,"version":"Vals AI"},"vals:code-migration":{"value":25.77,"norm":0.2577,"version":"Vals AI"},"vals:lcb":{"value":81.38,"norm":0.8138,"version":"Vals AI"},"vals:medscribe":{"value":72.27,"norm":0.7227,"version":"Vals AI"},"x:Chess Puzzles":{"value":0.2,"norm":0.2,"version":"glm-5.1"},"vals:legal_research":{"value":27.89,"norm":0.2789,"version":"Vals AI"},"fm":{"value":0.3684210526315789,"norm":0.3684,"version":"glm-5.1"},"vals:gpqa":{"value":84.52,"norm":0.8452,"version":"Vals AI"}},"variant":"GLM-5.1","match":"exact"},"api":{"or_id":"z-ai/glm-5.1","in":1.4,"out":4.4,"context":204800,"hf":"zai-org/GLM-5.1","created":"2026-04-07"}},{"id":"muse-spark","name":"Muse Spark","lab":"Meta","date":"2026-04-08","type":"flagship","major":true,"open":false,"release":{"title":"Muse Spark (proprietary; MSL)","source":"https://about.fb.com/news/2026/04/introducing-muse-spark-meta-superintelligence-labs/","verified":true,"confidence":"high"},"index":{"score":159.7,"lo":142.7,"hi":176.7,"se":4.24,"solve":49,"status":"rated","areas":["coding","maths","reasoning","work"],"core_areas":["reasoning"],"testers":["Epoch AI","Scale AI","Vals AI"],"tests":{"vals:medcode":{"value":51.31,"norm":0.5131,"version":"Vals AI"},"aime":{"value":0.889,"norm":0.889,"version":"muse-spark"},"vals:swebench":{"value":74.4,"norm":0.744,"version":"Vals AI"},"vals:vibe-code":{"value":19.67,"norm":0.1967,"version":"Vals AI"},"x:FrontierMath-Tier-4-2025-07-01-Private":{"value":0.146,"norm":0.146,"version":"muse-spark"},"vals:mmmu":{"value":87.4,"norm":0.874,"version":"Vals AI"},"hle":{"value":0.4056,"norm":0.4056,"version":"muse-spark"},"vals:medscribe":{"value":85.9,"norm":0.859,"version":"Vals AI"},"vals:gpqa":{"value":89.65,"norm":0.8965,"version":"Vals AI"},"gpqa":{"value":0.898,"norm":0.898,"version":"muse-spark"},"vals:legal_bench":{"value":84.22,"norm":0.8422,"version":"Vals AI"},"vals:mmlu_pro":{"value":87.32,"norm":0.8732,"version":"Vals AI"},"scale:swe_bench_pro":{"value":55,"norm":0.55,"version":"Scale AI"},"vals:corp_fin_v2":{"value":65.11,"norm":0.6511,"version":"Vals AI"},"x:FrontierMath-2025-02-28-Private":{"value":0.39,"norm":0.39,"version":"muse-spark"},"vals:tax_eval_v2":{"value":77.68,"norm":0.7768,"version":"Vals AI"}},"variant":"Muse Spark","match":"exact"}},{"id":"claude-opus-4-7","name":"Claude Opus 4.7","lab":"Anthropic","date":"2026-04-16","type":"flagship","major":true,"open":false,"release":{"title":"Claude Opus 4.7","source":"https://www.anthropic.com/news/claude-opus-4-7","verified":true,"confidence":"high"},"index":{"score":166.8,"lo":150.8,"hi":182.7,"se":2.3,"solve":56.4,"status":"rated","areas":["agents","coding","maths","reasoning","work"],"core_areas":["agents","coding","maths","reasoning","work"],"testers":["Epoch AI","GSO team","Havard T. Ihle","LiveBench","Mercor","Redwood Research","Scale AI","SimpleBench team","Vals AI"],"tests":{"vals:mmmu":{"value":85.55,"norm":0.8555,"version":"Vals AI"},"livebench:coding":{"value":82.09,"norm":0.8209,"version":"LiveBench"},"x:FrontierMath-Tier-4-v2-Private":{"value":0.3170731707317073,"norm":0.3171,"version":"claude-opus-4-7_max"},"aime":{"value":0.978,"norm":0.978,"version":"claude-opus-4-7_xhigh"},"fm":{"value":0.7017543859649122,"norm":0.7018,"version":"claude-opus-4-7_max"},"livebench:agentic-coding":{"value":50.66,"norm":0.5066,"version":"LiveBench"},"x:FrontierMath-2025-02-28-Private":{"value":0.4379310344827586,"norm":0.4379,"version":"claude-opus-4-7_xhigh"},"x:Chess Puzzles":{"value":0.41,"norm":0.41,"version":"claude-opus-4-7_max"},"x:EBR-bench":{"value":0.19047619047619047,"norm":0.1905,"version":"claude-opus-4-7_max"},"vals:tax_eval_v2":{"value":75.27,"norm":0.7527,"version":"Vals AI"},"hle":{"value":0.36200000000000004,"norm":0.362,"version":"claude-opus-4-7_unknown"},"x:SimpleQA Verified":{"value":0.517,"norm":0.517,"version":"claude-opus-4-7_xhigh"},"vals:medcode":{"value":54.86,"norm":0.5486,"version":"Vals AI"},"swe":{"value":0.8347107438016529,"norm":0.8347,"version":"claude-opus-4-7_max"},"vals:emb":{"value":63.73,"norm":0.6373,"version":"Vals AI"},"x:Mystery Game Puzzles":{"value":0.28,"norm":0.28,"version":"claude-opus-4-7_max"},"vals:swebench":{"value":82,"norm":0.82,"version":"Vals AI"},"vals:gpqa":{"value":90.15,"norm":0.9015,"version":"Vals AI"},"x:SimpleBench":{"value":0.617,"norm":0.617,"version":"claude-opus-4-7_unknown"},"livebench:if":{"value":66.74,"norm":0.6674,"version":"LiveBench"},"x:WeirdML":{"value":0.7644,"norm":0.7644,"version":"claude-opus-4-7_high"},"x:FrontierMath-Tier-4-2025-07-01-Private":{"value":0.22916666666666666,"norm":0.2292,"version":"claude-opus-4-7_xhigh"},"livebench:data-analysis":{"value":78.26,"norm":0.7826,"version":"LiveBench"},"vals:tax_agent_bench":{"value":23.74,"norm":0.2374,"version":"Vals AI"},"livebench:language":{"value":77.91,"norm":0.7791,"version":"LiveBench"},"vals:lcb":{"value":85.07,"norm":0.8507,"version":"Vals AI"},"vals:vibe-code":{"value":71,"norm":0.71,"version":"Vals AI"},"x:GSO-Bench":{"value":0.4412,"norm":0.4412,"version":"claude-opus-4-7"},"vals:programbench":{"value":0,"norm":0,"version":"Vals AI"},"x:LMCA":{"value":52.2,"norm":0.522,"version":"claude-opus-4-7_max"},"livebench:reasoning":{"value":87.19,"norm":0.8719,"version":"LiveBench"},"apex":{"value":0.49200000000000005,"norm":0.492,"version":"claude-opus-4-7_max"},"vals:medscribe":{"value":82.95,"norm":0.8295,"version":"Vals AI"},"x:Furniture Assembly":{"value":0.3333333333333333,"norm":0.3333,"version":"claude-opus-4-7_max"},"vals:fabv2":{"value":51.51,"norm":0.5151,"version":"Vals AI"},"vals:corp_fin_v2":{"value":66.08,"norm":0.6608,"version":"Vals AI"},"vals:legal_research":{"value":38.46,"norm":0.3846,"version":"Vals AI"},"vals:mmlu_pro":{"value":89.87,"norm":0.8987,"version":"Vals AI"},"vals:mortgage_tax":{"value":70.27,"norm":0.7027,"version":"Vals AI"},"x:DTBench":{"value":0.9465000000000001,"norm":0.9465,"version":"claude-opus-4-7_max"},"x:MirrorCode":{"value":0.43333333333333335,"norm":0.4333,"version":"claude-opus-4-7_max"},"livebench:mathematics":{"value":92.85,"norm":0.9285,"version":"LiveBench"},"gpqa":{"value":0.9015151515151515,"norm":0.9015,"version":"claude-opus-4-7_xhigh"},"vals:legal_bench":{"value":85.25,"norm":0.8525,"version":"Vals AI"},"vals:code-migration":{"value":43.88,"norm":0.4388,"version":"Vals AI"}},"variant":"Claude Opus 4.7","match":"exact"},"api":{"or_id":"anthropic/claude-opus-4.7","in":5,"out":25,"context":1000000,"hf":null,"created":"2026-04-16"}},{"id":"qwen3-6-35b-a3b","name":"Qwen3.6-35B-A3B","lab":"Alibaba","date":"2026-04-16","type":"open-weights","major":true,"open":true,"release":{"title":"Qwen3.6-35B-A3B (open)","source":null,"verified":false,"confidence":"high"},"index":{"score":138.7,"lo":122.4,"hi":155.1,"se":3.11,"solve":29.5,"status":"rated","areas":["agents","coding","maths","reasoning"],"core_areas":["maths","reasoning"],"testers":["Epoch AI","Havard T. Ihle","Redwood Research","Yann Henon"],"tests":{"x:LMCA":{"value":29.7,"norm":0.297,"version":"qwen3.6-35b-a3b"},"x:Surface Evolver Bench":{"value":0.44375000000000003,"norm":0.4438,"version":"qwen3.6-35b-a3b"},"fm":{"value":0.20350877192982456,"norm":0.2035,"version":"qwen3.6-35b-a3b_none"},"x:DTBench":{"value":0.7387,"norm":0.7387,"version":"qwen3.6-35b-a3b"},"aime":{"value":0.8666666666666667,"norm":0.8667,"version":"qwen3.6-35b-a3b"},"x:Mystery Game Puzzles":{"value":0.22,"norm":0.22,"version":"qwen3.6-35b-a3b_none"},"gpqa":{"value":0.8484848484848485,"norm":0.8485,"version":"qwen3.6-35b-a3b_none"},"x:WeirdML":{"value":0.34490000000000004,"norm":0.3449,"version":"qwen3.6-35b-a3b"}},"variant":"Qwen 3.6 35B-A3B","match":"exact"},"api":{"or_id":"qwen/qwen3.6-35b-a3b","in":0.1,"out":0.9,"context":262144,"hf":"Qwen/Qwen3.6-35B-A3B","created":"2026-04-27"}},{"id":"grok-4-3","name":"Grok 4.3","lab":"xAI","date":"2026-04-17","type":"flagship","major":true,"open":false,"release":{"title":"Grok 4.3 (beta; API/GA 04-30)","source":"https://docs.x.ai/developers/models/grok-4.3","verified":false,"confidence":"high"},"index":{"score":135.1,"lo":114.5,"hi":155.8,"se":8.31,"solve":27.1,"status":"rated","areas":["agents","coding","maths","reasoning","work"],"core_areas":["work"],"testers":["LiveBench","Vals AI"],"tests":{"vals:medcode":{"value":38.07,"norm":0.3807,"version":"Vals AI"},"livebench:mathematics":{"value":84.34,"norm":0.8434,"version":"LiveBench"},"vals:tax_agent_bench":{"value":6.18,"norm":0.0618,"version":"Vals AI"},"vals:skillsbench":{"value":40.64,"norm":0.4064,"version":"Vals AI"},"vals:code-migration":{"value":6.79,"norm":0.0679,"version":"Vals AI"},"livebench:data-analysis":{"value":55.77,"norm":0.5577,"version":"LiveBench"},"vals:lcb":{"value":84.49,"norm":0.8449,"version":"Vals AI"},"vals:swebench":{"value":71.4,"norm":0.714,"version":"Vals AI"},"livebench:if":{"value":62.75,"norm":0.6275,"version":"LiveBench"},"vals:mmmu":{"value":83.06,"norm":0.8306,"version":"Vals AI"},"vals:medscribe":{"value":74.4,"norm":0.744,"version":"Vals AI"},"livebench:language":{"value":73.58,"norm":0.7358,"version":"LiveBench"},"vals:legal_bench":{"value":84.46,"norm":0.8446,"version":"Vals AI"},"vals:gpqa":{"value":91.41,"norm":0.9141,"version":"Vals AI"},"vals:mmlu_pro":{"value":85.84,"norm":0.8584,"version":"Vals AI"},"vals:programbench":{"value":0,"norm":0,"version":"Vals AI"},"livebench:reasoning":{"value":70.82,"norm":0.7082,"version":"LiveBench"},"vals:emb":{"value":18.39,"norm":0.1839,"version":"Vals AI"},"vals:tax_eval_v2":{"value":70.81,"norm":0.7081,"version":"Vals AI"},"vals:vibe-code":{"value":19.4,"norm":0.194,"version":"Vals AI"},"vals:mortgage_tax":{"value":48.25,"norm":0.4825,"version":"Vals AI"},"vals:fabv2":{"value":37.73,"norm":0.3773,"version":"Vals AI"},"livebench:coding":{"value":69.93,"norm":0.6993,"version":"LiveBench"},"livebench:agentic-coding":{"value":18.54,"norm":0.1854,"version":"LiveBench"},"vals:corp_fin_v2":{"value":68.53,"norm":0.6853,"version":"Vals AI"},"vals:public-benefits-bench":{"value":51.69,"norm":0.5169,"version":"Vals AI"},"vals:legal_research":{"value":15.38,"norm":0.1538,"version":"Vals AI"}},"variant":"Grok 4.3","match":"exact"},"api":{"or_id":"x-ai/grok-4.3","in":1.25,"out":2.5,"context":1000000,"hf":null,"created":"2026-05-01"}},{"id":"grok-4-3-beta","name":"Grok 4.3 Beta","lab":"xAI","date":"2026-04-17","type":"flagship","major":false,"open":false,"index":{"score":145.9,"lo":129.1,"hi":162.8,"se":4.02,"solve":35,"status":"rated","areas":["agents","coding","maths","reasoning"],"core_areas":["maths","reasoning"],"testers":["Epoch AI","GeoBench team","Havard T. Ihle","Redwood Research"],"tests":{"x:GeoBench":{"value":0.56,"norm":0.56,"version":"grok-4.3_unknown"},"aime":{"value":0.9333333333333333,"norm":0.9333,"version":"grok-4.3_high"},"fm":{"value":0.4280701754385965,"norm":0.4281,"version":"grok-4.3_high"},"gpqa":{"value":0.8882575757575758,"norm":0.8883,"version":"grok-4.3_high"},"x:FrontierMath-Tier-4-v2-Private":{"value":0.14634146341463414,"norm":0.1463,"version":"grok-4.3_high"},"x:Chess Puzzles":{"value":0.18,"norm":0.18,"version":"grok-4.3_high"},"x:SimpleQA Verified":{"value":0.332,"norm":0.332,"version":"grok-4.3_high"},"x:WeirdML":{"value":0.4989,"norm":0.4989,"version":"grok-4.3_unknown"},"x:LMCA":{"value":38.33,"norm":0.3833,"version":"grok-4.3_high"},"x:DTBench":{"value":0.9067000000000001,"norm":0.9067,"version":"grok-4.3_high"}},"variant":"Grok 4.3 Beta","match":"self"},"date_from":"epoch"},{"id":"kimi-k2-6","name":"Kimi K2.6","lab":"Moonshot","date":"2026-04-20","type":"open-weights","major":true,"open":true,"release":{"title":"Kimi K2.6","source":"https://www.kimi.com/en/blog/kimi-k2-6","verified":true,"confidence":"high"},"index":{"score":152.2,"lo":135.5,"hi":168.9,"se":3.76,"solve":41,"status":"rated","areas":["agents","coding","maths","reasoning","work"],"core_areas":["coding","maths","reasoning","work"],"testers":["Epoch AI","Havard T. Ihle","LiveBench","Redwood Research","Vals AI"],"tests":{"vals:legal_bench":{"value":84.74,"norm":0.8474,"version":"Vals AI"},"vals:gpqa":{"value":89.14,"norm":0.8914,"version":"Vals AI"},"x:EBR-bench":{"value":0.023809523809523808,"norm":0.0238,"version":"kimi-k2.6"},"swe":{"value":0.7665289256198347,"norm":0.7665,"version":"kimi-k2.6"},"vals:medscribe":{"value":78.15,"norm":0.7815,"version":"Vals AI"},"x:Furniture Assembly":{"value":0.21666666666666667,"norm":0.2167,"version":"kimi-k2.6"},"livebench:coding":{"value":78.57,"norm":0.7857,"version":"LiveBench"},"fm":{"value":0.5719298245614035,"norm":0.5719,"version":"kimi-k2.6"},"livebench:mathematics":{"value":84.28,"norm":0.8428,"version":"LiveBench"},"vals:corp_fin_v2":{"value":66.74,"norm":0.6674,"version":"Vals AI"},"vals:swebench":{"value":76.2,"norm":0.762,"version":"Vals AI"},"vals:vibe-code":{"value":37.89,"norm":0.3789,"version":"Vals AI"},"vals:tax_eval_v2":{"value":74.65,"norm":0.7465,"version":"Vals AI"},"x:WeirdML":{"value":0.5586,"norm":0.5586,"version":"kimi-k2.6"},"x:FrontierMath-Tier-4-2025-07-01-Private":{"value":0.1458,"norm":0.1458,"version":"kimi-k2.6"},"livebench:agentic-coding":{"value":46.92,"norm":0.4692,"version":"LiveBench"},"vals:tax_agent_bench":{"value":12.4,"norm":0.124,"version":"Vals AI"},"x:Mystery Game Puzzles":{"value":0.18,"norm":0.18,"version":"kimi-k2.6"},"gpqa":{"value":0.9078282828282829,"norm":0.9078,"version":"kimi-k2.6"},"x:FrontierMath-Tier-4-v2-Private":{"value":0.2564102564102564,"norm":0.2564,"version":"kimi-k2.6"},"x:Chess Puzzles":{"value":0.2,"norm":0.2,"version":"kimi-k2.6"},"vals:code-migration":{"value":27.77,"norm":0.2777,"version":"Vals AI"},"x:LMCA":{"value":37.26,"norm":0.3726,"version":"kimi-k2.6"},"livebench:reasoning":{"value":79.37,"norm":0.7937,"version":"LiveBench"},"vals:mortgage_tax":{"value":65.82,"norm":0.6582,"version":"Vals AI"},"livebench:if":{"value":64.36,"norm":0.6436,"version":"LiveBench"},"x:DTBench":{"value":0.9093000000000001,"norm":0.9093,"version":"kimi-k2.6"},"vals:medcode":{"value":40.14,"norm":0.4014,"version":"Vals AI"},"aime":{"value":0.9611111111111111,"norm":0.9611,"version":"kimi-k2.6"},"vals:fabv2":{"value":44.9,"norm":0.449,"version":"Vals AI"},"vals:lcb":{"value":86.77,"norm":0.8677,"version":"Vals AI"},"vals:mmlu_pro":{"value":87.57,"norm":0.8757,"version":"Vals AI"},"vals:mmmu":{"value":86.3,"norm":0.863,"version":"Vals AI"},"livebench:data-analysis":{"value":65.13,"norm":0.6513,"version":"LiveBench"},"vals:emb":{"value":57.86,"norm":0.5786,"version":"Vals AI"},"livebench:language":{"value":75.14,"norm":0.7514,"version":"LiveBench"},"x:FrontierMath-2025-02-28-Private":{"value":0.3896551724137931,"norm":0.3897,"version":"kimi-k2.6"},"vals:public-benefits-bench":{"value":56.63,"norm":0.5663,"version":"Vals AI"},"vals:legal_research":{"value":15.87,"norm":0.1587,"version":"Vals AI"},"vals:programbench":{"value":0,"norm":0,"version":"Vals AI"},"x:SimpleQA Verified":{"value":0.349,"norm":0.349,"version":"kimi-k2.6"}},"variant":"Kimi K2.6","match":"exact"},"api":{"or_id":"moonshotai/kimi-k2.6","in":0.65,"out":3.41,"context":262144,"hf":"moonshotai/Kimi-K2.6","created":"2026-04-20"}},{"id":"qwen3-6-max-preview","name":"Qwen3.6-Max-Preview","lab":"Alibaba","date":"2026-04-20","type":"flagship","major":true,"open":false,"release":{"title":"Qwen3.6-Max-Preview","source":null,"verified":false,"confidence":"high"},"index":{"score":155.2,"lo":138,"hi":172.4,"se":4.53,"solve":44.2,"status":"rated","areas":["agents","coding","maths","reasoning","work"],"core_areas":["coding","reasoning"],"testers":["Epoch AI","Redwood Research","SimpleBench team","Vals AI"],"tests":{"swe":{"value":0.7665289256198347,"norm":0.7665,"version":"qwen3.6-max-preview"},"gpqa":{"value":0.8737373737373737,"norm":0.8737,"version":"qwen3.6-max-preview"},"vals:swebench":{"value":72.8,"norm":0.728,"version":"Vals AI"},"x:LMCA":{"value":42.53,"norm":0.4253,"version":"qwen3.6-max-preview"},"x:Mystery Game Puzzles":{"value":0.19,"norm":0.19,"version":"qwen3.6-max-preview_none"},"x:SimpleQA Verified":{"value":0.5200400801603207,"norm":0.52,"version":"qwen3.6-max-preview"},"x:DTBench":{"value":0.872,"norm":0.872,"version":"qwen3.6-max-preview"},"x:SimpleBench":{"value":0.63,"norm":0.63,"version":"qwen3.6-max-preview"},"aime":{"value":0.9111111111111111,"norm":0.9111,"version":"qwen3.6-max-preview"},"x:FrontierMath-Tier-4-2025-07-01-Private":{"value":0.041666666666666664,"norm":0.0417,"version":"qwen3.6-max-preview"},"x:FrontierMath-2025-02-28-Private":{"value":0.23103448275862068,"norm":0.231,"version":"qwen3.6-max-preview"},"vals:corp_fin_v2":{"value":66.47,"norm":0.6647,"version":"Vals AI"}},"variant":"Qwen 3.6 Max (Preview)","match":"exact"}},{"id":"mimo-v2-5-pro","name":"MiMo-V2.5-Pro","lab":"Xiaomi","date":"2026-04-22","type":"open-weights","major":true,"open":true,"release":{"title":"MiMo-V2.5-Pro","source":"https://huggingface.co/XiaomiMiMo/MiMo-V2.5-Pro","verified":false,"confidence":"high"},"api":{"or_id":"xiaomi/mimo-v2.5-pro","in":0.435,"out":0.87,"context":1024000,"hf":"XiaomiMiMo/MiMo-V2.5-Pro","created":"2026-04-22"},"open_evidence":"https://huggingface.co/XiaomiMiMo"},{"id":"qwen3-6-27b","name":"Qwen3.6 27B","lab":"Alibaba","date":"2026-04-22","type":"open-weights","major":false,"open":true,"index":{"score":141.1,"lo":124.5,"hi":157.8,"se":3.66,"solve":31.2,"status":"rated","areas":["agents","coding","maths","reasoning","work"],"core_areas":["maths","reasoning"],"testers":["Epoch AI","LiveBench","Redwood Research","Vals AI"],"tests":{"livebench:data-analysis":{"value":70.43,"norm":0.7043,"version":"LiveBench"},"fm":{"value":0.3508771929824561,"norm":0.3509,"version":"qwen3.6-27b"},"livebench:mathematics":{"value":79.87,"norm":0.7987,"version":"LiveBench"},"vals:vibe-code":{"value":11.94,"norm":0.1194,"version":"Vals AI"},"livebench:if":{"value":53.23,"norm":0.5323,"version":"LiveBench"},"x:LMCA":{"value":34.53,"norm":0.3453,"version":"qwen3.6-27b"},"vals:corp_fin_v2":{"value":62.31,"norm":0.6231,"version":"Vals AI"},"vals:swebench":{"value":70,"norm":0.7,"version":"Vals AI"},"livebench:coding":{"value":71.78,"norm":0.7178,"version":"LiveBench"},"livebench:agentic-coding":{"value":39.29,"norm":0.3929,"version":"LiveBench"},"gpqa":{"value":0.8585858585858586,"norm":0.8586,"version":"qwen3.6-27b"},"livebench:language":{"value":63.3,"norm":0.633,"version":"LiveBench"},"vals:tax_eval_v2":{"value":71.26,"norm":0.7126,"version":"Vals AI"},"livebench:reasoning":{"value":70.28,"norm":0.7028,"version":"LiveBench"},"vals:mortgage_tax":{"value":68.28,"norm":0.6828,"version":"Vals AI"},"x:Mystery Game Puzzles":{"value":0.07,"norm":0.07,"version":"qwen3.6-27b_none"},"aime":{"value":0.9111111111111111,"norm":0.9111,"version":"qwen3.6-27b"},"x:DTBench":{"value":0.7813,"norm":0.7813,"version":"qwen3.6-27b"}},"variant":"Qwen3.6 27B","match":"self"},"api":{"or_id":"qwen/qwen3.6-27b","in":0.32,"out":3.2,"context":262144,"hf":"Qwen/Qwen3.6-27B","created":"2026-04-27"},"date_from":"epoch"},{"id":"gpt-5-5","name":"GPT-5.5","lab":"OpenAI","date":"2026-04-23","type":"flagship","major":false,"open":false,"index":{"score":172,"lo":156.2,"hi":187.9,"se":1.97,"solve":61.2,"status":"rated","areas":["agents","coding","maths","novel","reasoning","work"],"core_areas":["agents","coding","maths","novel","reasoning","work"],"testers":["ARC Prize","Datacurve","Epoch AI","FutureSearch","GSO team","Havard T. Ihle","LiveBench","Mercor","Redwood Research","SimpleBench team","Vals AI","Yann Henon"],"tests":{"x:FrontierMath-Tier-4-v2-Private":{"value":0.725,"norm":0.725,"version":"gpt-5.5_xhigh"},"vals:fabv2":{"value":51.76,"norm":0.5176,"version":"Vals AI"},"vals:legal_bench":{"value":86.51,"norm":0.8651,"version":"Vals AI"},"x:GSO-Bench":{"value":0.402,"norm":0.402,"version":"gpt-5.5_xhigh"},"vals:programbench":{"value":0.5,"norm":0.005,"version":"Vals AI"},"x:Surface Evolver Bench":{"value":0.88125,"norm":0.8812,"version":"gpt-5.5_high"},"vals:vibe-code":{"value":69.85,"norm":0.6985,"version":"Vals AI"},"apex":{"value":0.551,"norm":0.551,"version":"gpt-5.5_unknown"},"vals:mmmu":{"value":88.27,"norm":0.8827,"version":"Vals AI"},"x:DeepResearch Bench":{"value":0.54,"norm":0.54,"version":"gpt-5.5_high"},"livebench:coding":{"value":82.15,"norm":0.8215,"version":"LiveBench"},"aime":{"value":1,"norm":1,"version":"gpt-5.5-pre-release_xhigh"},"livebench:mathematics":{"value":95.86,"norm":0.9586,"version":"LiveBench"},"vals:code-migration":{"value":45.16,"norm":0.4516,"version":"Vals AI"},"livebench:if":{"value":70.73,"norm":0.7073,"version":"LiveBench"},"arc:arc_agi_3":{"value":0.4,"norm":0.004,"version":"ARC Prize"},"x:DTBench":{"value":0.96,"norm":0.96,"version":"gpt-5.5_xhigh"},"x:SimpleBench":{"value":0.6900000000000001,"norm":0.69,"version":"gpt-5.5_unknown"},"vals:corp_fin_v2":{"value":68.42,"norm":0.6842,"version":"Vals AI"},"x:Mystery Game Puzzles":{"value":0.56,"norm":0.56,"version":"gpt-5.5_xhigh"},"vals:mmlu_pro":{"value":88.14,"norm":0.8814,"version":"Vals AI"},"vals:public-benefits-bench":{"value":60.89,"norm":0.6089,"version":"Vals AI"},"vals:lcb":{"value":85.3,"norm":0.853,"version":"Vals AI"},"x:SimpleQA Verified":{"value":0.63,"norm":0.63,"version":"gpt-5.5_xhigh"},"arc:arc_agi_2":{"value":85,"norm":0.85,"version":"ARC Prize"},"vals:tax_eval_v2":{"value":74.98,"norm":0.7498,"version":"Vals AI"},"x:LMCA":{"value":54.27,"norm":0.5427,"version":"gpt-5.5_xhigh"},"vals:medcode":{"value":49.1,"norm":0.491,"version":"Vals AI"},"vals:srebench":{"value":3.82,"norm":0.0382,"version":"Vals AI"},"x:MirrorCode":{"value":0.23333333333333334,"norm":0.2333,"version":"gpt-5.5_xhigh"},"vals:medscribe":{"value":86.87,"norm":0.8687,"version":"Vals AI"},"x:Furniture Assembly":{"value":0.44166666666666665,"norm":0.4417,"version":"gpt-5.5_xhigh"},"vals:emb":{"value":64.54,"norm":0.6454,"version":"Vals AI"},"fm":{"value":0.8526315789473684,"norm":0.8526,"version":"gpt-5.5_xhigh"},"vals:legal_research":{"value":40.38,"norm":0.4038,"version":"Vals AI"},"vals:gpqa":{"value":93.18,"norm":0.9318,"version":"Vals AI"},"vals:swebench":{"value":82.6,"norm":0.826,"version":"Vals AI"},"vals:skillsbench":{"value":62.21,"norm":0.6221,"version":"Vals AI"},"x:DeepSWE":{"value":0.6703539823008849,"norm":0.6704,"version":"gpt-5.5_xhigh"},"x:EBR-bench":{"value":0.3428571428571428,"norm":0.3429,"version":"gpt-5.5_xhigh"},"livebench:reasoning":{"value":89.65,"norm":0.8965,"version":"LiveBench"},"x:WeirdML":{"value":0.8491,"norm":0.8491,"version":"gpt-5.5_xhigh"},"vals:mortgage_tax":{"value":68.76,"norm":0.6876,"version":"Vals AI"},"x:FrontierMath-2025-02-28-Private":{"value":0.517,"norm":0.517,"version":"gpt-5.5-pre-release_xhigh"},"x:Chess Puzzles":{"value":0.63,"norm":0.63,"version":"gpt-5.5_xhigh"},"vals:tax_agent_bench":{"value":21.76,"norm":0.2176,"version":"Vals AI"},"gpqa":{"value":0.9400252525252525,"norm":0.94,"version":"gpt-5.5-pre-release_xhigh"},"livebench:language":{"value":87.36,"norm":0.8736,"version":"LiveBench"},"arc:arc_agi_1":{"value":95,"norm":0.95,"version":"ARC Prize"},"swe":{"value":0.8057851239669421,"norm":0.8058,"version":"gpt-5.5-pre-release_xhigh"},"x:FrontierMath-Tier-4-2025-07-01-Private":{"value":0.354,"norm":0.354,"version":"gpt-5.5-pre-release_xhigh"},"livebench:data-analysis":{"value":81.58,"norm":0.8158,"version":"LiveBench"},"livebench:agentic-coding":{"value":53.99,"norm":0.5399,"version":"LiveBench"}},"variant":"GPT-5.5","match":"self"},"api":{"or_id":"openai/gpt-5.5","in":5,"out":30,"context":1050000,"hf":null,"created":"2026-04-24"},"modes":[{"label":"Pro","name":"GPT-5.5 Pro","index":{"score":176.6,"lo":160.1,"hi":193.1,"se":3.44,"solve":64.7,"status":"rated","areas":["agents","maths","novel","reasoning"],"core_areas":["maths","novel","reasoning"],"testers":["ARC Prize","Epoch AI","Redwood Research","SimpleBench team"],"tests":{"arc:arc_agi_2":{"value":84.6,"norm":0.846,"version":"ARC Prize"},"fm":{"value":0.8771929824561403,"norm":0.8772,"version":"gpt-5.5-pro_xhigh"},"x:FrontierMath-Tier-4-v2-Private":{"value":0.7804878048780488,"norm":0.7805,"version":"gpt-5.5-pro_xhigh"},"x:Chess Puzzles":{"value":0.62,"norm":0.62,"version":"gpt-5.5-pro_xhigh"},"x:FrontierMath-Tier-4-2025-07-01-Private":{"value":0.396,"norm":0.396,"version":"gpt-5.5-pro-pre-release_high"},"gpqa":{"value":0.9392135642135643,"norm":0.9392,"version":"gpt-5.5-pro-pre-release_xhigh"},"x:LMCA":{"value":53.94,"norm":0.5394,"version":"gpt-5.5-pro_xhigh"},"arc:arc_agi_1":{"value":96.5,"norm":0.965,"version":"ARC Prize"},"aime":{"value":1,"norm":1,"version":"gpt-5.5-pro-pre-release_xhigh"},"x:FrontierMath-2025-02-28-Private":{"value":0.524,"norm":0.524,"version":"gpt-5.5-pro-pre-release_high"},"x:DTBench":{"value":0.96,"norm":0.96,"version":"gpt-5.5-pro_xhigh"},"x:SimpleBench":{"value":0.7690000000000001,"norm":0.769,"version":"gpt-5.5-pro_high"}}},"api":{"or_id":"openai/gpt-5.5-pro","in":30,"out":180,"context":1050000,"hf":null,"created":"2026-04-24"}}],"aliases":["gpt-5-5-pro"],"successor":"gpt-5-6-sol","date_from":"epoch"},{"id":"gpt-5-5-thinking","name":"GPT-5.5 Thinking","lab":"OpenAI","date":"2026-04-23","type":"flagship","major":true,"open":false,"release":{"title":"GPT-5.5 (Thinking/Pro)","source":"https://openai.com/index/introducing-gpt-5-5/","verified":true,"confidence":"high"}},{"id":"hy3-preview","name":"Hy3 preview","lab":"Tencent","date":"2026-04-23","type":"open-weights","major":true,"open":true,"release":{"title":"Hy3 preview","source":"https://huggingface.co/tencent/Hy3-preview","verified":true,"confidence":"high"},"api":{"or_id":"tencent/hy3-preview","in":0.18,"out":0.6,"context":262144,"hf":"tencent/Hy3-preview","created":"2026-04-22"}},{"id":"deepseek-v4-flash","name":"DeepSeek-V4-Flash (Apr 2026)","lab":"DeepSeek","date":"2026-04-24","type":"open-weights","major":true,"open":true,"release":{"title":"DeepSeek-V4 preview (V4-Pro/V4-Flash)","source":null,"verified":false,"confidence":"high"},"index":{"score":141.8,"lo":124.6,"hi":159,"se":4.59,"solve":31.7,"status":"rated","areas":["agents","coding","maths","reasoning"],"core_areas":[],"testers":["Havard T. Ihle","LiveBench","Redwood Research","SimpleBench team"],"tests":{"livebench:mathematics":{"value":79.65,"norm":0.7965,"version":"LiveBench"},"x:LMCA":{"value":35.88,"norm":0.3588,"version":"deepseek-v4-flash_max"},"x:DTBench":{"value":0.8640000000000001,"norm":0.864,"version":"deepseek-v4-flash_max"},"livebench:coding":{"value":69.23,"norm":0.6923,"version":"LiveBench"},"livebench:language":{"value":70.12,"norm":0.7012,"version":"LiveBench"},"livebench:if":{"value":63.14,"norm":0.6314,"version":"LiveBench"},"x:WeirdML":{"value":0.45630000000000004,"norm":0.4563,"version":"deepseek-v4-flash_max"},"livebench:agentic-coding":{"value":37.63,"norm":0.3763,"version":"LiveBench"},"x:SimpleBench":{"value":0.46299999999999997,"norm":0.463,"version":"deepseek-v4-flash_unknown"},"livebench:data-analysis":{"value":68.02,"norm":0.6802,"version":"LiveBench"},"livebench:reasoning":{"value":70.58,"norm":0.7058,"version":"LiveBench"}},"variant":"DeepSeek-V4-Flash","match":"exact"},"api":{"or_id":"deepseek/deepseek-v4-flash","in":0.09,"out":0.18,"context":1048576,"hf":"deepseek-ai/DeepSeek-V4-Flash","created":"2026-04-24"}},{"id":"deepseek-v4-pro","name":"DeepSeek-V4-Pro (Apr 2026)","lab":"DeepSeek","date":"2026-04-24","type":"open-weights","major":true,"open":true,"release":{"title":"DeepSeek-V4 preview (V4-Pro/V4-Flash)","source":null,"verified":false,"confidence":"high"},"index":{"score":147.1,"lo":130.7,"hi":163.5,"se":3.28,"solve":36.1,"status":"rated","areas":["agents","coding","maths","reasoning"],"core_areas":["coding","maths","reasoning"],"testers":["Epoch AI","Havard T. Ihle","LiveBench","Redwood Research","Yann Henon"],"tests":{"gpqa":{"value":0.9090909090909091,"norm":0.9091,"version":"deepseek-v4-pro_high"},"x:Surface Evolver Bench":{"value":0.4,"norm":0.4,"version":"deepseek-v4-pro_high"},"livebench:mathematics":{"value":90.68,"norm":0.9068,"version":"LiveBench"},"x:SimpleQA Verified":{"value":0.46993987975951906,"norm":0.4699,"version":"deepseek-v4-pro_max"},"x:LMCA":{"value":41.2,"norm":0.412,"version":"deepseek-v4-pro_max"},"livebench:coding":{"value":69.99,"norm":0.6999,"version":"LiveBench"},"fm":{"value":0.45263157894736844,"norm":0.4526,"version":"deepseek-v4-pro_max"},"x:DTBench":{"value":0.9067000000000001,"norm":0.9067,"version":"deepseek-v4-pro_max"},"livebench:agentic-coding":{"value":42.63,"norm":0.4263,"version":"LiveBench"},"x:Mystery Game Puzzles":{"value":0.17,"norm":0.17,"version":"deepseek-v4-pro_none"},"livebench:language":{"value":78.13,"norm":0.7813,"version":"LiveBench"},"swe":{"value":0.7763975155279503,"norm":0.7764,"version":"deepseek-v4-pro_max"},"livebench:data-analysis":{"value":74.54,"norm":0.7454,"version":"LiveBench"},"x:Chess Puzzles":{"value":0.19,"norm":0.19,"version":"deepseek-v4-pro_max"},"livebench:reasoning":{"value":82.69,"norm":0.8269,"version":"LiveBench"},"livebench:if":{"value":62.35,"norm":0.6235,"version":"LiveBench"},"x:FrontierMath-Tier-4-v2-Private":{"value":0.024390243902439025,"norm":0.0244,"version":"deepseek-v4-pro_max"},"aime":{"value":0.9666666666666667,"norm":0.9667,"version":"deepseek-v4-pro_max"},"x:WeirdML":{"value":0.48610000000000003,"norm":0.4861,"version":"deepseek-v4-pro_max"}},"variant":"DeepSeek-V4-Pro","match":"exact"},"api":{"or_id":"deepseek/deepseek-v4-pro","in":0.9,"out":1.5,"context":1048576,"hf":"deepseek-ai/DeepSeek-V4-Pro","created":"2026-04-24"}},{"id":"qwen-3-6-flash","name":"Qwen 3.6 Flash","lab":"Alibaba","date":"2026-04-27","type":"flagship","major":false,"open":false,"index":{"score":135.5,"lo":118.5,"hi":152.4,"se":4.14,"solve":27.3,"status":"rated","areas":["agents","maths","reasoning"],"core_areas":["maths","reasoning"],"testers":["Epoch AI","Redwood Research","SimpleBench team"],"tests":{"gpqa":{"value":0.8333333333333334,"norm":0.8333,"version":"qwen3.6-flash"},"x:Chess Puzzles":{"value":0.12,"norm":0.12,"version":"qwen3.6-flash"},"x:SimpleBench":{"value":0.35200000000000004,"norm":0.352,"version":"qwen3.6-flash"},"x:SimpleQA Verified":{"value":0.15915915915915915,"norm":0.1592,"version":"qwen3.6-flash"},"x:LMCA":{"value":30.98,"norm":0.3098,"version":"qwen3.6-flash"},"x:FrontierMath-Tier-4-2025-07-01-Private":{"value":0,"norm":0,"version":"qwen3.6-flash"},"x:Mystery Game Puzzles":{"value":0.18,"norm":0.18,"version":"qwen3.6-flash_none"},"x:DTBench":{"value":0.7706999999999999,"norm":0.7707,"version":"qwen3.6-flash"},"aime":{"value":0.8444444444444444,"norm":0.8444,"version":"qwen3.6-flash"},"fm":{"value":0.22456140350877193,"norm":0.2246,"version":"qwen3.6-flash_none"},"x:FrontierMath-2025-02-28-Private":{"value":0.10344827586206896,"norm":0.1034,"version":"qwen3.6-flash"}},"variant":"Qwen 3.6 Flash","match":"self"},"date_from":"epoch"},{"id":"doubao-seed-2-0-lite-apr-2026","name":"Doubao Seed 2.0 Lite (Apr 2026)","lab":"ByteDance","date":"2026-04-28","type":"flagship","major":false,"open":false,"index":{"score":143.9,"lo":125,"hi":162.8,"se":6.58,"solve":33.3,"status":"provisional","areas":["maths","reasoning"],"core_areas":["reasoning"],"testers":["Epoch AI"],"tests":{"aime":{"value":0.9111111111111111,"norm":0.9111,"version":"doubao-seed-2.0-lite-260428_high"},"gpqa":{"value":0.8813131313131313,"norm":0.8813,"version":"doubao-seed-2.0-lite-260428_high"},"x:Chess Puzzles":{"value":0.2,"norm":0.2,"version":"doubao-seed-2.0-lite-260428_high"},"x:SimpleQA Verified":{"value":0.315,"norm":0.315,"version":"doubao-seed-2.0-lite-260428_high"}},"variant":"Doubao Seed 2.0 Lite (Apr 2026)","match":"self"},"date_from":"epoch"},{"id":"mistral-medium-3-5","name":"Mistral Medium 3.5","lab":"Mistral","date":"2026-04-28","type":"open-weights","major":true,"open":true,"release":{"title":"Mistral Medium 3.5","source":"https://mistral.ai/news/vibe-remote-agents-mistral-medium-3-5/","verified":false,"confidence":"high"},"index":{"score":131.4,"lo":114.4,"hi":148.4,"se":4.21,"solve":24.8,"status":"rated","areas":["agents","coding","maths","reasoning","work"],"core_areas":["work"],"testers":["Havard T. Ihle","Redwood Research","Vals AI","Yann Henon"],"tests":{"vals:medcode":{"value":33.75,"norm":0.3375,"version":"Vals AI"},"vals:mortgage_tax":{"value":28.89,"norm":0.2889,"version":"Vals AI"},"vals:corp_fin_v2":{"value":58.78,"norm":0.5878,"version":"Vals AI"},"x:DTBench":{"value":0.7547,"norm":0.7547,"version":"mistral-medium-2604"},"vals:medscribe":{"value":67.73,"norm":0.6773,"version":"Vals AI"},"vals:sage":{"value":71.97,"norm":0.7197,"version":"Vals AI"},"vals:legal_research":{"value":9.14,"norm":0.0914,"version":"Vals AI"},"vals:tax_agent_bench":{"value":4.48,"norm":0.0448,"version":"Vals AI"},"vals:mmlu_pro":{"value":75.33,"norm":0.7533,"version":"Vals AI"},"vals:code-migration":{"value":5.13,"norm":0.0513,"version":"Vals AI"},"x:Surface Evolver Bench":{"value":0.26875,"norm":0.2687,"version":"mistral-medium-2604"},"vals:vibe-code":{"value":2.89,"norm":0.0289,"version":"Vals AI"},"x:WeirdML":{"value":0.43710000000000004,"norm":0.4371,"version":"mistral-medium-2604"},"vals:proof_bench":{"value":9,"norm":0.09,"version":"Vals AI"},"vals:emb":{"value":11.19,"norm":0.1119,"version":"Vals AI"},"vals:swebench":{"value":66.4,"norm":0.664,"version":"Vals AI"},"vals:tax_eval_v2":{"value":67.99,"norm":0.6799,"version":"Vals AI"},"x:LMCA":{"value":26.06,"norm":0.2606,"version":"mistral-medium-2604"},"vals:fabv2":{"value":32.1,"norm":0.321,"version":"Vals AI"},"vals:gpqa":{"value":34.85,"norm":0.3485,"version":"Vals AI"}},"variant":"Mistral Medium 3.5","match":"exact"},"api":{"or_id":"mistralai/mistral-medium-3-5","in":1.5,"out":7.5,"context":262144,"hf":"mistralai/Mistral-Medium-3.5-128B","created":"2026-04-30"}},{"id":"gpt-5-5-instant","name":"GPT-5.5 Instant","lab":"OpenAI","date":"2026-05-05","type":"flagship","major":false,"open":false,"index":{"score":134,"lo":117.2,"hi":150.7,"se":3.88,"solve":26.3,"status":"provisional","areas":["maths","reasoning"],"core_areas":["maths","reasoning"],"testers":["Epoch AI"],"tests":{"x:FrontierMath-Tier-4-v2-Private":{"value":0.024390243902439025,"norm":0.0244,"version":"gpt-5.5-instant"},"fm":{"value":0.2631578947368421,"norm":0.2632,"version":"gpt-5.5-instant"},"aime":{"value":0.6805555555555556,"norm":0.6806,"version":"gpt-5.5-instant"},"gpqa":{"value":0.8251262626262627,"norm":0.8251,"version":"gpt-5.5-instant"}},"variant":"GPT-5.5 Instant","match":"self"},"date_from":"epoch"},{"id":"gemini-3-5-flash","name":"Gemini 3.5 Flash","lab":"Google","date":"2026-05-19","type":"flagship","major":true,"open":false,"release":{"title":"Gemini 3.5 Flash (3.5 Pro announced; unreleased)","source":"https://blog.google/innovation-and-ai/models-and-research/gemini-models/gemini-3-5/?_bhlid=25af37ed31cbc0806de888e382fe92e66a1318da&trk=article-ssr-frontend-pulse_little-text-block","verified":true,"confidence":"high"},"index":{"score":161.7,"lo":145.9,"hi":177.4,"se":1.75,"solve":51.2,"status":"rated","areas":["agents","coding","maths","novel","reasoning","work"],"core_areas":["agents","coding","maths","novel","reasoning","work"],"testers":["ARC Prize","Datacurve","Epoch AI","GeoBench team","Havard T. Ihle","Mercor","Redwood Research","SimpleBench team","Vals AI","Yann Henon"],"tests":{"aime":{"value":0.9555555555555556,"norm":0.9556,"version":"gemini-3.5-flash_high"},"x:SimpleBench":{"value":0.767,"norm":0.767,"version":"gemini-3.5-flash_unknown"},"vals:programbench":{"value":0,"norm":0,"version":"Vals AI"},"vals:lcb":{"value":87.6,"norm":0.876,"version":"Vals AI"},"x:EBR-bench":{"value":0.047619047619047616,"norm":0.0476,"version":"gemini-3.5-flash_high"},"apex":{"value":0.275,"norm":0.275,"version":"gemini-3.5-flash_unknown"},"vals:proof_bench":{"value":31,"norm":0.31,"version":"Vals AI"},"vals:tax_eval_v2":{"value":74.37,"norm":0.7437,"version":"Vals AI"},"arc:arc_agi_1":{"value":92.5,"norm":0.925,"version":"ARC Prize"},"vals:medcode":{"value":55.83,"norm":0.5583,"version":"Vals AI"},"vals:fabv2":{"value":57.86,"norm":0.5786,"version":"Vals AI"},"gpqa":{"value":0.928030303030303,"norm":0.928,"version":"gemini-3.5-flash_high"},"vals:skillsbench":{"value":52.74,"norm":0.5274,"version":"Vals AI"},"vals:public-benefits-bench":{"value":59.47,"norm":0.5947,"version":"Vals AI"},"x:Chess Puzzles":{"value":0.45,"norm":0.45,"version":"gemini-3.5-flash_high"},"vals:tax_agent_bench":{"value":21.65,"norm":0.2165,"version":"Vals AI"},"vals:mmlu_pro":{"value":89.52,"norm":0.8952,"version":"Vals AI"},"x:DeepSWE":{"value":0.37389380530973454,"norm":0.3739,"version":"gemini-3.5-flash_medium"},"x:FrontierMath-Tier-4-2025-07-01-Private":{"value":0.14583333333333334,"norm":0.1458,"version":"gemini-3.5-flash_high"},"fm":{"value":0.6280701754385964,"norm":0.6281,"version":"gemini-3.5-flash_high"},"x:FrontierMath-Tier-4-v2-Private":{"value":0.2682926829268293,"norm":0.2683,"version":"gemini-3.5-flash_high"},"vals:gpqa":{"value":92.68,"norm":0.9268,"version":"Vals AI"},"x:Mystery Game Puzzles":{"value":0.32,"norm":0.32,"version":"gemini-3.5-flash_high"},"vals:swebench":{"value":78.8,"norm":0.788,"version":"Vals AI"},"vals:legal_bench":{"value":83.6,"norm":0.836,"version":"Vals AI"},"x:WeirdML":{"value":0.6264000000000001,"norm":0.6264,"version":"gemini-3.5-flash_high"},"vals:mortgage_tax":{"value":68.12,"norm":0.6812,"version":"Vals AI"},"swe":{"value":0.7933884297520661,"norm":0.7934,"version":"gemini-3.5-flash_high"},"vals:medscribe":{"value":76.57,"norm":0.7657,"version":"Vals AI"},"x:GeoBench":{"value":0.85,"norm":0.85,"version":"gemini-3.5-flash_unknown"},"arc:arc_agi_2":{"value":72.1,"norm":0.721,"version":"ARC Prize"},"x:DTBench":{"value":0.9467,"norm":0.9467,"version":"gemini-3.5-flash_high"},"x:LMCA":{"value":47.15,"norm":0.4715,"version":"gemini-3.5-flash_high"},"vals:emb":{"value":63.55,"norm":0.6355,"version":"Vals AI"},"vals:corp_fin_v2":{"value":64.69,"norm":0.6469,"version":"Vals AI"},"x:FrontierMath-2025-02-28-Private":{"value":0.3896551724137931,"norm":0.3897,"version":"gemini-3.5-flash_high"},"vals:legal_research":{"value":30.77,"norm":0.3077,"version":"Vals AI"},"vals:terminal-bench-science":{"value":5.71,"norm":0.0571,"version":"Vals AI"},"vals:terminal-bench-4":{"value":6.06,"norm":0.0606,"version":"Vals AI"},"vals:code-migration":{"value":26.75,"norm":0.2675,"version":"Vals AI"},"x:Surface Evolver Bench":{"value":0.58125,"norm":0.5813,"version":"gemini-3.5-flash_medium"},"vals:vibe-code":{"value":48.68,"norm":0.4868,"version":"Vals AI"},"x:SimpleQA Verified":{"value":0.662,"norm":0.662,"version":"gemini-3.5-flash_high"},"vals:mmmu":{"value":88.27,"norm":0.8827,"version":"Vals AI"}},"variant":"Gemini 3.5 Flash","match":"exact"},"api":{"or_id":"google/gemini-3.5-flash","in":1.5,"out":9,"context":1048576,"hf":null,"created":"2026-05-19"}},{"id":"qwen3-7-max","name":"Qwen3.7-Max","lab":"Alibaba","date":"2026-05-19","type":"flagship","major":true,"open":false,"release":{"title":"Qwen3.7-Max","source":null,"verified":false,"confidence":"high"},"index":{"score":160.2,"lo":143.7,"hi":176.7,"se":3.45,"solve":49.6,"status":"rated","areas":["agents","coding","maths","reasoning","work"],"core_areas":["coding","maths","reasoning","work"],"testers":["Epoch AI","LiveBench","Redwood Research","SimpleBench team","Vals AI"],"tests":{"vals:medscribe":{"value":79.4,"norm":0.794,"version":"Vals AI"},"aime":{"value":0.9555555555555556,"norm":0.9556,"version":"qwen3.7-max"},"vals:emb":{"value":57.04,"norm":0.5704,"version":"Vals AI"},"fm":{"value":0.6456140350877193,"norm":0.6456,"version":"qwen3.7-max"},"livebench:mathematics":{"value":85.25,"norm":0.8525,"version":"LiveBench"},"gpqa":{"value":0.9090909090909091,"norm":0.9091,"version":"qwen3.7-max"},"vals:legal_bench":{"value":84.91,"norm":0.8491,"version":"Vals AI"},"x:Chess Puzzles":{"value":0.27,"norm":0.27,"version":"qwen3.7-max"},"livebench:reasoning":{"value":83.34,"norm":0.8334,"version":"LiveBench"},"vals:medcode":{"value":38.75,"norm":0.3875,"version":"Vals AI"},"livebench:coding":{"value":74.22,"norm":0.7422,"version":"LiveBench"},"x:Mystery Game Puzzles":{"value":0.32,"norm":0.32,"version":"qwen3.7-max"},"x:FrontierMath-Tier-4-v2-Private":{"value":0.34146341463414637,"norm":0.3415,"version":"qwen3.7-max"},"vals:gpqa":{"value":90.15,"norm":0.9015,"version":"Vals AI"},"x:SimpleBench":{"value":0.7040000000000001,"norm":0.704,"version":"qwen3.7-max"},"livebench:data-analysis":{"value":71.79,"norm":0.7179,"version":"LiveBench"},"vals:legal_research":{"value":25.48,"norm":0.2548,"version":"Vals AI"},"vals:tax_agent_bench":{"value":15.83,"norm":0.1583,"version":"Vals AI"},"livebench:if":{"value":74.04,"norm":0.7404,"version":"LiveBench"},"x:DTBench":{"value":0.9227,"norm":0.9227,"version":"qwen3.7-max"},"x:LMCA":{"value":44.02,"norm":0.4402,"version":"qwen3.7-max"},"swe":{"value":0.7727272727272727,"norm":0.7727,"version":"qwen3.7-max"},"livebench:language":{"value":79.74,"norm":0.7974,"version":"LiveBench"},"vals:lcb":{"value":87.06,"norm":0.8706,"version":"Vals AI"},"vals:mmlu_pro":{"value":89.31,"norm":0.8931,"version":"Vals AI"},"vals:code-migration":{"value":13.07,"norm":0.1307,"version":"Vals AI"},"x:SimpleQA Verified":{"value":0.5576730190571715,"norm":0.5577,"version":"qwen3.7-max"},"vals:tax_eval_v2":{"value":75.31,"norm":0.7531,"version":"Vals AI"},"vals:fabv2":{"value":47.78,"norm":0.4778,"version":"Vals AI"},"livebench:agentic-coding":{"value":43.59,"norm":0.4359,"version":"LiveBench"},"vals:corp_fin_v2":{"value":63.71,"norm":0.6371,"version":"Vals AI"},"vals:swebench":{"value":68.8,"norm":0.688,"version":"Vals AI"},"x:EBR-bench":{"value":0.09523809523809523,"norm":0.0952,"version":"qwen3.7-max"},"vals:vibe-code":{"value":47.67,"norm":0.4767,"version":"Vals AI"}},"variant":"Qwen3.7-Max","match":"exact"},"api":{"or_id":"qwen/qwen3.7-max","in":1.475,"out":4.425,"context":1000000,"hf":null,"created":"2026-05-21"}},{"id":"grok-build-0-1","name":"Grok Build 0.1","lab":"xAI","date":"2026-05-20","type":"reasoning","major":true,"open":false,"release":{"title":"Grok Build 0.1 (coding)","source":"https://x.ai/news/grok-build-0-1","verified":false,"confidence":"med"},"index":{"score":141.3,"lo":114,"hi":168.5,"se":13.63,"solve":31.3,"status":"provisional","areas":["coding","maths","reasoning"],"core_areas":[],"testers":["LiveBench"],"tests":{"livebench:if":{"value":65.22,"norm":0.6522,"version":"LiveBench"},"livebench:reasoning":{"value":76.37,"norm":0.7637,"version":"LiveBench"},"livebench:data-analysis":{"value":70.79,"norm":0.7079,"version":"LiveBench"},"livebench:language":{"value":72.46,"norm":0.7246,"version":"LiveBench"},"livebench:agentic-coding":{"value":45.81,"norm":0.4581,"version":"LiveBench"},"livebench:coding":{"value":65.39,"norm":0.6539,"version":"LiveBench"},"livebench:mathematics":{"value":78.43,"norm":0.7843,"version":"LiveBench"}},"variant":"Grok Build 0.1","match":"exact"},"api":{"or_id":"x-ai/grok-build-0.1","in":1,"out":2,"context":256000,"hf":null,"created":"2026-05-20"}},{"id":"claude-opus-4-8","name":"Claude Opus 4.8","lab":"Anthropic","date":"2026-05-28","type":"flagship","major":true,"open":false,"release":{"title":"Claude Opus 4.8","source":"https://www.anthropic.com/research/claude-opus-4-8","verified":true,"confidence":"high"},"index":{"score":170.2,"lo":154.4,"hi":186,"se":1.9,"solve":59.6,"status":"rated","areas":["agents","coding","maths","novel","reasoning","work"],"core_areas":["agents","coding","maths","novel","reasoning","work"],"testers":["ARC Prize","Datacurve","Epoch AI","FutureSearch","GSO team","GeoBench team","Havard T. Ihle","LiveBench","Mercor","Redwood Research","SimpleBench team","Vals AI","Yann Henon"],"tests":{"x:DeepResearch Bench":{"value":0.502,"norm":0.502,"version":"claude-opus-4-8_high"},"x:FrontierMath-Tier-4-2025-07-01-Private":{"value":0.3125,"norm":0.3125,"version":"claude-opus-4-8_max"},"x:Furniture Assembly":{"value":0.425,"norm":0.425,"version":"claude-opus-4-8_max"},"vals:fabv2":{"value":53.92,"norm":0.5392,"version":"Vals AI"},"vals:corp_fin_v2":{"value":66.71,"norm":0.6671,"version":"Vals AI"},"x:Mystery Game Puzzles":{"value":0.36,"norm":0.36,"version":"claude-opus-4-8_max"},"vals:skillsbench":{"value":59.23,"norm":0.5923,"version":"Vals AI"},"arc:arc_agi_2":{"value":72.1,"norm":0.721,"version":"ARC Prize"},"livebench:if":{"value":72.03,"norm":0.7203,"version":"LiveBench"},"livebench:data-analysis":{"value":66.03,"norm":0.6603,"version":"LiveBench"},"livebench:coding":{"value":81.83,"norm":0.8183,"version":"LiveBench"},"fm":{"value":0.8,"norm":0.8,"version":"claude-opus-4-8_max"},"livebench:agentic-coding":{"value":50.5,"norm":0.505,"version":"LiveBench"},"vals:swebench":{"value":88.6,"norm":0.886,"version":"Vals AI"},"x:FrontierMath-2025-02-28-Private":{"value":0.4724137931034483,"norm":0.4724,"version":"claude-opus-4-8_max"},"vals:legal_research":{"value":43.75,"norm":0.4375,"version":"Vals AI"},"vals:lcb":{"value":87.82,"norm":0.8782,"version":"Vals AI"},"x:GeoBench":{"value":0.81,"norm":0.81,"version":"claude-opus-4-8_high"},"vals:terminal-bench-science":{"value":4.29,"norm":0.0429,"version":"Vals AI"},"x:GSO-Bench":{"value":0.4706,"norm":0.4706,"version":"claude-opus-4-8_unknown"},"vals:tax_eval_v2":{"value":75.63,"norm":0.7563,"version":"Vals AI"},"livebench:reasoning":{"value":89.19,"norm":0.8919,"version":"LiveBench"},"apex":{"value":0.489,"norm":0.489,"version":"claude-opus-4-8_max"},"aime":{"value":0.9833333333333333,"norm":0.9833,"version":"claude-opus-4-8_max"},"vals:emb":{"value":69.37,"norm":0.6937,"version":"Vals AI"},"livebench:mathematics":{"value":94.32,"norm":0.9432,"version":"LiveBench"},"gpqa":{"value":0.9103535353535354,"norm":0.9104,"version":"claude-opus-4-8_max"},"vals:legal_bench":{"value":83.57,"norm":0.8357,"version":"Vals AI"},"x:Chess Puzzles":{"value":0.52,"norm":0.52,"version":"claude-opus-4-8_max"},"vals:gpqa":{"value":92.42,"norm":0.9242,"version":"Vals AI"},"x:SimpleBench":{"value":0.648,"norm":0.648,"version":"claude-opus-4-8_unknown"},"x:SimpleQA Verified":{"value":0.53,"norm":0.53,"version":"claude-opus-4-8_max"},"x:WeirdML":{"value":0.8289,"norm":0.8289,"version":"claude-opus-4-8_xhigh"},"vals:mmmu":{"value":86.59,"norm":0.8659,"version":"Vals AI"},"x:DTBench":{"value":0.9493,"norm":0.9493,"version":"claude-opus-4-8_max"},"vals:medcode":{"value":53.22,"norm":0.5322,"version":"Vals AI"},"x:MirrorCode":{"value":0.6555555555555556,"norm":0.6556,"version":"claude-opus-4-8_max"},"livebench:language":{"value":79.66,"norm":0.7966,"version":"LiveBench"},"x:FrontierMath-Tier-4-v2-Private":{"value":0.5609756097560976,"norm":0.561,"version":"claude-opus-4-8_max"},"vals:mmlu_pro":{"value":89.58,"norm":0.8958,"version":"Vals AI"},"vals:programbench":{"value":1,"norm":0.01,"version":"Vals AI"},"vals:code-migration":{"value":47.25,"norm":0.4725,"version":"Vals AI"},"x:Surface Evolver Bench":{"value":0.875,"norm":0.875,"version":"claude-opus-4-8_high"},"vals:medscribe":{"value":85.75,"norm":0.8575,"version":"Vals AI"},"arc:arc_agi_3":{"value":1.5,"norm":0.015,"version":"ARC Prize"},"vals:tax_agent_bench":{"value":30,"norm":0.3,"version":"Vals AI"},"vals:public-benefits-bench":{"value":68.13,"norm":0.6813,"version":"Vals AI"},"x:DeepSWE":{"value":0.5897435897435898,"norm":0.5897,"version":"claude-opus-4-8_max"},"vals:vibe-code":{"value":82.72,"norm":0.8272,"version":"Vals AI"},"vals:terminal-bench-4":{"value":23.23,"norm":0.2323,"version":"Vals AI"},"x:LMCA":{"value":57.51,"norm":0.5751,"version":"claude-opus-4-8_max"},"arc:arc_agi_1":{"value":92.5,"norm":0.925,"version":"ARC Prize"},"x:EBR-bench":{"value":0.2857142857142857,"norm":0.2857,"version":"claude-opus-4-8_max"},"vals:mortgage_tax":{"value":69.91,"norm":0.6991,"version":"Vals AI"}},"variant":"Claude Opus 4.8","match":"exact"},"api":{"or_id":"anthropic/claude-opus-4.8","in":5,"out":25,"context":1000000,"hf":null,"created":"2026-05-27"}},{"id":"step-3-7-flash","name":"Step 3.7 Flash","lab":"StepFun","date":"2026-05-29","type":"open-weights","major":true,"open":true,"release":{"title":"Step 3.7 Flash","source":"https://huggingface.co/stepfun-ai/Step-3.7-Flash","verified":false,"confidence":"high"},"api":{"or_id":"stepfun/step-3.7-flash","in":0.2,"out":1.15,"context":262144,"hf":"stepfun-ai/Step-3.7-Flash","created":"2026-05-28"}},{"id":"minimax-m3","name":"MiniMax-M3","lab":"MiniMax","date":"2026-06-01","type":"open-weights","major":true,"open":true,"release":{"title":"MiniMax-M3","source":"https://huggingface.co/MiniMaxAI/MiniMax-M3","verified":true,"confidence":"high"},"index":{"score":147.3,"lo":130.9,"hi":163.6,"se":3.09,"solve":36.2,"status":"rated","areas":["agents","coding","maths","reasoning","work"],"core_areas":["agents","coding","reasoning","work"],"testers":["Epoch AI","LiveBench","Mercor","Redwood Research","SimpleBench team","Vals AI","Yann Henon"],"tests":{"x:Surface Evolver Bench":{"value":0.5499999999999999,"norm":0.55,"version":"MiniMax-M3"},"vals:medcode":{"value":46.29,"norm":0.4629,"version":"Vals AI"},"vals:medscribe":{"value":87.25,"norm":0.8725,"version":"Vals AI"},"x:Mystery Game Puzzles":{"value":0.08,"norm":0.08,"version":"MiniMax-M3_none"},"gpqa":{"value":0.9090909090909091,"norm":0.9091,"version":"MiniMax-M3"},"vals:legal_research":{"value":29.81,"norm":0.2981,"version":"Vals AI"},"vals:terminal-bench-4":{"value":1.01,"norm":0.0101,"version":"Vals AI"},"vals:sage":{"value":85.53,"norm":0.8553,"version":"Vals AI"},"vals:fabv2":{"value":48.27,"norm":0.4827,"version":"Vals AI"},"vals:corp_fin_v2":{"value":68.1,"norm":0.681,"version":"Vals AI"},"vals:gpqa":{"value":92.68,"norm":0.9268,"version":"Vals AI"},"x:SimpleBench":{"value":0.45799999999999996,"norm":0.458,"version":"MiniMax-M3"},"vals:vcb-1-100":{"value":9.22,"norm":0.0922,"version":"Vals AI"},"vals:mmmu":{"value":81.16,"norm":0.8116,"version":"Vals AI"},"livebench:language":{"value":76.84,"norm":0.7684,"version":"LiveBench"},"vals:swebench":{"value":75,"norm":0.75,"version":"Vals AI"},"vals:lcb":{"value":82.15,"norm":0.8215,"version":"Vals AI"},"vals:vibe-code":{"value":47.57,"norm":0.4757,"version":"Vals AI"},"vals:proof_bench":{"value":18,"norm":0.18,"version":"Vals AI"},"vals:mortgage_tax":{"value":68.36,"norm":0.6836,"version":"Vals AI"},"livebench:if":{"value":57.51,"norm":0.5751,"version":"LiveBench"},"livebench:coding":{"value":68.2,"norm":0.682,"version":"LiveBench"},"aime":{"value":0.7111111111111111,"norm":0.7111,"version":"MiniMax-M3"},"livebench:data-analysis":{"value":76.17,"norm":0.7617,"version":"LiveBench"},"livebench:mathematics":{"value":76.95,"norm":0.7695,"version":"LiveBench"},"vals:tax_agent_bench":{"value":20.59,"norm":0.2059,"version":"Vals AI"},"vals:legal_bench":{"value":85.42,"norm":0.8542,"version":"Vals AI"},"vals:public-benefits-bench":{"value":64.14,"norm":0.6414,"version":"Vals AI"},"x:Chess Puzzles":{"value":0.17,"norm":0.17,"version":"MiniMax-M3"},"vals:mmlu_pro":{"value":84.22,"norm":0.8422,"version":"Vals AI"},"vals:tax_eval_v2":{"value":72.73,"norm":0.7273,"version":"Vals AI"},"x:LMCA":{"value":33.7,"norm":0.337,"version":"MiniMax-M3"},"livebench:reasoning":{"value":74.48,"norm":0.7448,"version":"LiveBench"},"apex":{"value":0.37700000000000006,"norm":0.377,"version":"MiniMax-M3"},"x:DTBench":{"value":0.7893000000000001,"norm":0.7893,"version":"MiniMax-M3"},"vals:emb":{"value":47.76,"norm":0.4776,"version":"Vals AI"},"livebench:agentic-coding":{"value":40.66,"norm":0.4066,"version":"LiveBench"},"vals:skillsbench":{"value":51.5,"norm":0.515,"version":"Vals AI"},"vals:cyber":{"value":65,"norm":0.65,"version":"Vals AI"},"vals:code-migration":{"value":19.93,"norm":0.1993,"version":"Vals AI"}},"variant":"MiniMax-M3","match":"exact"},"api":{"or_id":"minimax/minimax-m3","in":0.3,"out":1.2,"context":1048576,"hf":"MiniMaxAI/Minimax-M3","created":"2026-05-31"}},{"id":"qwen3-7-plus","name":"Qwen3.7-Plus","lab":"Alibaba","date":"2026-06-01","type":"flagship","major":true,"open":false,"release":{"title":"Qwen3.7-Plus","source":null,"verified":false,"confidence":"med"},"index":{"score":147.9,"lo":131.1,"hi":164.7,"se":3.93,"solve":36.8,"status":"rated","areas":["agents","coding","maths","reasoning","work"],"core_areas":["maths","reasoning","work"],"testers":["Epoch AI","Redwood Research","Vals AI"],"tests":{"vals:sage":{"value":89.15,"norm":0.8915,"version":"Vals AI"},"vals:tax_agent_bench":{"value":10.47,"norm":0.1047,"version":"Vals AI"},"vals:skillsbench":{"value":54.3,"norm":0.543,"version":"Vals AI"},"vals:vibe-code":{"value":46.39,"norm":0.4639,"version":"Vals AI"},"vals:emb":{"value":49.34,"norm":0.4934,"version":"Vals AI"},"x:Mystery Game Puzzles":{"value":0.17,"norm":0.17,"version":"qwen3.7-plus_none"},"vals:mortgage_tax":{"value":66.18,"norm":0.6618,"version":"Vals AI"},"vals:fabv2":{"value":38.22,"norm":0.3822,"version":"Vals AI"},"gpqa":{"value":0.8787878787878788,"norm":0.8788,"version":"qwen3.7-plus"},"vals:code-migration":{"value":12.86,"norm":0.1286,"version":"Vals AI"},"x:LMCA":{"value":37.58,"norm":0.3758,"version":"qwen3.7-plus"},"x:DTBench":{"value":0.84,"norm":0.84,"version":"qwen3.7-plus"},"aime":{"value":0.9333333333333333,"norm":0.9333,"version":"qwen3.7-plus"},"fm":{"value":0.34385964912280703,"norm":0.3439,"version":"qwen3.7-plus_none"},"vals:legal_research":{"value":16.35,"norm":0.1635,"version":"Vals AI"}},"variant":"Qwen3.7-Plus","match":"exact"},"api":{"or_id":"qwen/qwen3.7-plus","in":0.32,"out":1.28,"context":1000000,"hf":null,"created":"2026-06-03"}},{"id":"nemotron-3-ultra","name":"Nemotron 3 Ultra","lab":"NVIDIA","date":"2026-06-04","type":"open-weights","major":true,"open":true,"release":{"title":"Nemotron 3 Ultra","source":"https://developer.nvidia.com/blog/nvidia-nemotron-3-ultra-powers-faster-more-efficient-reasoning-for-long-running-agents/","verified":true,"confidence":"high"},"index":{"score":140.6,"lo":124.2,"hi":156.9,"se":3.22,"solve":30.8,"status":"rated","areas":["agents","coding","maths","reasoning","work"],"core_areas":["agents","reasoning","work"],"testers":["Epoch AI","Havard T. Ihle","Mercor","Redwood Research","SimpleBench team","Vals AI"],"tests":{"vals:vibe-code":{"value":7.64,"norm":0.0764,"version":"Vals AI"},"x:DTBench":{"value":0.9013,"norm":0.9013,"version":"nemotron-3-ultra"},"aime":{"value":0.8666666666666667,"norm":0.8667,"version":"nemotron-3-ultra"},"vals:mmlu_pro":{"value":85.76,"norm":0.8576,"version":"Vals AI"},"vals:swebench":{"value":69,"norm":0.69,"version":"Vals AI"},"vals:lcb":{"value":85.98,"norm":0.8598,"version":"Vals AI"},"x:LMCA":{"value":36.9,"norm":0.369,"version":"nemotron-3-ultra"},"vals:medcode":{"value":38.62,"norm":0.3862,"version":"Vals AI"},"x:SimpleBench":{"value":0.41700000000000004,"norm":0.417,"version":"nemotron-3-ultra"},"vals:programbench":{"value":0,"norm":0,"version":"Vals AI"},"apex":{"value":0.227,"norm":0.227,"version":"nemotron-3-ultra"},"vals:tax_eval_v2":{"value":73.1,"norm":0.731,"version":"Vals AI"},"vals:legal_research":{"value":15.38,"norm":0.1538,"version":"Vals AI"},"x:Mystery Game Puzzles":{"value":0.2,"norm":0.2,"version":"nemotron-3-ultra"},"vals:code-migration":{"value":4.91,"norm":0.0491,"version":"Vals AI"},"vals:emb":{"value":31.74,"norm":0.3174,"version":"Vals AI"},"vals:fabv2":{"value":37.67,"norm":0.3767,"version":"Vals AI"},"vals:corp_fin_v2":{"value":65.46,"norm":0.6546,"version":"Vals AI"},"vals:legal_bench":{"value":82.07,"norm":0.8207,"version":"Vals AI"},"x:WeirdML":{"value":0.43450000000000005,"norm":0.4345,"version":"nemotron-3-ultra"},"gpqa":{"value":0.8535353535353535,"norm":0.8535,"version":"nemotron-3-ultra"},"vals:gpqa":{"value":86.11,"norm":0.8611,"version":"Vals AI"}},"variant":"Nemotron 3 Ultra","match":"exact"},"api":{"or_id":"nvidia/nemotron-3-ultra-550b-a55b","in":0.5,"out":2.2,"context":262144,"hf":"nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-BF16","created":"2026-06-04"}},{"id":"claude-fable-5","name":"Claude Fable 5","lab":"Anthropic","date":"2026-06-09","type":"flagship","major":true,"open":false,"release":{"title":"Claude Fable 5 + Mythos 5 (restricted)","source":"https://www.anthropic.com/news/claude-fable-5-mythos-5","verified":true,"confidence":"high"},"index":{"score":183.1,"lo":167.2,"hi":199,"se":2.24,"solve":69,"status":"rated","areas":["agents","coding","maths","novel","reasoning","work"],"core_areas":["agents","coding","maths","novel","reasoning","work"],"testers":["ARC Prize","Datacurve","Epoch AI","GSO team","GeoBench team","Havard T. Ihle","LiveBench","Mercor","Proximal","Redwood Research","SimpleBench team","Vals AI","Yann Henon"],"tests":{"x:Surface Evolver Bench":{"value":0.9500000000000001,"norm":0.95,"version":"claude-fable-5_high"},"livebench:if":{"value":75.77,"norm":0.7577,"version":"LiveBench"},"x:LMCA":{"value":61.15,"norm":0.6115,"version":"claude-fable-5_high"},"vals:medcode":{"value":56.07,"norm":0.5607,"version":"Vals AI"},"livebench:data-analysis":{"value":80.54,"norm":0.8054,"version":"LiveBench"},"x:Furniture Assembly":{"value":0.35833333333333334,"norm":0.3583,"version":"claude-fable-5_max"},"x:DTBench":{"value":0.9840000000000001,"norm":0.984,"version":"claude-fable-5_max"},"vals:public-benefits-bench":{"value":70.43,"norm":0.7043,"version":"Vals AI"},"vals:gpqa":{"value":93.18,"norm":0.9318,"version":"Vals AI"},"vals:fabv2":{"value":56.31,"norm":0.5631,"version":"Vals AI"},"x:SimpleBench":{"value":0.8190000000000001,"norm":0.819,"version":"claude-fable-5_max"},"x:Mystery Game Puzzles":{"value":0.52,"norm":0.52,"version":"claude-fable-5_max"},"x:DeepSWE":{"value":0.6991150442477876,"norm":0.6991,"version":"claude-fable-5_xhigh"},"vals:programbench":{"value":2,"norm":0.02,"version":"Vals AI"},"vals:code-migration":{"value":55.06,"norm":0.5506,"version":"Vals AI"},"livebench:reasoning":{"value":89.65,"norm":0.8965,"version":"LiveBench"},"x:SimpleQA Verified":{"value":0.707,"norm":0.707,"version":"claude-fable-5_xhigh"},"vals:tax_eval_v2":{"value":76.94,"norm":0.7694,"version":"Vals AI"},"vals:medscribe":{"value":88.52,"norm":0.8852,"version":"Vals AI"},"vals:emb":{"value":73.67,"norm":0.7367,"version":"Vals AI"},"fm":{"value":0.8701754385964913,"norm":0.8702,"version":"claude-fable-5_max"},"vals:legal_research":{"value":49.52,"norm":0.4952,"version":"Vals AI"},"x:Chess Puzzles":{"value":0.47,"norm":0.47,"version":"claude-fable-5_max"},"vals:terminal-bench-science":{"value":15.71,"norm":0.1571,"version":"Vals AI"},"gpqa":{"value":0.9387626262626263,"norm":0.9388,"version":"claude-fable-5-fallback_max"},"arc:arc_agi_2":{"value":89.2,"norm":0.892,"version":"ARC Prize"},"x:GSO-Bench":{"value":0.7843000000000001,"norm":0.7843,"version":"claude-fable-5_unknown"},"x:EBR-bench":{"value":0.395238,"norm":0.3952,"version":"claude-fable-5_max"},"vals:vibe-code":{"value":90.35,"norm":0.9035,"version":"Vals AI"},"apex":{"value":0.636,"norm":0.636,"version":"claude-fable-5"},"vals:proof_bench":{"value":95,"norm":0.95,"version":"Vals AI"},"arc:arc_agi_1":{"value":98.5,"norm":0.985,"version":"ARC Prize"},"livebench:agentic-coding":{"value":62.17,"norm":0.6217,"version":"LiveBench"},"livebench:language":{"value":90.68,"norm":0.9068,"version":"LiveBench"},"aime":{"value":1,"norm":1,"version":"claude-fable-5_high"},"x:GeoBench":{"value":0.89,"norm":0.89,"version":"claude-fable-5_high"},"vals:legal_bench":{"value":88.56,"norm":0.8856,"version":"Vals AI"},"vals:terminal-bench-4":{"value":41.41,"norm":0.4141,"version":"Vals AI"},"x:WeirdML":{"value":0.9194,"norm":0.9194,"version":"claude-fable-5_max"},"vals:mmmu":{"value":89.31,"norm":0.8931,"version":"Vals AI"},"vals:corp_fin_v2":{"value":71.83,"norm":0.7183,"version":"Vals AI"},"x:FrontierMath-Tier-4-v2-Private":{"value":0.902,"norm":0.902,"version":"claude-fable-5_max"},"livebench:mathematics":{"value":95.99,"norm":0.9599,"version":"LiveBench"},"vals:tax_agent_bench":{"value":38.28,"norm":0.3828,"version":"Vals AI"},"vals:mmlu_pro":{"value":91.5,"norm":0.915,"version":"Vals AI"},"vals:lcb":{"value":89.78,"norm":0.8978,"version":"Vals AI"},"vals:mortgage_tax":{"value":68.92,"norm":0.6892,"version":"Vals AI"},"x:MirrorCode":{"value":0.7,"norm":0.7,"version":"claude-fable-5_max"},"x:FrontierSWE":{"value":0.4696449352941177,"norm":0.4696,"version":"claude-fable-5_max"},"livebench:coding":{"value":85.99,"norm":0.8599,"version":"LiveBench"},"vals:swebench":{"value":95,"norm":0.95,"version":"Vals AI"}},"variant":"Claude Fable 5","match":"exact"},"api":{"or_id":"anthropic/claude-fable-5","in":10,"out":50,"context":1000000,"hf":null,"created":"2026-06-09"}},{"id":"claude-mythos-5","name":"Claude Mythos 5","lab":"Anthropic","date":"2026-06-09","type":"flagship","major":true,"open":false,"release":{"title":"Claude Fable 5 + Mythos 5 (restricted)","source":"https://www.anthropic.com/news/claude-fable-5-mythos-5","verified":true,"confidence":"high"}},{"id":"kimi-k2-7-code","name":"Kimi K2.7 Code","lab":"Moonshot","date":"2026-06-12","type":"open-weights","major":true,"open":true,"release":{"title":"Kimi K2.7 Code","source":"https://www.kimi.com/resources/kimi-k2-7-code","verified":true,"confidence":"high"},"index":{"score":150.6,"lo":134.5,"hi":166.6,"se":2.48,"solve":39.4,"status":"rated","areas":["agents","coding","maths","reasoning"],"core_areas":["agents","maths","reasoning"],"testers":["Datacurve","Epoch AI","Havard T. Ihle","LiveBench","Mercor","SimpleBench team","Vals AI","Yann Henon"],"tests":{"vals:swebench":{"value":78.2,"norm":0.782,"version":"Vals AI"},"livebench:reasoning":{"value":82.81,"norm":0.8281,"version":"LiveBench"},"x:Surface Evolver Bench":{"value":0.4875,"norm":0.4875,"version":"kimi-k2.7-code"},"apex":{"value":0.376,"norm":0.376,"version":"kimi-k2.7-code"},"x:WeirdML":{"value":0.5412,"norm":0.5412,"version":"kimi-k2.7-code"},"aime":{"value":0.9555555555555556,"norm":0.9556,"version":"kimi-k2.7-code"},"livebench:agentic-coding":{"value":45.66,"norm":0.4566,"version":"LiveBench"},"x:Chess Puzzles":{"value":0.14,"norm":0.14,"version":"kimi-k2.7-code"},"livebench:language":{"value":77.91,"norm":0.7791,"version":"LiveBench"},"vals:skillsbench":{"value":50.04,"norm":0.5004,"version":"Vals AI"},"x:FrontierMath-Tier-4-v2-Private":{"value":0.12195121951219512,"norm":0.122,"version":"kimi-k2.7-code"},"x:SimpleBench":{"value":0.579,"norm":0.579,"version":"kimi-k2.7-code"},"vals:programbench":{"value":0,"norm":0,"version":"Vals AI"},"x:SimpleQA Verified":{"value":0.365,"norm":0.365,"version":"kimi-k2.7-code"},"livebench:data-analysis":{"value":62.66,"norm":0.6266,"version":"LiveBench"},"x:DeepSWE":{"value":0.3053097345132743,"norm":0.3053,"version":"kimi-k2.7-code"},"vals:code-migration":{"value":25.39,"norm":0.2539,"version":"Vals AI"},"vals:vibe-code":{"value":47.21,"norm":0.4721,"version":"Vals AI"},"livebench:coding":{"value":73.96,"norm":0.7396,"version":"LiveBench"},"fm":{"value":0.5403508771929825,"norm":0.5404,"version":"kimi-k2.7-code"},"livebench:mathematics":{"value":79.6,"norm":0.796,"version":"LiveBench"},"gpqa":{"value":0.8787878787878788,"norm":0.8788,"version":"kimi-k2.7-code"},"vals:lcb":{"value":82.05,"norm":0.8205,"version":"Vals AI"},"livebench:if":{"value":56.29,"norm":0.5629,"version":"LiveBench"}},"variant":"Kimi K2.7 Code","match":"exact"},"api":{"or_id":"moonshotai/kimi-k2.7-code","in":0.7,"out":3.75,"context":262144,"hf":"moonshotai/Kimi-K2.7-Code","created":"2026-06-12"}},{"id":"glm-5-2","name":"GLM-5.2","lab":"Zhipu","date":"2026-06-16","type":"open-weights","major":true,"open":true,"release":{"title":"GLM-5.2","source":"https://z.ai/blog/glm-5.2","verified":true,"confidence":"high"},"index":{"score":153.6,"lo":137.9,"hi":169.4,"se":1.73,"solve":42.5,"status":"rated","areas":["agents","coding","maths","novel","reasoning","work"],"core_areas":["agents","coding","maths","novel","reasoning","work"],"testers":["ARC Prize","Datacurve","Epoch AI","Havard T. Ihle","LiveBench","Mercor","Redwood Research","SimpleBench team","Vals AI","Yann Henon"],"tests":{"aime":{"value":0.8638888888888889,"norm":0.8639,"version":"glm-5.2_max"},"x:Mystery Game Puzzles":{"value":0.19,"norm":0.19,"version":"glm-5.2_low"},"x:DeepSWE":{"value":0.4377777777777778,"norm":0.4378,"version":"glm-5.2_max"},"vals:programbench":{"value":0.5,"norm":0.005,"version":"Vals AI"},"x:EBR-bench":{"value":0.09523809523809523,"norm":0.0952,"version":"glm-5.2_max"},"livebench:if":{"value":62.29,"norm":0.6229,"version":"LiveBench"},"x:MirrorCode":{"value":0.36666666666666664,"norm":0.3667,"version":"glm-5.2_max"},"swe":{"value":0.787,"norm":0.787,"version":"glm-5.2_max"},"x:Chess Puzzles":{"value":0.22,"norm":0.22,"version":"glm-5.2_max"},"vals:fabv2":{"value":49.7,"norm":0.497,"version":"Vals AI"},"livebench:mathematics":{"value":89.78,"norm":0.8978,"version":"LiveBench"},"vals:mmlu_pro":{"value":86.71,"norm":0.8671,"version":"Vals AI"},"arc:arc_agi_2":{"value":22.8,"norm":0.228,"version":"ARC Prize"},"vals:legal_bench":{"value":84.07,"norm":0.8407,"version":"Vals AI"},"x:Surface Evolver Bench":{"value":0.55625,"norm":0.5563,"version":"glm-5.2_high"},"vals:medcode":{"value":40.77,"norm":0.4077,"version":"Vals AI"},"vals:medscribe":{"value":83.53,"norm":0.8353,"version":"Vals AI"},"livebench:agentic-coding":{"value":51.77,"norm":0.5177,"version":"LiveBench"},"vals:swebench":{"value":82.8,"norm":0.828,"version":"Vals AI"},"vals:code-migration":{"value":37.87,"norm":0.3787,"version":"Vals AI"},"vals:lcb":{"value":69.5,"norm":0.695,"version":"Vals AI"},"x:SimpleQA Verified":{"value":0.342,"norm":0.342,"version":"glm-5.2_max"},"x:WeirdML":{"value":0.7012,"norm":0.7012,"version":"glm-5.2_max"},"x:LMCA":{"value":45.82,"norm":0.4582,"version":"glm-5.2_max"},"vals:srebench":{"value":0,"norm":0,"version":"Vals AI"},"fm":{"value":0.592057761732852,"norm":0.5921,"version":"glm-5.2_max"},"x:SimpleBench":{"value":0.588,"norm":0.588,"version":"glm-5.2_unknown"},"gpqa":{"value":0.9185606060606061,"norm":0.9186,"version":"glm-5.2_max"},"livebench:language":{"value":76.24,"norm":0.7624,"version":"LiveBench"},"apex":{"value":0.452,"norm":0.452,"version":"glm-5.2_unknown"},"vals:tax_eval_v2":{"value":73.34,"norm":0.7334,"version":"Vals AI"},"arc:arc_agi_1":{"value":77,"norm":0.77,"version":"ARC Prize"},"livebench:data-analysis":{"value":73.74,"norm":0.7374,"version":"LiveBench"},"vals:emb":{"value":61.53,"norm":0.6153,"version":"Vals AI"},"vals:legal_research":{"value":31.25,"norm":0.3125,"version":"Vals AI"},"x:FrontierMath-Tier-4-v2-Private":{"value":0.2926829268292683,"norm":0.2927,"version":"glm-5.2_max"},"vals:gpqa":{"value":85.61,"norm":0.8561,"version":"Vals AI"},"vals:skillsbench":{"value":45.08,"norm":0.4508,"version":"Vals AI"},"livebench:reasoning":{"value":78.63,"norm":0.7863,"version":"LiveBench"},"vals:vibe-code":{"value":63.96,"norm":0.6396,"version":"Vals AI"},"livebench:coding":{"value":79.65,"norm":0.7965,"version":"LiveBench"},"x:DTBench":{"value":0.9359999999999999,"norm":0.936,"version":"glm-5.2_max"},"vals:corp_fin_v2":{"value":66.12,"norm":0.6612,"version":"Vals AI"}},"variant":"GLM-5.2","match":"exact"},"api":{"or_id":"z-ai/glm-5.2","in":0.007,"out":6.8,"context":1048576,"hf":"zai-org/GLM-5.2","created":"2026-06-16"}},{"id":"doubao-seed-2-1-pro","name":"Doubao Seed 2.1 Pro","lab":"ByteDance","date":"2026-06-23","type":"flagship","major":false,"open":false,"index":{"score":149.9,"lo":129.7,"hi":170.1,"se":7.83,"solve":38.7,"status":"provisional","areas":["maths","reasoning"],"core_areas":["reasoning"],"testers":["Epoch AI"],"tests":{"x:SimpleQA Verified":{"value":0.447,"norm":0.447,"version":"doubao-seed-2.1-pro-260628_high"},"x:Chess Puzzles":{"value":0.21,"norm":0.21,"version":"doubao-seed-2.1-pro-260628_high"},"aime":{"value":0.9555555555555556,"norm":0.9556,"version":"doubao-seed-2.1-pro-260628_high"},"gpqa":{"value":0.8933080808080808,"norm":0.8933,"version":"doubao-seed-2.1-pro-260628_high"}},"variant":"Doubao Seed 2.1 Pro","match":"self"},"date_from":"epoch"},{"id":"doubao-seed-2-1-turbo","name":"Doubao Seed 2.1 Turbo","lab":"ByteDance","date":"2026-06-23","type":"flagship","major":false,"open":false,"index":{"score":146.7,"lo":127.1,"hi":166.3,"se":7.29,"solve":35.8,"status":"provisional","areas":["maths","reasoning"],"core_areas":["reasoning"],"testers":["Epoch AI"],"tests":{"gpqa":{"value":0.8958333333333334,"norm":0.8958,"version":"doubao-seed-2.1-turbo-260628_high"},"aime":{"value":0.9611111111111111,"norm":0.9611,"version":"doubao-seed-2.1-turbo-260628_high"},"x:SimpleQA Verified":{"value":0.297,"norm":0.297,"version":"doubao-seed-2.1-turbo-260628_high"},"x:Chess Puzzles":{"value":0.26,"norm":0.26,"version":"doubao-seed-2.1-turbo-260628_high"}},"variant":"Doubao Seed 2.1 Turbo","match":"self"},"date_from":"epoch"},{"id":"claude-sonnet-5","name":"Claude Sonnet 5","lab":"Anthropic","date":"2026-06-30","type":"flagship","major":true,"open":false,"release":{"title":"Claude Sonnet 5","source":"https://www.anthropic.com/news/claude-sonnet-5","verified":true,"confidence":"high"},"index":{"score":164.1,"lo":148.3,"hi":180,"se":2.07,"solve":53.8,"status":"rated","areas":["agents","coding","maths","reasoning","work"],"core_areas":["agents","coding","maths","reasoning","work"],"testers":["Datacurve","Epoch AI","GSO team","GeoBench team","Havard T. Ihle","LiveBench","Mercor","Redwood Research","SimpleBench team","Vals AI","Yann Henon"],"tests":{"livebench:language":{"value":74.97,"norm":0.7497,"version":"LiveBench"},"vals:swebench":{"value":79.6,"norm":0.796,"version":"Vals AI"},"vals:skillsbench":{"value":46.48,"norm":0.4648,"version":"Vals AI"},"livebench:reasoning":{"value":88.69,"norm":0.8869,"version":"LiveBench"},"apex":{"value":0.545,"norm":0.545,"version":"claude-sonnet-5_unknown"},"vals:medcode":{"value":47.54,"norm":0.4754,"version":"Vals AI"},"livebench:data-analysis":{"value":71.74,"norm":0.7174,"version":"LiveBench"},"vals:emb":{"value":66.32,"norm":0.6632,"version":"Vals AI"},"x:DTBench":{"value":0.9248000000000001,"norm":0.9248,"version":"claude-sonnet-5_max"},"livebench:agentic-coding":{"value":59.39,"norm":0.5939,"version":"LiveBench"},"vals:legal_research":{"value":41.83,"norm":0.4183,"version":"Vals AI"},"x:Chess Puzzles":{"value":0.45,"norm":0.45,"version":"claude-sonnet-5_max"},"x:GeoBench":{"value":0.74,"norm":0.74,"version":"claude-sonnet-5_high"},"vals:tax_agent_bench":{"value":29.37,"norm":0.2937,"version":"Vals AI"},"gpqa":{"value":0.9053030303030303,"norm":0.9053,"version":"claude-sonnet-5_xhigh"},"vals:legal_bench":{"value":83.92,"norm":0.8392,"version":"Vals AI"},"vals:terminal-bench-4":{"value":10.61,"norm":0.1061,"version":"Vals AI"},"vals:lcb":{"value":82.43,"norm":0.8243,"version":"Vals AI"},"vals:ioi":{"value":45,"norm":0.45,"version":"Vals AI"},"livebench:if":{"value":63.86,"norm":0.6386,"version":"LiveBench"},"vals:cyber":{"value":70.3,"norm":0.703,"version":"Vals AI"},"vals:mmmu":{"value":83.01,"norm":0.8301,"version":"Vals AI"},"x:MirrorCode":{"value":0.6333333333333333,"norm":0.6333,"version":"claude-sonnet-5_max"},"livebench:coding":{"value":80.68,"norm":0.8068,"version":"LiveBench"},"x:FrontierMath-Tier-4-v2-Private":{"value":0.2926829268292683,"norm":0.2927,"version":"claude-sonnet-5_max"},"vals:gpqa":{"value":88.89,"norm":0.8889,"version":"Vals AI"},"aime":{"value":0.9472222222222222,"norm":0.9472,"version":"claude-sonnet-5_xhigh"},"x:SimpleBench":{"value":0.606,"norm":0.606,"version":"claude-sonnet-5_unknown"},"x:Mystery Game Puzzles":{"value":0.35,"norm":0.35,"version":"claude-sonnet-5_max"},"vals:mmlu_pro":{"value":87.55,"norm":0.8755,"version":"Vals AI"},"x:GSO-Bench":{"value":0.3725,"norm":0.3725,"version":"claude-sonnet-5_unknown"},"x:Surface Evolver Bench":{"value":0.6,"norm":0.6,"version":"claude-sonnet-5_medium"},"vals:vibe-code":{"value":81.33,"norm":0.8133,"version":"Vals AI"},"x:SimpleQA Verified":{"value":0.337,"norm":0.337,"version":"claude-sonnet-5_max"},"x:WeirdML":{"value":0.6898000000000001,"norm":0.6898,"version":"claude-sonnet-5_high"},"vals:medscribe":{"value":76.05,"norm":0.7605,"version":"Vals AI"},"vals:corp_fin_v2":{"value":66.98,"norm":0.6698,"version":"Vals AI"},"vals:terminal-bench-science":{"value":5.71,"norm":0.0571,"version":"Vals AI"},"livebench:mathematics":{"value":92.94,"norm":0.9294,"version":"LiveBench"},"vals:public-benefits-bench":{"value":66.03,"norm":0.6603,"version":"Vals AI"},"x:DeepSWE":{"value":0.5384615384615384,"norm":0.5385,"version":"claude-sonnet-5_max"},"vals:programbench":{"value":0,"norm":0,"version":"Vals AI"},"vals:code-migration":{"value":44.39,"norm":0.4439,"version":"Vals AI"},"vals:proof_bench":{"value":77,"norm":0.77,"version":"Vals AI"},"vals:tax_eval_v2":{"value":75.63,"norm":0.7563,"version":"Vals AI"},"x:LMCA":{"value":50.02,"norm":0.5002,"version":"claude-sonnet-5_high"},"vals:srebench":{"value":0.38,"norm":0.0038,"version":"Vals AI"},"vals:fabv2":{"value":53.91,"norm":0.5391,"version":"Vals AI"},"vals:vcb-1-100":{"value":13.82,"norm":0.1382,"version":"Vals AI"},"vals:mortgage_tax":{"value":70.03,"norm":0.7003,"version":"Vals AI"},"fm":{"value":0.656140350877193,"norm":0.6561,"version":"claude-sonnet-5_max"}},"variant":"Claude Sonnet 5","match":"exact"},"api":{"or_id":"anthropic/claude-sonnet-5","in":2,"out":10,"context":1000000,"hf":null,"created":"2026-06-30"}},{"id":"grok-4-5","name":"Grok 4.5","lab":"xAI","date":"2026-07-08","type":"flagship","major":true,"open":false,"release":{"title":"Grok 4.5","source":"https://x.ai/news/grok-4-5","verified":false,"confidence":"high"},"index":{"score":160.7,"lo":144.9,"hi":176.5,"se":1.88,"solve":50.1,"status":"rated","areas":["agents","coding","maths","novel","reasoning","work"],"core_areas":["agents","coding","maths","novel","reasoning","work"],"testers":["ARC Prize","Datacurve","Epoch AI","Havard T. Ihle","LiveBench","Mercor","Redwood Research","SimpleBench team","Vals AI","Yann Henon"],"tests":{"livebench:agentic-coding":{"value":56.46,"norm":0.5646,"version":"LiveBench"},"x:Chess Puzzles":{"value":0.35,"norm":0.35,"version":"grok-4.5_high"},"x:SimpleBench":{"value":0.7000000000000001,"norm":0.7,"version":"grok-4.5_unknown"},"vals:swebench":{"value":86.6,"norm":0.866,"version":"Vals AI"},"vals:lcb":{"value":87.35,"norm":0.8735,"version":"Vals AI"},"livebench:reasoning":{"value":87.17,"norm":0.8717,"version":"LiveBench"},"apex":{"value":0.562,"norm":0.562,"version":"grok-4.5_unknown"},"vals:cyber":{"value":71.01,"norm":0.7101,"version":"Vals AI"},"x:WeirdML":{"value":0.4643,"norm":0.4643,"version":"grok-4.5_unknown"},"vals:emb":{"value":52.91,"norm":0.5291,"version":"Vals AI"},"aime":{"value":0.9777777777777777,"norm":0.9778,"version":"grok-4.5_high"},"vals:skillsbench":{"value":66.03,"norm":0.6603,"version":"Vals AI"},"vals:code-migration":{"value":36.6,"norm":0.366,"version":"Vals AI"},"vals:mmlu_pro":{"value":89.22,"norm":0.8922,"version":"Vals AI"},"vals:ioi":{"value":40.56,"norm":0.4056,"version":"Vals AI"},"livebench:if":{"value":71.53,"norm":0.7153,"version":"LiveBench"},"livebench:coding":{"value":68.59,"norm":0.6859,"version":"LiveBench"},"vals:medcode":{"value":43.29,"norm":0.4329,"version":"Vals AI"},"x:FrontierMath-Tier-4-v2-Private":{"value":0.24390243902439024,"norm":0.2439,"version":"grok-4.5_high"},"livebench:data-analysis":{"value":73.04,"norm":0.7304,"version":"LiveBench"},"vals:fabv2":{"value":48.35,"norm":0.4835,"version":"Vals AI"},"livebench:mathematics":{"value":90.82,"norm":0.9082,"version":"LiveBench"},"vals:tax_agent_bench":{"value":24.85,"norm":0.2485,"version":"Vals AI"},"vals:programbench":{"value":0,"norm":0,"version":"Vals AI"},"x:Surface Evolver Bench":{"value":0.74375,"norm":0.7438,"version":"grok-4.5_high"},"vals:tax_eval_v2":{"value":71.67,"norm":0.7167,"version":"Vals AI"},"x:LMCA":{"value":45.2,"norm":0.452,"version":"grok-4.5_high"},"vals:srebench":{"value":0.76,"norm":0.0076,"version":"Vals AI"},"x:MirrorCode":{"value":0.32222222222222224,"norm":0.3222,"version":"grok-4.5_high"},"x:Furniture Assembly":{"value":0.225,"norm":0.225,"version":"grok-4.5_high"},"arc:arc_agi_3":{"value":0.3,"norm":0.003,"version":"ARC Prize"},"x:DTBench":{"value":0.9653,"norm":0.9653,"version":"grok-4.5_high"},"vals:corp_fin_v2":{"value":67.41,"norm":0.6741,"version":"Vals AI"},"gpqa":{"value":0.9343434343434344,"norm":0.9343,"version":"grok-4.5_high"},"livebench:language":{"value":82.8,"norm":0.828,"version":"LiveBench"},"vals:legal_research":{"value":37.98,"norm":0.3798,"version":"Vals AI"},"vals:terminal-bench-4":{"value":8.59,"norm":0.0859,"version":"Vals AI"},"vals:mortgage_tax":{"value":61.8,"norm":0.618,"version":"Vals AI"},"vals:medscribe":{"value":86.88,"norm":0.8688,"version":"Vals AI"},"fm":{"value":0.5719298245614035,"norm":0.5719,"version":"grok-4.5_high"},"vals:gpqa":{"value":92.93,"norm":0.9293,"version":"Vals AI"},"vals:legal_bench":{"value":85.97,"norm":0.8597,"version":"Vals AI"},"x:DeepSWE":{"value":0.5376106194690266,"norm":0.5376,"version":"grok-4.5_high"},"vals:vibe-code":{"value":69,"norm":0.69,"version":"Vals AI"},"x:SimpleQA Verified":{"value":0.483,"norm":0.483,"version":"grok-4.5_high"},"vals:mmmu":{"value":61.79,"norm":0.6179,"version":"Vals AI"},"vals:proof_bench":{"value":31,"norm":0.31,"version":"Vals AI"},"arc:arc_agi_2":{"value":52.6,"norm":0.526,"version":"ARC Prize"},"arc:arc_agi_1":{"value":87.2,"norm":0.872,"version":"ARC Prize"}},"variant":"Grok 4.5","match":"exact"},"api":{"or_id":"x-ai/grok-4.5","in":2,"out":6,"context":500000,"hf":null,"created":"2026-07-08"}},{"id":"gpt-5-6-luna","name":"GPT-5.6 Luna","lab":"OpenAI","date":"2026-07-09","type":"flagship","major":true,"open":false,"release":{"title":"GPT-5.6 (Sol/Terra/Luna; preview 06-26)","source":"https://openai.com/index/improving-gpt-5-6-sol-in-chatgpt/","verified":false,"confidence":"high"},"index":{"score":162.2,"lo":146.4,"hi":178,"se":1.9,"solve":51.7,"status":"rated","areas":["agents","coding","maths","novel","reasoning","work"],"core_areas":["agents","coding","maths","novel","reasoning","work"],"testers":["ARC Prize","BALROG team","Datacurve","Epoch AI","Havard T. Ihle","LiveBench","Mercor","Redwood Research","SimpleBench team","Vals AI","Yann Henon"],"tests":{"vals:emb":{"value":67.12,"norm":0.6712,"version":"Vals AI"},"fm":{"value":0.8210526315789474,"norm":0.8211,"version":"gpt-5.6-luna_max"},"gpqa":{"value":0.9160353535353535,"norm":0.916,"version":"gpt-5.6-luna_max"},"livebench:language":{"value":72.57,"norm":0.7257,"version":"LiveBench"},"vals:terminal-bench-science":{"value":0,"norm":0,"version":"Vals AI"},"x:SimpleBench":{"value":0.468,"norm":0.468,"version":"gpt-5.6-luna_unknown"},"vals:tax_agent_bench":{"value":24.82,"norm":0.2482,"version":"Vals AI"},"vals:vcb-1-100":{"value":22.59,"norm":0.2259,"version":"Vals AI"},"livebench:reasoning":{"value":85.64,"norm":0.8564,"version":"LiveBench"},"x:Surface Evolver Bench":{"value":0.61875,"norm":0.6188,"version":"gpt-5.6-luna_medium"},"x:SimpleQA Verified":{"value":0.41,"norm":0.41,"version":"gpt-5.6-luna_max"},"vals:proof_bench":{"value":60,"norm":0.6,"version":"Vals AI"},"x:DTBench":{"value":0.8906999999999999,"norm":0.8907,"version":"gpt-5.6-luna_xhigh"},"arc:arc_agi_1":{"value":90.7,"norm":0.907,"version":"ARC Prize"},"vals:biomysterybench":{"value":61.48,"norm":0.6148,"version":"Vals AI"},"vals:mortgage_tax":{"value":67.29,"norm":0.6729,"version":"Vals AI"},"x:Furniture Assembly":{"value":0.425,"norm":0.425,"version":"gpt-5.6-luna_max"},"arc:arc_agi_3":{"value":0.2,"norm":0.002,"version":"ARC Prize"},"x:Mystery Game Puzzles":{"value":0.21,"norm":0.21,"version":"gpt-5.6-luna_max"},"vals:mmlu_pro":{"value":86.04,"norm":0.8604,"version":"Vals AI"},"vals:ioi":{"value":61.78,"norm":0.6178,"version":"Vals AI"},"apex":{"value":0.43,"norm":0.43,"version":"gpt-5.6-luna_unknown"},"x:LMCA":{"value":48.54,"norm":0.4854,"version":"gpt-5.6-luna_xhigh"},"livebench:coding":{"value":82.91,"norm":0.8291,"version":"LiveBench"},"livebench:mathematics":{"value":87.2,"norm":0.872,"version":"LiveBench"},"vals:corp_fin_v2":{"value":64.22,"norm":0.6422,"version":"Vals AI"},"vals:legal_bench":{"value":84.03,"norm":0.8403,"version":"Vals AI"},"vals:skillsbench":{"value":60.45,"norm":0.6045,"version":"Vals AI"},"x:DeepSWE":{"value":0.671875,"norm":0.6719,"version":"gpt-5.6-luna_max"},"livebench:if":{"value":60.12,"norm":0.6012,"version":"LiveBench"},"x:WeirdML":{"value":0.6086,"norm":0.6086,"version":"gpt-5.6-luna_high"},"vals:mmmu":{"value":85.03,"norm":0.8503,"version":"Vals AI"},"vals:medcode":{"value":42.39,"norm":0.4239,"version":"Vals AI"},"livebench:data-analysis":{"value":78.03,"norm":0.7803,"version":"LiveBench"},"vals:fabv2":{"value":55.04,"norm":0.5504,"version":"Vals AI"},"vals:public-benefits-bench":{"value":61.16,"norm":0.6116,"version":"Vals AI"},"vals:legal_research":{"value":36.54,"norm":0.3654,"version":"Vals AI"},"vals:cyber":{"value":73.63,"norm":0.7363,"version":"Vals AI"},"vals:swebench":{"value":93,"norm":0.93,"version":"Vals AI"},"vals:terminal-bench-4":{"value":11.62,"norm":0.1162,"version":"Vals AI"},"vals:programbench":{"value":0,"norm":0,"version":"Vals AI"},"vals:code-migration":{"value":44.55,"norm":0.4455,"version":"Vals AI"},"vals:vibe-code":{"value":77.06,"norm":0.7706,"version":"Vals AI"},"vals:srebench":{"value":1.53,"norm":0.0153,"version":"Vals AI"},"x:Balrog":{"value":0.456,"norm":0.456,"version":"gpt-5.6-luna_max"},"vals:mysterymechanism":{"value":14.41,"norm":0.1441,"version":"Vals AI"},"vals:tax_eval_v2":{"value":76.17,"norm":0.7617,"version":"Vals AI"},"x:MirrorCode":{"value":0.2111111111111111,"norm":0.2111,"version":"gpt-5.6-luna_max"},"vals:medscribe":{"value":84.39,"norm":0.8439,"version":"Vals AI"},"aime":{"value":0.9833333333333333,"norm":0.9833,"version":"gpt-5.6-luna_max"},"livebench:agentic-coding":{"value":48.43,"norm":0.4843,"version":"LiveBench"},"x:FrontierMath-Tier-4-v2-Private":{"value":0.6097560975609756,"norm":0.6098,"version":"gpt-5.6-luna_max"},"x:Chess Puzzles":{"value":0.43,"norm":0.43,"version":"gpt-5.6-luna_max"},"vals:gpqa":{"value":91.67,"norm":0.9167,"version":"Vals AI"},"arc:arc_agi_2":{"value":59.6,"norm":0.596,"version":"ARC Prize"}},"variant":"GPT-5.6 Luna","match":"exact"},"api":{"or_id":"openai/gpt-5.6-luna","in":0.2,"out":1.2,"context":1050000,"hf":null,"created":"2026-07-09"}},{"id":"gpt-5-6-sol","name":"GPT-5.6 Sol","lab":"OpenAI","date":"2026-07-09","type":"flagship","major":true,"open":false,"release":{"title":"GPT-5.6 (Sol/Terra/Luna; preview 06-26)","source":"https://openai.com/index/improving-gpt-5-6-sol-in-chatgpt/","verified":false,"confidence":"high"},"index":{"score":179.4,"lo":163.5,"hi":195.2,"se":1.98,"solve":66.6,"status":"rated","areas":["agents","coding","maths","novel","reasoning","work"],"core_areas":["coding","maths","novel","reasoning","work"],"testers":["ARC Prize","BALROG team","Datacurve","Epoch AI","GSO team","GeoBench team","Havard T. Ihle","LiveBench","Proximal","Redwood Research","SimpleBench team","Vals AI","Yann Henon"],"tests":{"aime":{"value":1,"norm":1,"version":"gpt-5.6-sol_max"},"x:SimpleBench":{"value":0.648,"norm":0.648,"version":"gpt-5.6-sol_xhigh"},"vals:tax_agent_bench":{"value":31.09,"norm":0.3109,"version":"Vals AI"},"vals:vcb-1-100":{"value":20.01,"norm":0.2001,"version":"Vals AI"},"x:Mystery Game Puzzles":{"value":0.58,"norm":0.58,"version":"gpt-5.6-sol_max"},"vals:mmlu_pro":{"value":89.1,"norm":0.891,"version":"Vals AI"},"vals:legal_bench":{"value":86.97,"norm":0.8697,"version":"Vals AI"},"vals:terminal-bench-4":{"value":37.88,"norm":0.3788,"version":"Vals AI"},"vals:vibe-code":{"value":80.5,"norm":0.805,"version":"Vals AI"},"vals:srebench":{"value":30.53,"norm":0.3053,"version":"Vals AI"},"livebench:if":{"value":71.85,"norm":0.7185,"version":"LiveBench"},"vals:tax_eval_v2":{"value":74.78,"norm":0.7478,"version":"Vals AI"},"vals:biomysterybench":{"value":71.11,"norm":0.7111,"version":"Vals AI"},"vals:emb":{"value":72.34,"norm":0.7234,"version":"Vals AI"},"livebench:mathematics":{"value":96.2,"norm":0.962,"version":"LiveBench"},"gpqa":{"value":0.9349747474747475,"norm":0.935,"version":"gpt-5.6-sol_max"},"vals:terminal-bench-science":{"value":20,"norm":0.2,"version":"Vals AI"},"vals:cyber":{"value":76.31,"norm":0.7631,"version":"Vals AI"},"vals:programbench":{"value":1.5,"norm":0.015,"version":"Vals AI"},"x:Surface Evolver Bench":{"value":0.93125,"norm":0.9313,"version":"gpt-5.6-sol_xhigh"},"x:LMCA":{"value":58.49,"norm":0.5849,"version":"gpt-5.6-sol_xhigh"},"vals:mortgage_tax":{"value":67.29,"norm":0.6729,"version":"Vals AI"},"x:FrontierSWE":{"value":0.3220246705882353,"norm":0.322,"version":"gpt-5.6-sol_max"},"vals:fabv2":{"value":53.76,"norm":0.5376,"version":"Vals AI"},"livebench:coding":{"value":83.94,"norm":0.8394,"version":"LiveBench"},"livebench:agentic-coding":{"value":56.21,"norm":0.5621,"version":"LiveBench"},"vals:gpqa":{"value":95.2,"norm":0.952,"version":"Vals AI"},"x:GeoBench":{"value":0.73,"norm":0.73,"version":"gpt-5.6-sol_high"},"arc:arc_agi_2":{"value":92.5,"norm":0.925,"version":"ARC Prize"},"vals:swebench":{"value":96.2,"norm":0.962,"version":"Vals AI"},"vals:code-migration":{"value":52.92,"norm":0.5292,"version":"Vals AI"},"vals:ioi":{"value":91.17,"norm":0.9117,"version":"Vals AI"},"x:WeirdML":{"value":0.8876000000000001,"norm":0.8876,"version":"gpt-5.6-sol_high"},"vals:mmmu":{"value":88.84,"norm":0.8884,"version":"Vals AI"},"vals:mysterymechanism":{"value":33.33,"norm":0.3333,"version":"Vals AI"},"vals:proof_bench":{"value":83,"norm":0.83,"version":"Vals AI"},"x:DTBench":{"value":0.9573,"norm":0.9573,"version":"gpt-5.6-sol_xhigh"},"arc:arc_agi_1":{"value":97.5,"norm":0.975,"version":"ARC Prize"},"x:Furniture Assembly":{"value":0.5666666666666667,"norm":0.5667,"version":"gpt-5.6-sol_max"},"livebench:language":{"value":87.68,"norm":0.8768,"version":"LiveBench"},"vals:skillsbench":{"value":54.1,"norm":0.541,"version":"Vals AI"},"x:FrontierMath-Tier-4-v2-Private":{"value":0.8292682926829268,"norm":0.8293,"version":"gpt-5.6-sol_max"},"x:GSO-Bench":{"value":0.7647,"norm":0.7647,"version":"gpt-5.6-sol_unknown"},"x:DeepSWE":{"value":0.7266666666666667,"norm":0.7267,"version":"gpt-5.6-sol_max"},"vals:lcb":{"value":82.6,"norm":0.826,"version":"Vals AI"},"livebench:reasoning":{"value":91.65,"norm":0.9165,"version":"LiveBench"},"x:SimpleQA Verified":{"value":0.697,"norm":0.697,"version":"gpt-5.6-sol_max"},"vals:medcode":{"value":43.97,"norm":0.4397,"version":"Vals AI"},"livebench:data-analysis":{"value":79.84,"norm":0.7984,"version":"LiveBench"},"vals:public-benefits-bench":{"value":66.51,"norm":0.6651,"version":"Vals AI"},"vals:legal_research":{"value":48.08,"norm":0.4808,"version":"Vals AI"},"x:Chess Puzzles":{"value":0.59,"norm":0.59,"version":"gpt-5.6-sol_max"},"x:EBR-bench":{"value":0.4476190476190475,"norm":0.4476,"version":"gpt-5.6-sol_max"},"x:Balrog":{"value":0.6,"norm":0.6,"version":"gpt-5.6-sol_max"},"arc:arc_agi_3":{"value":7.8,"norm":0.078,"version":"ARC Prize"},"x:MirrorCode":{"value":0.2,"norm":0.2,"version":"gpt-5.6-sol_high"},"vals:medscribe":{"value":85.23,"norm":0.8523,"version":"Vals AI"},"fm":{"value":0.8912280701754386,"norm":0.8912,"version":"gpt-5.6-sol_max"},"vals:corp_fin_v2":{"value":64.38,"norm":0.6438,"version":"Vals AI"}},"variant":"GPT-5.6 Sol","match":"exact"},"api":{"or_id":"openai/gpt-5.6-sol","in":4,"out":20,"context":1050000,"hf":null,"created":"2026-07-09"},"modes":[{"label":"Pro","name":"GPT-5.6 Sol Pro","index":{"score":181.5,"lo":165.2,"hi":197.9,"se":3.12,"solve":68,"status":"rated","areas":["agents","coding","maths","reasoning"],"core_areas":["agents"],"testers":["Epoch AI","Havard T. Ihle","Mercor","Redwood Research","SimpleBench team"],"tests":{"x:DTBench":{"value":0.96,"norm":0.96,"version":"gpt-5.6-sol_promax"},"x:Chess Puzzles":{"value":0.68,"norm":0.68,"version":"gpt-5.6-sol_promax"},"x:WeirdML":{"value":0.8943000000000001,"norm":0.8943,"version":"gpt-5.6-sol_promax"},"apex":{"value":0.514,"norm":0.514,"version":"gpt-5.6-sol_promax"},"x:FrontierMath-Tier-4-v2-Private":{"value":0.8048780487804879,"norm":0.8049,"version":"gpt-5.6-sol_promax"},"x:SimpleBench":{"value":0.7170000000000001,"norm":0.717,"version":"gpt-5.6-sol_proxhigh"},"x:LMCA":{"value":59.21,"norm":0.5921,"version":"gpt-5.6-sol_promax"}}},"api":{"or_id":"openai/gpt-5.6-sol-pro","in":4,"out":20,"context":1050000,"hf":null,"created":"2026-07-09"}}],"aliases":["gpt-5-6-sol-pro"]},{"id":"gpt-5-6-terra","name":"GPT-5.6 Terra","lab":"OpenAI","date":"2026-07-09","type":"flagship","major":true,"open":false,"release":{"title":"GPT-5.6 (Sol/Terra/Luna; preview 06-26)","source":"https://openai.com/index/improving-gpt-5-6-sol-in-chatgpt/","verified":false,"confidence":"high"},"index":{"score":171.7,"lo":155.8,"hi":187.7,"se":2.26,"solve":60.9,"status":"rated","areas":["agents","coding","maths","novel","reasoning","work"],"core_areas":["agents","coding","maths","novel","reasoning","work"],"testers":["ARC Prize","BALROG team","Datacurve","Epoch AI","Havard T. Ihle","LiveBench","Mercor","Redwood Research","SimpleBench team","Vals AI","Yann Henon"],"tests":{"vals:swebench":{"value":95.4,"norm":0.954,"version":"Vals AI"},"vals:legal_research":{"value":41.35,"norm":0.4135,"version":"Vals AI"},"vals:terminal-bench-science":{"value":10,"norm":0.1,"version":"Vals AI"},"x:WeirdML":{"value":0.7827,"norm":0.7827,"version":"gpt-5.6-terra_high"},"vals:mmmu":{"value":86.47,"norm":0.8647,"version":"Vals AI"},"vals:vcb-1-100":{"value":14.82,"norm":0.1482,"version":"Vals AI"},"x:MirrorCode":{"value":0.4111111111111111,"norm":0.4111,"version":"gpt-5.6-terra_max"},"fm":{"value":0.8596491228070176,"norm":0.8596,"version":"gpt-5.6-terra_max"},"livebench:agentic-coding":{"value":54.95,"norm":0.5495,"version":"LiveBench"},"vals:legal_bench":{"value":85.11,"norm":0.8511,"version":"Vals AI"},"vals:public-benefits-bench":{"value":62.38,"norm":0.6238,"version":"Vals AI"},"livebench:reasoning":{"value":90.63,"norm":0.9063,"version":"LiveBench"},"apex":{"value":0.5820000000000001,"norm":0.582,"version":"gpt-5.6-terra_max"},"vals:proof_bench":{"value":74,"norm":0.74,"version":"Vals AI"},"arc:arc_agi_1":{"value":96.5,"norm":0.965,"version":"ARC Prize"},"vals:srebench":{"value":9.54,"norm":0.0954,"version":"Vals AI"},"vals:medscribe":{"value":82.87,"norm":0.8287,"version":"Vals AI"},"livebench:if":{"value":64.62,"norm":0.6462,"version":"LiveBench"},"vals:gpqa":{"value":90.91,"norm":0.9091,"version":"Vals AI"},"livebench:mathematics":{"value":94.91,"norm":0.9491,"version":"LiveBench"},"vals:corp_fin_v2":{"value":65.31,"norm":0.6531,"version":"Vals AI"},"gpqa":{"value":0.9330808080808081,"norm":0.9331,"version":"gpt-5.6-terra_max"},"vals:programbench":{"value":0.5,"norm":0.005,"version":"Vals AI"},"x:SimpleQA Verified":{"value":0.432,"norm":0.432,"version":"gpt-5.6-terra_max"},"x:LMCA":{"value":54.95,"norm":0.5495,"version":"gpt-5.6-terra_max"},"vals:mortgage_tax":{"value":67.33,"norm":0.6733,"version":"Vals AI"},"x:Furniture Assembly":{"value":0.5416666666666666,"norm":0.5417,"version":"gpt-5.6-terra_max"},"vals:emb":{"value":66.21,"norm":0.6621,"version":"Vals AI"},"x:DTBench":{"value":0.9333,"norm":0.9333,"version":"gpt-5.6-terra_max"},"x:Mystery Game Puzzles":{"value":0.35,"norm":0.35,"version":"gpt-5.6-terra_max"},"livebench:language":{"value":82.89,"norm":0.8289,"version":"LiveBench"},"x:DeepSWE":{"value":0.6962305986696231,"norm":0.6962,"version":"gpt-5.6-terra_max"},"vals:code-migration":{"value":47.8,"norm":0.478,"version":"Vals AI"},"vals:ioi":{"value":87.61,"norm":0.8761,"version":"Vals AI"},"vals:vibe-code":{"value":74.59,"norm":0.7459,"version":"Vals AI"},"vals:mmlu_pro":{"value":86.66,"norm":0.8666,"version":"Vals AI"},"arc:arc_agi_2":{"value":83.9,"norm":0.839,"version":"ARC Prize"},"vals:terminal-bench-4":{"value":22.73,"norm":0.2273,"version":"Vals AI"},"vals:sage":{"value":87.04,"norm":0.8704,"version":"Vals AI"},"livebench:coding":{"value":78.25,"norm":0.7825,"version":"LiveBench"},"arc:arc_agi_3":{"value":0.8,"norm":0.008,"version":"ARC Prize"},"x:FrontierMath-Tier-4-v2-Private":{"value":0.7073170731707317,"norm":0.7073,"version":"gpt-5.6-terra_max"},"aime":{"value":0.9972222222222222,"norm":0.9972,"version":"gpt-5.6-terra_max"},"livebench:data-analysis":{"value":79.31,"norm":0.7931,"version":"LiveBench"},"x:SimpleBench":{"value":0.489,"norm":0.489,"version":"gpt-5.6-terra_xhigh"},"vals:tax_agent_bench":{"value":30.04,"norm":0.3004,"version":"Vals AI"},"vals:skillsbench":{"value":58.9,"norm":0.589,"version":"Vals AI"},"vals:lcb":{"value":85.93,"norm":0.8593,"version":"Vals AI"},"x:Surface Evolver Bench":{"value":0.8375,"norm":0.8375,"version":"gpt-5.6-terra_xhigh"},"x:Balrog":{"value":0.532,"norm":0.532,"version":"gpt-5.6-terra_max"},"vals:cyber":{"value":72.08,"norm":0.7208,"version":"Vals AI"},"vals:tax_eval_v2":{"value":76.17,"norm":0.7617,"version":"Vals AI"},"vals:medcode":{"value":43.41,"norm":0.4341,"version":"Vals AI"},"x:Chess Puzzles":{"value":0.47,"norm":0.47,"version":"gpt-5.6-terra_max"},"vals:fabv2":{"value":54.44,"norm":0.5444,"version":"Vals AI"}},"variant":"GPT-5.6 Terra","match":"exact"},"api":{"or_id":"openai/gpt-5.6-terra","in":2,"out":12,"context":1050000,"hf":null,"created":"2026-07-09"}},{"id":"muse-spark-1-1","name":"Muse Spark 1.1","lab":"Meta","date":"2026-07-09","type":"flagship","major":true,"open":false,"release":{"title":"Muse Spark 1.1 (Meta Model API preview)","source":"https://about.fb.com/news/2026/07/meta-ai-muse-spark-doesnt-just-think-it-acts/","verified":false,"confidence":"high"},"index":{"score":161,"lo":144.9,"hi":177.2,"se":2.71,"solve":50.5,"status":"rated","areas":["agents","coding","maths","reasoning","work"],"core_areas":["agents","work"],"testers":["Datacurve","Epoch AI","LiveBench","Mercor","Redwood Research","Scale AI","Vals AI","Yann Henon"],"tests":{"x:Surface Evolver Bench":{"value":0.525,"norm":0.525,"version":"muse-spark-1.1_high"},"apex":{"value":0.318,"norm":0.318,"version":"muse-spark-1.1"},"livebench:if":{"value":69.64,"norm":0.6964,"version":"LiveBench"},"vals:mmmu":{"value":86.59,"norm":0.8659,"version":"Vals AI"},"livebench:data-analysis":{"value":72.55,"norm":0.7255,"version":"LiveBench"},"vals:fabv2":{"value":57.21,"norm":0.5721,"version":"Vals AI"},"livebench:coding":{"value":77.16,"norm":0.7716,"version":"LiveBench"},"vals:vibe-code":{"value":72.16,"norm":0.7216,"version":"Vals AI"},"vals:code-migration":{"value":31.11,"norm":0.3111,"version":"Vals AI"},"livebench:agentic-coding":{"value":58.54,"norm":0.5854,"version":"LiveBench"},"vals:skillsbench":{"value":59.19,"norm":0.5919,"version":"Vals AI"},"vals:legal_research":{"value":37.98,"norm":0.3798,"version":"Vals AI"},"x:Chess Puzzles":{"value":0.34,"norm":0.34,"version":"muse-spark-1.1_xhigh"},"vals:mmlu_pro":{"value":88.73,"norm":0.8873,"version":"Vals AI"},"vals:tax_eval_v2":{"value":79.72,"norm":0.7972,"version":"Vals AI"},"vals:programbench":{"value":0,"norm":0,"version":"Vals AI"},"x:LMCA":{"value":49.91,"norm":0.4991,"version":"muse-spark-1.1_high"},"vals:mortgage_tax":{"value":66.14,"norm":0.6614,"version":"Vals AI"},"x:SimpleQA Verified":{"value":0.5778894472361809,"norm":0.5779,"version":"muse-spark-1.1"},"x:DTBench":{"value":0.9440000000000001,"norm":0.944,"version":"muse-spark-1.1_high"},"vals:medscribe":{"value":88.89,"norm":0.8889,"version":"Vals AI"},"vals:emb":{"value":56.36,"norm":0.5636,"version":"Vals AI"},"vals:corp_fin_v2":{"value":71.29,"norm":0.7129,"version":"Vals AI"},"livebench:language":{"value":74.34,"norm":0.7434,"version":"LiveBench"},"vals:swebench":{"value":82,"norm":0.82,"version":"Vals AI"},"vals:legal_bench":{"value":84.98,"norm":0.8498,"version":"Vals AI"},"vals:public-benefits-bench":{"value":61.3,"norm":0.613,"version":"Vals AI"},"vals:lcb":{"value":85.91,"norm":0.8591,"version":"Vals AI"},"vals:gpqa":{"value":91.16,"norm":0.9116,"version":"Vals AI"},"scale:swe_bench_pro":{"value":61.5,"norm":0.615,"version":"Scale AI"},"livebench:reasoning":{"value":87.73,"norm":0.8773,"version":"LiveBench"},"livebench:mathematics":{"value":87.14,"norm":0.8714,"version":"LiveBench"},"x:DeepSWE":{"value":0.5331858407079646,"norm":0.5332,"version":"muse-spark-1.1"}},"variant":"Muse Spark 1.1","match":"exact"},"api":{"or_id":"meta/muse-spark-1.1","in":1.25,"out":4.25,"context":1048576,"hf":null,"created":"2026-07-16"}},{"id":"inkling","name":"Inkling","lab":"Thinking Machines","date":"2026-07-15","type":"open-weights","major":true,"open":true,"release":{"title":"Inkling","source":"https://thinkingmachines.ai/news/introducing-inkling/","verified":true,"confidence":"high"},"index":{"score":149,"lo":133.2,"hi":164.9,"se":1.95,"solve":37.9,"status":"rated","areas":["agents","coding","maths","novel","reasoning","work"],"core_areas":["agents","coding","maths","novel","reasoning","work"],"testers":["ARC Prize","Epoch AI","Havard T. Ihle","LiveBench","Mercor","Proximal","Redwood Research","SimpleBench team","Vals AI"],"tests":{"vals:ioi":{"value":14.94,"norm":0.1494,"version":"Vals AI"},"x:SimpleQA Verified":{"value":0.403,"norm":0.403,"version":"Inkling_xhigh"},"x:WeirdML":{"value":0.3229,"norm":0.3229,"version":"Inkling_high"},"x:DTBench":{"value":0.8747,"norm":0.8747,"version":"Inkling_xhigh"},"vals:medscribe":{"value":85.4,"norm":0.854,"version":"Vals AI"},"fm":{"value":0.3333333333333333,"norm":0.3333,"version":"Inkling_xhigh"},"livebench:mathematics":{"value":88.36,"norm":0.8836,"version":"LiveBench"},"vals:skillsbench":{"value":26.09,"norm":0.2609,"version":"Vals AI"},"vals:terminal-bench-4":{"value":0.5,"norm":0.005,"version":"Vals AI"},"livebench:reasoning":{"value":78.35,"norm":0.7835,"version":"LiveBench"},"vals:srebench":{"value":0,"norm":0,"version":"Vals AI"},"apex":{"value":0.33799999999999997,"norm":0.338,"version":"Inkling"},"x:LMCA":{"value":37.58,"norm":0.3758,"version":"Inkling_xhigh"},"vals:mortgage_tax":{"value":63.99,"norm":0.6399,"version":"Vals AI"},"aime":{"value":0.8888888888888888,"norm":0.8889,"version":"Inkling_xhigh"},"livebench:data-analysis":{"value":72.78,"norm":0.7278,"version":"LiveBench"},"vals:emb":{"value":38.36,"norm":0.3836,"version":"Vals AI"},"vals:cyber":{"value":66.79,"norm":0.6679,"version":"Vals AI"},"vals:tax_agent_bench":{"value":15.29,"norm":0.1529,"version":"Vals AI"},"vals:vcb-1-100":{"value":7.33,"norm":0.0733,"version":"Vals AI"},"vals:mmmu":{"value":78.5,"norm":0.785,"version":"Vals AI"},"vals:medcode":{"value":41.19,"norm":0.4119,"version":"Vals AI"},"livebench:agentic-coding":{"value":49.39,"norm":0.4939,"version":"LiveBench"},"vals:corp_fin_v2":{"value":68.57,"norm":0.6857,"version":"Vals AI"},"vals:public-benefits-bench":{"value":58.46,"norm":0.5846,"version":"Vals AI"},"vals:gpqa":{"value":87.12,"norm":0.8712,"version":"Vals AI"},"vals:hlab":{"value":3.75,"norm":0.0375,"version":"Vals AI"},"x:SimpleBench":{"value":0.5,"norm":0.5,"version":"Inkling"},"vals:mmlu_pro":{"value":86.29,"norm":0.8629,"version":"Vals AI"},"vals:programbench":{"value":0,"norm":0,"version":"Vals AI"},"vals:vibe-code":{"value":19.21,"norm":0.1921,"version":"Vals AI"},"livebench:if":{"value":70.1,"norm":0.701,"version":"LiveBench"},"vals:proof_bench":{"value":0,"norm":0,"version":"Vals AI"},"vals:tax_eval_v2":{"value":75.31,"norm":0.7531,"version":"Vals AI"},"livebench:coding":{"value":71.02,"norm":0.7102,"version":"LiveBench"},"gpqa":{"value":0.8825757575757576,"norm":0.8826,"version":"Inkling_xhigh"},"livebench:language":{"value":73.46,"norm":0.7346,"version":"LiveBench"},"x:FrontierMath-Tier-4-v2-Private":{"value":0.04878048780487805,"norm":0.0488,"version":"Inkling_xhigh"},"arc:arc_agi_2":{"value":36.5,"norm":0.365,"version":"ARC Prize"},"vals:code-migration":{"value":11.79,"norm":0.1179,"version":"Vals AI"},"arc:arc_agi_1":{"value":79.5,"norm":0.795,"version":"ARC Prize"},"x:FrontierSWE":{"value":0.04118886470588236,"norm":0.0412,"version":"Inkling_xhigh"},"vals:fabv2":{"value":46.6,"norm":0.466,"version":"Vals AI"},"vals:legal_bench":{"value":82.95,"norm":0.8295,"version":"Vals AI"},"vals:legal_research":{"value":28.36,"norm":0.2836,"version":"Vals AI"},"vals:lcb":{"value":85.52,"norm":0.8552,"version":"Vals AI"},"x:Chess Puzzles":{"value":0.21,"norm":0.21,"version":"Inkling_xhigh"},"vals:swebench":{"value":77.6,"norm":0.776,"version":"Vals AI"}},"variant":"Inkling","match":"exact"},"api":{"or_id":"thinkingmachines/inkling","in":1,"out":4.05,"context":524288,"hf":"thinkingmachines/Inkling","created":"2026-07-18"},"open_evidence":"https://huggingface.co/thinkingmachines/Inkling"},{"id":"kimi-k3","name":"Kimi K3","lab":"Moonshot","date":"2026-07-16","type":"open-weights","major":true,"open":true,"open_from":"2026-07-27","release":{"title":"Kimi K3 (weights 07-27)","source":"https://www.kimi.com/blog/kimi-k3?ref=apifox.com","verified":true,"confidence":"high"},"index":{"score":167.5,"lo":151.7,"hi":183.3,"se":1.92,"solve":57.1,"status":"rated","areas":["agents","coding","maths","novel","reasoning","work"],"core_areas":["agents","coding","maths","novel","reasoning","work"],"testers":["ARC Prize","Datacurve","Epoch AI","Havard T. Ihle","LiveBench","Mercor","Proximal","Redwood Research","SimpleBench team","Vals AI","Yann Henon"],"tests":{"x:LMCA":{"value":52.74,"norm":0.5274,"version":"kimi-k3_max"},"vals:sage":{"value":86.77,"norm":0.8677,"version":"Vals AI"},"aime":{"value":0.9722222222222222,"norm":0.9722,"version":"kimi-k3_max"},"vals:corp_fin_v2":{"value":71.56,"norm":0.7156,"version":"Vals AI"},"x:Mystery Game Puzzles":{"value":0.26,"norm":0.26,"version":"kimi-k3_max"},"vals:public-benefits-bench":{"value":68.2,"norm":0.682,"version":"Vals AI"},"vals:lcb":{"value":87.19,"norm":0.8719,"version":"Vals AI"},"vals:hlab":{"value":12.92,"norm":0.1292,"version":"Vals AI"},"vals:terminal-bench-science":{"value":1.43,"norm":0.0143,"version":"Vals AI"},"vals:tax_agent_bench":{"value":33.37,"norm":0.3337,"version":"Vals AI"},"vals:mmlu_pro":{"value":87.97,"norm":0.8797,"version":"Vals AI"},"arc:arc_agi_1":{"value":94.5,"norm":0.945,"version":"ARC Prize"},"x:Surface Evolver Bench":{"value":0.9500000000000001,"norm":0.95,"version":"kimi-k3_unknown"},"x:SimpleQA Verified":{"value":0.506,"norm":0.506,"version":"kimi-k3_max"},"x:DTBench":{"value":0.912,"norm":0.912,"version":"kimi-k3_max"},"vals:medcode":{"value":49.36,"norm":0.4936,"version":"Vals AI"},"vals:medscribe":{"value":88.05,"norm":0.8805,"version":"Vals AI"},"livebench:data-analysis":{"value":78.73,"norm":0.7873,"version":"LiveBench"},"fm":{"value":0.721830985915493,"norm":0.7218,"version":"kimi-k3_max"},"gpqa":{"value":0.9311868686868687,"norm":0.9312,"version":"kimi-k3_max"},"vals:legal_bench":{"value":86.21,"norm":0.8621,"version":"Vals AI"},"vals:legal_research":{"value":46.15,"norm":0.4615,"version":"Vals AI"},"vals:gpqa":{"value":92.93,"norm":0.9293,"version":"Vals AI"},"x:SimpleBench":{"value":0.6070000000000001,"norm":0.607,"version":"kimi-k3_max"},"vals:vcb-1-100":{"value":18.24,"norm":0.1824,"version":"Vals AI"},"vals:swebench":{"value":93.4,"norm":0.934,"version":"Vals AI"},"vals:code-migration":{"value":16.1,"norm":0.161,"version":"Vals AI"},"vals:vibe-code":{"value":84.97,"norm":0.8497,"version":"Vals AI"},"vals:mmmu":{"value":88.15,"norm":0.8815,"version":"Vals AI"},"vals:biomysterybench":{"value":72.59,"norm":0.7259,"version":"Vals AI"},"x:Furniture Assembly":{"value":0.3416666666666667,"norm":0.3417,"version":"kimi-k3_max"},"livebench:mathematics":{"value":84.44,"norm":0.8444,"version":"LiveBench"},"livebench:language":{"value":85.53,"norm":0.8553,"version":"LiveBench"},"x:DeepSWE":{"value":0.6851441241685144,"norm":0.6851,"version":"kimi-k3_max"},"arc:arc_agi_2":{"value":60.4,"norm":0.604,"version":"ARC Prize"},"livebench:if":{"value":71.36,"norm":0.7136,"version":"LiveBench"},"vals:proof_bench":{"value":87,"norm":0.87,"version":"Vals AI"},"vals:tax_eval_v2":{"value":75.72,"norm":0.7572,"version":"Vals AI"},"vals:mortgage_tax":{"value":66.34,"norm":0.6634,"version":"Vals AI"},"x:MirrorCode":{"value":0.5444444444444444,"norm":0.5444,"version":"kimi-k3_max"},"x:FrontierSWE":{"value":0.25865032058823534,"norm":0.2587,"version":"kimi-k3_max"},"vals:emb":{"value":66.68,"norm":0.6668,"version":"Vals AI"},"livebench:coding":{"value":81.45,"norm":0.8145,"version":"LiveBench"},"x:Chess Puzzles":{"value":0.37,"norm":0.37,"version":"kimi-k3_high"},"vals:programbench":{"value":2,"norm":0.02,"version":"Vals AI"},"vals:fabv2":{"value":53.11,"norm":0.5311,"version":"Vals AI"},"livebench:agentic-coding":{"value":62.17,"norm":0.6217,"version":"LiveBench"},"x:FrontierMath-Tier-4-v2-Private":{"value":0.3902439024390244,"norm":0.3902,"version":"kimi-k3_max"},"vals:cyber":{"value":75.24,"norm":0.7524,"version":"Vals AI"},"vals:terminal-bench-4":{"value":17.17,"norm":0.1717,"version":"Vals AI"},"vals:ioi":{"value":48.94,"norm":0.4894,"version":"Vals AI"},"livebench:reasoning":{"value":90.67,"norm":0.9067,"version":"LiveBench"},"apex":{"value":0.478,"norm":0.478,"version":"kimi-k3_unknown"},"x:WeirdML":{"value":0.8257,"norm":0.8257,"version":"kimi-k3_max"}},"variant":"Kimi K3","match":"exact"},"api":{"or_id":"moonshotai/kimi-k3","in":0.192,"out":16,"context":1048576,"hf":"moonshotai/Kimi-K3","created":"2026-07-16"}},{"id":"longcat-2-0","name":"LongCat 2.0","lab":"Meituan","date":"2026-07-20","type":"open-weights","major":true,"open":true,"release":{"title":"LongCat 2.0","source":null,"verified":false,"confidence":"high"},"api":{"or_id":"meituan/longcat-2.0","in":0.3,"out":1.2,"context":1048756,"hf":"meituan-longcat/LongCat-2.0","created":"2026-07-20"}},{"id":"gemini-3-5-flash-lite","name":"Gemini 3.5 Flash-Lite","lab":"Google","date":"2026-07-21","type":"flagship","major":true,"open":false,"release":{"title":"Gemini 3.6 Flash + 3.5 Flash-Lite","source":"https://blog.google/innovation-and-ai/models-and-research/gemini-models/gemini-3-6-flash-3-5-flash-lite-3-5-flash-cyber/","verified":true,"confidence":"high"},"index":{"score":140.5,"lo":124.6,"hi":156.4,"se":2.15,"solve":30.7,"status":"rated","areas":["agents","coding","maths","novel","reasoning","work"],"core_areas":["agents","maths","novel","reasoning","work"],"testers":["ARC Prize","Epoch AI","Havard T. Ihle","Mercor","Redwood Research","Vals AI"],"tests":{"x:Chess Puzzles":{"value":0.21,"norm":0.21,"version":"gemini-3.5-flash-lite_low"},"aime":{"value":0.7111111111111111,"norm":0.7111,"version":"gemini-3.5-flash-lite_high"},"vals:corp_fin_v2":{"value":60.68,"norm":0.6068,"version":"Vals AI"},"x:Mystery Game Puzzles":{"value":0.19,"norm":0.19,"version":"gemini-3.5-flash-lite_low"},"vals:public-benefits-bench":{"value":51.83,"norm":0.5183,"version":"Vals AI"},"vals:programbench":{"value":0,"norm":0,"version":"Vals AI"},"apex":{"value":0.29300000000000004,"norm":0.293,"version":"gemini-3.5-flash-lite"},"vals:legal_bench":{"value":84.07,"norm":0.8407,"version":"Vals AI"},"x:WeirdML":{"value":0.39,"norm":0.39,"version":"gemini-3.5-flash-lite_high"},"x:LMCA":{"value":37.6,"norm":0.376,"version":"gemini-3.5-flash-lite_high"},"x:DTBench":{"value":0.8347,"norm":0.8347,"version":"gemini-3.5-flash-lite_high"},"vals:gpqa":{"value":83.84,"norm":0.8384,"version":"Vals AI"},"vals:fabv2":{"value":47.44,"norm":0.4744,"version":"Vals AI"},"vals:swebench":{"value":75,"norm":0.75,"version":"Vals AI"},"vals:code-migration":{"value":6.15,"norm":0.0615,"version":"Vals AI"},"vals:vibe-code":{"value":37.16,"norm":0.3716,"version":"Vals AI"},"vals:mmmu":{"value":83.58,"norm":0.8358,"version":"Vals AI"},"arc:arc_agi_2":{"value":10.3,"norm":0.103,"version":"ARC Prize"},"arc:arc_agi_1":{"value":53.5,"norm":0.535,"version":"ARC Prize"},"vals:sage":{"value":79.52,"norm":0.7952,"version":"Vals AI"},"vals:emb":{"value":43.23,"norm":0.4323,"version":"Vals AI"},"vals:medcode":{"value":43.49,"norm":0.4349,"version":"Vals AI"},"x:FrontierMath-Tier-4-v2-Private":{"value":0,"norm":0,"version":"gemini-3.5-flash-lite_high"},"vals:tax_agent_bench":{"value":7.25,"norm":0.0725,"version":"Vals AI"},"vals:legal_research":{"value":13.94,"norm":0.1394,"version":"Vals AI"},"vals:lcb":{"value":79.01,"norm":0.7901,"version":"Vals AI"},"fm":{"value":0.2596491228070175,"norm":0.2596,"version":"gemini-3.5-flash-lite_high"},"gpqa":{"value":0.8333333333333334,"norm":0.8333,"version":"gemini-3.5-flash-lite_high"},"vals:cyber":{"value":59.11,"norm":0.5911,"version":"Vals AI"},"vals:mmlu_pro":{"value":85.84,"norm":0.8584,"version":"Vals AI"},"vals:tax_eval_v2":{"value":72.61,"norm":0.7261,"version":"Vals AI"},"vals:mortgage_tax":{"value":68.68,"norm":0.6868,"version":"Vals AI"},"vals:medscribe":{"value":70.89,"norm":0.7089,"version":"Vals AI"}},"variant":"Gemini 3.5 Flash-Lite","match":"exact"},"api":{"or_id":"google/gemini-3.5-flash-lite","in":0.3,"out":2.5,"context":1048576,"hf":null,"created":"2026-07-21"}},{"id":"gemini-3-6-flash","name":"Gemini 3.6 Flash","lab":"Google","date":"2026-07-21","type":"flagship","major":true,"open":false,"release":{"title":"Gemini 3.6 Flash + 3.5 Flash-Lite","source":"https://blog.google/innovation-and-ai/models-and-research/gemini-models/gemini-3-6-flash-3-5-flash-lite-3-5-flash-cyber/","verified":true,"confidence":"high"},"index":{"score":160.5,"lo":144.7,"hi":176.3,"se":1.87,"solve":49.9,"status":"rated","areas":["agents","coding","maths","novel","reasoning","work"],"core_areas":["agents","maths","novel","reasoning","work"],"testers":["ARC Prize","Datacurve","Epoch AI","Havard T. Ihle","Mercor","Redwood Research","Vals AI"],"tests":{"vals:medscribe":{"value":79.66,"norm":0.7966,"version":"Vals AI"},"x:Furniture Assembly":{"value":0.23333333333333334,"norm":0.2333,"version":"gemini-3.6-flash_high"},"vals:emb":{"value":65.41,"norm":0.6541,"version":"Vals AI"},"vals:cyber":{"value":45.36,"norm":0.4536,"version":"Vals AI"},"vals:fabv2":{"value":56.3,"norm":0.563,"version":"Vals AI"},"vals:mmlu_pro":{"value":89.28,"norm":0.8928,"version":"Vals AI"},"vals:legal_research":{"value":25,"norm":0.25,"version":"Vals AI"},"aime":{"value":0.9416666666666667,"norm":0.9417,"version":"gemini-3.6-flash_high"},"vals:tax_agent_bench":{"value":18.04,"norm":0.1804,"version":"Vals AI"},"x:Mystery Game Puzzles":{"value":0.3,"norm":0.3,"version":"gemini-3.6-flash_high"},"arc:arc_agi_2":{"value":60.4,"norm":0.604,"version":"ARC Prize"},"vals:swebench":{"value":79.6,"norm":0.796,"version":"Vals AI"},"vals:programbench":{"value":0,"norm":0,"version":"Vals AI"},"vals:code-migration":{"value":30.93,"norm":0.3093,"version":"Vals AI"},"vals:tax_eval_v2":{"value":74.86,"norm":0.7486,"version":"Vals AI"},"vals:biomysterybench":{"value":58.52,"norm":0.5852,"version":"Vals AI"},"vals:mortgage_tax":{"value":67.61,"norm":0.6761,"version":"Vals AI"},"vals:corp_fin_v2":{"value":63.33,"norm":0.6333,"version":"Vals AI"},"x:Chess Puzzles":{"value":0.42,"norm":0.42,"version":"gemini-3.6-flash_low"},"vals:ioi":{"value":35.06,"norm":0.3506,"version":"Vals AI"},"vals:vibe-code":{"value":64.01,"norm":0.6401,"version":"Vals AI"},"x:SimpleQA Verified":{"value":0.662,"norm":0.662,"version":"gemini-3.6-flash_high"},"x:WeirdML":{"value":0.561,"norm":0.561,"version":"gemini-3.6-flash_high"},"vals:mmmu":{"value":88.38,"norm":0.8838,"version":"Vals AI"},"arc:arc_agi_1":{"value":91.2,"norm":0.912,"version":"ARC Prize"},"vals:medcode":{"value":53.15,"norm":0.5315,"version":"Vals AI"},"x:MirrorCode":{"value":0.23333333333333334,"norm":0.2333,"version":"gemini-3.6-flash_high"},"fm":{"value":0.5894736842105263,"norm":0.5895,"version":"gemini-3.6-flash_high"},"gpqa":{"value":0.9412878787878788,"norm":0.9413,"version":"gemini-3.6-flash_high"},"vals:public-benefits-bench":{"value":56.56,"norm":0.5656,"version":"Vals AI"},"x:FrontierMath-Tier-4-v2-Private":{"value":0.21951219512195122,"norm":0.2195,"version":"gemini-3.6-flash_high"},"vals:gpqa":{"value":93.43,"norm":0.9343,"version":"Vals AI"},"vals:terminal-bench-science":{"value":4.29,"norm":0.0429,"version":"Vals AI"},"vals:legal_bench":{"value":86.7,"norm":0.867,"version":"Vals AI"},"x:DeepSWE":{"value":0.4668141592920354,"norm":0.4668,"version":"gemini-3.6-flash_high"},"vals:lcb":{"value":88.08,"norm":0.8808,"version":"Vals AI"},"apex":{"value":0.469,"norm":0.469,"version":"gemini-3.6-flash_unknown"},"x:DTBench":{"value":0.9547,"norm":0.9547,"version":"gemini-3.6-flash_high"},"x:LMCA":{"value":44.93,"norm":0.4493,"version":"gemini-3.6-flash_high"}},"variant":"Gemini 3.6 Flash","match":"exact"},"api":{"or_id":"google/gemini-3.6-flash","in":0.75,"out":3.75,"context":1048576,"hf":null,"created":"2026-07-21"}},{"id":"ling-3-0","name":"Ling 3.0","lab":"Ant Group","date":"2026-07-23","type":"open-weights","major":true,"open":true,"release":{"title":"Ling 3.0","source":null,"verified":false,"confidence":"high"}},{"id":"claude-opus-5","name":"Claude Opus 5","lab":"Anthropic","date":"2026-07-24","type":"flagship","major":true,"open":false,"release":{"title":"Claude Opus 5","source":"https://www.anthropic.com/news/claude-opus-5","verified":true,"confidence":"high"},"index":{"score":184.7,"lo":168.8,"hi":200.6,"se":2.13,"solve":69.9,"status":"rated","areas":["agents","coding","maths","novel","reasoning","work"],"core_areas":["agents","coding","maths","novel","reasoning","work"],"testers":["ARC Prize","BALROG team","Datacurve","Epoch AI","Havard T. Ihle","LiveBench","Mercor","Proximal","Redwood Research","SimpleBench team","Vals AI"],"tests":{"vals:legal_research":{"value":55.29,"norm":0.5529,"version":"Vals AI"},"aime":{"value":0.9888888888888889,"norm":0.9889,"version":"claude-opus-5_max"},"gpqa":{"value":0.9387626262626263,"norm":0.9388,"version":"claude-opus-5_max"},"vals:mmlu_pro":{"value":91.59,"norm":0.9159,"version":"Vals AI"},"x:DeepSWE":{"value":0.7364864864864865,"norm":0.7365,"version":"claude-opus-5_max"},"x:EBR-bench":{"value":0.45714285714285713,"norm":0.4571,"version":"claude-opus-5_max"},"x:Balrog":{"value":0.634,"norm":0.634,"version":"claude-opus-5_max"},"vals:vcb-1-100":{"value":28.53,"norm":0.2853,"version":"Vals AI"},"vals:medcode":{"value":63.57,"norm":0.6357,"version":"Vals AI"},"vals:biomysterybench":{"value":79.26,"norm":0.7926,"version":"Vals AI"},"x:Chess Puzzles":{"value":0.43,"norm":0.43,"version":"claude-opus-5_max"},"vals:gpqa":{"value":93.43,"norm":0.9343,"version":"Vals AI"},"x:SimpleBench":{"value":0.8059999999999999,"norm":0.806,"version":"claude-opus-5_unknown"},"vals:corp_fin_v2":{"value":73.19,"norm":0.7319,"version":"Vals AI"},"vals:legal_bench":{"value":86.97,"norm":0.8697,"version":"Vals AI"},"vals:lcb":{"value":89.03,"norm":0.8903,"version":"Vals AI"},"livebench:if":{"value":63.77,"norm":0.6377,"version":"LiveBench"},"vals:tax_eval_v2":{"value":75.14,"norm":0.7514,"version":"Vals AI"},"x:LMCA":{"value":64.45,"norm":0.6445,"version":"claude-opus-5_xhigh"},"vals:mortgage_tax":{"value":72.06,"norm":0.7206,"version":"Vals AI"},"vals:medscribe":{"value":90.99,"norm":0.9099,"version":"Vals AI"},"livebench:data-analysis":{"value":74.55,"norm":0.7455,"version":"LiveBench"},"x:FrontierMath-Tier-4-v2-Private":{"value":0.7317073170731707,"norm":0.7317,"version":"claude-opus-5_max"},"livebench:mathematics":{"value":95.73,"norm":0.9573,"version":"LiveBench"},"vals:tax_agent_bench":{"value":47.1,"norm":0.471,"version":"Vals AI"},"x:Mystery Game Puzzles":{"value":0.59,"norm":0.59,"version":"claude-opus-5_max"},"vals:swebench":{"value":97,"norm":0.97,"version":"Vals AI"},"vals:public-benefits-bench":{"value":76.93,"norm":0.7693,"version":"Vals AI"},"vals:vibe-code":{"value":88.4,"norm":0.884,"version":"Vals AI"},"x:SimpleQA Verified":{"value":0.599,"norm":0.599,"version":"claude-opus-5_max"},"vals:mmmu":{"value":89.88,"norm":0.8988,"version":"Vals AI"},"vals:mysterymechanism":{"value":37.39,"norm":0.3739,"version":"Vals AI"},"vals:srebench":{"value":12.21,"norm":0.1221,"version":"Vals AI"},"x:Furniture Assembly":{"value":0.6083333333333333,"norm":0.6083,"version":"claude-opus-5_max"},"fm":{"value":0.856140350877193,"norm":0.8561,"version":"claude-opus-5_max"},"arc:arc_agi_3":{"value":30.2,"norm":0.302,"version":"ARC Prize"},"x:DTBench":{"value":0.9787,"norm":0.9787,"version":"claude-opus-5_high"},"livebench:agentic-coding":{"value":65.2,"norm":0.652,"version":"LiveBench"},"vals:programbench":{"value":3,"norm":0.03,"version":"Vals AI"},"vals:ioi":{"value":84.33,"norm":0.8433,"version":"Vals AI"},"livebench:reasoning":{"value":91.21,"norm":0.9121,"version":"LiveBench"},"apex":{"value":0.658,"norm":0.658,"version":"claude-opus-5_max"},"vals:terminal-bench-science":{"value":27.14,"norm":0.2714,"version":"Vals AI"},"vals:cyber":{"value":65.36,"norm":0.6536,"version":"Vals AI"},"x:WeirdML":{"value":0.9178000000000001,"norm":0.9178,"version":"claude-opus-5_max"},"vals:proof_bench":{"value":99,"norm":0.99,"version":"Vals AI"},"x:FrontierSWE":{"value":0.5201176764705882,"norm":0.5201,"version":"claude-opus-5_max"},"vals:fabv2":{"value":58.63,"norm":0.5863,"version":"Vals AI"},"livebench:language":{"value":88.69,"norm":0.8869,"version":"LiveBench"},"vals:code-migration":{"value":57.47,"norm":0.5747,"version":"Vals AI"},"vals:skillsbench":{"value":60.44,"norm":0.6044,"version":"Vals AI"},"arc:arc_agi_2":{"value":90.4,"norm":0.904,"version":"ARC Prize"},"vals:terminal-bench-4":{"value":53.53,"norm":0.5353,"version":"Vals AI"},"arc:arc_agi_1":{"value":97.5,"norm":0.975,"version":"ARC Prize"},"x:MirrorCode":{"value":0.8777777777777778,"norm":0.8778,"version":"claude-opus-5_max"},"vals:emb":{"value":73.56,"norm":0.7356,"version":"Vals AI"},"livebench:coding":{"value":81.45,"norm":0.8145,"version":"LiveBench"}},"variant":"Claude Opus 5","match":"exact"},"api":{"or_id":"anthropic/claude-opus-5","in":5,"out":25,"context":1000000,"hf":null,"created":"2026-07-24"}},{"id":"qwen3-7-flash","name":"Qwen3.7 Flash","lab":"Alibaba","date":"2026-07-27","type":"flagship","major":false,"open":false,"index":{"score":138.1,"lo":120.9,"hi":155.3,"se":4.56,"solve":29,"status":"provisional","areas":["agents","maths","reasoning"],"core_areas":["maths","reasoning"],"testers":["Epoch AI"],"tests":{"x:Mystery Game Puzzles":{"value":0.15,"norm":0.15,"version":"qwen3.7-flash"},"fm":{"value":0.19298245614035087,"norm":0.193,"version":"qwen3.7-flash"},"gpqa":{"value":0.8232323232323232,"norm":0.8232,"version":"qwen3.7-flash"},"aime":{"value":0.8666666666666667,"norm":0.8667,"version":"qwen3.7-flash"}},"variant":"Qwen3.7 Flash","match":"self"},"api":{"or_id":"qwen/qwen3.7-flash","in":0.03,"out":0.13,"context":1000000,"hf":null,"created":"2026-07-28"},"date_from":"epoch"},{"id":"inkling-small","name":"Inkling-Small","lab":"Thinking Machines","date":"2026-07-30","type":"open-weights","major":true,"open":true,"release":{"title":"Inkling-Small","source":"https://thinkingmachines.ai/news/inkling-small/","verified":true,"confidence":"high"},"index":{"score":153.1,"lo":137.2,"hi":168.9,"se":2.12,"solve":41.9,"status":"rated","areas":["agents","coding","maths","novel","reasoning","work"],"core_areas":["coding","maths","novel","reasoning","work"],"testers":["ARC Prize","Epoch AI","Vals AI"],"tests":{"vals:medcode":{"value":37.89,"norm":0.3789,"version":"Vals AI"},"vals:tax_agent_bench":{"value":15.39,"norm":0.1539,"version":"Vals AI"},"vals:public-benefits-bench":{"value":59.4,"norm":0.594,"version":"Vals AI"},"vals:legal_research":{"value":25.48,"norm":0.2548,"version":"Vals AI"},"vals:emb":{"value":31.76,"norm":0.3176,"version":"Vals AI"},"gpqa":{"value":0.88510101010101,"norm":0.8851,"version":"inkling-small_xhigh"},"vals:skillsbench":{"value":33.58,"norm":0.3358,"version":"Vals AI"},"arc:arc_agi_2":{"value":40.1,"norm":0.401,"version":"ARC Prize"},"vals:terminal-bench-4":{"value":0,"norm":0,"version":"Vals AI"},"vals:corp_fin_v2":{"value":69.62,"norm":0.6962,"version":"Vals AI"},"vals:swebench":{"value":82.2,"norm":0.822,"version":"Vals AI"},"vals:lcb":{"value":85.93,"norm":0.8593,"version":"Vals AI"},"x:SimpleQA Verified":{"value":0.191,"norm":0.191,"version":"inkling-small_xhigh"},"aime":{"value":0.9,"norm":0.9,"version":"inkling-small_xhigh"},"vals:gpqa":{"value":83.59,"norm":0.8359,"version":"Vals AI"},"vals:legal_bench":{"value":82.99,"norm":0.8299,"version":"Vals AI"},"vals:hlab":{"value":2.92,"norm":0.0292,"version":"Vals AI"},"vals:mmlu_pro":{"value":85.57,"norm":0.8557,"version":"Vals AI"},"vals:tax_eval_v2":{"value":75.51,"norm":0.7551,"version":"Vals AI"},"vals:ioi":{"value":9.33,"norm":0.0933,"version":"Vals AI"},"fm":{"value":0.4631578947368421,"norm":0.4632,"version":"inkling-small_xhigh"},"x:FrontierMath-Tier-4-v2-Private":{"value":0.17073170731707318,"norm":0.1707,"version":"inkling-small_xhigh"},"vals:fabv2":{"value":41.26,"norm":0.4126,"version":"Vals AI"},"x:Mystery Game Puzzles":{"value":0.06,"norm":0.06,"version":"inkling-small_xhigh"},"vals:programbench":{"value":0.5,"norm":0.005,"version":"Vals AI"},"vals:vibe-code":{"value":19.05,"norm":0.1905,"version":"Vals AI"},"vals:cyber":{"value":64.11,"norm":0.6411,"version":"Vals AI"},"vals:proof_bench":{"value":6,"norm":0.06,"version":"Vals AI"},"vals:code-migration":{"value":13.68,"norm":0.1368,"version":"Vals AI"},"arc:arc_agi_1":{"value":84,"norm":0.84,"version":"ARC Prize"},"vals:mortgage_tax":{"value":62.28,"norm":0.6228,"version":"Vals AI"},"vals:medscribe":{"value":84.11,"norm":0.8411,"version":"Vals AI"}},"variant":"Inkling-Small","match":"exact"},"api":{"or_id":"thinkingmachines/inkling-small","in":0.45,"out":1.2,"context":524288,"hf":"thinkingmachines/Inkling-Small","created":"2026-07-30"},"open_evidence":"https://huggingface.co/thinkingmachines/Inkling","date_epoch":"2026-07-30","date_evidence":"https://thinkingmachines.ai/news/inkling-small/"},{"id":"deepseek-v4-flash-2026-07","name":"DeepSeek-V4-Flash","lab":"DeepSeek","date":"2026-07-31","type":"open-weights","major":true,"open":true,"release":{"title":"DeepSeek-V4-Flash (official release; 0731 weights)","source":"https://api-docs.deepseek.com/updates/","verified":true,"confidence":"high"},"index":{"score":163.7,"lo":132.8,"hi":194.6,"se":16.25,"solve":53.3,"status":"provisional","areas":["coding","maths","reasoning"],"core_areas":[],"testers":["LiveBench"],"tests":{"livebench:data-analysis":{"value":79.48,"norm":0.7948,"version":"LiveBench"},"livebench:agentic-coding":{"value":65.1,"norm":0.651,"version":"LiveBench"},"livebench:mathematics":{"value":87.81,"norm":0.8781,"version":"LiveBench"},"livebench:reasoning":{"value":85.4,"norm":0.854,"version":"LiveBench"},"livebench:if":{"value":70.96,"norm":0.7096,"version":"LiveBench"},"livebench:coding":{"value":68.2,"norm":0.682,"version":"LiveBench"},"livebench:language":{"value":80.36,"norm":0.8036,"version":"LiveBench"}},"variant":"DeepSeek-V4-Flash-Vision-Exp","match":"family"}},{"id":"qwen3-8-max","name":"Qwen3.8-Max","lab":"Alibaba","date":"2026-08-03","type":"flagship","major":true,"open":false,"release":{"title":"Qwen3.8-Max (preview 07-19; weights 08-12)","source":"https://qwenlm.github.io/qwen-code-docs/en/blog/cases/qwencode-bailian-skill-openai-cover-gen/","verified":false,"confidence":"high"},"index":{"score":167.1,"lo":150.8,"hi":183.3,"se":2.93,"solve":56.7,"status":"rated","areas":["agents","coding","maths","reasoning"],"core_areas":["agents","maths","reasoning"],"testers":["Datacurve","Epoch AI","Mercor","Proximal","Redwood Research"],"tests":{"x:MirrorCode":{"value":0.5222222222222223,"norm":0.5222,"version":"qwen3.8-max_xhigh"},"x:FrontierSWE":{"value":0.15809350588235294,"norm":0.1581,"version":"qwen3.8-max_xhigh"},"fm":{"value":0.7473684210526316,"norm":0.7474,"version":"qwen3.8-max_xhigh"},"x:FrontierMath-Tier-4-v2-Private":{"value":0.4634146341463415,"norm":0.4634,"version":"qwen3.8-max_xhigh"},"x:Mystery Game Puzzles":{"value":0.38,"norm":0.38,"version":"qwen3.8-max_xhigh"},"x:SimpleQA Verified":{"value":0.458,"norm":0.458,"version":"qwen3.8-max_xhigh"},"x:DTBench":{"value":0.92,"norm":0.92,"version":"qwen3.8-max_xhigh"},"x:LMCA":{"value":46.24,"norm":0.4624,"version":"qwen3.8-max_xhigh"},"x:Chess Puzzles":{"value":0.46,"norm":0.46,"version":"qwen3.8-max_xhigh"},"aime":{"value":0.9944444444444445,"norm":0.9944,"version":"qwen3.8-max_xhigh"},"gpqa":{"value":0.9267676767676768,"norm":0.9268,"version":"qwen3.8-max_xhigh"},"x:DeepSWE":{"value":0.5746102449888641,"norm":0.5746,"version":"qwen3.8-max_xhigh"},"apex":{"value":0.633,"norm":0.633,"version":"qwen3.8-max_unknown"}},"variant":"Qwen 3.8 Max","match":"exact"},"open_evidence":"Hosted model; the open weights are Qwen3.8-2.4T-A95B, a separate artifact (https://huggingface.co/Qwen/Qwen3.8-2.4T-A95B)"},{"id":"muse-spark-1-2","name":"Muse Spark 1.2","lab":"Meta","date":"2026-08-05","type":"flagship","major":true,"open":false,"release":{"title":"Muse Spark 1.2 (+ Muse Code)","source":null,"verified":false,"confidence":"high"},"index":{"score":162.2,"lo":146.1,"hi":178.3,"se":2.63,"solve":51.7,"status":"rated","areas":["agents","coding","maths","reasoning","work"],"core_areas":["agents","coding","work"],"testers":["Datacurve","Epoch AI","Havard T. Ihle","LiveBench","Mercor","Proximal","Redwood Research","SimpleBench team","Vals AI"],"tests":{"vals:medcode":{"value":49.35,"norm":0.4935,"version":"Vals AI"},"vals:fabv2":{"value":60.6,"norm":0.606,"version":"Vals AI"},"vals:corp_fin_v2":{"value":70.94,"norm":0.7094,"version":"Vals AI"},"x:SimpleQA Verified":{"value":0.6030150753768844,"norm":0.603,"version":"muse-spark-1.2_xhigh"},"x:WeirdML":{"value":0.6038,"norm":0.6038,"version":"muse-spark-1.2_xhigh"},"vals:mmmu":{"value":86.13,"norm":0.8613,"version":"Vals AI"},"vals:code-migration":{"value":29.95,"norm":0.2995,"version":"Vals AI"},"x:DTBench":{"value":0.9467,"norm":0.9467,"version":"muse-spark-1.2_xhigh"},"vals:emb":{"value":56.98,"norm":0.5698,"version":"Vals AI"},"vals:tax_agent_bench":{"value":32.8,"norm":0.328,"version":"Vals AI"},"vals:proof_bench":{"value":43,"norm":0.43,"version":"Vals AI"},"vals:terminal-bench-4":{"value":6.06,"norm":0.0606,"version":"Vals AI"},"vals:tax_eval_v2":{"value":80.38,"norm":0.8038,"version":"Vals AI"},"vals:srebench":{"value":0,"norm":0,"version":"Vals AI"},"vals:medscribe":{"value":90.06,"norm":0.9006,"version":"Vals AI"},"livebench:agentic-coding":{"value":57.58,"norm":0.5758,"version":"LiveBench"},"x:Chess Puzzles":{"value":0.27,"norm":0.27,"version":"muse-spark-1.2_xhigh"},"vals:swebench":{"value":86.6,"norm":0.866,"version":"Vals AI"},"vals:skillsbench":{"value":53.04,"norm":0.5304,"version":"Vals AI"},"vals:public-benefits-bench":{"value":68.47,"norm":0.6847,"version":"Vals AI"},"vals:legal_research":{"value":43.75,"norm":0.4375,"version":"Vals AI"},"livebench:reasoning":{"value":90,"norm":0.9,"version":"LiveBench"},"vals:vibe-code":{"value":79.1,"norm":0.791,"version":"Vals AI"},"vals:mmlu_pro":{"value":88.28,"norm":0.8828,"version":"Vals AI"},"x:LMCA":{"value":48.35,"norm":0.4835,"version":"muse-spark-1.2_xhigh"},"x:FrontierSWE":{"value":0.11970256470588238,"norm":0.1197,"version":"muse-spark-1.2_xhigh"},"livebench:language":{"value":78.57,"norm":0.7857,"version":"LiveBench"},"vals:programbench":{"value":0.5,"norm":0.005,"version":"Vals AI"},"apex":{"value":0.454,"norm":0.454,"version":"muse-spark-1.2"},"vals:cyber":{"value":69.52,"norm":0.6952,"version":"Vals AI"},"vals:ioi":{"value":21.78,"norm":0.2178,"version":"Vals AI"},"vals:biomysterybench":{"value":64.82,"norm":0.6482,"version":"Vals AI"},"vals:mortgage_tax":{"value":65.42,"norm":0.6542,"version":"Vals AI"},"livebench:if":{"value":74.33,"norm":0.7433,"version":"LiveBench"},"livebench:coding":{"value":77.54,"norm":0.7754,"version":"LiveBench"},"livebench:data-analysis":{"value":76.46,"norm":0.7646,"version":"LiveBench"},"livebench:mathematics":{"value":91.2,"norm":0.912,"version":"LiveBench"},"x:SimpleBench":{"value":0.745,"norm":0.745,"version":"muse-spark-1.2"},"vals:legal_bench":{"value":85.26,"norm":0.8526,"version":"Vals AI"},"x:DeepSWE":{"value":0.5486725663716814,"norm":0.5487,"version":"muse-spark-1.2_xhigh"}},"variant":"Muse Spark 1.2","match":"exact"},"api":{"or_id":"meta/muse-spark-1.2","in":1.25,"out":4.25,"context":1048576,"hf":null,"created":"2026-08-05"}},{"id":"muse-glimmer-30b","name":"Muse Glimmer 30B","lab":"Meta","date":"2026-08-10","type":"open-weights","major":true,"open":true,"release":{"title":"Muse Glimmer 30B","source":null,"verified":false,"confidence":"high"},"api":{"or_id":"meta/muse-glimmer-30b","in":0.35,"out":1.5,"context":131072,"hf":"meta-models/Muse-Glimmer-30B","created":"2026-08-09"}},{"id":"grok-4-6","name":"Grok 4.6","lab":"xAI","date":"2026-08-12","type":"flagship","major":true,"open":false,"release":{"title":"Grok 4.6","source":"https://x.ai/news/grok-4-6","verified":true,"confidence":"high"},"index":{"score":167.2,"lo":151.3,"hi":183.1,"se":2.11,"solve":56.8,"status":"rated","areas":["agents","coding","maths","novel","reasoning","work"],"core_areas":["agents","coding","maths","novel","reasoning","work"],"testers":["ARC Prize","Datacurve","Epoch AI","Havard T. Ihle","LiveBench","Mercor","Proximal","Redwood Research","SimpleBench team","Vals AI"],"tests":{"vals:vcb-1-100":{"value":14.75,"norm":0.1475,"version":"Vals AI"},"vals:terminal-bench-4":{"value":17.17,"norm":0.1717,"version":"Vals AI"},"vals:tax_eval_v2":{"value":71.1,"norm":0.711,"version":"Vals AI"},"x:LMCA":{"value":48.45,"norm":0.4845,"version":"grok-4.6_xhigh"},"vals:ioi":{"value":47.61,"norm":0.4761,"version":"Vals AI"},"x:MirrorCode":{"value":0.4111111111111111,"norm":0.4111,"version":"grok-4.6_xhigh"},"vals:medscribe":{"value":86.53,"norm":0.8653,"version":"Vals AI"},"livebench:agentic-coding":{"value":57.02,"norm":0.5702,"version":"LiveBench"},"vals:corp_fin_v2":{"value":66.16,"norm":0.6616,"version":"Vals AI"},"vals:skillsbench":{"value":55.77,"norm":0.5577,"version":"Vals AI"},"vals:legal_research":{"value":48.08,"norm":0.4808,"version":"Vals AI"},"x:FrontierMath-Tier-4-v2-Private":{"value":0.3170731707317073,"norm":0.3171,"version":"grok-4.6_xhigh"},"vals:lcb":{"value":88.22,"norm":0.8822,"version":"Vals AI"},"vals:gpqa":{"value":94.7,"norm":0.947,"version":"Vals AI"},"vals:vibe-code":{"value":76.24,"norm":0.7624,"version":"Vals AI"},"arc:arc_agi_2":{"value":67.1,"norm":0.671,"version":"ARC Prize"},"vals:mysterymechanism":{"value":30.63,"norm":0.3063,"version":"Vals AI"},"x:DTBench":{"value":0.9733,"norm":0.9733,"version":"grok-4.6_xhigh"},"vals:emb":{"value":62.73,"norm":0.6273,"version":"Vals AI"},"livebench:coding":{"value":76.78,"norm":0.7678,"version":"LiveBench"},"arc:arc_agi_3":{"value":2.1,"norm":0.021,"version":"ARC Prize"},"gpqa":{"value":0.9400252525252525,"norm":0.94,"version":"grok-4.6_high"},"vals:mmlu_pro":{"value":89.4,"norm":0.894,"version":"Vals AI"},"x:EBR-bench":{"value":0.30476190476190473,"norm":0.3048,"version":"grok-4.6_xhigh"},"livebench:reasoning":{"value":90.51,"norm":0.9051,"version":"LiveBench"},"apex":{"value":0.653,"norm":0.653,"version":"grok-4.6_unknown"},"livebench:data-analysis":{"value":73.86,"norm":0.7386,"version":"LiveBench"},"x:SimpleBench":{"value":0.7590000000000001,"norm":0.759,"version":"grok-4.6_unknown"},"vals:code-migration":{"value":44.57,"norm":0.4457,"version":"Vals AI"},"arc:arc_agi_1":{"value":87.5,"norm":0.875,"version":"ARC Prize"},"vals:mortgage_tax":{"value":64.19,"norm":0.6419,"version":"Vals AI"},"x:SimpleQA Verified":{"value":0.493,"norm":0.493,"version":"grok-4.6_high"},"livebench:if":{"value":71.87,"norm":0.7187,"version":"LiveBench"},"vals:medcode":{"value":44.71,"norm":0.4471,"version":"Vals AI"},"aime":{"value":0.9916666666666667,"norm":0.9917,"version":"grok-4.6_xhigh"},"x:Furniture Assembly":{"value":0.4,"norm":0.4,"version":"grok-4.6_xhigh"},"fm":{"value":0.6596491228070176,"norm":0.6596,"version":"grok-4.6_xhigh"},"livebench:mathematics":{"value":92.57,"norm":0.9257,"version":"LiveBench"},"vals:swebench":{"value":95.6,"norm":0.956,"version":"Vals AI"},"vals:terminal-bench-science":{"value":11.43,"norm":0.1143,"version":"Vals AI"},"vals:cyber":{"value":67.68,"norm":0.6768,"version":"Vals AI"},"vals:programbench":{"value":1,"norm":0.01,"version":"Vals AI"},"x:WeirdML":{"value":0.6729,"norm":0.6729,"version":"grok-4.6_high"},"vals:proof_bench":{"value":51,"norm":0.51,"version":"Vals AI"},"vals:biomysterybench":{"value":72.22,"norm":0.7222,"version":"Vals AI"},"x:FrontierSWE":{"value":0.25286195882352946,"norm":0.2529,"version":"grok-4.6_xhigh"},"vals:fabv2":{"value":53.68,"norm":0.5368,"version":"Vals AI"},"vals:tax_agent_bench":{"value":33.66,"norm":0.3366,"version":"Vals AI"},"x:Mystery Game Puzzles":{"value":0.34,"norm":0.34,"version":"grok-4.6_xhigh"},"livebench:language":{"value":83.7,"norm":0.837,"version":"LiveBench"},"vals:legal_bench":{"value":86.31,"norm":0.8631,"version":"Vals AI"},"vals:public-benefits-bench":{"value":66.85,"norm":0.6685,"version":"Vals AI"},"x:DeepSWE":{"value":0.6747787610619469,"norm":0.6748,"version":"grok-4.6_medium"},"x:Chess Puzzles":{"value":0.33,"norm":0.33,"version":"grok-4.6_high"}},"variant":"Grok 4.6","match":"exact"},"api":{"or_id":"x-ai/grok-4.6","in":2,"out":6,"context":500000,"hf":null,"created":"2026-08-12"}},{"id":"qwen3-8-2-4t-a95b","name":"Qwen3.8-2.4T-A95B","lab":"Alibaba","date":"2026-08-12","type":"open-weights","major":true,"open":true,"release":{"title":"Qwen3.8-2.4T-A95B","source":"https://huggingface.co/Qwen/Qwen3.8-2.4T-A95B","verified":false,"confidence":"high"},"api":{"or_id":"qwen/qwen3.8-2.4t-a95b","in":2,"out":6,"context":1048576,"hf":"Qwen/Qwen3.8-2.4T-A95B","created":"2026-08-12"}},{"id":"seed-2-1","name":"Seed 2.1","lab":"ByteDance","date":"2026-08-12","type":"flagship","major":true,"open":false,"release":{"title":"Seed 2.1","source":null,"verified":false,"confidence":"high"}},{"id":"deepseek-v4-pro-2026-08","name":"DeepSeek-V4-Pro","lab":"DeepSeek","date":"2026-08-13","type":"flagship","major":true,"open":true,"release":{"title":"DeepSeek-V4-Pro (GA)","source":"https://api-docs.deepseek.com/updates/","verified":false,"confidence":"high"},"index":{"score":162.5,"lo":146.6,"hi":178.3,"se":1.97,"solve":52,"status":"rated","areas":["agents","coding","maths","novel","reasoning","work"],"core_areas":["agents","coding","maths","novel","reasoning","work"],"testers":["ARC Prize","Epoch AI","Havard T. Ihle","LiveBench","Mercor","Redwood Research","Vals AI"],"tests":{"vals:medscribe":{"value":80.17,"norm":0.8017,"version":"Vals AI"},"livebench:coding":{"value":77.16,"norm":0.7716,"version":"LiveBench"},"vals:corp_fin_v2":{"value":65.42,"norm":0.6542,"version":"Vals AI"},"vals:legal_bench":{"value":82.36,"norm":0.8236,"version":"Vals AI"},"x:FrontierMath-Tier-4-v2-Private":{"value":0.2682926829268293,"norm":0.2683,"version":"deepseek-v4-pro-0813_max"},"vals:gpqa":{"value":92.42,"norm":0.9242,"version":"Vals AI"},"vals:cyber":{"value":72.86,"norm":0.7286,"version":"Vals AI"},"vals:vcb-1-100":{"value":17.53,"norm":0.1753,"version":"Vals AI"},"vals:programbench":{"value":0,"norm":0,"version":"Vals AI"},"vals:ioi":{"value":51.61,"norm":0.5161,"version":"Vals AI"},"livebench:if":{"value":67.7,"norm":0.677,"version":"LiveBench"},"fm":{"value":0.6456140350877193,"norm":0.6456,"version":"deepseek-v4-pro-0813_max"},"livebench:mathematics":{"value":95.09,"norm":0.9509,"version":"LiveBench"},"livebench:agentic-coding":{"value":54.95,"norm":0.5495,"version":"LiveBench"},"livebench:language":{"value":82.07,"norm":0.8207,"version":"LiveBench"},"vals:skillsbench":{"value":53.83,"norm":0.5383,"version":"Vals AI"},"vals:terminal-bench-science":{"value":4.29,"norm":0.0429,"version":"Vals AI"},"livebench:reasoning":{"value":85.84,"norm":0.8584,"version":"LiveBench"},"vals:vibe-code":{"value":82.3,"norm":0.823,"version":"Vals AI"},"x:LMCA":{"value":45.45,"norm":0.4545,"version":"deepseek-v4-pro-0813_unknown"},"vals:emb":{"value":52.8,"norm":0.528,"version":"Vals AI"},"vals:fabv2":{"value":50.39,"norm":0.5039,"version":"Vals AI"},"apex":{"value":0.473,"norm":0.473,"version":"deepseek-v4-pro-0813_unknown"},"x:WeirdML":{"value":0.6622,"norm":0.6622,"version":"deepseek-v4-pro-0813_max"},"x:DTBench":{"value":0.9387000000000001,"norm":0.9387,"version":"deepseek-v4-pro-0813_unknown"},"vals:tax_eval_v2":{"value":73.06,"norm":0.7306,"version":"Vals AI"},"vals:medcode":{"value":42.47,"norm":0.4247,"version":"Vals AI"},"aime":{"value":0.9861111111111112,"norm":0.9861,"version":"deepseek-v4-pro-0813_max"},"livebench:data-analysis":{"value":79.24,"norm":0.7924,"version":"LiveBench"},"gpqa":{"value":0.9166666666666666,"norm":0.9167,"version":"deepseek-v4-pro-0813_max"},"vals:legal_research":{"value":40.87,"norm":0.4087,"version":"Vals AI"},"vals:tax_agent_bench":{"value":26.76,"norm":0.2676,"version":"Vals AI"},"arc:arc_agi_2":{"value":61.3,"norm":0.613,"version":"ARC Prize"},"vals:swebench":{"value":96.4,"norm":0.964,"version":"Vals AI"},"vals:code-migration":{"value":41.54,"norm":0.4154,"version":"Vals AI"},"arc:arc_agi_1":{"value":90.5,"norm":0.905,"version":"ARC Prize"},"vals:srebench":{"value":1.91,"norm":0.0191,"version":"Vals AI"},"vals:proof_bench":{"value":50,"norm":0.5,"version":"Vals AI"},"x:MirrorCode":{"value":0.3333333333333333,"norm":0.3333,"version":"deepseek-v4-pro-0813_max"},"x:Mystery Game Puzzles":{"value":0.43,"norm":0.43,"version":"deepseek-v4-pro-0813_max"},"vals:lcb":{"value":87.53,"norm":0.8753,"version":"Vals AI"},"x:Chess Puzzles":{"value":0.46,"norm":0.46,"version":"deepseek-v4-pro-0813_max"},"vals:hlab":{"value":10.83,"norm":0.1083,"version":"Vals AI"},"vals:mmlu_pro":{"value":86.97,"norm":0.8697,"version":"Vals AI"},"vals:terminal-bench-4":{"value":14.14,"norm":0.1414,"version":"Vals AI"},"x:SimpleQA Verified":{"value":0.5290581162324649,"norm":0.5291,"version":"deepseek-v4-pro-0813_max"}},"variant":"DeepSeek V4 Pro 0813","match":"family"},"api":{"or_id":"deepseek/deepseek-v4-pro-0813","in":0.111,"out":4.44,"context":1048576,"hf":"deepseek-ai/DeepSeek-V4-Pro-0813","created":"2026-08-12"}},{"id":"gemini-3-7-flash","name":"Gemini 3.7 Flash","lab":"Google","date":"2026-08-13","type":"flagship","major":true,"open":false,"release":{"title":"Gemini 3.7 Flash","source":"https://blog.google/innovation-and-ai/models-and-research/gemini-models/introducing-gemini-3-7-flash/","verified":true,"confidence":"high"},"index":{"score":170.3,"lo":154.3,"hi":186.2,"se":2.25,"solve":59.6,"status":"rated","areas":["agents","coding","maths","novel","reasoning","work"],"core_areas":["agents","coding","maths","novel","reasoning","work"],"testers":["ARC Prize","Datacurve","Epoch AI","Mercor","Proximal","Redwood Research","Vals AI"],"tests":{"gpqa":{"value":0.9482323232323232,"norm":0.9482,"version":"gemini-3.7-flash_high"},"x:FrontierMath-Tier-4-v2-Private":{"value":0.36585365853658536,"norm":0.3659,"version":"gemini-3.7-flash_high"},"x:Mystery Game Puzzles":{"value":0.37,"norm":0.37,"version":"gemini-3.7-flash_high"},"vals:terminal-bench-4":{"value":12.12,"norm":0.1212,"version":"Vals AI"},"x:DeepSWE":{"value":0.6548672566371682,"norm":0.6549,"version":"gemini-3.7-flash_medium"},"x:DTBench":{"value":0.968,"norm":0.968,"version":"gemini-3.7-flash_high"},"arc:arc_agi_1":{"value":95.5,"norm":0.955,"version":"ARC Prize"},"x:Chess Puzzles":{"value":0.66,"norm":0.66,"version":"gemini-3.7-flash_high"},"aime":{"value":0.9722222222222222,"norm":0.9722,"version":"gemini-3.7-flash_high"},"vals:terminal-bench-science":{"value":7.14,"norm":0.0714,"version":"Vals AI"},"vals:cyber":{"value":43.75,"norm":0.4375,"version":"Vals AI"},"vals:legal_bench":{"value":87.26,"norm":0.8726,"version":"Vals AI"},"vals:programbench":{"value":0,"norm":0,"version":"Vals AI"},"vals:ioi":{"value":67.83,"norm":0.6783,"version":"Vals AI"},"vals:tax_eval_v2":{"value":74.73,"norm":0.7473,"version":"Vals AI"},"x:LMCA":{"value":50.38,"norm":0.5038,"version":"gemini-3.7-flash_high"},"vals:medcode":{"value":53.39,"norm":0.5339,"version":"Vals AI"},"vals:emb":{"value":71.33,"norm":0.7133,"version":"Vals AI"},"vals:skillsbench":{"value":65.89,"norm":0.6589,"version":"Vals AI"},"vals:legal_research":{"value":34.62,"norm":0.3462,"version":"Vals AI"},"vals:fabv2":{"value":59.04,"norm":0.5904,"version":"Vals AI"},"vals:tax_agent_bench":{"value":22.8,"norm":0.228,"version":"Vals AI"},"vals:mmlu_pro":{"value":90.12,"norm":0.9012,"version":"Vals AI"},"vals:lcb":{"value":88.65,"norm":0.8865,"version":"Vals AI"},"vals:vibe-code":{"value":70.39,"norm":0.7039,"version":"Vals AI"},"x:SimpleQA Verified":{"value":0.692,"norm":0.692,"version":"gemini-3.7-flash_high"},"vals:proof_bench":{"value":58,"norm":0.58,"version":"Vals AI"},"vals:mortgage_tax":{"value":66.65,"norm":0.6665,"version":"Vals AI"},"x:MirrorCode":{"value":0.4111111111111111,"norm":0.4111,"version":"gemini-3.7-flash_high"},"x:Furniture Assembly":{"value":0.26666666666666666,"norm":0.2667,"version":"gemini-3.7-flash_high"},"arc:arc_agi_2":{"value":84.6,"norm":0.846,"version":"ARC Prize"},"vals:swebench":{"value":80.8,"norm":0.808,"version":"Vals AI"},"vals:srebench":{"value":4.58,"norm":0.0458,"version":"Vals AI"},"fm":{"value":0.7157894736842105,"norm":0.7158,"version":"gemini-3.7-flash_high"},"vals:gpqa":{"value":93.94,"norm":0.9394,"version":"Vals AI"},"vals:code-migration":{"value":34.8,"norm":0.348,"version":"Vals AI"},"apex":{"value":0.6779999999999999,"norm":0.678,"version":"gemini-3.7-flash_unknown"},"vals:mmmu":{"value":88.96,"norm":0.8896,"version":"Vals AI"},"vals:medscribe":{"value":83.94,"norm":0.8394,"version":"Vals AI"},"x:FrontierSWE":{"value":0.20260590588235297,"norm":0.2026,"version":"gemini-3.7-flash_high"}},"variant":"Gemini 3.7 Flash","match":"exact"},"api":{"or_id":"google/gemini-3.7-flash","in":0.75,"out":3.75,"context":1048576,"hf":null,"created":"2026-08-13"}},{"id":"glm-5-3","name":"GLM-5.3","lab":"Zhipu","date":"2026-08-14","type":"open-weights","major":true,"open":true,"release":{"title":"GLM-5.3","source":"https://z.ai/blog/glm-5.3","verified":true,"confidence":"high"},"index":{"score":168.2,"lo":152.2,"hi":184.2,"se":2.47,"solve":57.8,"status":"rated","areas":["agents","coding","maths","novel","reasoning","work"],"core_areas":["agents","coding","maths","reasoning","work"],"testers":["Datacurve","Epoch AI","Havard T. Ihle","LiveBench","Mercor","Proximal","Redwood Research","SimpleBench team","Vals AI"],"tests":{"vals:srebench":{"value":9.16,"norm":0.0916,"version":"Vals AI"},"fm":{"value":0.6877192982456141,"norm":0.6877,"version":"glm-5.3_max"},"x:DTBench":{"value":0.8767,"norm":0.8767,"version":"glm-5.3_unknown"},"vals:skillsbench":{"value":47.51,"norm":0.4751,"version":"Vals AI"},"vals:legal_research":{"value":49.04,"norm":0.4904,"version":"Vals AI"},"x:FrontierMath-Tier-4-v2-Private":{"value":0.2926829268292683,"norm":0.2927,"version":"glm-5.3_max"},"vals:fabv2":{"value":55.84,"norm":0.5584,"version":"Vals AI"},"livebench:mathematics":{"value":87.9,"norm":0.879,"version":"LiveBench"},"vals:swebench":{"value":95.4,"norm":0.954,"version":"Vals AI"},"vals:programbench":{"value":1.5,"norm":0.015,"version":"Vals AI"},"livebench:reasoning":{"value":85.8,"norm":0.858,"version":"LiveBench"},"x:SimpleQA Verified":{"value":0.41,"norm":0.41,"version":"glm-5.3_max"},"vals:proof_bench":{"value":49,"norm":0.49,"version":"Vals AI"},"vals:medcode":{"value":42.86,"norm":0.4286,"version":"Vals AI"},"livebench:coding":{"value":78.95,"norm":0.7895,"version":"LiveBench"},"aime":{"value":0.9111111111111111,"norm":0.9111,"version":"glm-5.3_max"},"vals:terminal-bench-science":{"value":5.71,"norm":0.0571,"version":"Vals AI"},"vals:tax_agent_bench":{"value":43.33,"norm":0.4333,"version":"Vals AI"},"vals:vcb-1-100":{"value":19.99,"norm":0.1999,"version":"Vals AI"},"vals:legal_bench":{"value":84.84,"norm":0.8484,"version":"Vals AI"},"vals:emb":{"value":56.34,"norm":0.5634,"version":"Vals AI"},"x:Chess Puzzles":{"value":0.37,"norm":0.37,"version":"glm-5.3_max"},"x:SimpleBench":{"value":0.662,"norm":0.662,"version":"glm-5.3_unknown"},"vals:mmlu_pro":{"value":86.77,"norm":0.8677,"version":"Vals AI"},"vals:lcb":{"value":80.54,"norm":0.8054,"version":"Vals AI"},"vals:vibe-code":{"value":78.12,"norm":0.7812,"version":"Vals AI"},"livebench:data-analysis":{"value":70.24,"norm":0.7024,"version":"LiveBench"},"livebench:agentic-coding":{"value":60.91,"norm":0.6091,"version":"LiveBench"},"vals:public-benefits-bench":{"value":68.54,"norm":0.6854,"version":"Vals AI"},"vals:hlab":{"value":14.58,"norm":0.1458,"version":"Vals AI"},"x:Mystery Game Puzzles":{"value":0.33,"norm":0.33,"version":"glm-5.3_max"},"gpqa":{"value":0.9090909090909091,"norm":0.9091,"version":"glm-5.3_max"},"vals:terminal-bench-4":{"value":38.89,"norm":0.3889,"version":"Vals AI"},"x:DeepSWE":{"value":0.6895787139689579,"norm":0.6896,"version":"glm-5.3_max"},"vals:ioi":{"value":68.44,"norm":0.6844,"version":"Vals AI"},"livebench:if":{"value":69.3,"norm":0.693,"version":"LiveBench"},"vals:mysterymechanism":{"value":22.97,"norm":0.2297,"version":"Vals AI"},"vals:tax_eval_v2":{"value":72.36,"norm":0.7236,"version":"Vals AI"},"x:LMCA":{"value":55.51,"norm":0.5551,"version":"glm-5.3_unknown"},"vals:medscribe":{"value":88.81,"norm":0.8881,"version":"Vals AI"},"x:FrontierSWE":{"value":0.3018186411764705,"norm":0.3018,"version":"glm-5.3_max"},"livebench:language":{"value":79.86,"norm":0.7986,"version":"LiveBench"},"vals:gpqa":{"value":88.13,"norm":0.8813,"version":"Vals AI"},"vals:cyber":{"value":72.08,"norm":0.7208,"version":"Vals AI"},"vals:code-migration":{"value":44.22,"norm":0.4422,"version":"Vals AI"},"apex":{"value":0.5660000000000001,"norm":0.566,"version":"glm-5.3_unknown"},"x:WeirdML":{"value":0.7540000000000001,"norm":0.754,"version":"glm-5.3_max"}},"variant":"GLM-5.3","match":"exact"},"api":{"or_id":"z-ai/glm-5.3","in":0.0699,"out":1.4,"context":1048576,"hf":"zai-org/GLM-5.3","created":"2026-08-18"},"open_evidence":"https://huggingface.co/zai-org/GLM-5.3 (weights published 25 Aug 2026, as the launch post promised)","open_from":"2026-08-25"},{"id":"qwen3-8-27b","name":"Qwen3.8-27B","lab":"Alibaba","date":"2026-08-14","type":"open-weights","major":true,"open":true,"release":{"title":"Qwen3.8-27B","source":"https://qwen.ai/blog?id=qwen3.8-flash-next&ref=upstract.com","verified":false,"confidence":"high"},"index":{"score":155.1,"lo":139.3,"hi":170.9,"se":1.9,"solve":44.1,"status":"rated","areas":["agents","coding","maths","novel","reasoning","work"],"core_areas":["agents","novel","work"],"testers":["ARC Prize","GeoBench team","LiveBench","Mercor","Redwood Research","SimpleBench team","Vals AI","Yann Henon"],"tests":{"arc:arc_agi_1":{"value":87.5,"norm":0.875,"version":"ARC Prize"},"vals:swebench":{"value":86,"norm":0.86,"version":"Vals AI"},"vals:code-migration":{"value":14.16,"norm":0.1416,"version":"Vals AI"},"vals:terminal-bench-science":{"value":0,"norm":0,"version":"Vals AI"},"vals:cyber":{"value":67.8,"norm":0.678,"version":"Vals AI"},"vals:mmlu_pro":{"value":84.34,"norm":0.8434,"version":"Vals AI"},"livebench:agentic-coding":{"value":61.36,"norm":0.6136,"version":"LiveBench"},"livebench:data-analysis":{"value":76.59,"norm":0.7659,"version":"LiveBench"},"vals:fabv2":{"value":48.55,"norm":0.4855,"version":"Vals AI"},"x:SimpleBench":{"value":0.6020000000000001,"norm":0.602,"version":"qwen3.8-27b"},"livebench:language":{"value":74.35,"norm":0.7435,"version":"LiveBench"},"vals:legal_bench":{"value":82.43,"norm":0.8243,"version":"Vals AI"},"vals:skillsbench":{"value":38.11,"norm":0.3811,"version":"Vals AI"},"vals:lcb":{"value":84,"norm":0.84,"version":"Vals AI"},"vals:ioi":{"value":39.06,"norm":0.3906,"version":"Vals AI"},"apex":{"value":0.47500000000000003,"norm":0.475,"version":"qwen3.8-27b"},"x:GeoBench":{"value":0.38,"norm":0.38,"version":"qwen3.8-27b"},"x:LMCA":{"value":41.36,"norm":0.4136,"version":"qwen3.8-27b"},"vals:medscribe":{"value":83.85,"norm":0.8385,"version":"Vals AI"},"livebench:coding":{"value":75.69,"norm":0.7569,"version":"LiveBench"},"vals:gpqa":{"value":88.89,"norm":0.8889,"version":"Vals AI"},"arc:arc_agi_2":{"value":42.4,"norm":0.424,"version":"ARC Prize"},"vals:tax_eval_v2":{"value":70.85,"norm":0.7085,"version":"Vals AI"},"livebench:if":{"value":72.66,"norm":0.7266,"version":"LiveBench"},"vals:emb":{"value":59.66,"norm":0.5966,"version":"Vals AI"},"livebench:mathematics":{"value":86.21,"norm":0.8621,"version":"LiveBench"},"vals:legal_research":{"value":36.06,"norm":0.3606,"version":"Vals AI"},"livebench:reasoning":{"value":80.03,"norm":0.8003,"version":"LiveBench"},"x:Surface Evolver Bench":{"value":0.45,"norm":0.45,"version":"qwen3.8-27b_xhigh"},"vals:vibe-code":{"value":64.85,"norm":0.6485,"version":"Vals AI"},"vals:mmmu":{"value":83.93,"norm":0.8393,"version":"Vals AI"},"vals:proof_bench":{"value":16,"norm":0.16,"version":"Vals AI"},"vals:mortgage_tax":{"value":64.94,"norm":0.6494,"version":"Vals AI"},"x:DTBench":{"value":0.88,"norm":0.88,"version":"qwen3.8-27b"},"vals:medcode":{"value":28.7,"norm":0.287,"version":"Vals AI"},"vals:tax_agent_bench":{"value":30.89,"norm":0.3089,"version":"Vals AI"},"vals:programbench":{"value":0,"norm":0,"version":"Vals AI"}},"variant":"Qwen 3.8 27B","match":"exact"},"api":{"or_id":"qwen/qwen3.8-27b","in":0.06,"out":2.2,"context":1024000,"hf":"Qwen/Qwen3.8-27B","created":"2026-08-14"}},{"id":"glm-5-3-flash","name":"GLM-5.3-Flash","lab":"Zhipu","date":"2026-08-20","type":"open-weights","major":true,"open":true,"release":{"title":"GLM-5.3-Flash","source":"https://huggingface.co/zai-org/GLM-5.3-Flash","verified":false,"confidence":"high"},"index":{"score":161,"lo":145.2,"hi":176.7,"se":1.77,"solve":50.4,"status":"rated","areas":["agents","coding","maths","novel","reasoning","work"],"core_areas":["agents","coding","maths","novel","reasoning","work"],"testers":["ARC Prize","Datacurve","Epoch AI","LiveBench","Mercor","Proximal","Vals AI","Yann Henon"],"tests":{"arc:arc_agi_2":{"value":65.8,"norm":0.658,"version":"ARC Prize"},"vals:tax_eval_v2":{"value":75.59,"norm":0.7559,"version":"Vals AI"},"arc:arc_agi_1":{"value":91,"norm":0.91,"version":"ARC Prize"},"vals:srebench":{"value":0.76,"norm":0.0076,"version":"Vals AI"},"livebench:mathematics":{"value":81.24,"norm":0.8124,"version":"LiveBench"},"vals:tax_agent_bench":{"value":31.55,"norm":0.3155,"version":"Vals AI"},"vals:legal_research":{"value":45.19,"norm":0.4519,"version":"Vals AI"},"livebench:reasoning":{"value":77.64,"norm":0.7764,"version":"LiveBench"},"apex":{"value":0.528,"norm":0.528,"version":"glm-5.3-flash_unknown"},"vals:cyber":{"value":67.86,"norm":0.6786,"version":"Vals AI"},"vals:proof_bench":{"value":21,"norm":0.21,"version":"Vals AI"},"vals:vcb-1-100":{"value":16.03,"norm":0.1603,"version":"Vals AI"},"vals:terminal-bench-4":{"value":37.37,"norm":0.3737,"version":"Vals AI"},"vals:code-migration":{"value":20.51,"norm":0.2051,"version":"Vals AI"},"livebench:if":{"value":52.82,"norm":0.5282,"version":"LiveBench"},"x:FrontierMath-Tier-4-v2-Private":{"value":0.17073170731707318,"norm":0.1707,"version":"glm-5.3-flash_max"},"x:Mystery Game Puzzles":{"value":0.08,"norm":0.08,"version":"glm-5.3-flash_max"},"livebench:language":{"value":77.31,"norm":0.7731,"version":"LiveBench"},"vals:swebench":{"value":92,"norm":0.92,"version":"Vals AI"},"vals:legal_bench":{"value":83.93,"norm":0.8393,"version":"Vals AI"},"vals:skillsbench":{"value":40.16,"norm":0.4016,"version":"Vals AI"},"vals:lcb":{"value":80.51,"norm":0.8051,"version":"Vals AI"},"vals:terminal-bench-science":{"value":2.86,"norm":0.0286,"version":"Vals AI"},"vals:mmmu":{"value":86.01,"norm":0.8601,"version":"Vals AI"},"vals:ioi":{"value":52.5,"norm":0.525,"version":"Vals AI"},"vals:medscribe":{"value":88.94,"norm":0.8894,"version":"Vals AI"},"livebench:coding":{"value":78.95,"norm":0.7895,"version":"LiveBench"},"x:Chess Puzzles":{"value":0.21,"norm":0.21,"version":"glm-5.3-flash_max"},"x:FrontierSWE":{"value":0.18125205882352943,"norm":0.1813,"version":"glm-5.3-flash_max"},"livebench:data-analysis":{"value":76.4,"norm":0.764,"version":"LiveBench"},"vals:fabv2":{"value":57.85,"norm":0.5785,"version":"Vals AI"},"gpqa":{"value":0.9015151515151515,"norm":0.9015,"version":"glm-5.3-flash_max"},"x:Surface Evolver Bench":{"value":0.525,"norm":0.525,"version":"glm-5.3-flash_max"},"vals:mmlu_pro":{"value":86.06,"norm":0.8606,"version":"Vals AI"},"aime":{"value":0.9388888888888889,"norm":0.9389,"version":"glm-5.3-flash_max"},"vals:vibe-code":{"value":30.76,"norm":0.3076,"version":"Vals AI"},"vals:sage":{"value":89.25,"norm":0.8925,"version":"Vals AI"},"fm":{"value":0.5578947368421052,"norm":0.5579,"version":"glm-5.3-flash_max"},"livebench:agentic-coding":{"value":56.77,"norm":0.5677,"version":"LiveBench"},"vals:gpqa":{"value":86.36,"norm":0.8636,"version":"Vals AI"},"vals:emb":{"value":55.93,"norm":0.5593,"version":"Vals AI"},"x:DeepSWE":{"value":0.6339285714285714,"norm":0.6339,"version":"glm-5.3-flash_max"},"vals:programbench":{"value":0,"norm":0,"version":"Vals AI"},"vals:hlab":{"value":10.42,"norm":0.1042,"version":"Vals AI"}},"variant":"GLM-5.3-Flash","match":"exact"},"api":{"or_id":"z-ai/glm-5.3-flash","in":0.15,"out":0.5,"context":1048576,"hf":"zai-org/GLM-5.3-Flash","created":"2026-08-26"}},{"id":"deepseek-v4-flash-vision-exp","name":"DeepSeek-V4-Flash-Vision-Exp","lab":"DeepSeek","date":"2026-08-21","type":"open-weights","major":true,"open":true,"open_from":"2026-08-31","release":{"title":"DeepSeek-V4-Flash-Vision-Exp (weights 08-31)","source":"https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash-Vision-Exp","verified":false,"confidence":"high"},"index":{"score":163.7,"lo":132.8,"hi":194.6,"se":16.25,"solve":53.3,"status":"provisional","areas":["coding","maths","reasoning"],"core_areas":[],"testers":["LiveBench"],"tests":{"livebench:data-analysis":{"value":79.48,"norm":0.7948,"version":"LiveBench"},"livebench:agentic-coding":{"value":65.1,"norm":0.651,"version":"LiveBench"},"livebench:mathematics":{"value":87.81,"norm":0.8781,"version":"LiveBench"},"livebench:reasoning":{"value":85.4,"norm":0.854,"version":"LiveBench"},"livebench:if":{"value":70.96,"norm":0.7096,"version":"LiveBench"},"livebench:coding":{"value":68.2,"norm":0.682,"version":"LiveBench"},"livebench:language":{"value":80.36,"norm":0.8036,"version":"LiveBench"}},"variant":"DeepSeek-V4-Flash-Vision-Exp","match":"exact"},"api":{"or_id":"deepseek/deepseek-v4-flash-vision-exp","in":0.44,"out":1.32,"context":1048576,"hf":"deepseek-ai/DeepSeek-V4-Flash-Vision-Exp","created":"2026-08-21"}},{"id":"qwen3-8-flash-next","name":"Qwen3.8-Flash-Next","lab":"Alibaba","date":"2026-08-24","type":"open-weights","major":true,"open":true,"release":{"title":"Qwen3.8-Flash-Next","source":"https://huggingface.co/Qwen/Qwen3.8-Flash-Next","verified":true,"confidence":"high"},"index":{"score":161.9,"lo":131.4,"hi":192.5,"se":16.01,"solve":51.5,"status":"provisional","areas":["coding","maths","reasoning"],"core_areas":[],"testers":["LiveBench"],"tests":{"livebench:language":{"value":74.64,"norm":0.7464,"version":"LiveBench"},"livebench:if":{"value":77.11,"norm":0.7711,"version":"LiveBench"},"livebench:data-analysis":{"value":74.24,"norm":0.7424,"version":"LiveBench"},"livebench:mathematics":{"value":85.82,"norm":0.8582,"version":"LiveBench"},"livebench:agentic-coding":{"value":61.62,"norm":0.6162,"version":"LiveBench"},"livebench:coding":{"value":72.55,"norm":0.7255,"version":"LiveBench"},"livebench:reasoning":{"value":87.38,"norm":0.8738,"version":"LiveBench"}},"variant":"Qwen3.8-Flash-Next","match":"exact"}},{"id":"qwen3-8-flash","name":"Qwen3.8-Flash","lab":"Alibaba","date":"2026-08-26","type":"small","major":true,"open":true,"release":{"title":"Qwen3.8-Flash","source":"https://qwenlm.github.io/qwen-code-docs/en/blog/updates/weekly-update-2026-09-03/","verified":false,"confidence":"med"},"index":{"score":161.9,"lo":131.4,"hi":192.5,"se":16.01,"solve":51.5,"status":"provisional","areas":["coding","maths","reasoning"],"core_areas":[],"testers":["LiveBench"],"tests":{"livebench:language":{"value":74.64,"norm":0.7464,"version":"LiveBench"},"livebench:if":{"value":77.11,"norm":0.7711,"version":"LiveBench"},"livebench:data-analysis":{"value":74.24,"norm":0.7424,"version":"LiveBench"},"livebench:mathematics":{"value":85.82,"norm":0.8582,"version":"LiveBench"},"livebench:agentic-coding":{"value":61.62,"norm":0.6162,"version":"LiveBench"},"livebench:coding":{"value":72.55,"norm":0.7255,"version":"LiveBench"},"livebench:reasoning":{"value":87.38,"norm":0.8738,"version":"LiveBench"}},"variant":"Qwen3.8-Flash-Next","match":"family"}},{"id":"hy4-preview","name":"Hy4 preview","lab":"Tencent","date":"2026-08-28","type":"open-weights","major":true,"open":true,"release":{"title":"Hy4 preview","source":"https://simonwillison.net/2026/Aug/29/hy4/","verified":false,"confidence":"high"},"index":{"score":172.8,"lo":153.9,"hi":191.8,"se":6.65,"solve":61.8,"status":"provisional","areas":["agents","coding","maths","work"],"core_areas":["coding","work"],"testers":["Vals AI"],"tests":{"vals:medscribe":{"value":83.6,"norm":0.836,"version":"Vals AI"},"vals:legal_bench":{"value":83.76,"norm":0.8376,"version":"Vals AI"},"vals:srebench":{"value":2.29,"norm":0.0229,"version":"Vals AI"},"vals:medcode":{"value":43.25,"norm":0.4325,"version":"Vals AI"},"vals:public-benefits-bench":{"value":68.61,"norm":0.6861,"version":"Vals AI"},"vals:programbench":{"value":0,"norm":0,"version":"Vals AI"},"vals:code-migration":{"value":47.43,"norm":0.4743,"version":"Vals AI"},"vals:hlab":{"value":12.5,"norm":0.125,"version":"Vals AI"},"vals:cyber":{"value":65.36,"norm":0.6536,"version":"Vals AI"},"vals:ioi":{"value":59.33,"norm":0.5933,"version":"Vals AI"},"vals:biomysterybench":{"value":69.26,"norm":0.6926,"version":"Vals AI"},"vals:legal_research":{"value":45.19,"norm":0.4519,"version":"Vals AI"},"vals:vibe-code":{"value":81.01,"norm":0.8101,"version":"Vals AI"},"vals:proof_bench":{"value":75,"norm":0.75,"version":"Vals AI"},"vals:terminal-bench-4":{"value":39.39,"norm":0.3939,"version":"Vals AI"},"vals:emb":{"value":58.08,"norm":0.5808,"version":"Vals AI"},"vals:fabv2":{"value":55.06,"norm":0.5506,"version":"Vals AI"},"vals:tax_agent_bench":{"value":34.4,"norm":0.344,"version":"Vals AI"}},"variant":"Hy4 preview","match":"exact"},"api":{"or_id":"tencent/hy4-preview","in":0.834,"out":2.501,"context":1048576,"hf":"tencent/Hy4-preview","created":"2026-08-28"},"open_evidence":"https://www.tencent.com/tencent-releases-and-open-sources-tencent-hy4-preview/","open_from":"2026-08-28"},{"id":"claude-fable-5-1","name":"Claude Fable 5.1","lab":"Anthropic","date":"2026-09-01","type":"flagship","major":true,"open":false,"release":{"title":"Claude Fable 5.1 + Mythos 5.1","source":"https://www.anthropic.com/claude-fable-and-mythos-5-1","verified":false,"confidence":"high"},"index":{"score":188.8,"lo":172.8,"hi":204.8,"se":2.5,"solve":72.2,"status":"rated","areas":["agents","coding","maths","novel","reasoning","work"],"core_areas":["agents","coding","maths","novel","reasoning","work"],"testers":["ARC Prize","Epoch AI","GSO team","GeoBench team","Havard T. Ihle","LiveBench","Mercor","Proximal","Redwood Research","Scale AI","SimpleBench team","Vals AI"],"tests":{"x:Chess Puzzles":{"value":0.48,"norm":0.48,"version":"claude-fable-5-1_max"},"vals:gpqa":{"value":93.43,"norm":0.9343,"version":"Vals AI"},"x:FrontierSWE":{"value":0.5629078999999999,"norm":0.5629,"version":"claude-fable-5-1_max"},"vals:legal_bench":{"value":88.51,"norm":0.8851,"version":"Vals AI"},"vals:programbench":{"value":7,"norm":0.07,"version":"Vals AI"},"x:EBR-bench":{"value":0.5714285714285714,"norm":0.5714,"version":"claude-fable-5-1_max"},"vals:mysterymechanism":{"value":47.75,"norm":0.4775,"version":"Vals AI"},"vals:vcb-1-100":{"value":28,"norm":0.28,"version":"Vals AI"},"arc:arc_agi_2":{"value":90,"norm":0.9,"version":"ARC Prize"},"vals:tax_eval_v2":{"value":75.96,"norm":0.7596,"version":"Vals AI"},"livebench:coding":{"value":86.38,"norm":0.8638,"version":"LiveBench"},"vals:fabv2":{"value":58.88,"norm":0.5888,"version":"Vals AI"},"x:Mystery Game Puzzles":{"value":0.58,"norm":0.58,"version":"claude-fable-5-1_max"},"livebench:language":{"value":89.5,"norm":0.895,"version":"LiveBench"},"vals:lcb":{"value":90.52,"norm":0.9052,"version":"Vals AI"},"livebench:reasoning":{"value":91.69,"norm":0.9169,"version":"LiveBench"},"vals:vibe-code":{"value":90.26,"norm":0.9026,"version":"Vals AI"},"x:SimpleQA Verified":{"value":0.708,"norm":0.708,"version":"claude-fable-5-1_max"},"vals:ioi":{"value":90.78,"norm":0.9078,"version":"Vals AI"},"x:MirrorCode":{"value":0.9333333333333333,"norm":0.9333,"version":"claude-fable-5-1_max"},"vals:medscribe":{"value":91.29,"norm":0.9129,"version":"Vals AI"},"fm":{"value":0.9017543859649123,"norm":0.9018,"version":"claude-fable-5-1_max"},"vals:emb":{"value":76.67,"norm":0.7667,"version":"Vals AI"},"x:SimpleBench":{"value":0.866,"norm":0.866,"version":"claude-fable-5-1_unknown"},"x:GSO-Bench":{"value":0.8824,"norm":0.8824,"version":"claude-fable-5-1_unknown"},"apex":{"value":0.6859999999999999,"norm":0.686,"version":"claude-fable-5-1_unknown"},"vals:mmmu":{"value":90.64,"norm":0.9064,"version":"Vals AI"},"vals:terminal-bench-4":{"value":58.08,"norm":0.5808,"version":"Vals AI"},"vals:code-migration":{"value":54.61,"norm":0.5461,"version":"Vals AI"},"arc:arc_agi_1":{"value":97.5,"norm":0.975,"version":"ARC Prize"},"vals:srebench":{"value":22.9,"norm":0.229,"version":"Vals AI"},"livebench:if":{"value":72.99,"norm":0.7299,"version":"LiveBench"},"x:Furniture Assembly":{"value":0.7,"norm":0.7,"version":"claude-fable-5-1_max"},"x:DTBench":{"value":0.976,"norm":0.976,"version":"claude-fable-5-1_xhigh"},"vals:medcode":{"value":53.51,"norm":0.5351,"version":"Vals AI"},"aime":{"value":1,"norm":1,"version":"claude-fable-5-1_max"},"livebench:data-analysis":{"value":80.28,"norm":0.8028,"version":"LiveBench"},"livebench:mathematics":{"value":97.01,"norm":0.9701,"version":"LiveBench"},"vals:tax_agent_bench":{"value":50.75,"norm":0.5075,"version":"Vals AI"},"vals:skillsbench":{"value":61.55,"norm":0.6155,"version":"Vals AI"},"x:GeoBench":{"value":0.84,"norm":0.84,"version":"claude-fable-5-1_high"},"x:LMCA":{"value":65.46,"norm":0.6546,"version":"claude-fable-5-1_unknown"},"hle":{"value":0.465,"norm":0.465,"version":"claude-fable-5-1_xhigh"},"livebench:agentic-coding":{"value":66.06,"norm":0.6606,"version":"LiveBench"},"x:FrontierMath-Tier-4-v2-Private":{"value":0.8780487804878049,"norm":0.878,"version":"claude-fable-5-1_max"},"vals:public-benefits-bench":{"value":74.9,"norm":0.749,"version":"Vals AI"},"vals:legal_research":{"value":55.29,"norm":0.5529,"version":"Vals AI"},"vals:terminal-bench-science":{"value":40,"norm":0.4,"version":"Vals AI"},"vals:cyber":{"value":70.42,"norm":0.7042,"version":"Vals AI"},"x:WeirdML":{"value":0.929,"norm":0.929,"version":"claude-fable-5-1_max"},"vals:proof_bench":{"value":100,"norm":1,"version":"Vals AI"},"vals:mmlu_pro":{"value":92.37,"norm":0.9237,"version":"Vals AI"},"vals:mortgage_tax":{"value":70.79,"norm":0.7079,"version":"Vals AI"}},"variant":"Claude Fable 5.1","match":"exact"},"api":{"or_id":"anthropic/claude-fable-5.1","in":10,"out":50,"context":1000000,"hf":null,"created":"2026-09-01"}},{"id":"claude-mythos-5-1","name":"Claude Mythos 5.1","lab":"Anthropic","date":"2026-09-01","type":"flagship","major":true,"open":false,"release":{"title":"Claude Fable 5.1 + Mythos 5.1","source":"https://www.anthropic.com/claude-fable-and-mythos-5-1","verified":false,"confidence":"high"}},{"id":"qwen3-8-max-0902","name":"Qwen3.8 Max (0902)","lab":"Alibaba","date":"2026-09-01","type":"flagship","major":false,"open":false,"index":{"score":161.3,"lo":144.2,"hi":178.4,"se":4.38,"solve":50.8,"status":"rated","areas":["agents","coding","maths","reasoning"],"core_areas":["maths","reasoning"],"testers":["Epoch AI","Proximal"],"tests":{"aime":{"value":1,"norm":1,"version":"qwen3.8-max-0902_xhigh"},"x:FrontierSWE":{"value":0.17809649411764705,"norm":0.1781,"version":"qwen3.8-max-0902_xhigh"},"x:Furniture Assembly":{"value":0.2,"norm":0.2,"version":"qwen3.8-max-0902_xhigh"},"x:FrontierMath-Tier-4-v2-Private":{"value":0.34146341463414637,"norm":0.3415,"version":"qwen3.8-max-0902_xhigh"},"x:MirrorCode":{"value":0.4777777777777778,"norm":0.4778,"version":"qwen3.8-max-0902_xhigh"},"x:Chess Puzzles":{"value":0.37,"norm":0.37,"version":"qwen3.8-max-0902_xhigh"},"x:SimpleQA Verified":{"value":0.473,"norm":0.473,"version":"qwen3.8-max-0902_xhigh"},"gpqa":{"value":0.922979797979798,"norm":0.923,"version":"qwen3.8-max-0902_xhigh"},"fm":{"value":0.656140350877193,"norm":0.6561,"version":"qwen3.8-max-0902_xhigh"}},"variant":"Qwen3.8 Max (0902)","match":"self"},"api":{"or_id":"qwen/qwen3.8-max-0902","in":2,"out":6,"context":1000000,"hf":null,"created":"2026-09-03"},"open_evidence":"Hosted Max snapshot","date_from":"epoch"},{"id":"gemini-3-8-flash","name":"Gemini 3.8 Flash","lab":"Google","date":"2026-09-02","type":"flagship","major":true,"open":false,"release":{"title":"Gemini 3.8 Flash","source":"https://blog.google/innovation-and-ai/models-and-research/gemini-models/3-8-flash-and-3-8-flash-cyber/","verified":true,"confidence":"high"},"index":{"score":172.6,"lo":156.7,"hi":188.5,"se":2.14,"solve":61.6,"status":"rated","areas":["agents","coding","maths","novel","reasoning","work"],"core_areas":["agents","coding","maths","novel","reasoning","work"],"testers":["ARC Prize","Datacurve","Epoch AI","GeoBench team","Havard T. Ihle","Mercor","Proximal","Redwood Research","Scale AI","SimpleBench team","Vals AI","Yann Henon"],"tests":{"vals:public-benefits-bench":{"value":65.29,"norm":0.6529,"version":"Vals AI"},"vals:hlab":{"value":12.08,"norm":0.1208,"version":"Vals AI"},"x:SimpleQA Verified":{"value":0.697,"norm":0.697,"version":"gemini-3.8-flash_high"},"x:GeoBench":{"value":0.86,"norm":0.86,"version":"gemini-3.8-flash_unknown"},"vals:ioi":{"value":56.94,"norm":0.5694,"version":"Vals AI"},"vals:biomysterybench":{"value":62.22,"norm":0.6222,"version":"Vals AI"},"vals:medcode":{"value":48.14,"norm":0.4814,"version":"Vals AI"},"x:FrontierMath-Tier-4-v2-Private":{"value":0.21951219512195122,"norm":0.2195,"version":"gemini-3.8-flash_high"},"x:Chess Puzzles":{"value":0.58,"norm":0.58,"version":"gemini-3.8-flash_high"},"vals:fabv2":{"value":61.44,"norm":0.6144,"version":"Vals AI"},"vals:tax_agent_bench":{"value":32.12,"norm":0.3212,"version":"Vals AI"},"vals:programbench":{"value":1,"norm":0.01,"version":"Vals AI"},"vals:lcb":{"value":89.48,"norm":0.8948,"version":"Vals AI"},"vals:vibe-code":{"value":78.65,"norm":0.7865,"version":"Vals AI"},"apex":{"value":0.643,"norm":0.643,"version":"gemini-3.8-flash_unknown"},"vals:mmmu":{"value":89.08,"norm":0.8908,"version":"Vals AI"},"vals:proof_bench":{"value":48,"norm":0.48,"version":"Vals AI"},"vals:vcb-1-100":{"value":18.77,"norm":0.1877,"version":"Vals AI"},"arc:arc_agi_2":{"value":89.2,"norm":0.892,"version":"ARC Prize"},"vals:terminal-bench-4":{"value":19.7,"norm":0.197,"version":"Vals AI"},"arc:arc_agi_1":{"value":98.5,"norm":0.985,"version":"ARC Prize"},"vals:mortgage_tax":{"value":65.34,"norm":0.6534,"version":"Vals AI"},"vals:srebench":{"value":11.07,"norm":0.1107,"version":"Vals AI"},"hle":{"value":0.44520000000000004,"norm":0.4452,"version":"gemini-3.8-flash_unknown"},"vals:medscribe":{"value":84.5,"norm":0.845,"version":"Vals AI"},"x:DTBench":{"value":0.9572,"norm":0.9572,"version":"gemini-3.8-flash_unknown"},"gpqa":{"value":0.9539141414141414,"norm":0.9539,"version":"gemini-3.8-flash_high"},"vals:legal_bench":{"value":86.99,"norm":0.8699,"version":"Vals AI"},"vals:mmlu_pro":{"value":90.22,"norm":0.9022,"version":"Vals AI"},"x:LMCA":{"value":52.9,"norm":0.529,"version":"gemini-3.8-flash_medium"},"x:MirrorCode":{"value":0.3888888888888889,"norm":0.3889,"version":"gemini-3.8-flash_high"},"x:Furniture Assembly":{"value":0.31666666666666665,"norm":0.3167,"version":"gemini-3.8-flash_high"},"arc:arc_agi_3":{"value":10.4,"norm":0.104,"version":"ARC Prize"},"vals:gpqa":{"value":94.44,"norm":0.9444,"version":"Vals AI"},"x:FrontierSWE":{"value":0.19626902941176463,"norm":0.1963,"version":"gemini-3.8-flash_high"},"vals:swebench":{"value":80,"norm":0.8,"version":"Vals AI"},"vals:skillsbench":{"value":57.98,"norm":0.5798,"version":"Vals AI"},"vals:terminal-bench-science":{"value":8.57,"norm":0.0857,"version":"Vals AI"},"x:WeirdML":{"value":0.8481000000000001,"norm":0.8481,"version":"gemini-3.8-flash_high"},"vals:mysterymechanism":{"value":36.49,"norm":0.3649,"version":"Vals AI"},"vals:tax_eval_v2":{"value":74.45,"norm":0.7445,"version":"Vals AI"},"aime":{"value":0.9888888888888889,"norm":0.9889,"version":"gemini-3.8-flash_high"},"x:SimpleBench":{"value":0.8240000000000001,"norm":0.824,"version":"gemini-3.8-flash_unknown"},"x:Mystery Game Puzzles":{"value":0.47,"norm":0.47,"version":"gemini-3.8-flash_high"},"x:DeepSWE":{"value":0.738255033557047,"norm":0.7383,"version":"gemini-3.8-flash_high"},"vals:legal_research":{"value":38.94,"norm":0.3894,"version":"Vals AI"},"x:Surface Evolver Bench":{"value":0.76875,"norm":0.7688,"version":"gemini-3.8-flash_high"},"vals:cyber":{"value":43.75,"norm":0.4375,"version":"Vals AI"},"vals:code-migration":{"value":36.55,"norm":0.3655,"version":"Vals AI"},"vals:sage":{"value":88.24,"norm":0.8824,"version":"Vals AI"},"fm":{"value":0.6842105263157895,"norm":0.6842,"version":"gemini-3.8-flash_high"},"vals:emb":{"value":72.2,"norm":0.722,"version":"Vals AI"}},"variant":"Gemini 3.8 Flash","match":"exact"},"api":{"or_id":"google/gemini-3.8-flash","in":0.825,"out":4.125,"context":1048576,"hf":null,"created":"2026-09-02"}},{"id":"muse-spark-1-3","name":"Muse Spark 1.3","lab":"Meta","date":"2026-09-02","type":"flagship","major":true,"open":false,"release":{"title":"Muse Spark 1.3","source":null,"verified":false,"confidence":"high"},"index":{"score":169.4,"lo":153.4,"hi":185.5,"se":2.5,"solve":58.9,"status":"rated","areas":["agents","coding","maths","reasoning","work"],"core_areas":["agents","coding","maths","work"],"testers":["Epoch AI","GeoBench team","LiveBench","Mercor","Proximal","Redwood Research","SimpleBench team","Vals AI"],"tests":{"livebench:reasoning":{"value":89.65,"norm":0.8965,"version":"LiveBench"},"vals:vibe-code":{"value":82.86,"norm":0.8286,"version":"Vals AI"},"x:DTBench":{"value":0.9653,"norm":0.9653,"version":"muse-spark-1.3_unknown"},"aime":{"value":0.9916666666666667,"norm":0.9917,"version":"muse-spark-1.3_xhigh"},"livebench:language":{"value":82.79,"norm":0.8279,"version":"LiveBench"},"vals:code-migration":{"value":27.58,"norm":0.2758,"version":"Vals AI"},"vals:ioi":{"value":43.94,"norm":0.4394,"version":"Vals AI"},"vals:proof_bench":{"value":55,"norm":0.55,"version":"Vals AI"},"livebench:data-analysis":{"value":79.57,"norm":0.7957,"version":"LiveBench"},"vals:legal_research":{"value":40.87,"norm":0.4087,"version":"Vals AI"},"x:FrontierMath-Tier-4-v2-Private":{"value":0.4634146341463415,"norm":0.4634,"version":"muse-spark-1.3_max"},"x:Chess Puzzles":{"value":0.37,"norm":0.37,"version":"muse-spark-1.3_max"},"x:GeoBench":{"value":0.86,"norm":0.86,"version":"muse-spark-1.3_unknown"},"vals:terminal-bench-science":{"value":4.29,"norm":0.0429,"version":"Vals AI"},"x:SimpleBench":{"value":0.818,"norm":0.818,"version":"muse-spark-1.3_unknown"},"vals:terminal-bench-4":{"value":10.61,"norm":0.1061,"version":"Vals AI"},"livebench:if":{"value":78,"norm":0.78,"version":"LiveBench"},"x:FrontierSWE":{"value":0.2580214823529412,"norm":0.258,"version":"muse-spark-1.3_max"},"livebench:coding":{"value":81.06,"norm":0.8106,"version":"LiveBench"},"x:Mystery Game Puzzles":{"value":0.25,"norm":0.25,"version":"muse-spark-1.3_max"},"vals:cyber":{"value":69.41,"norm":0.6941,"version":"Vals AI"},"apex":{"value":0.5860000000000001,"norm":0.586,"version":"muse-spark-1.3_unknown"},"vals:emb":{"value":62.71,"norm":0.6271,"version":"Vals AI"},"vals:fabv2":{"value":58.9,"norm":0.589,"version":"Vals AI"},"fm":{"value":0.743859649122807,"norm":0.7439,"version":"muse-spark-1.3_xhigh"},"livebench:mathematics":{"value":95.95,"norm":0.9595,"version":"LiveBench"},"livebench:agentic-coding":{"value":64.09,"norm":0.6409,"version":"LiveBench"},"vals:programbench":{"value":0.5,"norm":0.005,"version":"Vals AI"},"x:LMCA":{"value":53.87,"norm":0.5387,"version":"muse-spark-1.3_unknown"},"x:MirrorCode":{"value":0.5444444444444444,"norm":0.5444,"version":"muse-spark-1.3_max"},"vals:tax_agent_bench":{"value":37.89,"norm":0.3789,"version":"Vals AI"}},"variant":"Muse Spark 1.3","match":"exact"},"api":{"or_id":"meta/muse-spark-1.3","in":1.25,"out":4.25,"context":1048576,"hf":null,"created":"2026-09-02"}},{"id":"gpt-6-astra","name":"GPT-6 Astra","lab":"OpenAI","date":"2026-09-04","type":"flagship","major":true,"open":false,"release":{"title":"GPT-6 Astra (preview 09-03)","source":"https://openai.com/index/gpt-6-astra-next-generation-work/","verified":false,"confidence":"high"},"index":{"score":193.4,"lo":177.6,"hi":209.2,"se":1.93,"solve":74.4,"status":"rated","areas":["agents","coding","maths","novel","reasoning","work"],"core_areas":["agents","coding","maths","novel","reasoning","work"],"testers":["ARC Prize","BALROG team","Datacurve","Epoch AI","GSO team","Havard T. Ihle","LiveBench","Mercor","Proximal","Redwood Research","Scale AI","SimpleBench team","Vals AI"],"tests":{"vals:medscribe":{"value":87.91,"norm":0.8791,"version":"Vals AI"},"x:FrontierSWE":{"value":0.6550757455882352,"norm":0.6551,"version":"gpt-6-astra_max"},"vals:fabv2":{"value":53.54,"norm":0.5354,"version":"Vals AI"},"fm":{"value":0.9368421052631579,"norm":0.9368,"version":"gpt-6-astra_max"},"livebench:mathematics":{"value":96.81,"norm":0.9681,"version":"LiveBench"},"livebench:agentic-coding":{"value":57.32,"norm":0.5732,"version":"LiveBench"},"vals:legal_research":{"value":39.42,"norm":0.3942,"version":"Vals AI"},"x:FrontierMath-Tier-4-v2-Private":{"value":0.976,"norm":0.976,"version":"gpt-6-astra_max"},"vals:cyber":{"value":41.07,"norm":0.4107,"version":"Vals AI"},"livebench:if":{"value":75.58,"norm":0.7558,"version":"LiveBench"},"vals:mysterymechanism":{"value":53.15,"norm":0.5315,"version":"Vals AI"},"x:LMCA":{"value":64.37,"norm":0.6437,"version":"gpt-6-astra_xhigh"},"vals:medcode":{"value":48.49,"norm":0.4849,"version":"Vals AI"},"x:MirrorCode":{"value":0.6,"norm":0.6,"version":"gpt-6-astra_max"},"x:Furniture Assembly":{"value":0.8,"norm":0.8,"version":"gpt-6-astra_max"},"vals:emb":{"value":71.7,"norm":0.717,"version":"Vals AI"},"livebench:coding":{"value":80.36,"norm":0.8036,"version":"LiveBench"},"livebench:language":{"value":89.43,"norm":0.8943,"version":"LiveBench"},"x:Chess Puzzles":{"value":0.74,"norm":0.74,"version":"gpt-6-astra_max"},"vals:terminal-bench-science":{"value":62.86,"norm":0.6286,"version":"Vals AI"},"x:SimpleBench":{"value":0.836,"norm":0.836,"version":"gpt-6-astra_unknown"},"vals:tax_agent_bench":{"value":20.72,"norm":0.2072,"version":"Vals AI"},"livebench:reasoning":{"value":92.65,"norm":0.9265,"version":"LiveBench"},"x:WeirdML":{"value":0.9326000000000001,"norm":0.9326,"version":"gpt-6-astra_max"},"aime":{"value":1,"norm":1,"version":"gpt-6-astra_max"},"vals:hlab":{"value":7.08,"norm":0.0708,"version":"Vals AI"},"vals:programbench":{"value":5.5,"norm":0.055,"version":"Vals AI"},"vals:code-migration":{"value":67.74,"norm":0.6774,"version":"Vals AI"},"hle":{"value":0.5479999999999999,"norm":0.548,"version":"gpt-6-astra_unknown"},"x:Balrog":{"value":0.6829999999999999,"norm":0.683,"version":"gpt-6-astra_max"},"vals:biomysterybench":{"value":79.26,"norm":0.7926,"version":"Vals AI"},"arc:arc_agi_3":{"value":62.7,"norm":0.627,"version":"ARC Prize"},"arc:arc_agi_2":{"value":95,"norm":0.95,"version":"ARC Prize"},"vals:terminal-bench-4":{"value":59.6,"norm":0.596,"version":"Vals AI"},"arc:arc_agi_1":{"value":98.5,"norm":0.985,"version":"ARC Prize"},"x:EBR-bench":{"value":0.7619047619047619,"norm":0.7619,"version":"gpt-6-astra_max"},"vals:ioi":{"value":100,"norm":1,"version":"Vals AI"},"vals:srebench":{"value":56.87,"norm":0.5687,"version":"Vals AI"},"x:DTBench":{"value":0.9733,"norm":0.9733,"version":"gpt-6-astra_xhigh"},"livebench:data-analysis":{"value":82.97,"norm":0.8297,"version":"LiveBench"},"x:Mystery Game Puzzles":{"value":0.84,"norm":0.84,"version":"gpt-6-astra_max"},"gpqa":{"value":0.9577020202020202,"norm":0.9577,"version":"gpt-6-astra_max"},"x:DeepSWE":{"value":0.7411504424778761,"norm":0.7412,"version":"gpt-6-astra_xhigh"},"vals:vcb-1-100":{"value":27.64,"norm":0.2764,"version":"Vals AI"},"x:GSO-Bench":{"value":0.7941,"norm":0.7941,"version":"gpt-6-astra_unknown"},"vals:vibe-code":{"value":89.59,"norm":0.8959,"version":"Vals AI"},"x:SimpleQA Verified":{"value":0.756,"norm":0.756,"version":"gpt-6-astra_max"},"apex":{"value":0.647,"norm":0.647,"version":"gpt-6-astra_unknown"},"vals:proof_bench":{"value":99,"norm":0.99,"version":"Vals AI"}},"variant":"GPT-6 Astra","match":"exact"},"api":{"or_id":"openai/gpt-6-astra","in":10,"out":50,"context":1050000,"hf":null,"created":"2026-09-04"},"modes":[{"label":"Pro","name":"GPT-6 Astra Pro","index":{"score":195.9,"lo":176.3,"hi":215.5,"se":7.29,"solve":75.6,"status":"rated","areas":["coding","maths","reasoning"],"core_areas":[],"testers":["Epoch AI","Havard T. Ihle","SimpleBench team"],"tests":{"x:SimpleBench":{"value":0.865,"norm":0.865,"version":"gpt-6-astra_promax"},"x:WeirdML":{"value":0.9357,"norm":0.9357,"version":"gpt-6-astra_promax"},"x:Chess Puzzles":{"value":0.76,"norm":0.76,"version":"gpt-6-astra_promax"}}},"api":{"or_id":"openai/gpt-6-astra-pro","in":10,"out":50,"context":1050000,"hf":null,"created":"2026-09-04"}}],"aliases":["gpt-6-astra-pro"]},{"id":"deepseek-v4-1-flash","name":"DeepSeek-V4.1-Flash","lab":"DeepSeek","date":"2026-09-10","type":"open-weights","major":true,"open":true,"release":{"title":"DeepSeek-V4.1-Flash","source":"https://api-docs.deepseek.com/news/news260910/","verified":false,"confidence":"high"},"index":{"score":163,"lo":147.2,"hi":178.9,"se":2.07,"solve":52.6,"status":"rated","areas":["agents","coding","maths","novel","reasoning","work"],"core_areas":["agents","coding","maths","novel","reasoning","work"],"testers":["ARC Prize","Epoch AI","LiveBench","Mercor","Redwood Research","SimpleBench team","Vals AI","Yann Henon"],"tests":{"livebench:agentic-coding":{"value":77.27,"norm":0.7727,"version":"LiveBench"},"livebench:language":{"value":81.19,"norm":0.8119,"version":"LiveBench"},"x:Mystery Game Puzzles":{"value":0.43,"norm":0.43,"version":"deepseek-v4.1-flash_max"},"vals:terminal-bench-4":{"value":19.7,"norm":0.197,"version":"Vals AI"},"vals:programbench":{"value":0.5,"norm":0.005,"version":"Vals AI"},"livebench:reasoning":{"value":86.69,"norm":0.8669,"version":"LiveBench"},"x:Surface Evolver Bench":{"value":0.4625,"norm":0.4625,"version":"deepseek-v4.1-flash_high"},"vals:vibe-code":{"value":84.74,"norm":0.8474,"version":"Vals AI"},"arc:arc_agi_1":{"value":94.5,"norm":0.945,"version":"ARC Prize"},"vals:emb":{"value":57.21,"norm":0.5721,"version":"Vals AI"},"arc:arc_agi_3":{"value":0.9,"norm":0.009,"version":"ARC Prize"},"vals:skillsbench":{"value":69.8,"norm":0.698,"version":"Vals AI"},"x:Chess Puzzles":{"value":0.48,"norm":0.48,"version":"deepseek-v4.1-flash_max"},"aime":{"value":0.9833333333333333,"norm":0.9833,"version":"deepseek-v4.1-flash_max"},"vals:cyber":{"value":73.69,"norm":0.7369,"version":"Vals AI"},"arc:arc_agi_2":{"value":72.9,"norm":0.729,"version":"ARC Prize"},"vals:legal_bench":{"value":83.28,"norm":0.8328,"version":"Vals AI"},"vals:code-migration":{"value":45.62,"norm":0.4562,"version":"Vals AI"},"vals:srebench":{"value":0.76,"norm":0.0076,"version":"Vals AI"},"x:SimpleQA Verified":{"value":0.429,"norm":0.429,"version":"deepseek-v4.1-flash_max"},"x:DTBench":{"value":0.8987,"norm":0.8987,"version":"deepseek-v4.1-flash_unknown"},"x:LMCA":{"value":46.97,"norm":0.4697,"version":"deepseek-v4.1-flash_unknown"},"livebench:data-analysis":{"value":79.25,"norm":0.7925,"version":"LiveBench"},"x:Furniture Assembly":{"value":0.3416666666666667,"norm":0.3417,"version":"deepseek-v4.1-flash_max"},"vals:public-benefits-bench":{"value":64.28,"norm":0.6428,"version":"Vals AI"},"vals:terminal-bench-science":{"value":0,"norm":0,"version":"Vals AI"},"vals:medcode":{"value":41.17,"norm":0.4117,"version":"Vals AI"},"x:MirrorCode":{"value":0.6444444444444445,"norm":0.6444,"version":"deepseek-v4.1-flash_max"},"vals:sage":{"value":85.38,"norm":0.8538,"version":"Vals AI"},"livebench:coding":{"value":80.04,"norm":0.8004,"version":"LiveBench"},"gpqa":{"value":0.8983585858585859,"norm":0.8984,"version":"deepseek-v4.1-flash_max"},"vals:legal_research":{"value":41.35,"norm":0.4135,"version":"Vals AI"},"x:SimpleBench":{"value":0.667,"norm":0.667,"version":"deepseek-v4.1-flash_unknown"},"vals:tax_agent_bench":{"value":26.63,"norm":0.2663,"version":"Vals AI"},"apex":{"value":0.395,"norm":0.395,"version":"deepseek-v4.1-flash_unknown"},"vals:mysterymechanism":{"value":21.17,"norm":0.2117,"version":"Vals AI"},"vals:proof_bench":{"value":54,"norm":0.54,"version":"Vals AI"},"vals:medscribe":{"value":85.5,"norm":0.855,"version":"Vals AI"},"fm":{"value":0.6736842105263158,"norm":0.6737,"version":"deepseek-v4.1-flash_max"},"x:FrontierMath-Tier-4-v2-Private":{"value":0.2682926829268293,"norm":0.2683,"version":"deepseek-v4.1-flash_max"},"vals:fabv2":{"value":53.48,"norm":0.5348,"version":"Vals AI"},"livebench:mathematics":{"value":93.29,"norm":0.9329,"version":"LiveBench"},"vals:vcb-1-100":{"value":16.38,"norm":0.1638,"version":"Vals AI"},"vals:ioi":{"value":40.28,"norm":0.4028,"version":"Vals AI"},"livebench:if":{"value":70.03,"norm":0.7003,"version":"LiveBench"},"vals:biomysterybench":{"value":67.78,"norm":0.6778,"version":"Vals AI"}},"variant":"DeepSeek V4.1 Flash","match":"exact"},"api":{"or_id":"deepseek/deepseek-v4.1-flash","in":0.051,"out":0.567,"context":1048576,"hf":"deepseek-ai/DeepSeek-V4.1-Flash","created":"2026-09-10"}},{"id":"kimi-k2-8-preview","name":"Kimi K2.8 Preview","lab":"Moonshot","date":"2026-09-11","type":"flagship","major":true,"open":false,"release":{"title":"Kimi K2.8 Preview","source":null,"verified":false,"confidence":"med"}},{"id":"step-5-preview","name":"Step 5 Preview","lab":"StepFun","date":"2026-09-18","type":"flagship","major":true,"open":false,"release":{"title":"Step 5 Preview","source":"https://platform.stepfun.ai/docs/en/guides/models/step-5-preview","verified":false,"confidence":"high"},"index":{"score":162.5,"lo":144.6,"hi":180.4,"se":5.45,"solve":52.1,"status":"provisional","areas":["agents","coding","maths","novel","work"],"core_areas":["coding","work"],"testers":["Vals AI"],"tests":{"vals:ioi":{"value":38.06,"norm":0.3806,"version":"Vals AI"},"vals:srebench":{"value":0.38,"norm":0.0038,"version":"Vals AI"},"vals:legal_research":{"value":40.38,"norm":0.4038,"version":"Vals AI"},"vals:proof_bench":{"value":42,"norm":0.42,"version":"Vals AI"},"vals:biomysterybench":{"value":67.41,"norm":0.6741,"version":"Vals AI"},"vals:code-migration":{"value":38.98,"norm":0.3898,"version":"Vals AI"},"vals:emb":{"value":57.12,"norm":0.5712,"version":"Vals AI"},"vals:vibe-code":{"value":70.4,"norm":0.704,"version":"Vals AI"},"vals:terminal-bench-4":{"value":31.82,"norm":0.3182,"version":"Vals AI"},"vals:fabv2":{"value":50.67,"norm":0.5067,"version":"Vals AI"},"vals:tax_agent_bench":{"value":27.27,"norm":0.2727,"version":"Vals AI"},"vals:public-benefits-bench":{"value":58.12,"norm":0.5812,"version":"Vals AI"},"vals:mysterymechanism":{"value":18.47,"norm":0.1847,"version":"Vals AI"},"vals:hlab":{"value":12.92,"norm":0.1292,"version":"Vals AI"},"vals:terminal-bench-science":{"value":2.86,"norm":0.0286,"version":"Vals AI"}},"variant":"Step 5 Preview","match":"exact"},"api":{"or_id":"stepfun/step-5-preview","in":1,"out":2.7,"context":1000000,"hf":"stepfun-ai/Step-5-Preview","created":"2026-10-08"}},{"id":"grok-4-7","name":"Grok 4.7","lab":"xAI","date":"2026-09-21","type":"flagship","major":true,"open":false,"release":{"title":"Grok 4.7","source":"https://x.ai/news/grok-4-7","verified":true,"confidence":"high"},"index":{"score":164.4,"lo":148.5,"hi":180.3,"se":2.25,"solve":54.1,"status":"rated","areas":["agents","coding","maths","novel","reasoning","work"],"core_areas":["agents","coding","maths","novel","reasoning","work"],"testers":["ARC Prize","Epoch AI","GeoBench team","Mercor","Proximal","Redwood Research","Vals AI"],"tests":{"gpqa":{"value":0.92739898989899,"norm":0.9274,"version":"grok-4.7_xhigh"},"vals:public-benefits-bench":{"value":65.63,"norm":0.6563,"version":"Vals AI"},"vals:legal_research":{"value":47.12,"norm":0.4712,"version":"Vals AI"},"x:MirrorCode":{"value":0.5444444444444444,"norm":0.5444,"version":"grok-4.7_xhigh"},"fm":{"value":0.5298245614035088,"norm":0.5298,"version":"grok-4.7_xhigh"},"vals:vibe-code":{"value":86.18,"norm":0.8618,"version":"Vals AI"},"vals:hlab":{"value":18.33,"norm":0.1833,"version":"Vals AI"},"x:GeoBench":{"value":0.61,"norm":0.61,"version":"grok-4.7_unknown"},"vals:proof_bench":{"value":26,"norm":0.26,"version":"Vals AI"},"arc:arc_agi_2":{"value":61.4,"norm":0.614,"version":"ARC Prize"},"vals:terminal-bench-4":{"value":28.79,"norm":0.2879,"version":"Vals AI"},"vals:programbench":{"value":0.5,"norm":0.005,"version":"Vals AI"},"vals:code-migration":{"value":44.82,"norm":0.4482,"version":"Vals AI"},"vals:biomysterybench":{"value":69.26,"norm":0.6926,"version":"Vals AI"},"vals:sage":{"value":88.71,"norm":0.8871,"version":"Vals AI"},"vals:medcode":{"value":49.55,"norm":0.4955,"version":"Vals AI"},"vals:fabv2":{"value":52.25,"norm":0.5225,"version":"Vals AI"},"x:Mystery Game Puzzles":{"value":0.29,"norm":0.29,"version":"grok-4.7_xhigh"},"x:Chess Puzzles":{"value":0.41,"norm":0.41,"version":"grok-4.7_xhigh"},"vals:cyber":{"value":69.46,"norm":0.6946,"version":"Vals AI"},"vals:srebench":{"value":3.82,"norm":0.0382,"version":"Vals AI"},"apex":{"value":0.7000000000000001,"norm":0.7,"version":"grok-4.7_unknown"},"x:DTBench":{"value":0.96,"norm":0.96,"version":"grok-4.7_xhigh"},"x:FrontierMath-Tier-4-v2-Private":{"value":0.17073170731707318,"norm":0.1707,"version":"grok-4.7_xhigh"},"aime":{"value":0.9805555555555555,"norm":0.9806,"version":"grok-4.7_xhigh"},"vals:tax_agent_bench":{"value":35.75,"norm":0.3575,"version":"Vals AI"},"vals:legal_bench":{"value":84.39,"norm":0.8439,"version":"Vals AI"},"x:SimpleQA Verified":{"value":0.56,"norm":0.56,"version":"grok-4.7_xhigh"},"x:LMCA":{"value":49.38,"norm":0.4938,"version":"grok-4.7_xhigh"},"arc:arc_agi_1":{"value":90.2,"norm":0.902,"version":"ARC Prize"},"vals:ioi":{"value":57.72,"norm":0.5772,"version":"Vals AI"},"vals:mysterymechanism":{"value":25.23,"norm":0.2523,"version":"Vals AI"},"arc:arc_agi_3":{"value":1.8,"norm":0.018,"version":"ARC Prize"},"vals:emb":{"value":66.99,"norm":0.6699,"version":"Vals AI"},"vals:terminal-bench-science":{"value":10,"norm":0.1,"version":"Vals AI"},"vals:medscribe":{"value":89.38,"norm":0.8938,"version":"Vals AI"},"x:FrontierSWE":{"value":0.2950476117647059,"norm":0.295,"version":"grok-4.7_xhigh"},"x:Furniture Assembly":{"value":0.20833333333333334,"norm":0.2083,"version":"grok-4.7_xhigh"}},"variant":"Grok 4.7","match":"exact"},"api":{"or_id":"x-ai/grok-4.7","in":2,"out":6,"context":500000,"hf":null,"created":"2026-09-21"}},{"id":"mimo-v2-6-flash","name":"MiMo-V2.6-Flash","lab":"Xiaomi","date":"2026-09-21","type":"open-weights","major":true,"open":true,"release":{"title":"MiMo-V2.6-Flash","source":"https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Flash-RL","verified":true,"confidence":"high"},"index":{"score":167.6,"lo":148.9,"hi":186.2,"se":6.32,"solve":57.2,"status":"provisional","areas":["agents","coding","maths","novel","work"],"core_areas":["coding","work"],"testers":["Vals AI"],"tests":{"vals:programbench":{"value":0.5,"norm":0.005,"version":"Vals AI"},"vals:medscribe":{"value":85.28,"norm":0.8528,"version":"Vals AI"},"vals:medcode":{"value":41.06,"norm":0.4106,"version":"Vals AI"},"vals:legal_research":{"value":37.98,"norm":0.3798,"version":"Vals AI"},"vals:terminal-bench-science":{"value":4.29,"norm":0.0429,"version":"Vals AI"},"vals:cyber":{"value":75.36,"norm":0.7536,"version":"Vals AI"},"vals:vibe-code":{"value":78.96,"norm":0.7896,"version":"Vals AI"},"vals:hlab":{"value":13.33,"norm":0.1333,"version":"Vals AI"},"vals:mysterymechanism":{"value":21.62,"norm":0.2162,"version":"Vals AI"},"vals:proof_bench":{"value":63,"norm":0.63,"version":"Vals AI"},"vals:fabv2":{"value":56.28,"norm":0.5628,"version":"Vals AI"},"vals:terminal-bench-4":{"value":24.24,"norm":0.2424,"version":"Vals AI"},"vals:ioi":{"value":47.67,"norm":0.4767,"version":"Vals AI"},"vals:emb":{"value":65.46,"norm":0.6546,"version":"Vals AI"},"vals:tax_agent_bench":{"value":28.51,"norm":0.2851,"version":"Vals AI"},"vals:public-benefits-bench":{"value":67.59,"norm":0.6759,"version":"Vals AI"},"vals:code-migration":{"value":40.93,"norm":0.4093,"version":"Vals AI"},"vals:biomysterybench":{"value":69.26,"norm":0.6926,"version":"Vals AI"},"vals:srebench":{"value":4.2,"norm":0.042,"version":"Vals AI"},"vals:sage":{"value":83.26,"norm":0.8326,"version":"Vals AI"}},"variant":"MiMo-V2.6-Flash","match":"exact"},"api":{"or_id":"xiaomi/mimo-v2.6-flash","in":0.14,"out":0.28,"context":1050000,"hf":"XiaomiMiMo/MiMo-V2.6-Flash-RL","created":"2026-09-21"},"open_evidence":"https://huggingface.co/XiaomiMiMo"},{"id":"mimo-v2-6-pro","name":"MiMo-V2.6-Pro","lab":"Xiaomi","date":"2026-09-21","type":"open-weights","major":true,"open":true,"release":{"title":"MiMo-V2.6-Pro","source":"https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Pro-RL","verified":true,"confidence":"high"},"api":{"or_id":"xiaomi/mimo-v2.6-pro","in":0.435,"out":0.87,"context":1050000,"hf":"XiaomiMiMo/MiMo-V2.6-Pro-RL","created":"2026-09-21"},"open_evidence":"https://huggingface.co/XiaomiMiMo"},{"id":"claude-opus-5-5","name":"Claude Opus 5.5","lab":"Anthropic","date":"2026-09-22","type":"flagship","major":true,"open":false,"release":{"title":"Claude Opus 5.5","source":"https://www.anthropic.com/claude-opus-5-5","verified":true,"confidence":"high"},"index":{"score":194.6,"lo":178.2,"hi":211,"se":3.31,"solve":75,"status":"rated","areas":["agents","coding","maths","novel","reasoning","work"],"core_areas":["agents","coding","maths","novel","reasoning","work"],"testers":["ARC Prize","Epoch AI","LiveBench","Mercor","Proximal","Redwood Research","SimpleBench team","Vals AI"],"tests":{"vals:tax_agent_bench":{"value":44.54,"norm":0.4454,"version":"Vals AI"},"livebench:language":{"value":86.27,"norm":0.8627,"version":"LiveBench"},"vals:programbench":{"value":18.5,"norm":0.185,"version":"Vals AI"},"apex":{"value":0.735,"norm":0.735,"version":"claude-opus-5-5_max"},"vals:biomysterybench":{"value":79.26,"norm":0.7926,"version":"Vals AI"},"vals:public-benefits-bench":{"value":70.64,"norm":0.7064,"version":"Vals AI"},"x:FrontierMath-Tier-4-v2-Private":{"value":0.95,"norm":0.95,"version":"claude-opus-5-5_max"},"vals:emb":{"value":75.94,"norm":0.7594,"version":"Vals AI"},"x:EBR-bench":{"value":0.7142857142857142,"norm":0.7143,"version":"claude-opus-5-5_max"},"livebench:reasoning":{"value":92.15,"norm":0.9215,"version":"LiveBench"},"x:SimpleQA Verified":{"value":0.722,"norm":0.722,"version":"claude-opus-5-5_max"},"arc:arc_agi_2":{"value":93.3,"norm":0.933,"version":"ARC Prize"},"vals:code-migration":{"value":66.65,"norm":0.6665,"version":"Vals AI"},"vals:medcode":{"value":49.8,"norm":0.498,"version":"Vals AI"},"x:Mystery Game Puzzles":{"value":0.71,"norm":0.71,"version":"claude-opus-5-5_max"},"vals:vibe-code":{"value":90.29,"norm":0.9029,"version":"Vals AI"},"vals:mysterymechanism":{"value":49.55,"norm":0.4955,"version":"Vals AI"},"vals:vcb-1-100":{"value":30.36,"norm":0.3036,"version":"Vals AI"},"x:LMCA":{"value":68.23,"norm":0.6823,"version":"claude-opus-5-5_max"},"arc:arc_agi_1":{"value":98.5,"norm":0.985,"version":"ARC Prize"},"vals:ioi":{"value":95.06,"norm":0.9506,"version":"Vals AI"},"vals:sage":{"value":91.3,"norm":0.913,"version":"Vals AI"},"vals:medscribe":{"value":91.43,"norm":0.9143,"version":"Vals AI"},"livebench:data-analysis":{"value":80.31,"norm":0.8031,"version":"LiveBench"},"livebench:mathematics":{"value":97.08,"norm":0.9708,"version":"LiveBench"},"gpqa":{"value":0.9406565656565656,"norm":0.9407,"version":"claude-opus-5-5-fallback_max"},"vals:terminal-bench-4":{"value":65.15,"norm":0.6515,"version":"Vals AI"},"x:MirrorCode":{"value":0.9555555555555556,"norm":0.9556,"version":"claude-opus-5-5_max"},"x:DTBench":{"value":0.9893000000000001,"norm":0.9893,"version":"claude-opus-5-5_max"},"livebench:agentic-coding":{"value":71.72,"norm":0.7172,"version":"LiveBench"},"vals:fabv2":{"value":58.59,"norm":0.5859,"version":"Vals AI"},"vals:legal_research":{"value":50.48,"norm":0.5048,"version":"Vals AI"},"vals:hlab":{"value":9.17,"norm":0.0917,"version":"Vals AI"},"vals:terminal-bench-science":{"value":47.14,"norm":0.4714,"version":"Vals AI"},"vals:cyber":{"value":74.58,"norm":0.7458,"version":"Vals AI"},"vals:proof_bench":{"value":100,"norm":1,"version":"Vals AI"},"vals:srebench":{"value":33.59,"norm":0.3359,"version":"Vals AI"},"livebench:if":{"value":67.05,"norm":0.6705,"version":"LiveBench"},"x:Furniture Assembly":{"value":0.8333333333333334,"norm":0.8333,"version":"claude-opus-5-5_max"},"livebench:coding":{"value":89.25,"norm":0.8925,"version":"LiveBench"},"fm":{"value":0.9122807017543859,"norm":0.9123,"version":"claude-opus-5-5_max"},"x:Chess Puzzles":{"value":0.67,"norm":0.67,"version":"claude-opus-5-5_max"},"aime":{"value":1,"norm":1,"version":"claude-opus-5-5_max"},"x:FrontierSWE":{"value":0.623309882352941,"norm":0.6233,"version":"claude-opus-5-5_max"},"x:SimpleBench":{"value":0.8840000000000001,"norm":0.884,"version":"claude-opus-5-5_unknown"}},"variant":"Claude Opus 5.5","match":"exact"},"api":{"or_id":"anthropic/claude-opus-5.5","in":4,"out":20,"context":1000000,"hf":null,"created":"2026-09-22"}},{"id":"gpt-6-luna","name":"GPT-6 Luna","lab":"OpenAI","date":"2026-09-22","type":"flagship","major":true,"open":false,"release":{"title":"GPT-6 Sol + GPT-6 Luna","source":"https://openai.com/index/introducing-gpt-6-sol-and-luna/","verified":false,"confidence":"high"},"index":{"score":162.3,"lo":146.4,"hi":178.2,"se":2.24,"solve":51.9,"status":"rated","areas":["agents","coding","maths","novel","reasoning","work"],"core_areas":["agents","coding","maths","novel","reasoning","work"],"testers":["ARC Prize","Epoch AI","LiveBench","Mercor","Redwood Research","Vals AI"],"tests":{"livebench:if":{"value":55.93,"norm":0.5593,"version":"LiveBench"},"livebench:data-analysis":{"value":73.37,"norm":0.7337,"version":"LiveBench"},"livebench:coding":{"value":78.95,"norm":0.7895,"version":"LiveBench"},"fm":{"value":0.7894736842105263,"norm":0.7895,"version":"gpt-6-luna_max"},"x:Chess Puzzles":{"value":0.36,"norm":0.36,"version":"gpt-6-luna_max"},"aime":{"value":0.9888888888888889,"norm":0.9889,"version":"gpt-6-luna_max"},"vals:fabv2":{"value":49.87,"norm":0.4987,"version":"Vals AI"},"vals:programbench":{"value":0.5,"norm":0.005,"version":"Vals AI"},"vals:cyber":{"value":76.25,"norm":0.7625,"version":"Vals AI"},"x:LMCA":{"value":44.49,"norm":0.4449,"version":"gpt-6-luna_max"},"x:MirrorCode":{"value":0.26666666666666666,"norm":0.2667,"version":"gpt-6-luna_max"},"x:Mystery Game Puzzles":{"value":0.07,"norm":0.07,"version":"gpt-6-luna_max"},"x:SimpleQA Verified":{"value":0.414,"norm":0.414,"version":"gpt-6-luna_max"},"vals:biomysterybench":{"value":61.48,"norm":0.6148,"version":"Vals AI"},"x:Furniture Assembly":{"value":0.44166666666666665,"norm":0.4417,"version":"gpt-6-luna_max"},"x:FrontierMath-Tier-4-v2-Private":{"value":0.5609756097560976,"norm":0.561,"version":"gpt-6-luna_max"},"livebench:language":{"value":73.83,"norm":0.7383,"version":"LiveBench"},"vals:public-benefits-bench":{"value":57.65,"norm":0.5765,"version":"Vals AI"},"apex":{"value":0.443,"norm":0.443,"version":"gpt-6-luna_max"},"vals:proof_bench":{"value":64,"norm":0.64,"version":"Vals AI"},"arc:arc_agi_2":{"value":59.3,"norm":0.593,"version":"ARC Prize"},"vals:medcode":{"value":44.69,"norm":0.4469,"version":"Vals AI"},"vals:srebench":{"value":2.67,"norm":0.0267,"version":"Vals AI"},"vals:sage":{"value":83.4,"norm":0.834,"version":"Vals AI"},"arc:arc_agi_3":{"value":0.2,"norm":0.002,"version":"ARC Prize"},"x:DTBench":{"value":0.9013,"norm":0.9013,"version":"gpt-6-luna_max"},"vals:legal_research":{"value":30.29,"norm":0.3029,"version":"Vals AI"},"vals:ioi":{"value":55.56,"norm":0.5556,"version":"Vals AI"},"vals:vibe-code":{"value":81.65,"norm":0.8165,"version":"Vals AI"},"vals:terminal-bench-4":{"value":13.64,"norm":0.1364,"version":"Vals AI"},"arc:arc_agi_1":{"value":86.7,"norm":0.867,"version":"ARC Prize"},"vals:medscribe":{"value":83.71,"norm":0.8371,"version":"Vals AI"},"vals:emb":{"value":68.52,"norm":0.6852,"version":"Vals AI"},"livebench:agentic-coding":{"value":51.21,"norm":0.5121,"version":"LiveBench"},"vals:hlab":{"value":4.17,"norm":0.0417,"version":"Vals AI"},"livebench:mathematics":{"value":89.12,"norm":0.8912,"version":"LiveBench"},"vals:tax_agent_bench":{"value":19.95,"norm":0.1995,"version":"Vals AI"},"gpqa":{"value":0.9046717171717171,"norm":0.9047,"version":"gpt-6-luna_max"},"vals:code-migration":{"value":42.55,"norm":0.4255,"version":"Vals AI"},"livebench:reasoning":{"value":81.77,"norm":0.8177,"version":"LiveBench"},"vals:terminal-bench-science":{"value":4.29,"norm":0.0429,"version":"Vals AI"},"vals:mysterymechanism":{"value":19.37,"norm":0.1937,"version":"Vals AI"}},"variant":"GPT-6 Luna","match":"exact"},"api":{"or_id":"openai/gpt-6-luna","in":0.1,"out":0.5,"context":1050000,"hf":null,"created":"2026-09-22"}},{"id":"gpt-6-sol","name":"GPT-6 Sol","lab":"OpenAI","date":"2026-09-22","type":"flagship","major":true,"open":false,"release":{"title":"GPT-6 Sol + GPT-6 Luna","source":"https://openai.com/index/introducing-gpt-6-sol-and-luna/","verified":false,"confidence":"high"},"index":{"score":177.6,"lo":161.5,"hi":193.7,"se":2.69,"solve":65.4,"status":"rated","areas":["agents","coding","maths","novel","reasoning","work"],"core_areas":["agents","coding","maths","novel","reasoning","work"],"testers":["ARC Prize","Epoch AI","LiveBench","Mercor","Redwood Research","SimpleBench team","Vals AI"],"tests":{"arc:arc_agi_3":{"value":4.6,"norm":0.046,"version":"ARC Prize"},"x:DTBench":{"value":0.9733,"norm":0.9733,"version":"gpt-6-sol_max"},"x:Chess Puzzles":{"value":0.56,"norm":0.56,"version":"gpt-6-sol_max"},"vals:fabv2":{"value":49.05,"norm":0.4905,"version":"Vals AI"},"vals:tax_agent_bench":{"value":15,"norm":0.15,"version":"Vals AI"},"livebench:language":{"value":85.3,"norm":0.853,"version":"LiveBench"},"vals:public-benefits-bench":{"value":56.63,"norm":0.5663,"version":"Vals AI"},"vals:ioi":{"value":82.61,"norm":0.8261,"version":"Vals AI"},"livebench:if":{"value":68.57,"norm":0.6857,"version":"LiveBench"},"vals:emb":{"value":71.53,"norm":0.7153,"version":"Vals AI"},"livebench:coding":{"value":81.77,"norm":0.8177,"version":"LiveBench"},"fm":{"value":0.8982456140350877,"norm":0.8982,"version":"gpt-6-sol_max"},"x:FrontierMath-Tier-4-v2-Private":{"value":0.9,"norm":0.9,"version":"gpt-6-sol_max"},"x:SimpleBench":{"value":0.731,"norm":0.731,"version":"gpt-6-sol_unknown"},"vals:code-migration":{"value":57.19,"norm":0.5719,"version":"Vals AI"},"x:EBR-bench":{"value":0.5333333333333333,"norm":0.5333,"version":"gpt-6-sol_max"},"x:LMCA":{"value":59.08,"norm":0.5908,"version":"gpt-6-sol_max"},"x:Furniture Assembly":{"value":0.5833333333333334,"norm":0.5833,"version":"gpt-6-sol_max"},"vals:legal_research":{"value":28.85,"norm":0.2885,"version":"Vals AI"},"aime":{"value":1,"norm":1,"version":"gpt-6-sol_max"},"x:Mystery Game Puzzles":{"value":0.56,"norm":0.56,"version":"gpt-6-sol_max"},"arc:arc_agi_2":{"value":89.6,"norm":0.896,"version":"ARC Prize"},"vals:terminal-bench-4":{"value":44.44,"norm":0.4444,"version":"Vals AI"},"vals:programbench":{"value":2,"norm":0.02,"version":"Vals AI"},"livebench:reasoning":{"value":88.65,"norm":0.8865,"version":"LiveBench"},"apex":{"value":0.543,"norm":0.543,"version":"gpt-6-sol_max"},"vals:proof_bench":{"value":83,"norm":0.83,"version":"Vals AI"},"vals:medcode":{"value":47.07,"norm":0.4707,"version":"Vals AI"},"vals:srebench":{"value":30.53,"norm":0.3053,"version":"Vals AI"},"x:MirrorCode":{"value":0.4888888888888889,"norm":0.4889,"version":"gpt-6-sol_max"},"vals:vibe-code":{"value":87.82,"norm":0.8782,"version":"Vals AI"},"vals:mysterymechanism":{"value":30.18,"norm":0.3018,"version":"Vals AI"},"vals:medscribe":{"value":82.03,"norm":0.8203,"version":"Vals AI"},"livebench:data-analysis":{"value":81.19,"norm":0.8119,"version":"LiveBench"},"livebench:agentic-coding":{"value":52.88,"norm":0.5288,"version":"LiveBench"},"vals:terminal-bench-science":{"value":30,"norm":0.3,"version":"Vals AI"},"livebench:mathematics":{"value":96.36,"norm":0.9636,"version":"LiveBench"},"gpqa":{"value":0.9431818181818182,"norm":0.9432,"version":"gpt-6-sol_max"},"x:SimpleQA Verified":{"value":0.607,"norm":0.607,"version":"gpt-6-sol_max"},"vals:cyber":{"value":77.98,"norm":0.7798,"version":"Vals AI"},"arc:arc_agi_1":{"value":95.5,"norm":0.955,"version":"ARC Prize"},"vals:biomysterybench":{"value":74.81,"norm":0.7481,"version":"Vals AI"}},"variant":"GPT-6 Sol","match":"exact"},"api":{"or_id":"openai/gpt-6-sol","in":2,"out":10,"context":1050000,"hf":null,"created":"2026-09-22"}},{"id":"claude-sonnet-5-5","name":"Claude Sonnet 5.5","lab":"Anthropic","date":"2026-09-28","type":"flagship","major":true,"open":false,"release":{"title":"Claude Sonnet 5.5","source":"https://www.anthropic.com/claude-sonnet-5-5","verified":true,"confidence":"high"},"index":{"score":186.7,"lo":170.8,"hi":202.6,"se":2.1,"solve":71,"status":"rated","areas":["agents","coding","maths","novel","reasoning","work"],"core_areas":["agents","coding","maths","novel","reasoning","work"],"testers":["ARC Prize","Epoch AI","LiveBench","Mercor","Proximal","Redwood Research","Scale AI","SimpleBench team","Vals AI"],"tests":{"vals:medscribe":{"value":91.1,"norm":0.911,"version":"Vals AI"},"livebench:data-analysis":{"value":78.62,"norm":0.7862,"version":"LiveBench"},"vals:fabv2":{"value":58.1,"norm":0.581,"version":"Vals AI"},"livebench:mathematics":{"value":96.69,"norm":0.9669,"version":"LiveBench"},"vals:tax_agent_bench":{"value":41.96,"norm":0.4196,"version":"Vals AI"},"livebench:language":{"value":83.43,"norm":0.8343,"version":"LiveBench"},"livebench:reasoning":{"value":91.63,"norm":0.9163,"version":"LiveBench"},"vals:proof_bench":{"value":100,"norm":1,"version":"Vals AI"},"x:MirrorCode":{"value":0.9777777777777777,"norm":0.9778,"version":"claude-sonnet-5-5_max"},"vals:sage":{"value":88.03,"norm":0.8803,"version":"Vals AI"},"x:FrontierSWE":{"value":0.618892994117647,"norm":0.6189,"version":"claude-sonnet-5-5_max"},"livebench:agentic-coding":{"value":56.31,"norm":0.5631,"version":"LiveBench"},"vals:legal_research":{"value":48.08,"norm":0.4808,"version":"Vals AI"},"x:SimpleBench":{"value":0.7590000000000001,"norm":0.759,"version":"claude-sonnet-5-5_unknown"},"arc:arc_agi_2":{"value":83.9,"norm":0.839,"version":"ARC Prize"},"vals:public-benefits-bench":{"value":67.19,"norm":0.6719,"version":"Vals AI"},"vals:programbench":{"value":6.5,"norm":0.065,"version":"Vals AI"},"vals:ioi":{"value":83.06,"norm":0.8306,"version":"Vals AI"},"vals:cyber":{"value":59.58,"norm":0.5958,"version":"Vals AI"},"vals:mysterymechanism":{"value":49.1,"norm":0.491,"version":"Vals AI"},"arc:arc_agi_1":{"value":96.5,"norm":0.965,"version":"ARC Prize"},"vals:biomysterybench":{"value":81.11,"norm":0.8111,"version":"Vals AI"},"vals:srebench":{"value":30.15,"norm":0.3015,"version":"Vals AI"},"vals:emb":{"value":75.71,"norm":0.7571,"version":"Vals AI"},"fm":{"value":0.887719298245614,"norm":0.8877,"version":"claude-sonnet-5-5_max"},"x:FrontierMath-Tier-4-v2-Private":{"value":0.8048780487804879,"norm":0.8049,"version":"claude-sonnet-5-5_max"},"x:Mystery Game Puzzles":{"value":0.65,"norm":0.65,"version":"claude-sonnet-5-5_max"},"gpqa":{"value":0.9558080808080808,"norm":0.9558,"version":"claude-sonnet-5-5_max"},"vals:code-migration":{"value":69.83,"norm":0.6983,"version":"Vals AI"},"x:SimpleQA Verified":{"value":0.465,"norm":0.465,"version":"claude-sonnet-5-5_max"},"apex":{"value":0.755,"norm":0.755,"version":"claude-sonnet-5-5_max"},"x:Furniture Assembly":{"value":0.75,"norm":0.75,"version":"claude-sonnet-5-5_max"},"x:DTBench":{"value":0.9733,"norm":0.9733,"version":"claude-sonnet-5-5_max"},"aime":{"value":1,"norm":1,"version":"claude-sonnet-5-5_max"},"vals:hlab":{"value":10.42,"norm":0.1042,"version":"Vals AI"},"vals:vibe-code":{"value":92.39,"norm":0.9239,"version":"Vals AI"},"livebench:if":{"value":70.52,"norm":0.7052,"version":"LiveBench"},"vals:terminal-bench-4":{"value":64.14,"norm":0.6414,"version":"Vals AI"},"x:LMCA":{"value":68.26,"norm":0.6826,"version":"claude-sonnet-5-5_max"},"vals:medcode":{"value":52.92,"norm":0.5292,"version":"Vals AI"},"hle":{"value":0.4612,"norm":0.4612,"version":"claude-sonnet-5-5_xhigh"},"livebench:coding":{"value":91.37,"norm":0.9137,"version":"LiveBench"},"arc:arc_agi_3":{"value":24.3,"norm":0.243,"version":"ARC Prize"},"x:Chess Puzzles":{"value":0.59,"norm":0.59,"version":"claude-sonnet-5-5_max"},"vals:terminal-bench-science":{"value":47.14,"norm":0.4714,"version":"Vals AI"}},"variant":"Claude Sonnet 5.5","match":"exact"},"api":{"or_id":"anthropic/claude-sonnet-5.5","in":2,"out":10,"context":1000000,"hf":null,"created":"2026-09-28"}},{"id":"gpt-6-1-sol","name":"GPT-6.1 Sol","lab":"OpenAI","date":"2026-09-29","type":"flagship","major":true,"open":false,"release":{"title":"GPT-6.1 Sol","source":"https://openai.com/index/introducing-gpt-6-1-sol/","verified":false,"confidence":"high"},"index":{"score":190.9,"lo":174.9,"hi":206.8,"se":2.24,"solve":73.2,"status":"rated","areas":["agents","coding","maths","novel","reasoning","work"],"core_areas":["agents","coding","maths","novel","reasoning","work"],"testers":["ARC Prize","Epoch AI","LiveBench","Mercor","Redwood Research","SimpleBench team","Vals AI"],"tests":{"arc:arc_agi_2":{"value":94.2,"norm":0.942,"version":"ARC Prize"},"livebench:reasoning":{"value":92.63,"norm":0.9263,"version":"LiveBench"},"vals:mysterymechanism":{"value":46.4,"norm":0.464,"version":"Vals AI"},"vals:emb":{"value":70.77,"norm":0.7077,"version":"Vals AI"},"livebench:coding":{"value":80.74,"norm":0.8074,"version":"LiveBench"},"gpqa":{"value":0.9539141414141414,"norm":0.9539,"version":"gpt-6.1-sol_max"},"x:Chess Puzzles":{"value":0.68,"norm":0.68,"version":"gpt-6.1-sol_max"},"aime":{"value":1,"norm":1,"version":"gpt-6.1-sol_max"},"vals:cyber":{"value":39.29,"norm":0.3929,"version":"Vals AI"},"vals:programbench":{"value":3,"norm":0.03,"version":"Vals AI"},"vals:code-migration":{"value":65.12,"norm":0.6512,"version":"Vals AI"},"vals:srebench":{"value":50.76,"norm":0.5076,"version":"Vals AI"},"livebench:if":{"value":74.15,"norm":0.7415,"version":"LiveBench"},"arc:arc_agi_1":{"value":98.5,"norm":0.985,"version":"ARC Prize"},"vals:medcode":{"value":48.84,"norm":0.4884,"version":"Vals AI"},"vals:biomysterybench":{"value":79.63,"norm":0.7963,"version":"Vals AI"},"x:MirrorCode":{"value":0.6111111111111112,"norm":0.6111,"version":"gpt-6.1-sol_max"},"vals:medscribe":{"value":86.45,"norm":0.8645,"version":"Vals AI"},"livebench:language":{"value":90.13,"norm":0.9013,"version":"LiveBench"},"x:FrontierMath-Tier-4-v2-Private":{"value":1,"norm":1,"version":"gpt-6.1-sol_max"},"vals:fabv2":{"value":52.03,"norm":0.5203,"version":"Vals AI"},"livebench:mathematics":{"value":96.83,"norm":0.9683,"version":"LiveBench"},"vals:tax_agent_bench":{"value":21.75,"norm":0.2175,"version":"Vals AI"},"vals:terminal-bench-4":{"value":55.05,"norm":0.5505,"version":"Vals AI"},"x:EBR-bench":{"value":0.5428571428571429,"norm":0.5429,"version":"gpt-6.1-sol_max"},"vals:ioi":{"value":96.89,"norm":0.9689,"version":"Vals AI"},"x:Furniture Assembly":{"value":0.8,"norm":0.8,"version":"gpt-6.1-sol_max"},"arc:arc_agi_3":{"value":52.7,"norm":0.527,"version":"ARC Prize"},"livebench:agentic-coding":{"value":56.77,"norm":0.5677,"version":"LiveBench"},"vals:hlab":{"value":6.25,"norm":0.0625,"version":"Vals AI"},"x:Mystery Game Puzzles":{"value":0.8,"norm":0.8,"version":"gpt-6.1-sol_max"},"vals:vibe-code":{"value":88.93,"norm":0.8893,"version":"Vals AI"},"vals:proof_bench":{"value":99,"norm":0.99,"version":"Vals AI"},"x:DTBench":{"value":0.9653,"norm":0.9653,"version":"gpt-6.1-sol_max"},"x:LMCA":{"value":62.78,"norm":0.6278,"version":"gpt-6.1-sol_max"},"vals:legal_research":{"value":38.46,"norm":0.3846,"version":"Vals AI"},"x:SimpleBench":{"value":0.8290000000000001,"norm":0.829,"version":"gpt-6.1-sol_unknown"},"x:SimpleQA Verified":{"value":0.739,"norm":0.739,"version":"gpt-6.1-sol_max"},"apex":{"value":0.6,"norm":0.6,"version":"gpt-6.1-sol_max"},"vals:sage":{"value":90.49,"norm":0.9049,"version":"Vals AI"},"livebench:data-analysis":{"value":82.66,"norm":0.8266,"version":"LiveBench"},"fm":{"value":0.9368421052631579,"norm":0.9368,"version":"gpt-6.1-sol_max"},"vals:public-benefits-bench":{"value":59.34,"norm":0.5934,"version":"Vals AI"},"vals:terminal-bench-science":{"value":52.86,"norm":0.5286,"version":"Vals AI"}},"variant":"GPT-6.1 Sol","match":"exact"},"api":{"or_id":"openai/gpt-6.1-sol","in":2,"out":10,"context":1050000,"hf":null,"created":"2026-09-29"}},{"id":"gemini-4-argon","name":"Gemini 4 Argon","lab":"Google","date":"2026-09-30","type":"flagship","major":true,"open":false,"release":{"title":"Gemini 4 Argon","source":"https://blog.google/innovation-and-ai/models-and-research/gemini-models/gemini-4-argon/","verified":false,"confidence":"auto"},"index":{"score":191.4,"lo":175.6,"hi":207.3,"se":2.1,"solve":73.5,"status":"rated","areas":["agents","coding","maths","novel","reasoning","work"],"core_areas":["agents","coding","maths","novel","reasoning","work"],"testers":["ARC Prize","Epoch AI","Mercor","Proximal","Redwood Research","SimpleBench team","Vals AI"],"tests":{"vals:terminal-bench-4":{"value":57.58,"norm":0.5758,"version":"Vals AI"},"x:EBR-bench":{"value":0.6476190476190476,"norm":0.6476,"version":"gemini-4-argon_high"},"x:SimpleQA Verified":{"value":0.76,"norm":0.76,"version":"gemini-4-argon_high"},"x:DTBench":{"value":0.976,"norm":0.976,"version":"gemini-4-argon_low"},"vals:biomysterybench":{"value":76.3,"norm":0.763,"version":"Vals AI"},"x:Furniture Assembly":{"value":0.725,"norm":0.725,"version":"gemini-4-argon_high"},"arc:arc_agi_3":{"value":49.8,"norm":0.498,"version":"ARC Prize"},"vals:public-benefits-bench":{"value":69.76,"norm":0.6976,"version":"Vals AI"},"vals:programbench":{"value":2.5,"norm":0.025,"version":"Vals AI"},"arc:arc_agi_1":{"value":98.5,"norm":0.985,"version":"ARC Prize"},"fm":{"value":0.8596491228070176,"norm":0.8596,"version":"gemini-4-argon_high"},"vals:hlab":{"value":27.08,"norm":0.2708,"version":"Vals AI"},"x:SimpleBench":{"value":0.917,"norm":0.917,"version":"gemini-4-argon_unknown"},"vals:tax_agent_bench":{"value":49.22,"norm":0.4922,"version":"Vals AI"},"vals:legal_bench":{"value":88.3,"norm":0.883,"version":"Vals AI"},"vals:vibe-code":{"value":91.91,"norm":0.9191,"version":"Vals AI"},"vals:proof_bench":{"value":99,"norm":0.99,"version":"Vals AI"},"x:LMCA":{"value":63.69,"norm":0.6369,"version":"gemini-4-argon_high"},"vals:sage":{"value":89.46,"norm":0.8946,"version":"Vals AI"},"vals:legal_research":{"value":54.81,"norm":0.5481,"version":"Vals AI"},"vals:terminal-bench-science":{"value":44.29,"norm":0.4429,"version":"Vals AI"},"x:Mystery Game Puzzles":{"value":0.64,"norm":0.64,"version":"gemini-4-argon_high"},"vals:srebench":{"value":44.27,"norm":0.4427,"version":"Vals AI"},"apex":{"value":0.8240000000000001,"norm":0.824,"version":"gemini-4-argon_unknown"},"vals:mysterymechanism":{"value":45.49,"norm":0.4549,"version":"Vals AI"},"vals:medcode":{"value":58.8,"norm":0.588,"version":"Vals AI"},"vals:medscribe":{"value":87.43,"norm":0.8743,"version":"Vals AI"},"vals:emb":{"value":75.24,"norm":0.7524,"version":"Vals AI"},"vals:cyber":{"value":77.86,"norm":0.7786,"version":"Vals AI"},"vals:vcb-1-100":{"value":21.2,"norm":0.212,"version":"Vals AI"},"arc:arc_agi_2":{"value":91.3,"norm":0.913,"version":"ARC Prize"},"vals:code-migration":{"value":68.17,"norm":0.6817,"version":"Vals AI"},"vals:ioi":{"value":100,"norm":1,"version":"Vals AI"},"x:MirrorCode":{"value":0.5888888888888889,"norm":0.5889,"version":"gemini-4-argon_high"},"x:FrontierSWE":{"value":0.5496118588235293,"norm":0.5496,"version":"gemini-4-argon_high"},"vals:fabv2":{"value":65.4,"norm":0.654,"version":"Vals AI"},"gpqa":{"value":0.9640151515151515,"norm":0.964,"version":"gemini-4-argon_high"},"x:FrontierMath-Tier-4-v2-Private":{"value":0.7804878048780488,"norm":0.7805,"version":"gemini-4-argon_high"},"x:Chess Puzzles":{"value":0.59,"norm":0.59,"version":"gemini-4-argon_high"},"aime":{"value":1,"norm":1,"version":"gemini-4-argon_high"}},"variant":"Gemini 4 Argon","match":"exact"},"api":{"or_id":"google/gemini-4-argon","in":2,"out":10,"context":1048576,"hf":null,"created":"2026-10-13"}},{"id":"mistral-large-4","name":"Mistral Large 4","lab":"Mistral","date":"2026-10-06","type":"open-weights","major":true,"open":true,"release":{"title":"Mistral Large 4","source":"https://mistral.ai/news/mistral-large-4/","verified":false,"confidence":"auto"},"index":{"score":148.1,"lo":131.5,"hi":164.6,"se":3.47,"solve":37,"status":"rated","areas":["agents","coding","maths","reasoning"],"core_areas":["agents","maths","reasoning"],"testers":["Epoch AI","Mercor","Redwood Research","Yann Henon"],"tests":{"x:DTBench":{"value":0.8824,"norm":0.8824,"version":"mistral-large-4-0_unknown"},"x:Furniture Assembly":{"value":0.20833333333333334,"norm":0.2083,"version":"mistral-large-4-0_high"},"x:FrontierMath-Tier-4-v2-Private":{"value":0.2682926829268293,"norm":0.2683,"version":"mistral-large-4-0_high"},"aime":{"value":0.9833333333333333,"norm":0.9833,"version":"mistral-large-4-0_high"},"x:Surface Evolver Bench":{"value":0.28125,"norm":0.2812,"version":"mistral-large-4-0_unknown"},"x:LMCA":{"value":35.79,"norm":0.3579,"version":"mistral-large-4-0_unknown"},"fm":{"value":0.5859649122807018,"norm":0.586,"version":"mistral-large-4-0_high"},"gpqa":{"value":0.8806818181818182,"norm":0.8807,"version":"mistral-large-4-0_high"},"x:SimpleQA Verified":{"value":0.2,"norm":0.2,"version":"mistral-large-4-0_high"},"apex":{"value":0.516,"norm":0.516,"version":"mistral-large-4-0_high"},"x:MirrorCode":{"value":0.4,"norm":0.4,"version":"mistral-large-4-0_high"},"x:Mystery Game Puzzles":{"value":0.17,"norm":0.17,"version":"mistral-large-4-0_high"},"x:Chess Puzzles":{"value":0.33,"norm":0.33,"version":"mistral-large-4-0_high"}},"variant":"Mistral Large 4 Preview","match":"exact"},"api":{"or_id":"mistralai/mistral-large-4-0","in":0.7,"out":2.1,"context":1048576,"hf":"mistralai/Mistral-Large-4-1T-A52B","created":"2026-10-06"}},{"id":"claude-haiku-5-5","name":"Claude Haiku 5.5","lab":"Anthropic","date":"2026-10-07","type":"small","major":true,"open":false,"release":{"title":"Claude Haiku 5.5","source":"https://www.anthropic.com/claude-haiku-5-5","verified":false,"confidence":"auto"},"index":{"score":167.9,"lo":152,"hi":183.8,"se":2.22,"solve":57.5,"status":"rated","areas":["agents","coding","maths","novel","reasoning","work"],"core_areas":["agents","coding","maths","novel","reasoning","work"],"testers":["ARC Prize","Epoch AI","LiveBench","Mercor","Proximal","Redwood Research","Scale AI","SimpleBench team","Vals AI"],"tests":{"x:Chess Puzzles":{"value":0.55,"norm":0.55,"version":"claude-haiku-5-5_max"},"aime":{"value":0.9888888888888889,"norm":0.9889,"version":"claude-haiku-5-5_xhigh"},"livebench:data-analysis":{"value":72.75,"norm":0.7275,"version":"LiveBench"},"vals:tax_agent_bench":{"value":31.92,"norm":0.3192,"version":"Vals AI"},"vals:legal_research":{"value":43.75,"norm":0.4375,"version":"Vals AI"},"x:EBR-bench":{"value":0.43809523809523804,"norm":0.4381,"version":"claude-haiku-5-5_max"},"vals:ioi":{"value":47.28,"norm":0.4728,"version":"Vals AI"},"vals:proof_bench":{"value":89,"norm":0.89,"version":"Vals AI"},"vals:terminal-bench-4":{"value":36.36,"norm":0.3636,"version":"Vals AI"},"x:LMCA":{"value":51.06,"norm":0.5106,"version":"claude-haiku-5-5_max"},"livebench:coding":{"value":77.86,"norm":0.7786,"version":"LiveBench"},"x:DTBench":{"value":0.9499,"norm":0.9499,"version":"claude-haiku-5-5_max"},"livebench:mathematics":{"value":92.75,"norm":0.9275,"version":"LiveBench"},"x:SimpleBench":{"value":0.5920000000000001,"norm":0.592,"version":"claude-haiku-5-5_unknown"},"gpqa":{"value":0.8964646464646465,"norm":0.8965,"version":"claude-haiku-5-5_max"},"vals:public-benefits-bench":{"value":63.19,"norm":0.6319,"version":"Vals AI"},"arc:arc_agi_1":{"value":89,"norm":0.89,"version":"ARC Prize"},"vals:medscribe":{"value":89.54,"norm":0.8954,"version":"Vals AI"},"x:Furniture Assembly":{"value":0.475,"norm":0.475,"version":"claude-haiku-5-5_max"},"x:FrontierMath-Tier-4-v2-Private":{"value":0.4634146341463415,"norm":0.4634,"version":"claude-haiku-5-5_max"},"vals:fabv2":{"value":54.14,"norm":0.5414,"version":"Vals AI"},"x:Mystery Game Puzzles":{"value":0.3,"norm":0.3,"version":"claude-haiku-5-5_max"},"livebench:reasoning":{"value":84.63,"norm":0.8463,"version":"LiveBench"},"x:SimpleQA Verified":{"value":0.238,"norm":0.238,"version":"claude-haiku-5-5_max"},"apex":{"value":0.561,"norm":0.561,"version":"claude-haiku-5-5_max"},"vals:terminal-bench-science":{"value":10,"norm":0.1,"version":"Vals AI"},"vals:mysterymechanism":{"value":35.13,"norm":0.3513,"version":"Vals AI"},"vals:biomysterybench":{"value":70.37,"norm":0.7037,"version":"Vals AI"},"hle":{"value":0.3012,"norm":0.3012,"version":"claude-haiku-5-5_unknown"},"vals:emb":{"value":59.01,"norm":0.5901,"version":"Vals AI"},"arc:arc_agi_3":{"value":0.5,"norm":0.005,"version":"ARC Prize"},"livebench:language":{"value":63.52,"norm":0.6352,"version":"LiveBench"},"vals:code-migration":{"value":44.57,"norm":0.4457,"version":"Vals AI"},"vals:vibe-code":{"value":90.44,"norm":0.9044,"version":"Vals AI"},"vals:hlab":{"value":4.17,"norm":0.0417,"version":"Vals AI"},"vals:cyber":{"value":75.48,"norm":0.7548,"version":"Vals AI"},"vals:vcb-1-100":{"value":20.41,"norm":0.2041,"version":"Vals AI"},"arc:arc_agi_2":{"value":61.3,"norm":0.613,"version":"ARC Prize"},"vals:sage":{"value":89.83,"norm":0.8983,"version":"Vals AI"},"livebench:if":{"value":66.47,"norm":0.6647,"version":"LiveBench"},"fm":{"value":0.7508771929824561,"norm":0.7509,"version":"claude-haiku-5-5_max"},"livebench:agentic-coding":{"value":57.17,"norm":0.5717,"version":"LiveBench"},"vals:medcode":{"value":51.54,"norm":0.5154,"version":"Vals AI"},"x:FrontierSWE":{"value":0.4376868294117648,"norm":0.4377,"version":"claude-haiku-5-5_max"},"vals:programbench":{"value":2.5,"norm":0.025,"version":"Vals AI"},"vals:srebench":{"value":5.72,"norm":0.0572,"version":"Vals AI"},"x:MirrorCode":{"value":0.7888888888888889,"norm":0.7889,"version":"claude-haiku-5-5_max"}},"variant":"Claude Haiku 5.5","match":"exact"},"api":{"or_id":"anthropic/claude-haiku-5.5","in":0.1,"out":0.5,"context":1000000,"hf":null,"created":"2026-10-07"}},{"id":"muse-wonder","name":"Muse Wonder","lab":"Meta","date":"2026-10-12","type":"flagship","major":true,"open":false,"release":{"title":"Muse Wonder + Muse Wonder Full","source":"https://about.fb.com/news/2026/10/introducing-muse-wonder-models/","verified":true,"confidence":"high"},"index":{"score":176.5,"lo":160,"hi":193,"se":3.46,"solve":64.6,"status":"rated","areas":["agents","coding","maths","reasoning"],"core_areas":["agents","maths"],"testers":["Epoch AI","Mercor","Redwood Research"],"tests":{"x:EBR-bench":{"value":0.3142857142857143,"norm":0.3143,"version":"muse-wonder-1.0_max"},"x:LMCA":{"value":56.84,"norm":0.5684,"version":"muse-wonder-1.0_unknown"},"fm":{"value":0.8210526315789474,"norm":0.8211,"version":"muse-wonder-1.0_max"},"x:DTBench":{"value":0.9520000000000001,"norm":0.952,"version":"muse-wonder-1.0_unknown"},"x:MirrorCode":{"value":0.45555555555555555,"norm":0.4556,"version":"muse-wonder-1.0_max"},"x:Furniture Assembly":{"value":0.5416666666666666,"norm":0.5417,"version":"muse-wonder-1.0_max"},"apex":{"value":0.6609999999999999,"norm":0.661,"version":"muse-wonder-1.0_max"},"aime":{"value":0.9944444444444445,"norm":0.9944,"version":"muse-wonder-1.0_max"},"x:Mystery Game Puzzles":{"value":0.51,"norm":0.51,"version":"muse-wonder-1.0_max"},"x:FrontierMath-Tier-4-v2-Private":{"value":0.7073170731707317,"norm":0.7073,"version":"muse-wonder-1.0_max"}},"variant":"Muse Wonder","match":"exact"},"api":{"or_id":"meta/muse-wonder","in":4,"out":20,"context":1048576,"hf":null,"created":"2026-10-12"}},{"id":"muse-wonder-full","name":"Muse Wonder Full","lab":"Meta","date":"2026-10-12","type":"flagship","major":true,"open":false,"release":{"title":"Muse Wonder + Muse Wonder Full","source":"https://about.fb.com/news/2026/10/introducing-muse-wonder-models/","verified":true,"confidence":"high"}},{"id":"muse-spark-open","name":"Muse Spark Open","lab":"Meta","date":"2026-10-15","type":"open-weights","major":false,"open":true,"index":{"score":161.1,"lo":145.2,"hi":177,"se":2.24,"solve":50.6,"status":"rated","areas":["agents","coding","maths","novel","reasoning","work"],"core_areas":["agents","coding","maths","novel","reasoning","work"],"testers":["ARC Prize","Epoch AI","Mercor","Vals AI"],"tests":{"vals:hlab":{"value":25,"norm":0.25,"version":"Vals AI"},"arc:arc_agi_2":{"value":57.5,"norm":0.575,"version":"ARC Prize"},"arc:arc_agi_3":{"value":0.1,"norm":0.001,"version":"ARC Prize"},"x:SimpleQA Verified":{"value":0.566,"norm":0.566,"version":"Muse-Spark-Open-816B-A42B_xhigh"},"vals:tax_agent_bench":{"value":26.32,"norm":0.2632,"version":"Vals AI"},"gpqa":{"value":0.8857323232323232,"norm":0.8857,"version":"Muse-Spark-Open-816B-A42B_xhigh"},"vals:terminal-bench-4":{"value":6.57,"norm":0.0657,"version":"Vals AI"},"arc:arc_agi_1":{"value":89,"norm":0.89,"version":"ARC Prize"},"vals:fabv2":{"value":61.34,"norm":0.6134,"version":"Vals AI"},"vals:legal_research":{"value":38.46,"norm":0.3846,"version":"Vals AI"},"vals:vibe-code":{"value":76.19,"norm":0.7619,"version":"Vals AI"},"vals:code-migration":{"value":21.17,"norm":0.2117,"version":"Vals AI"},"apex":{"value":0.334,"norm":0.334,"version":"Muse-Spark-Open-816B-A42B_unknown"},"x:FrontierMath-Tier-4-v2-Private":{"value":0.3170731707317073,"norm":0.3171,"version":"Muse-Spark-Open-816B-A42B_xhigh"},"vals:sage":{"value":85.77,"norm":0.8577,"version":"Vals AI"},"aime":{"value":0.9805555555555555,"norm":0.9806,"version":"Muse-Spark-Open-816B-A42B_xhigh"},"vals:emb":{"value":53.67,"norm":0.5367,"version":"Vals AI"},"fm":{"value":0.6982456140350877,"norm":0.6982,"version":"Muse-Spark-Open-816B-A42B_xhigh"}},"variant":"Muse Spark Open","match":"self"},"api":{"or_id":"meta/muse-spark-open","in":1.25,"out":4.25,"context":1048576,"hf":"meta-models/Muse-Spark-Open-816B-A42B","created":"2026-10-15"},"date_from":"epoch"},{"id":"deepseek-v4-1-flash-1017","name":"DeepSeek V4.1 Flash 1017","lab":"DeepSeek","date":"2026-10-17","type":"open-weights","major":false,"open":true,"index":{"score":171.9,"lo":155.7,"hi":188,"se":2.8,"solve":61,"status":"rated","areas":["agents","coding","maths","novel","reasoning","work"],"core_areas":["agents","coding","maths","novel","reasoning","work"],"testers":["ARC Prize","Epoch AI","Mercor","Vals AI"],"tests":{"x:Mystery Game Puzzles":{"value":0.49,"norm":0.49,"version":"deepseek-v4.1-flash-1017_max"},"gpqa":{"value":0.9318181818181818,"norm":0.9318,"version":"deepseek-v4.1-flash-1017_max"},"arc:arc_agi_1":{"value":94.5,"norm":0.945,"version":"ARC Prize"},"arc:arc_agi_3":{"value":0.2,"norm":0.002,"version":"ARC Prize"},"vals:tax_agent_bench":{"value":44.35,"norm":0.4435,"version":"Vals AI"},"x:Chess Puzzles":{"value":0.46,"norm":0.46,"version":"deepseek-v4.1-flash-1017_max"},"vals:hlab":{"value":14.17,"norm":0.1417,"version":"Vals AI"},"apex":{"value":0.698,"norm":0.698,"version":"deepseek-v4.1-flash-1017_max"},"vals:emb":{"value":61.8,"norm":0.618,"version":"Vals AI"},"fm":{"value":0.712280701754386,"norm":0.7123,"version":"deepseek-v4.1-flash-1017_max"},"vals:code-migration":{"value":44.21,"norm":0.4421,"version":"Vals AI"},"x:SimpleQA Verified":{"value":0.425,"norm":0.425,"version":"deepseek-v4.1-flash-1017_max"},"vals:fabv2":{"value":56.96,"norm":0.5696,"version":"Vals AI"},"vals:legal_research":{"value":45.19,"norm":0.4519,"version":"Vals AI"},"vals:terminal-bench-4":{"value":31.82,"norm":0.3182,"version":"Vals AI"},"x:FrontierMath-Tier-4-v2-Private":{"value":0.4634146341463415,"norm":0.4634,"version":"deepseek-v4.1-flash-1017_max"},"vals:vibe-code":{"value":87.73,"norm":0.8773,"version":"Vals AI"},"arc:arc_agi_2":{"value":78.1,"norm":0.781,"version":"ARC Prize"},"aime":{"value":0.9944444444444445,"norm":0.9944,"version":"deepseek-v4.1-flash-1017_max"}},"variant":"DeepSeek V4.1 Flash 1017","match":"self"},"api":{"or_id":"deepseek/deepseek-v4.1-flash-1017","in":0.036,"out":0.5,"context":1048576,"hf":"deepseek-ai/DeepSeek-V4.1-Flash-1017","created":"2026-10-17"},"date_from":"epoch"},{"id":"kimi-k3-1-pro","name":"Kimi K3.1 Pro","lab":"Moonshot","date":"2026-10-23","type":"open-weights","major":false,"open":true,"index":{"score":179,"lo":162.6,"hi":195.4,"se":3.27,"solve":66.3,"status":"rated","areas":["agents","coding","maths","novel","reasoning","work"],"core_areas":["agents","coding","maths","reasoning"],"testers":["Epoch AI","Mercor","Redwood Research","Vals AI"],"tests":{"x:DTBench":{"value":0.9333,"norm":0.9333,"version":"kimi-k3.1-pro_max"},"x:Chess Puzzles":{"value":0.45,"norm":0.45,"version":"kimi-k3.1-pro-together_max"},"vals:ioi":{"value":81.89,"norm":0.8189,"version":"Vals AI"},"x:SimpleQA Verified":{"value":0.622,"norm":0.622,"version":"kimi-k3.1-pro-together_max"},"vals:mysterymechanism":{"value":26.13,"norm":0.2613,"version":"Vals AI"},"vals:biomysterybench":{"value":75.56,"norm":0.7556,"version":"Vals AI"},"vals:legal_research":{"value":37.5,"norm":0.375,"version":"Vals AI"},"x:FrontierMath-Tier-4-v2-Private":{"value":0.6585365853658537,"norm":0.6585,"version":"kimi-k3.1-pro-together_max"},"vals:proof_bench":{"value":91,"norm":0.91,"version":"Vals AI"},"vals:terminal-bench-4":{"value":63.64,"norm":0.6364,"version":"Vals AI"},"vals:srebench":{"value":27.1,"norm":0.271,"version":"Vals AI"},"aime":{"value":1,"norm":1,"version":"kimi-k3.1-pro-together_max"},"vals:vibe-code":{"value":84.89,"norm":0.8489,"version":"Vals AI"},"x:Furniture Assembly":{"value":0.5583333333333333,"norm":0.5583,"version":"kimi-k3.1-pro-together_max"},"fm":{"value":0.7824561403508772,"norm":0.7825,"version":"kimi-k3.1-pro-together_max"},"apex":{"value":0.655,"norm":0.655,"version":"kimi-k3.1-pro_unknown"},"vals:terminal-bench-science":{"value":31.43,"norm":0.3143,"version":"Vals AI"},"x:LMCA":{"value":57.61,"norm":0.5761,"version":"kimi-k3.1-pro_max"},"vals:hlab":{"value":11.67,"norm":0.1167,"version":"Vals AI"},"vals:tax_agent_bench":{"value":24.87,"norm":0.2487,"version":"Vals AI"},"x:Mystery Game Puzzles":{"value":0.42,"norm":0.42,"version":"kimi-k3.1-pro-together_max"},"gpqa":{"value":0.9419191919191919,"norm":0.9419,"version":"kimi-k3.1-pro-together_max"}},"variant":"Kimi K3.1 Pro","match":"self"},"api":{"or_id":"moonshotai/kimi-k3.1-pro","in":0.2403,"out":8.4023,"context":1048576,"hf":"moonshotai/Kimi-K3.1-Pro","created":"2026-10-23"},"date_from":"epoch"},{"id":"gpt-6","name":"GPT-6","lab":"OpenAI","date":"2026-10-26","type":"flagship","major":true,"open":false,"release":{"title":"GPT-6","source":"https://openai.com/index/introducing-gpt-6-1-terra/","verified":false,"confidence":"auto"},"index":{"score":193.4,"lo":177.6,"hi":209.2,"se":1.93,"solve":74.4,"status":"rated","areas":["agents","coding","maths","novel","reasoning","work"],"core_areas":["agents","coding","maths","novel","reasoning","work"],"testers":["ARC Prize","BALROG team","Datacurve","Epoch AI","GSO team","Havard T. Ihle","LiveBench","Mercor","Proximal","Redwood Research","Scale AI","SimpleBench team","Vals AI"],"tests":{"vals:medscribe":{"value":87.91,"norm":0.8791,"version":"Vals AI"},"x:FrontierSWE":{"value":0.6550757455882352,"norm":0.6551,"version":"gpt-6-astra_max"},"vals:fabv2":{"value":53.54,"norm":0.5354,"version":"Vals AI"},"fm":{"value":0.9368421052631579,"norm":0.9368,"version":"gpt-6-astra_max"},"livebench:mathematics":{"value":96.81,"norm":0.9681,"version":"LiveBench"},"livebench:agentic-coding":{"value":57.32,"norm":0.5732,"version":"LiveBench"},"vals:legal_research":{"value":39.42,"norm":0.3942,"version":"Vals AI"},"x:FrontierMath-Tier-4-v2-Private":{"value":0.976,"norm":0.976,"version":"gpt-6-astra_max"},"vals:cyber":{"value":41.07,"norm":0.4107,"version":"Vals AI"},"livebench:if":{"value":75.58,"norm":0.7558,"version":"LiveBench"},"vals:mysterymechanism":{"value":53.15,"norm":0.5315,"version":"Vals AI"},"x:LMCA":{"value":64.37,"norm":0.6437,"version":"gpt-6-astra_xhigh"},"vals:medcode":{"value":48.49,"norm":0.4849,"version":"Vals AI"},"x:MirrorCode":{"value":0.6,"norm":0.6,"version":"gpt-6-astra_max"},"x:Furniture Assembly":{"value":0.8,"norm":0.8,"version":"gpt-6-astra_max"},"vals:emb":{"value":71.7,"norm":0.717,"version":"Vals AI"},"livebench:coding":{"value":80.36,"norm":0.8036,"version":"LiveBench"},"livebench:language":{"value":89.43,"norm":0.8943,"version":"LiveBench"},"x:Chess Puzzles":{"value":0.74,"norm":0.74,"version":"gpt-6-astra_max"},"vals:terminal-bench-science":{"value":62.86,"norm":0.6286,"version":"Vals AI"},"x:SimpleBench":{"value":0.836,"norm":0.836,"version":"gpt-6-astra_unknown"},"vals:tax_agent_bench":{"value":20.72,"norm":0.2072,"version":"Vals AI"},"livebench:reasoning":{"value":92.65,"norm":0.9265,"version":"LiveBench"},"x:WeirdML":{"value":0.9326000000000001,"norm":0.9326,"version":"gpt-6-astra_max"},"aime":{"value":1,"norm":1,"version":"gpt-6-astra_max"},"vals:hlab":{"value":7.08,"norm":0.0708,"version":"Vals AI"},"vals:programbench":{"value":5.5,"norm":0.055,"version":"Vals AI"},"vals:code-migration":{"value":67.74,"norm":0.6774,"version":"Vals AI"},"hle":{"value":0.5479999999999999,"norm":0.548,"version":"gpt-6-astra_unknown"},"x:Balrog":{"value":0.6829999999999999,"norm":0.683,"version":"gpt-6-astra_max"},"vals:biomysterybench":{"value":79.26,"norm":0.7926,"version":"Vals AI"},"arc:arc_agi_3":{"value":62.7,"norm":0.627,"version":"ARC Prize"},"arc:arc_agi_2":{"value":95,"norm":0.95,"version":"ARC Prize"},"vals:terminal-bench-4":{"value":59.6,"norm":0.596,"version":"Vals AI"},"arc:arc_agi_1":{"value":98.5,"norm":0.985,"version":"ARC Prize"},"x:EBR-bench":{"value":0.7619047619047619,"norm":0.7619,"version":"gpt-6-astra_max"},"vals:ioi":{"value":100,"norm":1,"version":"Vals AI"},"vals:srebench":{"value":56.87,"norm":0.5687,"version":"Vals AI"},"x:DTBench":{"value":0.9733,"norm":0.9733,"version":"gpt-6-astra_xhigh"},"livebench:data-analysis":{"value":82.97,"norm":0.8297,"version":"LiveBench"},"x:Mystery Game Puzzles":{"value":0.84,"norm":0.84,"version":"gpt-6-astra_max"},"gpqa":{"value":0.9577020202020202,"norm":0.9577,"version":"gpt-6-astra_max"},"x:DeepSWE":{"value":0.7411504424778761,"norm":0.7412,"version":"gpt-6-astra_xhigh"},"vals:vcb-1-100":{"value":27.64,"norm":0.2764,"version":"Vals AI"},"x:GSO-Bench":{"value":0.7941,"norm":0.7941,"version":"gpt-6-astra_unknown"},"vals:vibe-code":{"value":89.59,"norm":0.8959,"version":"Vals AI"},"x:SimpleQA Verified":{"value":0.756,"norm":0.756,"version":"gpt-6-astra_max"},"apex":{"value":0.647,"norm":0.647,"version":"gpt-6-astra_unknown"},"vals:proof_bench":{"value":99,"norm":0.99,"version":"Vals AI"}},"variant":"GPT-6 Astra","match":"family"}},{"id":"gpt-6-1-terra","name":"GPT-6.1 Terra","lab":"OpenAI","date":"2026-10-26","type":"flagship","major":true,"open":false,"release":{"title":"GPT-6.1 Terra","source":"https://openai.com/index/introducing-gpt-6-1-terra/","verified":false,"confidence":"auto"},"index":{"score":186.3,"lo":169.6,"hi":202.9,"se":3.67,"solve":70.8,"status":"rated","areas":["agents","coding","maths","reasoning","work"],"core_areas":["agents","coding","maths","reasoning"],"testers":["Epoch AI","Mercor","SimpleBench team","Vals AI"],"tests":{"vals:legal_research":{"value":25.96,"norm":0.2596,"version":"Vals AI"},"x:Mystery Game Puzzles":{"value":0.74,"norm":0.74,"version":"gpt-6.1-terra_max"},"vals:terminal-bench-4":{"value":49.5,"norm":0.495,"version":"Vals AI"},"x:Furniture Assembly":{"value":0.7916666666666666,"norm":0.7917,"version":"gpt-6.1-terra_max"},"fm":{"value":0.9087719298245615,"norm":0.9088,"version":"gpt-6.1-terra_max"},"vals:hlab":{"value":1.67,"norm":0.0167,"version":"Vals AI"},"vals:vibe-code":{"value":84,"norm":0.84,"version":"Vals AI"},"x:SimpleQA Verified":{"value":0.573,"norm":0.573,"version":"gpt-6.1-terra_max"},"gpqa":{"value":0.9431818181818182,"norm":0.9432,"version":"gpt-6.1-terra_max"},"apex":{"value":0.6829999999999999,"norm":0.683,"version":"gpt-6.1-terra_max"},"x:FrontierMath-Tier-4-v2-Private":{"value":0.926829268292683,"norm":0.9268,"version":"gpt-6.1-terra_max"},"aime":{"value":1,"norm":1,"version":"gpt-6.1-terra_max"},"vals:tax_agent_bench":{"value":17.08,"norm":0.1708,"version":"Vals AI"},"x:MirrorCode":{"value":0.7666666666666667,"norm":0.7667,"version":"gpt-6.1-terra_max"},"x:SimpleBench":{"value":0.7220000000000001,"norm":0.722,"version":"gpt-6.1-terra_unknown"},"x:EBR-bench":{"value":0.780952380952381,"norm":0.781,"version":"gpt-6.1-terra_max"}},"variant":"GPT-6.1 Terra","match":"exact"},"api":{"or_id":"openai/gpt-6.1-terra","in":1,"out":5,"context":1050000,"hf":null,"created":"2026-10-26"}},{"id":"gemini-4-carbon","name":"Gemini 4 Carbon","lab":"Google","date":"2026-10-28","type":"flagship","major":true,"open":false,"release":{"title":"Gemini 4 Carbon","source":"https://blog.google/innovation-and-ai/models-and-research/gemini-models/introducing-gemini-4-carbon/","verified":false,"confidence":"auto"},"index":{"score":198.2,"lo":181.5,"hi":214.9,"se":3.72,"solve":76.6,"status":"rated","areas":["agents","coding","maths","reasoning","work"],"core_areas":["agents","coding","maths","reasoning"],"testers":["Epoch AI","Mercor","Proximal","SimpleBench team","Vals AI"],"tests":{"x:Furniture Assembly":{"value":0.775,"norm":0.775,"version":"gemini-4-carbon_high"},"vals:hlab":{"value":17.5,"norm":0.175,"version":"Vals AI"},"x:SimpleQA Verified":{"value":0.777,"norm":0.777,"version":"gemini-4-carbon_high"},"apex":{"value":0.8420000000000001,"norm":0.842,"version":"gemini-4-carbon_unknown"},"x:SimpleBench":{"value":0.9259999999999999,"norm":0.926,"version":"gemini-4-carbon_unknown"},"aime":{"value":1,"norm":1,"version":"gemini-4-carbon_high"},"x:EBR-bench":{"value":0.8761904761904763,"norm":0.8762,"version":"gemini-4-carbon_high"},"vals:terminal-bench-4":{"value":63.13,"norm":0.6313,"version":"Vals AI"},"fm":{"value":0.9052631578947369,"norm":0.9053,"version":"gemini-4-carbon_high"},"x:FrontierMath-Tier-4-v2-Private":{"value":0.9512195121951219,"norm":0.9512,"version":"gemini-4-carbon_high"},"vals:legal_research":{"value":55.77,"norm":0.5577,"version":"Vals AI"},"vals:vibe-code":{"value":89.06,"norm":0.8906,"version":"Vals AI"},"x:Chess Puzzles":{"value":0.78,"norm":0.78,"version":"gemini-4-carbon_high"},"x:FrontierSWE":{"value":0.6119949235294118,"norm":0.612,"version":"gemini-4-carbon_high"},"vals:tax_agent_bench":{"value":51.78,"norm":0.5178,"version":"Vals AI"},"x:Mystery Game Puzzles":{"value":0.73,"norm":0.73,"version":"gemini-4-carbon_high"},"gpqa":{"value":0.9652777777777778,"norm":0.9653,"version":"gemini-4-carbon_high"}},"variant":"Gemini 4 Carbon","match":"exact"},"api":{"or_id":"google/gemini-4-carbon","in":2,"out":10,"context":1048576,"hf":null,"created":"2026-11-10"}},{"id":"doubao-seed-evolving-nov-2026","name":"Doubao Seed Evolving (Nov 2026)","lab":"ByteDance","date":"2026-11-02","type":"flagship","major":false,"open":false,"index":{"score":159,"lo":138.5,"hi":179.5,"se":8.14,"solve":48.3,"status":"provisional","areas":["maths","reasoning"],"core_areas":["reasoning"],"testers":["Epoch AI"],"tests":{"gpqa":{"value":0.9311868686868687,"norm":0.9312,"version":"doubao-seed-evolving_high"},"aime":{"value":0.9916666666666667,"norm":0.9917,"version":"doubao-seed-evolving_high"},"x:SimpleQA Verified":{"value":0.419,"norm":0.419,"version":"doubao-seed-evolving_high"},"x:Chess Puzzles":{"value":0.4,"norm":0.4,"version":"doubao-seed-evolving_high"}},"variant":"Doubao Seed Evolving (Nov 2026)","match":"self"},"date_from":"epoch"},{"id":"inkling-2","name":"Inkling 2","lab":"Thinking Machines","date":"2026-11-05","type":"open-weights","major":true,"open":true,"release":{"title":"Inkling 2","source":"https://thinkingmachines.ai/news/inkling-2-small/","verified":false,"confidence":"auto"},"open_evidence":"https://huggingface.co/thinkingmachines/Inkling"},{"id":"inkling-2-small","name":"Inkling 2 Small","lab":"Thinking Machines","date":"2026-11-05","type":"open-weights","major":false,"open":true,"index":{"score":166,"lo":148.8,"hi":183.1,"se":4.51,"solve":55.6,"status":"rated","areas":["agents","coding","novel","work"],"core_areas":["agents","coding"],"testers":["Mercor","Proximal","Vals AI"],"tests":{"vals:mysterymechanism":{"value":25.23,"norm":0.2523,"version":"Vals AI"},"vals:sage":{"value":88.57,"norm":0.8857,"version":"Vals AI"},"vals:tax_agent_bench":{"value":38.91,"norm":0.3891,"version":"Vals AI"},"vals:programbench":{"value":0.5,"norm":0.005,"version":"Vals AI"},"vals:srebench":{"value":1.91,"norm":0.0191,"version":"Vals AI"},"x:FrontierSWE":{"value":0.25069580588235296,"norm":0.2507,"version":"inkling-2-small_xhigh"},"vals:legal_research":{"value":46.63,"norm":0.4663,"version":"Vals AI"},"vals:terminal-bench-4":{"value":12.63,"norm":0.1263,"version":"Vals AI"},"vals:vibe-code":{"value":81.55,"norm":0.8155,"version":"Vals AI"},"apex":{"value":0.46799999999999997,"norm":0.468,"version":"inkling-2-small_unknown"},"vals:biomysterybench":{"value":70,"norm":0.7,"version":"Vals AI"},"vals:hlab":{"value":21.67,"norm":0.2167,"version":"Vals AI"},"vals:terminal-bench-science":{"value":2.86,"norm":0.0286,"version":"Vals AI"},"vals:cyber":{"value":72.74,"norm":0.7274,"version":"Vals AI"}},"variant":"Inkling 2 Small","match":"self"},"api":{"or_id":"thinkingmachines/inkling-2-small","in":0.3,"out":1,"context":1048576,"hf":"thinkingmachines/Inkling-2-Small","created":"2026-11-10"},"open_evidence":"https://huggingface.co/thinkingmachines/Inkling","date_from":"epoch"},{"id":"claude-fable-5-2","name":"Claude Fable 5.2","lab":"Anthropic","date":"2026-11-09","type":"flagship","major":true,"open":false,"release":{"title":"Claude Fable 5.2","source":"https://www.anthropic.com/claude-fable-and-mythos-5-2","verified":false,"confidence":"auto"},"index":{"score":201.5,"lo":184.8,"hi":218.2,"se":3.77,"solve":78,"status":"rated","areas":["agents","coding","maths","novel","reasoning","work"],"core_areas":["agents","coding","maths","reasoning"],"testers":["Epoch AI","Mercor","Proximal","Redwood Research","SimpleBench team","Vals AI"],"tests":{"vals:terminal-bench-science":{"value":55.71,"norm":0.5571,"version":"Vals AI"},"x:Mystery Game Puzzles":{"value":0.9,"norm":0.9,"version":"claude-fable-5-2_max"},"vals:terminal-bench-4":{"value":65.15,"norm":0.6515,"version":"Vals AI"},"vals:mysterymechanism":{"value":51.8,"norm":0.518,"version":"Vals AI"},"vals:proof_bench":{"value":100,"norm":1,"version":"Vals AI"},"vals:biomysterybench":{"value":83.7,"norm":0.837,"version":"Vals AI"},"vals:vibe-code":{"value":93.77,"norm":0.9377,"version":"Vals AI"},"x:Furniture Assembly":{"value":0.9,"norm":0.9,"version":"claude-fable-5-2_max"},"fm":{"value":0.9508771929824561,"norm":0.9509,"version":"claude-fable-5-2_max"},"x:FrontierMath-Tier-4-v2-Private":{"value":1,"norm":1,"version":"claude-fable-5-2_max"},"vals:hlab":{"value":5,"norm":0.05,"version":"Vals AI"},"x:EBR-bench":{"value":0.9047619047619048,"norm":0.9048,"version":"claude-fable-5-2_max"},"x:SimpleQA Verified":{"value":0.94,"norm":0.94,"version":"claude-fable-5-2_max"},"x:LMCA":{"value":70.61,"norm":0.7061,"version":"claude-fable-5-2_xhigh"},"x:FrontierSWE":{"value":0.6994018470588234,"norm":0.6994,"version":"claude-fable-5-2_max"},"x:DTBench":{"value":0.9920000000000001,"norm":0.992,"version":"claude-fable-5-2_unknown"},"vals:legal_research":{"value":46.15,"norm":0.4615,"version":"Vals AI"},"vals:programbench":{"value":19,"norm":0.19,"version":"Vals AI"},"aime":{"value":0.9972222222222222,"norm":0.9972,"version":"claude-fable-5-2_max"},"x:SimpleBench":{"value":0.9490000000000001,"norm":0.949,"version":"claude-fable-5-2_unknown"},"vals:tax_agent_bench":{"value":32.09,"norm":0.3209,"version":"Vals AI"},"gpqa":{"value":0.9602272727272727,"norm":0.9602,"version":"claude-fable-5-2-fallback_max"},"apex":{"value":0.8009999999999999,"norm":0.801,"version":"claude-fable-5-2_unknown"},"x:Chess Puzzles":{"value":0.79,"norm":0.79,"version":"claude-fable-5-2_max"}},"variant":"Claude Fable 5.2","match":"exact"},"api":{"or_id":"anthropic/claude-fable-5.2","in":8,"out":40,"context":1000000,"hf":null,"created":"2026-11-09"}},{"id":"claude-mythos-5-2","name":"Claude Mythos 5.2","lab":"Anthropic","date":"2026-11-09","type":"flagship","major":true,"open":false,"release":{"title":"Claude Mythos 5.2","source":"https://www.anthropic.com/claude-fable-and-mythos-5-2","verified":false,"confidence":"auto"}},{"id":"muse-ember","name":"Muse Ember","lab":"Meta","date":"2026-11-09","type":"flagship","major":false,"open":false,"index":{"score":178.5,"lo":161.1,"hi":195.9,"se":4.86,"solve":66,"status":"rated","areas":["agents","coding","reasoning","work"],"core_areas":["agents","coding"],"testers":["Mercor","SimpleBench team","Vals AI"],"tests":{"apex":{"value":0.643,"norm":0.643,"version":"muse-ember-1.0_unknown"},"vals:legal_research":{"value":49.04,"norm":0.4904,"version":"Vals AI"},"vals:hlab":{"value":25,"norm":0.25,"version":"Vals AI"},"vals:tax_agent_bench":{"value":38.23,"norm":0.3823,"version":"Vals AI"},"vals:terminal-bench-4":{"value":39.39,"norm":0.3939,"version":"Vals AI"},"vals:vibe-code":{"value":83.7,"norm":0.837,"version":"Vals AI"},"x:SimpleBench":{"value":0.7270000000000001,"norm":0.727,"version":"muse-ember-1.0_unknown"}},"variant":"Muse Ember","match":"self"},"api":{"or_id":"meta/muse-ember","in":0.1,"out":0.5,"context":1048576,"hf":null,"created":"2026-11-09"},"date_from":"epoch"},{"id":"gemini-4-hydrogen","name":"Gemini 4 Hydrogen","lab":"Google","date":"2026-11-12","type":"flagship","major":true,"open":false,"release":{"title":"Gemini 4 Hydrogen","source":"https://blog.google/innovation-and-ai/models-and-research/gemini-models/introducing-gemini-4-hydrogen/","verified":false,"confidence":"auto"},"index":{"score":190.9,"lo":173.9,"hi":207.8,"se":4.15,"solve":73.2,"status":"rated","areas":["agents","coding","work"],"core_areas":["agents","coding"],"testers":["Mercor","Proximal","Vals AI"],"tests":{"vals:hlab":{"value":11.25,"norm":0.1125,"version":"Vals AI"},"x:FrontierSWE":{"value":0.5311297705882353,"norm":0.5311,"version":"gemini-4-hydrogen_high"},"vals:tax_agent_bench":{"value":52.31,"norm":0.5231,"version":"Vals AI"},"vals:legal_research":{"value":53.85,"norm":0.5385,"version":"Vals AI"},"vals:vibe-code":{"value":87.78,"norm":0.8778,"version":"Vals AI"},"vals:terminal-bench-4":{"value":54.04,"norm":0.5404,"version":"Vals AI"},"apex":{"value":0.78,"norm":0.78,"version":"gemini-4-hydrogen_unknown"}},"variant":"Gemini 4 Hydrogen","match":"exact"},"api":{"or_id":"google/gemini-4-hydrogen","in":1.5,"out":7.5,"context":1048576,"hf":null,"created":"2026-11-12"}},{"id":"gpt-6-2","name":"GPT-6.2","lab":"OpenAI","date":"2026-11-17","type":"flagship","major":true,"open":false,"release":{"title":"GPT-6.2","source":"https://openai.com/index/introducing-gpt-6-2-astra-sol","verified":false,"confidence":"auto"},"index":{"score":201.9,"lo":184.2,"hi":219.6,"se":5.18,"solve":78.1,"status":"rated","areas":["agents","coding","maths","novel","reasoning","work"],"core_areas":["coding","maths","reasoning"],"testers":["Epoch AI","Vals AI"],"tests":{"vals:hlab":{"value":16.25,"norm":0.1625,"version":"Vals AI"},"x:Chess Puzzles":{"value":0.78,"norm":0.78,"version":"gpt-6.2-astra_max"},"vals:legal_research":{"value":42.79,"norm":0.4279,"version":"Vals AI"},"x:SimpleQA Verified":{"value":0.785,"norm":0.785,"version":"gpt-6.2-astra_max"},"fm":{"value":0.9263157894736842,"norm":0.9263,"version":"gpt-6.2-astra_max"},"aime":{"value":1,"norm":1,"version":"gpt-6.2-astra_max"},"x:Mystery Game Puzzles":{"value":0.95,"norm":0.95,"version":"gpt-6.2-astra_max"},"x:EBR-bench":{"value":0.9238095238095237,"norm":0.9238,"version":"gpt-6.2-astra_max"},"vals:terminal-bench-4":{"value":63.13,"norm":0.6313,"version":"Vals AI"},"vals:biomysterybench":{"value":80.74,"norm":0.8074,"version":"Vals AI"},"x:Furniture Assembly":{"value":0.8916666666666667,"norm":0.8917,"version":"gpt-6.2-astra_max"},"vals:tax_agent_bench":{"value":21.84,"norm":0.2184,"version":"Vals AI"},"vals:vibe-code":{"value":89.9,"norm":0.899,"version":"Vals AI"},"vals:terminal-bench-science":{"value":71.43,"norm":0.7143,"version":"Vals AI"},"x:MirrorCode":{"value":0.6777777777777778,"norm":0.6778,"version":"gpt-6.2-astra_max"},"x:FrontierMath-Tier-4-v2-Private":{"value":1,"norm":1,"version":"gpt-6.2-astra_max"},"gpqa":{"value":0.9684343434343434,"norm":0.9684,"version":"gpt-6.2-astra_max"},"vals:mysterymechanism":{"value":53.15,"norm":0.5315,"version":"Vals AI"}},"variant":"GPT-6.2 Astra","match":"family"},"api":{"or_id":"openai/gpt-6.2-astra","in":8,"out":40,"context":1050000,"hf":null,"created":"2026-11-17"}},{"id":"gpt-6-2-astra","name":"GPT-6.2 Astra","lab":"OpenAI","date":"2026-11-17","type":"flagship","major":true,"open":false,"release":{"title":"GPT-6.2 Astra","source":"https://openai.com/index/introducing-gpt-6-2-astra-sol","verified":false,"confidence":"auto"},"index":{"score":201.9,"lo":184.2,"hi":219.6,"se":5.18,"solve":78.1,"status":"rated","areas":["agents","coding","maths","novel","reasoning","work"],"core_areas":["coding","maths","reasoning"],"testers":["Epoch AI","Vals AI"],"tests":{"vals:hlab":{"value":16.25,"norm":0.1625,"version":"Vals AI"},"x:Chess Puzzles":{"value":0.78,"norm":0.78,"version":"gpt-6.2-astra_max"},"vals:legal_research":{"value":42.79,"norm":0.4279,"version":"Vals AI"},"x:SimpleQA Verified":{"value":0.785,"norm":0.785,"version":"gpt-6.2-astra_max"},"fm":{"value":0.9263157894736842,"norm":0.9263,"version":"gpt-6.2-astra_max"},"aime":{"value":1,"norm":1,"version":"gpt-6.2-astra_max"},"x:Mystery Game Puzzles":{"value":0.95,"norm":0.95,"version":"gpt-6.2-astra_max"},"x:EBR-bench":{"value":0.9238095238095237,"norm":0.9238,"version":"gpt-6.2-astra_max"},"vals:terminal-bench-4":{"value":63.13,"norm":0.6313,"version":"Vals AI"},"vals:biomysterybench":{"value":80.74,"norm":0.8074,"version":"Vals AI"},"x:Furniture Assembly":{"value":0.8916666666666667,"norm":0.8917,"version":"gpt-6.2-astra_max"},"vals:tax_agent_bench":{"value":21.84,"norm":0.2184,"version":"Vals AI"},"vals:vibe-code":{"value":89.9,"norm":0.899,"version":"Vals AI"},"vals:terminal-bench-science":{"value":71.43,"norm":0.7143,"version":"Vals AI"},"x:MirrorCode":{"value":0.6777777777777778,"norm":0.6778,"version":"gpt-6.2-astra_max"},"x:FrontierMath-Tier-4-v2-Private":{"value":1,"norm":1,"version":"gpt-6.2-astra_max"},"gpqa":{"value":0.9684343434343434,"norm":0.9684,"version":"gpt-6.2-astra_max"},"vals:mysterymechanism":{"value":53.15,"norm":0.5315,"version":"Vals AI"}},"variant":"GPT-6.2 Astra","match":"exact"},"api":{"or_id":"openai/gpt-6.2-astra","in":8,"out":40,"context":1050000,"hf":null,"created":"2026-11-17"}}]}