[{"id":"swe_bench_verified","name":"SWE-bench Verified","what":"Can the AI fix real bugs in real open-source Python projects, well enough that the project's own tests pass?","launched":"2024-08-13","human_baseline":null,"unit":"%","caveat":"Launch point is OpenAI's own figure (GPT-4o with the Agentless scaffold); every later point is Epoch AI's independent run with one standard scaffold, so it sits a few points below what labs announce. Labs claimed higher scores with their own setups: Claude Opus 4.5 80.9% (Nov 2025), Claude Mythos Preview 93.9% (Apr 2026, limited-access preview), Claude Opus 4.8 87.3% (May 2026). Aggregator sites list 95-96% for mid-2026 Claude models, but I could not confirm those in a primary source. In Feb 2026 OpenAI stopped reporting this benchmark because of training-data contamination and saturation, and it moved to SWE-bench Pro. Epoch's independent series stops at Claude Opus 4.7 (Apr 2026). Dates are model release dates.","points":[{"date":"2024-08-13","model":"GPT-4o (Agentless scaffold)","score":33.2,"source":"https://openai.com/index/introducing-swe-bench-verified/","independent":false},{"date":"2024-10-22","model":"Claude 3.5 Sonnet (new) + simple tools","score":49,"source":"https://www.swebench.com/","independent":false},{"date":"2025-02-24","model":"Claude 3.7 Sonnet","score":61,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2025-04-16","model":"o3 (medium)","score":62.3,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2025-05-22","model":"Claude Opus 4","score":70.7,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2025-08-05","model":"Claude Opus 4.1","score":73.3,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2025-08-07","model":"GPT-5 (high)","score":73.6,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2025-11-24","model":"Claude Opus 4.5","score":76.7,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2026-02-05","model":"Claude Opus 4.6","score":78.7,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2026-04-16","model":"Claude Opus 4.7 (max)","score":83.5,"source":"https://epoch.ai/benchmarks","independent":true}],"status":"active","status_note":"Still separating models: the record rose 10 points in the last 12 months to 83.5%."},{"id":"swe_bench_pro","name":"SWE-bench Pro (public set)","what":"A harder successor to SWE-bench: long, multi-file engineering tasks from many languages and codebases, designed to be hard to memorise.","launched":"2025-09-19","human_baseline":null,"unit":"%","caveat":"Scale AI runs every model independently with its SWE-Agent scaffold. The launch point (GPT-5 23.3%) used a 50-turn cost cap. The later points come from Scale's current leaderboard, which uses a 250-turn uncapped setup and re-ran older models, so the jump from the launch point is partly a methodology change. Later dates are model release dates, not the dates Scale posted results. Scale has not yet evaluated the newest models (Claude Opus 5/5.5, GPT-6 Astra). Lab-reported scores run much higher: Claude Mythos Preview 77.8% (Apr 2026), Claude Opus 4.8 75.4% (May 2026). Aggregators list 89.9% for Claude Opus 5.5, but I could not confirm that in a primary source.","points":[{"date":"2025-09-19","model":"GPT-5 (at launch, 50-turn cap)","score":23.3,"source":"https://labs.scale.com/leaderboard/swe_bench_pro","independent":true},{"date":"2025-09-29","model":"Claude Sonnet 4.5","score":43.6,"source":"https://labs.scale.com/leaderboard/swe_bench_pro","independent":true},{"date":"2025-11-24","model":"Claude Opus 4.5","score":45.9,"source":"https://labs.scale.com/leaderboard/swe_bench_pro","independent":true},{"date":"2026-02-05","model":"Claude Opus 4.6 (thinking)","score":51.9,"source":"https://labs.scale.com/leaderboard/swe_bench_pro","independent":true},{"date":"2026-03-05","model":"GPT-5.4 (xHigh)","score":59.1,"source":"https://labs.scale.com/leaderboard/swe_bench_pro","independent":true},{"date":"2026-07-09","model":"Meta Muse Spark 1.1","score":61.5,"source":"https://labs.scale.com/leaderboard/swe_bench_pro","independent":true}],"status":"active","status_note":"Still separating models: the record rose 18 points in the last 12 months to 61.5%."},{"id":"gpqa_diamond","name":"GPQA Diamond","what":"198 graduate-level biology, chemistry and physics questions written to be \"Google-proof\": even skilled non-experts with web access score barely above chance.","launched":"2023-11-20","human_baseline":{"value":69.7,"note":"PhD domain experts, as cited by OpenAI (o1 launch). The original paper reports ~65% for experts on full GPQA and ~34% for skilled non-experts with web access. Random guessing = 25%."},"unit":"%","caveat":"Epoch AI's independent runs with a consistent harness. Dates are model release dates; models released before launch are shown at the launch date. Now effectively saturated: the top models are within noise of each other at ~95%, and some questions are thought to have wrong or ambiguous answers.","points":[{"date":"2023-11-20","model":"GPT-4 Turbo (1106 preview)","score":42.4,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2024-02-29","model":"Claude 3 Opus","score":47.2,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2024-05-13","model":"GPT-4o","score":48.9,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2024-06-20","model":"Claude 3.5 Sonnet","score":54,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2024-09-12","model":"o1-mini (high)","score":62.4,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2024-12-17","model":"o1 (high)","score":76.8,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2025-01-31","model":"o3-mini (high)","score":77,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2025-02-24","model":"Claude 3.7 Sonnet (64K)","score":79.7,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2025-03-25","model":"Gemini 2.5 Pro (exp, Mar 2025)","score":83.8,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2025-06-05","model":"Gemini 2.5 Pro (preview, Jun 2025)","score":84.8,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2025-06-17","model":"Gemini 2.5 Pro","score":85.3,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2025-07-09","model":"Grok 4","score":87,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2025-11-13","model":"GPT-5.1 (high)","score":87.6,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2025-11-18","model":"Gemini 3 Pro (preview)","score":92.6,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2026-02-19","model":"Gemini 3.1 Pro Preview (high)","score":94.4,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2026-03-05","model":"GPT-5.4 Pro (xhigh)","score":94.6,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2026-08-13","model":"Gemini 3.7 Flash (high)","score":94.8,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2026-09-02","model":"Gemini 3.8 Flash (high)","score":95.4,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2026-09-03","model":"GPT-6 Astra (max)","score":95.8,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2026-09-30","model":"Gemini 4 Argon (high)","score":96.4,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2026-10-28","model":"Gemini 4 Carbon (high)","score":96.5,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2026-11-17","model":"GPT-6.2 Astra (max)","score":96.8,"source":"https://epoch.ai/benchmarks","independent":true}],"status":"saturated","status_note":"Passed 90% in Nov 2025 (Gemini 3 Pro (preview)); the record is now 96.8%. Too little headroom is left for it to separate the best models."},{"id":"hle","name":"Humanity's Last Exam","what":"2,500 expert-written questions at the frontier of human knowledge across dozens of fields, meant to be the \"final\" closed-ended academic exam for AI.","launched":"2025-01-23","human_baseline":{"value":null,"note":"No aggregate human score. Each question was written by a subject expert, and no single person could answer across all fields."},"unit":"%","caveat":"No-tools scores compiled by Epoch AI, mostly from the Scale AI / CAIS leaderboard (lastexam.ai). With tools (search, code) scores are much higher: Anthropic reports Claude Opus 5.5 at 67.7% with tools (Sep 2026), and Claude Mythos Preview at 56.8% without tools (Apr 2026, lab-reported, not in this series). Dates are model release dates.","points":[{"date":"2025-01-23","model":"o1","score":8,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2025-03-19","model":"o1 Pro","score":8.1,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2025-03-25","model":"Gemini 2.5 Pro (exp, Mar 2025)","score":18.2,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2025-04-16","model":"o3 (high)","score":20.3,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2025-06-05","model":"Gemini 2.5 Pro (preview, Jun 2025)","score":21.6,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2025-08-07","model":"GPT-5 (high)","score":25.3,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2025-10-07","model":"GPT-5 Pro","score":31.6,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2025-11-18","model":"Gemini 3 Pro (preview)","score":37.5,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2026-02-19","model":"Gemini 3.1 Pro Preview","score":46.4,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2026-09-01","model":"Claude Fable 5.1 (xhigh)","score":46.5,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2026-09-03","model":"GPT-6 Astra","score":54.8,"source":"https://epoch.ai/benchmarks","independent":true}],"status":"active","status_note":"Still separating models: the record rose 17 points in the last 12 months to 54.8%."},{"id":"arc_agi_1","name":"ARC-AGI-1","what":"Visual pattern puzzles that are easy for people but need the AI to spot a brand-new rule from 2-3 examples, testing learning on the fly rather than memory.","launched":"2019-11-05","human_baseline":{"value":98,"note":"ARC Prize human panel / STEM graduates on the semi-private set. Average Mechanical Turk worker: ~77%."},"unit":"%","caveat":"ARC Prize verified scores on the semi-private set, except: Icecuber 2020 (Kaggle private test, ~20%; date is the approximate competition close) and ARChitects (ARC Prize 2024 winner, 53.5% on the private set). o3-preview (Dec 2024) was an unreleased model and exceeded ARC Prize's cost limit ($200/task low compute; 87.5% at high compute). Third-party \"refinement\" harnesses on top of frontier models are left out. LLM dates are model release dates.","points":[{"date":"2020-05-27","model":"Icecuber (program search, Kaggle 2020 winner)","score":20.6,"source":"https://arcprize.org/history","independent":true},{"date":"2024-11-10","model":"ARChitects (ARC Prize 2024 winner)","score":53.5,"source":"https://arcprize.org/blog/arc-prize-2024-winners-technical-report","independent":true},{"date":"2024-12-20","model":"OpenAI o3-preview (low compute)","score":75.7,"source":"https://arcprize.org/leaderboard","independent":true},{"date":"2025-11-24","model":"Claude Opus 4.5 (Thinking 64K)","score":80,"source":"https://arcprize.org/leaderboard","independent":true},{"date":"2025-12-11","model":"GPT-5.2 Pro (xhigh)","score":90.5,"source":"https://arcprize.org/leaderboard","independent":true},{"date":"2026-02-05","model":"Claude Opus 4.6 (120K, high)","score":94,"source":"https://arcprize.org/leaderboard","independent":true},{"date":"2026-02-12","model":"Gemini 3 Deep Think","score":96,"source":"https://arcprize.org/leaderboard","independent":true},{"date":"2026-02-19","model":"Gemini 3.1 Pro (preview)","score":98,"source":"https://arcprize.org/leaderboard","independent":true},{"date":"2026-06-09","model":"Claude Fable 5 (max)","score":98.5,"source":"https://arcprize.org/leaderboard","independent":true}],"status":"superseded","status_note":"A newer version, ARC-AGI-2, arrived in Mar 2025. The record on this one is 98.5% (Claude Fable 5 (max), Jun 2026)."},{"id":"arc_agi_2","name":"ARC-AGI-2","what":"A harder second set of ARC puzzles needing multi-step, compositional rule-finding. Every task was solved by at least two people in testing.","launched":"2025-03-24","human_baseline":{"value":100,"note":"ARC Prize human panel: every task solved by at least 2 people within 2 attempts. ARC Prize says the average test-taker scores ~60%."},"unit":"%","caveat":"ARC Prize verified semi-private scores for frontier models. Kaggle competition entries and third-party refinement harnesses are left out: a Poetiq-style GPT-5.2 refinement scored 72.9% in early 2026. The launch point is o3-preview (unreleased, low compute). Dates are model release dates.","points":[{"date":"2025-03-24","model":"OpenAI o3-preview (low compute)","score":4,"source":"https://arcprize.org/leaderboard","independent":true},{"date":"2025-04-16","model":"o3 (high)","score":6.5,"source":"https://arcprize.org/leaderboard","independent":true},{"date":"2025-05-22","model":"Claude Opus 4 (Thinking 16K)","score":8.6,"source":"https://arcprize.org/leaderboard","independent":true},{"date":"2025-07-09","model":"Grok 4 (Thinking)","score":16,"source":"https://arcprize.org/leaderboard","independent":true},{"date":"2025-10-06","model":"GPT-5 Pro","score":18.3,"source":"https://arcprize.org/leaderboard","independent":true},{"date":"2025-11-18","model":"Gemini 3 Pro","score":31.1,"source":"https://arcprize.org/leaderboard","independent":true},{"date":"2025-11-24","model":"Claude Opus 4.5 (Thinking 64K)","score":37.6,"source":"https://arcprize.org/leaderboard","independent":true},{"date":"2025-12-11","model":"GPT-5.2 Pro (high)","score":54.2,"source":"https://arcprize.org/leaderboard","independent":true},{"date":"2026-02-05","model":"Claude Opus 4.6 (120K, high)","score":69.2,"source":"https://arcprize.org/leaderboard","independent":true},{"date":"2026-02-12","model":"Gemini 3 Deep Think","score":84.6,"source":"https://arcprize.org/leaderboard","independent":true},{"date":"2026-04-23","model":"GPT-5.5 (xhigh)","score":85,"source":"https://arcprize.org/leaderboard","independent":true},{"date":"2026-06-09","model":"Claude Fable 5 (max)","score":89.2,"source":"https://arcprize.org/leaderboard","independent":true},{"date":"2026-07-09","model":"GPT-5.6 Sol (max)","score":92.5,"source":"https://arcprize.org/leaderboard","independent":true},{"date":"2026-09-02","model":"GPT-6 Astra (max)","score":95,"source":"https://arcprize.org/results/openai-gpt-6-astra","independent":true}],"status":"saturated","status_note":"Passed 90% in Jul 2026 (GPT-5.6 Sol (max)); the record is now 95%. Too little headroom is left for it to separate the best models."},{"id":"arc_agi_3","name":"ARC-AGI-3","what":"Interactive, video-game-like puzzle worlds with no instructions: the AI has to explore, work out the rules and the goal, and win efficiently.","launched":"2026-03-25","human_baseline":{"value":100,"note":"Humans solved 100% of the environments in ARC Prize testing."},"unit":"%","caveat":"ARC Prize verified semi-private scores using its standard harness. With the \"provider adapter\" harness, which keeps the model's hidden reasoning state between turns, GPT-6 Astra scored 99.9% (Sep 2026) and Gemini 3.8 Flash 35.0%. I left these out of the staircase because the harness differs. The launch point is Claude Opus 4.6 (released Feb 2026), the best score at launch. Other dates are model release dates.","points":[{"date":"2026-03-25","model":"Claude Opus 4.6 (max)","score":0.5,"source":"https://arcprize.org/blog/arc-agi-3-launch","independent":true},{"date":"2026-05-28","model":"Claude Opus 4.8 (high)","score":1.5,"source":"https://arcprize.org/leaderboard","independent":true},{"date":"2026-07-09","model":"GPT-5.6 Sol (max)","score":7.8,"source":"https://arcprize.org/leaderboard","independent":true},{"date":"2026-07-24","model":"Claude Opus 5 (high)","score":30.2,"source":"https://arcprize.org/leaderboard","independent":true},{"date":"2026-09-02","model":"GPT-6 Astra (max)","score":62.7,"source":"https://arcprize.org/results/openai-gpt-6-astra","independent":true}],"status":"active","status_note":"Still separating models: the record has risen from 0.5% to 62.7% since launch in Mar 2026."},{"id":"frontiermath_t1_3","name":"FrontierMath (Tiers 1-3)","what":"Hundreds of unpublished, research-flavoured maths problems written by mathematicians, with answers a computer can check and that are practically impossible to guess.","launched":"2024-11-08","human_baseline":{"value":null,"note":"No human baseline. Problems take expert mathematicians hours to days each."},"unit":"%","caveat":"Epoch AI's own independent runs on the private set (v2, June 2026 revision, with older models re-scored). OpenAI funded FrontierMath and can see much of the problem set. OpenAI's Dec 2024 claim of ~25% for o3-preview is not in this series. Dates are model release dates; earlier models are shown at the launch date.","points":[{"date":"2024-11-08","model":"GPT-4o mini","score":0.7,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2024-12-17","model":"o1 (high)","score":14.7,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2025-01-31","model":"o3-mini (high)","score":18.6,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2025-04-16","model":"o4-mini (high)","score":36.1,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2025-08-07","model":"GPT-5 (high)","score":55.4,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2025-10-07","model":"GPT-5 Pro (high)","score":55.8,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2025-12-11","model":"GPT-5.2 Pro (xhigh)","score":74,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2026-03-05","model":"GPT-5.4 Pro (xhigh)","score":82.5,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2026-04-23","model":"GPT-5.5 Pro (xhigh)","score":87.7,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2026-07-09","model":"GPT-5.6 Sol (max)","score":89.1,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2026-09-01","model":"Claude Fable 5.1 (max)","score":90.2,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2026-09-03","model":"GPT-6 Astra (max)","score":93.7,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2026-11-09","model":"Claude Fable 5.2 (max)","score":95.1,"source":"https://epoch.ai/benchmarks","independent":true}],"status":"saturated","status_note":"Passed 90% in Sep 2026 (Claude Fable 5.1 (max)); the record is now 95.1%. Too little headroom is left for it to separate the best models."},{"id":"frontiermath_t4","name":"FrontierMath Tier 4","what":"The hardest FrontierMath tier: 50-ish problems that challenge research mathematicians.","launched":"2025-07-11","human_baseline":null,"unit":"%","caveat":"Epoch AI's independent runs (Tier 4 v2, June 2026 revision, with older models re-scored). The problem set is small (~40-50), so each record has a wide error bar (±4-6 points). Dates are model release dates; earlier models are shown at the launch date.","points":[{"date":"2025-07-11","model":"o4-mini (high)","score":4.9,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2025-08-07","model":"GPT-5 (high)","score":22,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2025-12-11","model":"GPT-5.2 Pro (xhigh)","score":46,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2026-03-05","model":"GPT-5.4 Pro (xhigh)","score":58.5,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2026-04-23","model":"GPT-5.5 Pro (xhigh)","score":78,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2026-06-09","model":"Claude Fable 5 (max)","score":90.2,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2026-09-03","model":"GPT-6 Astra (high)","score":97.6,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2026-09-29","model":"GPT-6.1 Sol (max)","score":100,"source":"https://epoch.ai/benchmarks","independent":true}],"status":"saturated","status_note":"Passed 90% in Jun 2026 (Claude Fable 5 (max)); the record is now 100%. Too little headroom is left for it to separate the best models."},{"id":"osworld","name":"OSWorld / OSWorld-Verified","what":"Can an AI use a real computer (Ubuntu desktop, browser, office apps) to finish everyday tasks by looking at the screen and clicking and typing?","launched":"2024-04-11","human_baseline":{"value":72.36,"note":"Human success rate reported in the OSWorld paper."},"unit":"%","caveat":"Two phases. Before 2025-07-28 the scores are on the original OSWorld and were mostly self-reported by agent developers, with varying step limits (15-200). From 2025-07-28 on they are OSWorld-Verified entries run through the maintainers' verified pipeline. The records are full agent systems, often with multiple models or best-of-N, not bare models. Anthropic reports Claude Sonnet 4.6 alone at 72.1% (Feb 2026) and Opus 4.8 at 83.8% (May 2026, lab-reported). The maintainers now also run OSWorld 2.x, a harder successor where the best binary accuracy was ~31% in Jul 2026 (Epoch). Labs quote OSWorld 2.1 partial-credit scores (e.g. Claude Opus 5.5 81.8%), which are not comparable.","points":[{"date":"2024-04-11","model":"GPT-4 (a11y tree, paper baseline)","score":12.24,"source":"https://arxiv.org/abs/2404.07972","independent":true},{"date":"2024-10-11","model":"Agent S w/ GPT-4o","score":20.58,"source":"https://www.simular.ai/agent-s","independent":false},{"date":"2024-10-12","model":"Learn-by-interact w/ Claude 3.5 Sonnet","score":22.5,"source":"https://arxiv.org/abs/2501.10893","independent":false},{"date":"2024-10-24","model":"OSCAR w/ GPT-4o","score":24.5,"source":"https://arxiv.org/pdf/2410.18963","independent":false},{"date":"2025-01-23","model":"OpenAI Computer-Using Agent (200 steps)","score":38.1,"source":"https://openai.com/index/computer-using-agent/","independent":false},{"date":"2025-04-17","model":"UI-TARS-1.5 (100 steps)","score":42.5,"source":"https://seed-tars.com/1.5","independent":false},{"date":"2025-05-23","model":"OpenAI CUA o3 (200 steps)","score":42.9,"source":"https://help.openai.com/en/articles/10561834-operator-release-notes","independent":false},{"date":"2025-07-07","model":"GTA1 w/ o3 (100 steps)","score":45.2,"source":"https://arxiv.org/abs/2507.05791","independent":false},{"date":"2025-07-28","model":"GTA1 w/ o3 [Verified]","score":53.1,"source":"https://os-world.github.io/","independent":true},{"date":"2025-07-31","model":"Agent S2.5 w/ o3 [Verified]","score":56,"source":"https://os-world.github.io/","independent":true},{"date":"2025-08-04","model":"CoACT-1 [Verified]","score":60.76,"source":"https://os-world.github.io/","independent":true},{"date":"2025-10-03","model":"GTA1 w/ GPT-5 [Verified]","score":63.41,"source":"https://os-world.github.io/","independent":true},{"date":"2025-10-04","model":"Agent S3 w/ GPT-5, best-of-10 [Verified]","score":69.9,"source":"https://os-world.github.io/","independent":true},{"date":"2025-12-11","model":"Agent S3 w/ Opus 4.5 + GPT-5, best-of-10 [Verified]","score":72.58,"source":"https://os-world.github.io/","independent":true},{"date":"2026-02-25","model":"HIPPO Agent w/ Claude Opus 4.5 [Verified]","score":74.48,"source":"https://os-world.github.io/","independent":true},{"date":"2026-04-17","model":"OpenAPA w/ Gemini 3.1 Pro [Verified]","score":78.34,"source":"https://os-world.github.io/","independent":true},{"date":"2026-04-20","model":"H Company Holo3-35B-A3B [Verified]","score":82.56,"source":"https://os-world.github.io/","independent":true},{"date":"2026-05-21","model":"Pointer Agent w/ Claude Opus 4.7 [Verified]","score":83.64,"source":"https://os-world.github.io/","independent":true},{"date":"2026-07-25","model":"Intelligence-Indeed Agent [Verified]","score":90.19,"source":"https://os-world.github.io/","independent":true}],"status":"superseded","status_note":"A newer version, OSWorld 2.0, arrived in Jun 2026. The record on this one is 90.19% (Intelligence-Indeed Agent [Verified], Jul 2026)."},{"id":"aime","name":"AIME (competition maths)","what":"Hard high-school olympiad-qualifier maths problems (American Invitational Mathematics Examination) with whole-number answers.","launched":"2024-12-19","human_baseline":null,"unit":"%","caveat":"This uses Epoch AI's \"OTIS Mock AIME 2024-2025\" set (fresh AIME-style problems, independent runs) rather than the official AIME papers, which leak into training data. Older models were scored retroactively, so dates are model release dates, including some before the set existed (launched = when Epoch added it). Labs' self-reported official-AIME 2025 scores reached ~100% by late 2025. Saturated.","points":[{"date":"2023-03-14","model":"GPT-4 (Mar 2023)","score":0.6,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2023-06-13","model":"GPT-4 (Jun 2023)","score":1.1,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2023-07-11","model":"Claude 2.0","score":2.5,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2024-02-29","model":"Claude 3 Opus","score":4.7,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2024-04-09","model":"GPT-4 Turbo","score":6.7,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2024-05-14","model":"Gemini 1.5 Pro","score":6.8,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2024-07-18","model":"GPT-4o mini","score":6.9,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2024-07-23","model":"Llama 3.1 405B Instruct","score":9.7,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2024-09-12","model":"o1-mini (high)","score":46.9,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2024-12-17","model":"o1 (high)","score":73.3,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2025-01-31","model":"o3-mini (high)","score":76.9,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2025-04-09","model":"Grok 3 mini (beta) (high)","score":77.8,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2025-04-16","model":"o3 (medium)","score":84.4,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2025-06-17","model":"Gemini 2.5 Pro","score":84.7,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2025-07-25","model":"Qwen3-235B-A22B Thinking (Jul 2025)","score":86.7,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2025-08-05","model":"gpt-oss-120b (high)","score":88.9,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2025-08-07","model":"GPT-5 (high)","score":91.4,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2025-12-11","model":"GPT-5.2 (high)","score":96.1,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2026-03-05","model":"GPT-5.4 (high)","score":97.8,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2026-04-23","model":"GPT-5.5 Pro, pre-release (xhigh)","score":100,"source":"https://epoch.ai/benchmarks","independent":true}],"status":"saturated","status_note":"Passed 90% in Aug 2025 (GPT-5 (high)); the record is now 100%. Too little headroom is left for it to separate the best models."},{"id":"mmlu","name":"MMLU","what":"57-subject multiple-choice exam (law, medicine, history, maths...) that was the standard \"general knowledge\" test for AI from 2020 to 2024.","launched":"2020-09-07","human_baseline":{"value":89.8,"note":"Estimated expert-level accuracy (Hendrycks et al. 2020). Random = 25%; non-expert crowd workers ~34.5%."},"unit":"%","caveat":"A mix of paper-reported and third-party (HELM) 5-shot scores compiled by Epoch AI, plus lab headline claims. Gemini Ultra's 90.0% used chain-of-thought with 32 samples, not standard 5-shot. By 2024 top models were at the benchmark's noise ceiling (a few % of questions have wrong labels), and labs stopped reporting it in favour of MMLU-Pro, GPQA and HLE. This is the saturation example.","points":[{"date":"2020-09-07","model":"GPT-3 175B (few-shot)","score":43.9,"source":"https://arxiv.org/abs/2009.03300","independent":true},{"date":"2021-12-08","model":"Gopher (280B)","score":60,"source":"https://epoch.ai/benchmarks","independent":false},{"date":"2022-03-15","model":"GPT-3.5 (text-davinci-002)","score":70,"source":"https://epoch.ai/benchmarks","independent":false},{"date":"2023-03-14","model":"GPT-4","score":86.4,"source":"https://epoch.ai/benchmarks","independent":false},{"date":"2023-12-06","model":"Gemini Ultra (CoT@32)","score":90,"source":"https://blog.google/technology/ai/google-gemini-ai/","independent":false},{"date":"2024-09-12","model":"OpenAI o1","score":92.3,"source":"https://openai.com/index/learning-to-reason-with-llms/","independent":false}],"status":"saturated","status_note":"Passed 90% in Dec 2023 (Gemini Ultra (CoT@32), reported by its own lab); the record is now 92.3%. Too little headroom is left for it to separate the best models."},{"id":"terminal_bench_2","name":"Terminal-Bench 2.0","what":"Can an AI agent get real jobs done in a computer terminal: compiling code, setting up servers, training models, fixing broken systems?","launched":"2025-11-07","human_baseline":null,"unit":"%","caveat":"Public leaderboard scores (via Epoch AI) for model + agent-harness pairs. Many were submitted by third-party agent builders rather than the maintainers, and harnesses vary widely. Dates are approximate (model release or submission). The benchmark is versioned fast: TB 2.1 (May 2026, fixed 28 tasks), 3.0 (Jul 2026) and 4.0 (Aug 28 2026). On TB 4.0, Anthropic reports Claude Opus 5.5 at 66.4% and GPT-6 Astra at 57.9% (Sep 2026, lab-reported). On the original TB 1.0 (May 2025) the record went from Claude Opus 4 (~43%) to Warp (52%, Jun 2025).","points":[{"date":"2025-11-07","model":"GPT-5 + Codex CLI","score":49.6,"source":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","independent":false},{"date":"2025-11-12","model":"GPT-5.1-Codex-mini + Hookele","score":61.6,"source":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","independent":true},{"date":"2025-11-18","model":"Gemini 3 Pro + Ante","score":69.4,"source":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","independent":true},{"date":"2026-02-05","model":"GPT-5.3-Codex + SageAgent","score":78.4,"source":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","independent":true},{"date":"2026-02-19","model":"Gemini 3.1 Pro + TongAgents","score":80.2,"source":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","independent":true},{"date":"2026-03-12","model":"GPT-5.4 + ForgeCode","score":81.8,"source":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","independent":true},{"date":"2026-04-23","model":"GPT-5.5 + NexAU-AHE","score":84.7,"source":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","independent":true}],"status":"superseded","status_note":"A newer version, Terminal-Bench 2.1 (followed by 3.0 and 4.0), arrived in May 2026. The record on this one is 84.7% (GPT-5.5 + NexAU-AHE, Apr 2026)."},{"id":"deepswe","name":"DeepSWE","what":"Large software-engineering tasks that span many files and need long, careful work.","launched":"2026-05-26","human_baseline":null,"unit":"%","caveat":"Record scores from the Epoch AI benchmarking hub, placed at each model's release date (or the test's launch, if later). Every record is an independent result.","points":[{"date":"2026-05-26","model":"GPT-5.5 (xhigh)","score":67,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2026-06-09","model":"Claude Fable 5 (xhigh)","score":69.9,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2026-07-09","model":"GPT-5.6 Sol (max)","score":72.7,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2026-07-24","model":"Claude Opus 5 (max)","score":73.6,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2026-09-02","model":"Gemini 3.8 Flash (high)","score":73.8,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2026-09-03","model":"GPT-6 Astra (xhigh)","score":74.1,"source":"https://epoch.ai/benchmarks","independent":true}],"status":"active","status_note":"Still separating models: the record has risen from 67% to 74.1% since launch in May 2026."},{"id":"math_level_5","name":"MATH (level 5)","what":"The hardest tier of a 2021 set of competition maths problems, from algebra to number theory.","launched":"2021-03-05","human_baseline":null,"unit":"%","caveat":"Record scores from the Epoch AI benchmarking hub, placed at each model's release date (or the test's launch, if later). Every record is an independent result.","points":[{"date":"2023-06-13","model":"GPT-4 (Jun 2023)","score":23,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2023-11-06","model":"GPT-4 Turbo Preview (Nov 2023)","score":40,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2024-04-09","model":"gpt-4-turbo-2024-04-09","score":46.7,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2024-05-13","model":"GPT-4o (May 2024)","score":51,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2024-06-20","model":"Claude 3.5 Sonnet (Jun 2024)","score":51.7,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2024-07-18","model":"gpt-4o-mini-2024-07-18","score":52.6,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2024-08-06","model":"GPT-4o (Aug 2024)","score":53.3,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2024-09-12","model":"o1-mini (high)","score":89.2,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2024-12-17","model":"o1 (high)","score":94.7,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2025-01-31","model":"o3-mini (high)","score":96.5,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2025-04-16","model":"o4-mini (high)","score":97.8,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2025-08-07","model":"GPT-5 (high)","score":98.1,"source":"https://epoch.ai/benchmarks","independent":true}],"status":"saturated","status_note":"Passed 90% in Dec 2024 (o1 (high)); the record is now 98.1%. Too little headroom is left for it to separate the best models."},{"id":"gsm8k","name":"GSM8K","what":"Grade-school maths word problems that need a few steps of arithmetic.","launched":"2021-10-27","human_baseline":null,"unit":"%","caveat":"Record scores from the Epoch AI benchmarking hub, placed at each model's release date (or the test's launch, if later). 3 of the 5 records were reported by the model's own lab and are marked as such.","points":[{"date":"2021-10-27","model":"babbage (1.3B)","score":0.7,"source":"https://crfm.stanford.edu/helm/classic/latest/#/groups/gsm","independent":true},{"date":"2022-03-15","model":"code-davinci-002","score":56.8,"source":"https://crfm.stanford.edu/helm/classic/latest/#/groups/gsm","independent":true},{"date":"2022-11-28","model":"GPT-3.5 (8-shot)","score":78.2,"source":"https://arxiv.org/abs/2309.16609","independent":false},{"date":"2023-03-14","model":"GPT-4 (Mar 2023)","score":92,"source":"http://arxiv.org/abs/2303.08774","independent":false},{"date":"2024-06-17","model":"DS-Coder-V2-Instruct","score":94.5,"source":"http://arxiv.org/abs/2409.12186","independent":false}],"status":"saturated","status_note":"Passed 90% in Mar 2023 (GPT-4 (Mar 2023), reported by its own lab); the record is now 94.5%. Too little headroom is left for it to separate the best models."},{"id":"aider_polyglot","name":"Aider polyglot","what":"Can a model edit code correctly across six programming languages, on 225 hard Exercism exercises?","launched":"2024-12-21","human_baseline":null,"unit":"%","caveat":"Record scores from the Epoch AI benchmarking hub, placed at each model's release date (or the test's launch, if later). Every record is an independent result.","points":[{"date":"2024-12-21","model":"o1 (high)","score":61.7,"source":"https://aider.chat/docs/leaderboards/#polyglot-leaderboard","independent":true},{"date":"2025-02-24","model":"Claude 3.7 Sonnet (32k thinking)","score":64.9,"source":"https://aider.chat/docs/leaderboards/#polyglot-leaderboard","independent":true},{"date":"2025-03-25","model":"Gemini 2.5 Pro Exp (Mar 2025)","score":72.9,"source":"https://aider.chat/docs/leaderboards/#polyglot-leaderboard","independent":true},{"date":"2025-04-16","model":"o3 (high)","score":81.3,"source":"https://aider.chat/docs/leaderboards/#polyglot-leaderboard","independent":true},{"date":"2025-06-05","model":"Gemini 2.5 Pro Preview (Jun 2025)","score":83.1,"source":"https://aider.chat/docs/leaderboards/#polyglot-leaderboard","independent":true},{"date":"2025-06-10","model":"o3-pro-2025-06-10_high","score":84.9,"source":"https://aider.chat/docs/leaderboards/#polyglot-leaderboard","independent":true},{"date":"2025-08-07","model":"GPT-5 (high)","score":88,"source":"https://aider.chat/docs/leaderboards/#polyglot-leaderboard","independent":true}],"status":"retired","status_note":"No new record since Aug 2025 (GPT-5 (high), 88%). Newer models are rarely reported on it."},{"id":"simpleqa_verified","name":"SimpleQA Verified","what":"Short factual questions with one checkable answer, answered from memory with no search: a test of knowing versus making things up.","launched":"2025-09-09","human_baseline":null,"unit":"%","caveat":"Record scores from the Epoch AI benchmarking hub, placed at each model's release date (or the test's launch, if later). Every record is an independent result.","points":[{"date":"2025-09-09","model":"GPT-5 (high)","score":50.1,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2025-12-17","model":"Gemini 3 Flash Preview","score":66.8,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2026-02-19","model":"Gemini 3.1 Pro Preview","score":73.5,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2026-09-03","model":"GPT-6 Astra (max)","score":75.6,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2026-10-13","model":"Gemini 4 Argon (high)","score":76,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2026-11-09","model":"Claude Fable 5.2 (max)","score":94,"source":"https://epoch.ai/benchmarks","independent":true}],"status":"saturated","status_note":"Passed 90% in Nov 2026 (Claude Fable 5.2 (max)); the record is now 94%. Too little headroom is left for it to separate the best models."},{"id":"simplebench","name":"SimpleBench","what":"Trick questions about everyday space, time and social situations that most people get right and language models get wrong.","launched":"2024-10-31","human_baseline":{"value":83.7,"note":"Average of nine non-specialist people, reported by the SimpleBench authors."},"unit":"%","caveat":"Record scores from the Epoch AI benchmarking hub, placed at each model's release date (or the test's launch, if later). Every record is an independent result.","points":[{"date":"2024-10-31","model":"o1-preview","score":41.7,"source":"https://simple-bench.com/","independent":true},{"date":"2025-02-24","model":"Claude 3.7 Sonnet (12k thinking)","score":46.4,"source":"https://simple-bench.com/","independent":true},{"date":"2025-03-25","model":"Gemini 2.5 Pro Exp (Mar 2025)","score":51.6,"source":"https://simple-bench.com/","independent":true},{"date":"2025-04-16","model":"o3 (high)","score":53.1,"source":"https://simple-bench.com/","independent":true},{"date":"2025-05-22","model":"Claude Opus 4 ","score":58.8,"source":"https://simple-bench.com/","independent":true},{"date":"2025-06-05","model":"Gemini 2.5 Pro Preview (Jun 2025)","score":62.4,"source":"https://simple-bench.com/","independent":true},{"date":"2025-11-18","model":"Gemini 3 Pro Preview","score":76.4,"source":"https://simple-bench.com/","independent":true},{"date":"2026-02-19","model":"Gemini 3.1 Pro Preview","score":79.6,"source":"https://simple-bench.com/","independent":true},{"date":"2026-06-09","model":"Claude Fable 5 (max)","score":81.9,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2026-09-01","model":"Claude Fable 5.1 (unknown)","score":86.6,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2026-09-22","model":"Claude Opus 5.5 (unknown)","score":88.4,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2026-10-13","model":"Gemini 4 Argon (unknown)","score":91.7,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2026-11-09","model":"Claude Fable 5.2 (unknown)","score":94.9,"source":"https://epoch.ai/benchmarks","independent":true}],"status":"saturated","status_note":"Passed 90% in Oct 2026 (Gemini 4 Argon (unknown)); the record is now 94.9%. Too little headroom is left for it to separate the best models."},{"id":"chess_puzzles","name":"Chess puzzles","what":"Find the best move in chess puzzles, a clean test of looking ahead.","launched":"2025-12-10","human_baseline":null,"unit":"%","caveat":"Record scores from the Epoch AI benchmarking hub, placed at each model's release date (or the test's launch, if later). Every record is an independent result.","points":[{"date":"2025-12-10","model":"GPT-5 Pro","score":48,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2025-12-11","model":"GPT-5.2 Pro","score":60,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2026-04-23","model":"GPT-5.5 (xhigh)","score":63,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2026-07-09","model":"GPT-5.6 Sol (pro, max)","score":68,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2026-09-03","model":"GPT-6 Astra (pro, max)","score":76,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2026-11-09","model":"Claude Fable 5.2 (max)","score":79,"source":"https://epoch.ai/benchmarks","independent":true}],"status":"active","status_note":"Still separating models: the record has risen from 48% to 79% since launch in Dec 2025."},{"id":"weirdml","name":"WeirdML","what":"Write working machine-learning code for unusual tasks that standard recipes do not solve.","launched":"2025-01-16","human_baseline":null,"unit":"%","caveat":"Record scores from the Epoch AI benchmarking hub, placed at each model's release date (or the test's launch, if later). Every record is an independent result.","points":[{"date":"2025-01-16","model":"o1-preview","score":47.6,"source":"https://htihle.github.io/weirdml.html","independent":true},{"date":"2025-04-16","model":"o4-mini (high)","score":52.6,"source":"https://htihle.github.io/weirdml.html","independent":true},{"date":"2025-06-10","model":"o3-pro-2025-06-10_high","score":58.2,"source":"https://htihle.github.io/weirdml.html","independent":true},{"date":"2025-08-07","model":"GPT-5 (high)","score":60.7,"source":"https://htihle.github.io/weirdml.html","independent":true},{"date":"2025-11-13","model":"GPT-5.1 (high)","score":60.8,"source":"https://htihle.github.io/weirdml.html","independent":true},{"date":"2025-11-18","model":"Gemini 3 Pro Preview","score":69.9,"source":"https://htihle.github.io/weirdml.html","independent":true},{"date":"2025-12-11","model":"GPT-5.2 (xhigh)","score":72.2,"source":"https://htihle.github.io/weirdml.html","independent":true},{"date":"2026-02-05","model":"GPT-5.3 Codex","score":79.3,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2026-04-23","model":"GPT-5.5 (xhigh)","score":84.9,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2026-06-09","model":"Claude Fable 5 (max)","score":91.9,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2026-09-01","model":"Claude Fable 5.1 (max)","score":92.9,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2026-09-03","model":"GPT-6 Astra (pro, max)","score":93.6,"source":"https://epoch.ai/benchmarks","independent":true}],"status":"superseded","status_note":"A newer version, WeirdML v3, arrived in Sep 2026. The record on this one is 93.6% (GPT-6 Astra (pro, max), Sep 2026)."},{"id":"apex_agents","name":"APEX-Agents","what":"Long, realistic professional tasks from investment banking, consulting and law, done by an agent with files and tools.","launched":"2026-01-21","human_baseline":null,"unit":"%","caveat":"Record scores from the Epoch AI benchmarking hub, placed at each model's release date (or the test's launch, if later). Every record is an independent result.","points":[{"date":"2026-01-21","model":"DeepSeek-V3.2 (Thinking; Novita)","score":21.3,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2026-02-05","model":"Claude Opus 4.6 (max)","score":46.3,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2026-03-05","model":"GPT-5.4 (unknown thinking)","score":52.4,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2026-04-23","model":"GPT-5.5 (unknown thinking)","score":55.1,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2026-06-09","model":"Claude Fable 5","score":63.6,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2026-07-24","model":"Claude Opus 5 (max)","score":65.8,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2026-08-13","model":"Gemini 3.7 Flash (unknown)","score":67.8,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2026-09-01","model":"Claude Fable 5.1 (unknown)","score":68.6,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2026-09-21","model":"Grok 4.7 (unknown)","score":70,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2026-09-22","model":"Claude Opus 5.5 (max)","score":73.5,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2026-09-28","model":"Claude Sonnet 5.5 (max)","score":75.5,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2026-10-13","model":"Gemini 4 Argon (unknown)","score":82.4,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2026-11-10","model":"Gemini 4 Carbon (unknown)","score":84.2,"source":"https://epoch.ai/benchmarks","independent":true}],"status":"active","status_note":"Still separating models: the record has risen from 21.3% to 84.2% since launch in Jan 2026."},{"id":"balrog","name":"BALROG","what":"Play text and grid games, from simple navigation to NetHack, measuring how far an agent progresses.","launched":"2024-11-20","human_baseline":null,"unit":"%","caveat":"Record scores from the Epoch AI benchmarking hub, placed at each model's release date (or the test's launch, if later). Every record is an independent result.","points":[{"date":"2024-11-20","model":"Claude 3.5 Sonnet (Oct 2024)","score":32.6,"source":"https://balrogai.com/","independent":true},{"date":"2025-01-20","model":"DeepSeek-R1","score":34.9,"source":"https://balrogai.com/","independent":true},{"date":"2025-03-25","model":"Gemini 2.5 Pro Exp (Mar 2025)","score":43.3,"source":"https://balrogai.com/","independent":true},{"date":"2025-07-09","model":"grok-4-0709","score":43.6,"source":"https://balrogai.com/","independent":true},{"date":"2025-11-18","model":"Gemini 3 Pro Preview","score":58.1,"source":"https://balrogai.com/","independent":true},{"date":"2026-07-09","model":"GPT-5.6 Sol (max)","score":60,"source":"https://balrogai.com/","independent":true},{"date":"2026-07-24","model":"Claude Opus 5 (max)","score":63.4,"source":"https://balrogai.com/","independent":true},{"date":"2026-09-03","model":"GPT-6 Astra (max)","score":68.3,"source":"https://balrogai.com/","independent":true}],"status":"active","status_note":"Still separating models: the record rose 10 points in the last 12 months to 68.3%."},{"id":"agent_company","name":"TheAgentCompany","what":"Act as an employee of a simulated software company: browse, code, and message colleagues to finish real work tasks.","launched":"2024-12-18","human_baseline":null,"unit":"%","caveat":"Record scores from the Epoch AI benchmarking hub, placed at each model's release date (or the test's launch, if later). Every record is an independent result.","points":[{"date":"2024-12-18","model":"Claude 3.5 Sonnet (Oct 2024)","score":24,"source":"https://github.com/TheAgentCompany/experiments/tree/main/evaluation/1.0.0/20241217_OpenHands-0.14.2-sonnet-20241022","independent":true},{"date":"2025-02-24","model":"OpenHands-Versa + Claude 3.7 Sonnet","score":30.9,"source":"https://the-agent-company.com/#/leaderboard","independent":true},{"date":"2025-05-22","model":"OpenHands-Versa + Claude Sonnet 4","score":33.1,"source":"https://the-agent-company.com/#/leaderboard","independent":true},{"date":"2025-09-25","model":"MUSE + Gemini 2.5 Flash","score":41.1,"source":"https://the-agent-company.com/#/leaderboard","independent":true},{"date":"2025-09-29","model":"TTE-MatrixAgent + Deepseek-V3.2","score":42.9,"source":"https://the-agent-company.com/#/leaderboard","independent":true}],"status":"retired","status_note":"No new record since Sep 2025 (TTE-MatrixAgent + Deepseek-V3.2, 42.9%). Newer models are rarely reported on it."},{"id":"cybench","name":"Cybench","what":"Professional capture-the-flag hacking challenges, solved without hints.","launched":"2024-08-15","human_baseline":null,"unit":"%","caveat":"Record scores from the Epoch AI benchmarking hub, placed at each model's release date (or the test's launch, if later). 4 of the 7 records were reported by the model's own lab and are marked as such.","points":[{"date":"2024-08-15","model":"Claude 3.5 Sonnet (Jun 2024)","score":17.5,"source":"https://cybench.github.io/","independent":true},{"date":"2025-01-31","model":"o3-mini (medium)","score":22.5,"source":"https://cybench.github.io/","independent":true},{"date":"2025-05-22","model":"Claude Opus 4","score":38,"source":"https://cybench.github.io/","independent":true},{"date":"2025-07-09","model":"grok-4-0709","score":43,"source":"https://data.x.ai/2025-08-20-grok-4-model-card.pdf","independent":false},{"date":"2025-09-29","model":"Claude Sonnet 4.5 (unknown thinking)","score":60,"source":"https://assets.anthropic.com/m/64823ba7485345a7/Claude-Opus-4-5-System-Card.pdf","independent":false},{"date":"2025-11-24","model":"Claude Opus 4.5 (unknown thinking)","score":82,"source":"https://assets.anthropic.com/m/64823ba7485345a7/Claude-Opus-4-5-System-Card.pdf","independent":false},{"date":"2026-02-05","model":"Claude Opus 4.6 (unknown thinking)","score":93,"source":"https://www-cdn.anthropic.com/0dd865075ad3132672ee0ab40b05a53f14cf5288.pdf","independent":false}],"status":"saturated","status_note":"Passed 90% in Feb 2026 (Claude Opus 4.6 (unknown thinking), reported by its own lab); the record is now 93%. Too little headroom is left for it to separate the best models."},{"id":"gdpval","name":"GDPval","what":"Real work products from 44 occupations, judged blind against an industry professional's version. The score is how often the model wins or ties.","launched":"2025-09-25","human_baseline":null,"unit":"%","caveat":"Record scores from the Epoch AI benchmarking hub, placed at each model's release date (or the test's launch, if later). Every record is an independent result.","points":[{"date":"2025-09-25","model":"Claude Opus 4.1","score":43.6,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2025-11-24","model":"Claude Opus 4.5 (no thinking)","score":45.5,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2025-12-11","model":"GPT-5.2 (none)","score":49.7,"source":"https://epoch.ai/benchmarks","independent":true}],"status":"active","status_note":"Still separating models: the record rose 6 points in the last 12 months to 49.7%."},{"id":"osworld_2","name":"OSWorld 2.0","what":"Operate a real desktop computer through the screen, mouse and keyboard to finish everyday tasks.","launched":"2026-06-26","human_baseline":null,"unit":"%","caveat":"Record scores from the Epoch AI benchmarking hub, placed at each model's release date (or the test's launch, if later). Every record is an independent result.","points":[{"date":"2026-06-26","model":"Claude Opus 4.8","score":20.6,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2026-07-09","model":"GPT-5.6 Sol (max)","score":27.3,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2026-07-24","model":"Claude Opus 5 (max)","score":31.4,"source":"https://epoch.ai/benchmarks","independent":true}],"status":"active","status_note":"Still separating models: the record has risen from 20.6% to 31.4% since launch in Jun 2026."},{"id":"frontiercode","name":"FrontierCode","what":"Hard, original programming problems written to be beyond today's coding benchmarks.","launched":"2026-06-08","human_baseline":null,"unit":"%","caveat":"Record scores from the Epoch AI benchmarking hub, placed at each model's release date (or the test's launch, if later). Every record is an independent result.","points":[{"date":"2026-06-08","model":"Claude Opus 4.8","score":46.5,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2026-06-09","model":"Claude Fable 5 (unknown)","score":53.5,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2026-09-22","model":"Claude Opus 5.5 (medium)","score":54.6,"source":"https://epoch.ai/benchmarks","independent":true}],"status":"active","status_note":"Still separating models: the record has risen from 46.5% to 54.6% since launch in Jun 2026."},{"id":"proofbench","name":"ProofBench","what":"Write complete mathematical proofs that are checked for correctness.","launched":"2026-01-30","human_baseline":null,"unit":"%","caveat":"Record scores from the Epoch AI benchmarking hub, placed at each model's release date (or the test's launch, if later). Every record is an independent result.","points":[{"date":"2026-01-30","model":"Claude Opus 4.5 (unknown thinking)","score":36,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2026-02-05","model":"Claude Opus 4.6 (max)","score":50,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2026-03-05","model":"GPT-5.4 (xhigh)","score":56,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2026-05-28","model":"Claude Opus 4.8","score":69,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2026-06-09","model":"Claude Fable 5 (max)","score":95,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2026-07-24","model":"Claude Opus 5 (max)","score":99,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2026-09-01","model":"Claude Fable 5.1 (max)","score":100,"source":"https://epoch.ai/benchmarks","independent":true}],"status":"saturated","status_note":"Passed 90% in Jun 2026 (Claude Fable 5 (max)); the record is now 100%. Too little headroom is left for it to separate the best models."},{"id":"rli","name":"Remote Labor Index","what":"Complete real freelance projects end to end, to a standard a paying client would accept.","launched":"2025-10-29","human_baseline":null,"unit":"%","caveat":"Record scores from the Epoch AI benchmarking hub, placed at each model's release date (or the test's launch, if later). Every record is an independent result.","points":[{"date":"2025-10-29","model":"Claude Sonnet 4.5 (unknown thinking)","score":2.1,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2025-11-24","model":"Claude Opus 4.5 (unknown thinking)","score":3.8,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2026-02-05","model":"Claude Opus 4.6 (unknown thinking)","score":4.2,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2026-04-23","model":"GPT-5.5 (unknown thinking)","score":6.2,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2026-05-28","model":"Claude Opus 4.8","score":8.3,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2026-06-09","model":"Claude Fable 5","score":16.1,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2026-09-01","model":"Claude Fable 5.1 (unknown)","score":17.9,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2026-09-03","model":"GPT-6 Astra (unknown thinking)","score":20.8,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2026-09-22","model":"Claude Opus 5.5 (unknown)","score":21.2,"source":"https://epoch.ai/benchmarks","independent":true},{"date":"2026-11-09","model":"Claude Fable 5.2 (unknown)","score":41.2,"source":"https://epoch.ai/benchmarks","independent":true}],"status":"active","status_note":"Still separating models: the record rose 39 points in the last 12 months to 41.2%."}]