{
 "chart": "verifiable_percentiles/percentiles",
 "title": "Current public frontier models outperform most humans (but not always the very top experts) on tasks with a large pool of participants",
 "subtitle": "Left: each point is a model's best run on one of 15 past AZsPCs contests, ranked among that contest's human participants. Agents had 48 hours on a 28-core machine, and participants had about three months. Right: each point is one agent run's speedup over record #78, ranked among the speedups of human records #79 to #92. Agents had 48 hours on 8xH100s, and contributors reported 16 to 40 hours per record.",
 "panels": [
  {
   "title": "AZsPCs: 15 past optimization contests",
   "x": {"label": "", "unit": "", "scale": "linear"},
   "y": {"label": "Human performance percentile", "unit": "", "scale": "linear"},
   "tables": [
    {
     "name": "best human",
     "rows": [
      {"reference": "Best human participant", "percentile_among_humans": 100}
     ]
    },
    {
     "name": "contest results",
     "rows": [
      {"model": "GPT-5", "contest": "azspcs/alphabet_city", "percentile_among_humans": 44.2308},
      {"model": "GPT-5", "contest": "azspcs/ap_math", "percentile_among_humans": 48.0},
      {"model": "GPT-5", "contest": "azspcs/baltos_puzzle", "percentile_among_humans": 19.5266},
      {"model": "GPT-5", "contest": "azspcs/non_coplanar_points", "percentile_among_humans": 53.5836},
      {"model": "GPT-5", "contest": "azspcs/point_packing", "percentile_among_humans": 64.4518},
      {"model": "GPT-5", "contest": "azspcs/polygonal_areas", "percentile_among_humans": 60.7903},
      {"model": "GPT-5", "contest": "azspcs/powerful_sums", "percentile_among_humans": 32.618},
      {"model": "GPT-5", "contest": "azspcs/primorial_soup", "percentile_among_humans": 16.3987},
      {"model": "GPT-5", "contest": "azspcs/retaining_water", "percentile_among_humans": 61.9718},
      {"model": "GPT-5", "contest": "azspcs/snakes_on_a_plane", "percentile_among_humans": 48.4536},
      {"model": "GPT-5", "contest": "azspcs/son_of_darts", "percentile_among_humans": 57.8692},
      {"model": "GPT-5", "contest": "azspcs/stepping_stones", "percentile_among_humans": 27.1889},
      {"model": "GPT-5", "contest": "azspcs/sums_of_powers", "percentile_among_humans": 26.935},
      {"model": "GPT-5", "contest": "azspcs/topswops", "percentile_among_humans": 47.3538},
      {"model": "GPT-5", "contest": "azspcs/words_search", "percentile_among_humans": 59.2105},
      {"model": "Claude Opus 4.5", "contest": "azspcs/alphabet_city", "percentile_among_humans": 62.5},
      {"model": "Claude Opus 4.5", "contest": "azspcs/ap_math", "percentile_among_humans": 69.7778},
      {"model": "Claude Opus 4.5", "contest": "azspcs/baltos_puzzle", "percentile_among_humans": 17.1598},
      {"model": "Claude Opus 4.5", "contest": "azspcs/non_coplanar_points", "percentile_among_humans": 67.5768},
      {"model": "Claude Opus 4.5", "contest": "azspcs/point_packing", "percentile_among_humans": 58.4718},
      {"model": "Claude Opus 4.5", "contest": "azspcs/polygonal_areas", "percentile_among_humans": 72.3404},
      {"model": "Claude Opus 4.5", "contest": "azspcs/powerful_sums", "percentile_among_humans": 42.4893},
      {"model": "Claude Opus 4.5", "contest": "azspcs/primorial_soup", "percentile_among_humans": 54.0193},
      {"model": "Claude Opus 4.5", "contest": "azspcs/retaining_water", "percentile_among_humans": 64.0845},
      {"model": "Claude Opus 4.5", "contest": "azspcs/snakes_on_a_plane", "percentile_among_humans": 61.8557},
      {"model": "Claude Opus 4.5", "contest": "azspcs/son_of_darts", "percentile_among_humans": 76.0291},
      {"model": "Claude Opus 4.5", "contest": "azspcs/stepping_stones", "percentile_among_humans": 29.4931},
      {"model": "Claude Opus 4.5", "contest": "azspcs/sums_of_powers", "percentile_among_humans": 26.935},
      {"model": "Claude Opus 4.5", "contest": "azspcs/topswops", "percentile_among_humans": 66.0167},
      {"model": "Claude Opus 4.5", "contest": "azspcs/words_search", "percentile_among_humans": 59.2105},
      {"model": "GPT-5.6 Sol", "contest": "azspcs/alphabet_city", "percentile_among_humans": 90.3846},
      {"model": "GPT-5.6 Sol", "contest": "azspcs/ap_math", "percentile_among_humans": 92.0},
      {"model": "GPT-5.6 Sol", "contest": "azspcs/baltos_puzzle", "percentile_among_humans": 78.1065},
      {"model": "GPT-5.6 Sol", "contest": "azspcs/non_coplanar_points", "percentile_among_humans": 97.2696},
      {"model": "GPT-5.6 Sol", "contest": "azspcs/point_packing", "percentile_among_humans": 96.6777},
      {"model": "GPT-5.6 Sol", "contest": "azspcs/polygonal_areas", "percentile_among_humans": 89.0578},
      {"model": "GPT-5.6 Sol", "contest": "azspcs/primorial_soup", "percentile_among_humans": 84.8875},
      {"model": "GPT-5.6 Sol", "contest": "azspcs/retaining_water", "percentile_among_humans": 88.7324},
      {"model": "GPT-5.6 Sol", "contest": "azspcs/snakes_on_a_plane", "percentile_among_humans": 84.5361},
      {"model": "GPT-5.6 Sol", "contest": "azspcs/son_of_darts", "percentile_among_humans": 93.9467},
      {"model": "GPT-5.6 Sol", "contest": "azspcs/stepping_stones", "percentile_among_humans": 88.9401},
      {"model": "GPT-5.6 Sol", "contest": "azspcs/sums_of_powers", "percentile_among_humans": 75.5418},
      {"model": "GPT-5.6 Sol", "contest": "azspcs/topswops", "percentile_among_humans": 77.4373},
      {"model": "GPT-5.6 Sol", "contest": "azspcs/words_search", "percentile_among_humans": 94.0789},
      {"model": "Claude Fable 5.1", "contest": "azspcs/alphabet_city", "percentile_among_humans": 94.2308},
      {"model": "Claude Fable 5.1", "contest": "azspcs/ap_math", "percentile_among_humans": 96.4444},
      {"model": "Claude Fable 5.1", "contest": "azspcs/baltos_puzzle", "percentile_among_humans": 95.2663},
      {"model": "Claude Fable 5.1", "contest": "azspcs/non_coplanar_points", "percentile_among_humans": 98.6348},
      {"model": "Claude Fable 5.1", "contest": "azspcs/point_packing", "percentile_among_humans": 95.0166},
      {"model": "Claude Fable 5.1", "contest": "azspcs/polygonal_areas", "percentile_among_humans": 94.2249},
      {"model": "Claude Fable 5.1", "contest": "azspcs/powerful_sums", "percentile_among_humans": 95.7082},
      {"model": "Claude Fable 5.1", "contest": "azspcs/primorial_soup", "percentile_among_humans": 88.1029},
      {"model": "Claude Fable 5.1", "contest": "azspcs/retaining_water", "percentile_among_humans": 100.0},
      {"model": "Claude Fable 5.1", "contest": "azspcs/snakes_on_a_plane", "percentile_among_humans": 97.9381},
      {"model": "Claude Fable 5.1", "contest": "azspcs/son_of_darts", "percentile_among_humans": 97.8208},
      {"model": "Claude Fable 5.1", "contest": "azspcs/stepping_stones", "percentile_among_humans": 94.47},
      {"model": "Claude Fable 5.1", "contest": "azspcs/sums_of_powers", "percentile_among_humans": 97.5232},
      {"model": "Claude Fable 5.1", "contest": "azspcs/topswops", "percentile_among_humans": 98.0501},
      {"model": "Claude Fable 5.1", "contest": "azspcs/words_search", "percentile_among_humans": 98.6842},
      {"model": "GPT-6 Astra", "contest": "azspcs/alphabet_city", "percentile_among_humans": 100.0},
      {"model": "GPT-6 Astra", "contest": "azspcs/ap_math", "percentile_among_humans": 100.0},
      {"model": "GPT-6 Astra", "contest": "azspcs/baltos_puzzle", "percentile_among_humans": 99.4083},
      {"model": "GPT-6 Astra", "contest": "azspcs/non_coplanar_points", "percentile_among_humans": 97.2696},
      {"model": "GPT-6 Astra", "contest": "azspcs/point_packing", "percentile_among_humans": 96.6777},
      {"model": "GPT-6 Astra", "contest": "azspcs/polygonal_areas", "percentile_among_humans": 98.4802},
      {"model": "GPT-6 Astra", "contest": "azspcs/powerful_sums", "percentile_among_humans": 94.8498},
      {"model": "GPT-6 Astra", "contest": "azspcs/primorial_soup", "percentile_among_humans": 89.3891},
      {"model": "GPT-6 Astra", "contest": "azspcs/retaining_water", "percentile_among_humans": 100.0},
      {"model": "GPT-6 Astra", "contest": "azspcs/snakes_on_a_plane", "percentile_among_humans": 100.0},
      {"model": "GPT-6 Astra", "contest": "azspcs/son_of_darts", "percentile_among_humans": 98.3051},
      {"model": "GPT-6 Astra", "contest": "azspcs/stepping_stones", "percentile_among_humans": 97.6959},
      {"model": "GPT-6 Astra", "contest": "azspcs/sums_of_powers", "percentile_among_humans": 98.452},
      {"model": "GPT-6 Astra", "contest": "azspcs/topswops", "percentile_among_humans": 97.493},
      {"model": "GPT-6 Astra", "contest": "azspcs/words_search", "percentile_among_humans": 98.6842},
      {"model": "Claude Opus 5.5", "contest": "azspcs/alphabet_city", "percentile_among_humans": 90.3846},
      {"model": "Claude Opus 5.5", "contest": "azspcs/ap_math", "percentile_among_humans": 95.5556},
      {"model": "Claude Opus 5.5", "contest": "azspcs/baltos_puzzle", "percentile_among_humans": 95.2663},
      {"model": "Claude Opus 5.5", "contest": "azspcs/non_coplanar_points", "percentile_among_humans": 98.9761},
      {"model": "Claude Opus 5.5", "contest": "azspcs/point_packing", "percentile_among_humans": 96.6777},
      {"model": "Claude Opus 5.5", "contest": "azspcs/polygonal_areas", "percentile_among_humans": 95.1368},
      {"model": "Claude Opus 5.5", "contest": "azspcs/powerful_sums", "percentile_among_humans": 97.4249},
      {"model": "Claude Opus 5.5", "contest": "azspcs/primorial_soup", "percentile_among_humans": 77.1704},
      {"model": "Claude Opus 5.5", "contest": "azspcs/retaining_water", "percentile_among_humans": 98.5915},
      {"model": "Claude Opus 5.5", "contest": "azspcs/snakes_on_a_plane", "percentile_among_humans": 100.0},
      {"model": "Claude Opus 5.5", "contest": "azspcs/son_of_darts", "percentile_among_humans": 98.063},
      {"model": "Claude Opus 5.5", "contest": "azspcs/stepping_stones", "percentile_among_humans": 98.1567},
      {"model": "Claude Opus 5.5", "contest": "azspcs/sums_of_powers", "percentile_among_humans": 97.8328},
      {"model": "Claude Opus 5.5", "contest": "azspcs/topswops", "percentile_among_humans": 97.493},
      {"model": "Claude Opus 5.5", "contest": "azspcs/words_search", "percentile_among_humans": 99.3421}
     ]
    },
    {
     "name": "model medians",
     "rows": [
      {"model": "GPT-5", "median_percentile_among_humans": 48.0},
      {"model": "Claude Opus 4.5", "median_percentile_among_humans": 61.8557},
      {"model": "GPT-5.6 Sol", "median_percentile_among_humans": 88.9989},
      {"model": "Claude Fable 5.1", "median_percentile_among_humans": 96.4444},
      {"model": "GPT-6 Astra", "median_percentile_among_humans": 98.452},
      {"model": "Claude Opus 5.5", "median_percentile_among_humans": 97.493}
     ]
    }
   ]
  },
  {
   "title": "Budget NanoGPT: human speedrun records #79 to #92",
   "x": {"label": "", "unit": "", "scale": "linear"},
   "y": {"label": "", "unit": "", "scale": "linear"},
   "tables": [
    {
     "name": "best human",
     "rows": [
      {"reference": "Best human record (#92, AI-assisted)", "percentile_among_humans": 100}
     ]
    },
    {
     "name": "agent runs",
     "rows": [
      {"model": "GPT-5", "speedup": 0.0, "percentile_among_humans": 0.0},
      {"model": "GPT-5", "speedup": 0.0, "percentile_among_humans": 0.0},
      {"model": "GPT-5", "speedup": 0.0, "percentile_among_humans": 0.0},
      {"model": "GPT-5", "speedup": 0.0, "percentile_among_humans": 0.0},
      {"model": "Claude Opus 4.5", "speedup": 0.0176211, "percentile_among_humans": 71.4286},
      {"model": "Claude Opus 4.5", "speedup": 0.0113515, "percentile_among_humans": 64.2857},
      {"model": "Claude Opus 4.5", "speedup": 0.0174458, "percentile_among_humans": 64.2857},
      {"model": "Claude Opus 4.5", "speedup": 0.00208261, "percentile_among_humans": 0.0},
      {"model": "GPT-5.6 Sol", "speedup": 0.0177572, "percentile_among_humans": 71.4286},
      {"model": "GPT-5.6 Sol", "speedup": 0.0110662, "percentile_among_humans": 64.2857},
      {"model": "GPT-5.6 Sol", "speedup": 0.00951078, "percentile_among_humans": 57.1429},
      {"model": "GPT-5.6 Sol", "speedup": 0.00747017, "percentile_among_humans": 42.8571},
      {"model": "Claude Fable 5.1", "speedup": 0.0347628, "percentile_among_humans": 85.7143},
      {"model": "Claude Fable 5.1", "speedup": 0.114952, "percentile_among_humans": 92.8571},
      {"model": "Claude Fable 5.1", "speedup": 0.0976075, "percentile_among_humans": 92.8571},
      {"model": "GPT-6 Astra", "speedup": 0.0961968, "percentile_among_humans": 92.8571},
      {"model": "GPT-6 Astra", "speedup": 0.157215, "percentile_among_humans": 92.8571},
      {"model": "Claude Opus 5.5", "speedup": 0.0846933, "percentile_among_humans": 92.8571},
      {"model": "Claude Opus 5.5", "speedup": 0.045241, "percentile_among_humans": 85.7143},
      {"model": "Claude Opus 5.5", "speedup": 0.116358, "percentile_among_humans": 92.8571}
     ]
    },
    {
     "name": "model medians",
     "rows": [
      {"model": "GPT-5", "median_percentile_among_humans": 0.0},
      {"model": "Claude Opus 4.5", "median_percentile_among_humans": 64.2857},
      {"model": "GPT-5.6 Sol", "median_percentile_among_humans": 60.7143},
      {"model": "Claude Fable 5.1", "median_percentile_among_humans": 92.8571},
      {"model": "GPT-6 Astra", "median_percentile_among_humans": 92.8571},
      {"model": "Claude Opus 5.5", "median_percentile_among_humans": 92.8571}
     ]
    }
   ]
  }
 ]
}
