benchmark_name: METR-Horizon-v1.1
doubling_time_in_days:
  all_time_stitched:
    point_estimate: 196.475
  from_2023_on:
    ci_high: 161.422
    ci_low: 107.334
    point_estimate: 130.801
long_tasks_version: 799cc9c4b4483a93fc3445623a49ea1bd74fdeb2
results:
  claude_3_5_sonnet_20240620_inspect:
    benchmark_name: METR-Horizon-v1.1
    metrics:
      average_score:
        estimate: 0.401502
      is_sota: true
      p50_horizon_length:
        ci_high: 20.808242
        ci_low: 5.025202
        estimate: 10.771603
      p80_horizon_length:
        ci_high: 3.616028
        ci_low: 0.439092
        estimate: 1.319663
      usage:
        usd: 0.0
        working_time: 20649.812666666672
    release_date: 2024-06-20
    scaffolds:
    - mtb/start_metr_task,metr_agents/react
    - ai_rd_fix_embedding/init_solver,use_tools,metr_agents/react
    - re_bench_common/init_solver,use_tools,metr_agents/react
    - metr_tasks_swaa/init_solver,generate
    - metr_agents/react
  claude_3_5_sonnet_20241022_inspect:
    benchmark_name: METR-Horizon-v1.1
    metrics:
      average_score:
        estimate: 0.452356
      is_sota: true
      p50_horizon_length:
        ci_high: 39.898341
        ci_low: 9.683132
        estimate: 19.776241
      p80_horizon_length:
        ci_high: 5.555657
        ci_low: 0.736051
        estimate: 2.110812
      usage:
        usd: 0.0
        working_time: 18941.195769444443
    release_date: 2024-10-22
    scaffolds:
    - mtb/start_metr_task,metr_agents/react
    - ai_rd_fix_embedding/init_solver,use_tools,metr_agents/react
    - re_bench_common/init_solver,use_tools,metr_agents/react
    - metr_tasks_swaa/init_solver,generate
    - metr_agents/react
  claude_3_7_sonnet_inspect:
    benchmark_name: METR-Horizon-v1.1
    metrics:
      average_score:
        estimate: 0.558217
      is_sota: true
      p50_horizon_length:
        ci_high: 105.940417
        ci_low: 31.750635
        estimate: 59.763762
      p80_horizon_length:
        ci_high: 21.817527
        ci_low: 3.672799
        estimate: 9.792108
      usage:
        usd: 0.0
        working_time: 16779.599820833333
    release_date: 2025-02-24
    scaffolds:
    - mtb/start_metr_task,metr_agents/react
    - ai_rd_fix_embedding/init_solver,use_tools,metr_agents/react
    - re_bench_common/init_solver,use_tools,metr_agents/react
    - metr_tasks_swaa/init_solver,generate
    - metr_agents/react
  claude_3_opus_inspect:
    benchmark_name: METR-Horizon-v1.1
    metrics:
      average_score:
        estimate: 0.294753
      is_sota: false
      p50_horizon_length:
        ci_high: 7.522352
        ci_low: 1.45372
        estimate: 3.575399
      p80_horizon_length:
        ci_high: 1.368064
        ci_low: 0.139029
        estimate: 0.476819
      usage:
        usd: 0.0
        working_time: 16045.181673611114
    release_date: 2024-03-04
    scaffolds:
    - mtb/start_metr_task,metr_agents/react
    - ai_rd_fix_embedding/init_solver,use_tools,metr_agents/react
    - re_bench_common/init_solver,use_tools,metr_agents/react
    - metr_tasks_swaa/init_solver,generate
    - metr_agents/react
  claude_4_opus_inspect:
    benchmark_name: METR-Horizon-v1.1
    metrics:
      average_score:
        estimate: 0.615452
      is_sota: false
      p50_horizon_length:
        ci_high: 169.789759
        ci_low: 58.072313
        estimate: 101.182643
      p80_horizon_length:
        ci_high: 33.556683
        ci_low: 7.159176
        estimate: 16.642056
      usage:
        usd: 0.0
        working_time: 14761.633798214285
    release_date: 2025-05-22
    scaffolds:
    - mtb/start_metr_task,metr_agents/react
    - ai_rd_fix_embedding/init_solver,use_tools,metr_agents/react
    - re_bench_common/init_solver,use_tools,metr_agents/react
    - metr_tasks_swaa/init_solver,generate
    - metr_agents/react
  claude_opus_4_5_inspect:
    benchmark_name: METR-Horizon-v1.1
    metrics:
      average_score:
        estimate: 0.73007
      is_sota: true
      p50_horizon_length:
        ci_high: 729.170929
        ci_low: 169.810681
        estimate: 320.422486
      p80_horizon_length:
        ci_high: 86.019974
        ci_low: 17.721124
        estimate: 41.551151
      usage:
        usd: 0.0
        working_time: 19769.594443055554
    release_date: 2025-11-24
    scaffolds:
    - mtb/start_metr_task,metr_agents/react
    - ai_rd_fix_embedding/init_solver,use_tools,metr_agents/react
    - re_bench_common/init_solver,use_tools,metr_agents/react
    - metr_tasks_swaa/init_solver,generate
    - metr_agents/react
  davinci_002:
    benchmark_name: METR-Horizon-v1.0
    metrics:
      average_score:
        estimate: 0.161869
      is_sota: true
      p50_horizon_length:
        ci_high: 0.246039
        ci_low: 0.07102
        estimate: 0.148793
      p80_horizon_length:
        ci_high: 0.062221
        ci_low: 0.013154
        estimate: 0.034139
      usage:
        usd: 66.11363827325752
        working_time: 8847.569692901172
    release_date: 2020-05-28
    scaffolds:
    - modular-public
    - null
  gpt2:
    benchmark_name: METR-Horizon-v1.0
    metrics:
      average_score:
        estimate: 0.101046
      is_sota: true
      p50_horizon_length:
        ci_high: 0.130397
        ci_low: 0.002019
        estimate: 0.039762
      p80_horizon_length:
        ci_high: 0.026215
        ci_low: 0.000151
        estimate: 0.005652
      usage:
        usd: 0.0001664748
        working_time: 0.0
    release_date: 2019-02-14
    scaffolds:
    - null
  gpt_3_5_turbo_instruct:
    benchmark_name: METR-Horizon-v1.0
    metrics:
      average_score:
        estimate: 0.214611
      is_sota: true
      p50_horizon_length:
        ci_high: 0.989884
        ci_low: 0.227348
        estimate: 0.604245
      p80_horizon_length:
        ci_high: 0.285643
        ci_low: 0.051881
        estimate: 0.1731
      usage:
        usd: 24.984426026785716
        working_time: 1381.0521758730158
    release_date: 2022-03-15
    scaffolds:
    - modular-public
    - null
  gpt_4:
    benchmark_name: METR-Horizon-v1.1
    metrics:
      average_score:
        estimate: 0.293044
      is_sota: true
      p50_horizon_length:
        ci_high: 6.944557
        ci_low: 1.63379
        estimate: 3.524514
      p80_horizon_length:
        ci_high: 1.528809
        ci_low: 0.21275
        estimate: 0.621658
      usage:
        usd: 4540.660434
        working_time: 2526.1250172222226
    release_date: 2023-03-14
    scaffolds:
    - modular-public
    - null
  gpt_4_1106_inspect:
    benchmark_name: METR-Horizon-v1.1
    metrics:
      average_score:
        estimate: 0.28905
      is_sota: true
      p50_horizon_length:
        ci_high: 7.487253
        ci_low: 1.601501
        estimate: 3.609578
      p80_horizon_length:
        ci_high: 1.502632
        ci_low: 0.181397
        estimate: 0.563384
      usage:
        usd: 0.0
        working_time: 32900.427695833336
    release_date: 2023-11-06
    scaffolds:
    - mtb/start_metr_task,metr_agents/react
    - ai_rd_fix_embedding/init_solver,use_tools,metr_agents/react
    - re_bench_common/init_solver,use_tools,metr_agents/react
    - metr_tasks_swaa/init_solver,generate
    - metr_agents/react
  gpt_4_turbo_inspect:
    benchmark_name: METR-Horizon-v1.1
    metrics:
      average_score:
        estimate: 0.271786
      is_sota: false
      p50_horizon_length:
        ci_high: 5.790891
        ci_low: 1.568921
        estimate: 3.225159
      p80_horizon_length:
        ci_high: 1.29209
        ci_low: 0.26029
        estimate: 0.617128
      usage:
        usd: 0.0
        working_time: 31032.684725000006
    release_date: 2024-04-09
    scaffolds:
    - mtb/start_metr_task,metr_agents/react
    - ai_rd_fix_embedding/init_solver,use_tools,metr_agents/react
    - re_bench_common/init_solver,use_tools,metr_agents/react
    - metr_tasks_swaa/init_solver,generate
    - metr_agents/react
  gpt_4o_inspect:
    benchmark_name: METR-Horizon-v1.1
    metrics:
      average_score:
        estimate: 0.338424
      is_sota: true
      p50_horizon_length:
        ci_high: 11.707363
        ci_low: 3.419852
        estimate: 6.404471
      p80_horizon_length:
        ci_high: 2.020488
        ci_low: 0.406565
        estimate: 0.947734
      usage:
        usd: 0.0
        working_time: 33212.51013333333
    release_date: 2024-05-13
    scaffolds:
    - mtb/start_metr_task,metr_agents/react
    - ai_rd_fix_embedding/init_solver,use_tools,metr_agents/react
    - re_bench_common/init_solver,use_tools,metr_agents/react
    - metr_tasks_swaa/init_solver,generate
    - metr_agents/react
  gpt_5_2025_08_07_inspect:
    benchmark_name: METR-Horizon-v1.1
    metrics:
      average_score:
        estimate: 0.693697
      is_sota: true
      p50_horizon_length:
        ci_high: 479.845586
        ci_low: 117.147709
        estimate: 213.954459
      p80_horizon_length:
        ci_high: 55.759355
        ci_low: 15.79307
        estimate: 31.643807
      usage:
        usd: 0.0
        working_time: 312206.48572429287
    release_date: 2025-08-07
    scaffolds:
    - mtb/start_metr_task,triframe_inspect/triframe_agent
    - ai_rd_fix_embedding/init_solver,use_tools,triframe_inspect/triframe_agent
    - re_bench_common/init_solver,use_tools,triframe_inspect/triframe_agent
    - metr_tasks_swaa/init_solver,generate
    - triframe_inspect/triframe_agent
  o1_inspect:
    benchmark_name: METR-Horizon-v1.1
    metrics:
      average_score:
        estimate: 0.510512
      is_sota: true
      p50_horizon_length:
        ci_high: 63.81793
        ci_low: 20.107361
        estimate: 37.937543
      p80_horizon_length:
        ci_high: 13.091987
        ci_low: 2.333383
        estimate: 5.720742
      usage:
        usd: 0.0
        working_time: 277917.33848472225
    release_date: 2024-12-05
    scaffolds:
    - mtb/start_metr_task,triframe_inspect/triframe_agent
    - ai_rd_fix_embedding/init_solver,use_tools,triframe_inspect/triframe_agent
    - re_bench_common/init_solver,use_tools,triframe_inspect/triframe_agent
    - metr_tasks_swaa/init_solver,generate
    - triframe_inspect/triframe_agent
  o1_preview:
    benchmark_name: METR-Horizon-v1.1
    metrics:
      average_score:
        estimate: 0.451441
      is_sota: true
      p50_horizon_length:
        ci_high: 32.305177
        ci_low: 11.143625
        estimate: 19.395096
      p80_horizon_length:
        ci_high: 6.815057
        ci_low: 1.560069
        estimate: 3.407526
      usage:
        usd: 4885.850659821428
        working_time: 5355.558304067461
    release_date: 2024-09-12
    scaffolds:
    - duet
    - null
  o3_inspect:
    benchmark_name: METR-Horizon-v1.1
    metrics:
      average_score:
        estimate: 0.636235
      is_sota: true
      p50_horizon_length:
        ci_high: 200.838465
        ci_low: 74.2793
        estimate: 120.730463
      p80_horizon_length:
        ci_high: 41.512986
        ci_low: 11.702685
        estimate: 23.909039
      usage:
        usd: 0.0
        working_time: 326482.30646666663
    release_date: 2025-04-16
    scaffolds:
    - mtb/start_metr_task,triframe_inspect/triframe_agent
    - ai_rd_fix_embedding/init_solver,use_tools,triframe_inspect/triframe_agent
    - re_bench_common/init_solver,use_tools,triframe_inspect/triframe_agent
    - metr_tasks_swaa/init_solver,generate
    - triframe_inspect/triframe_agent
swaa_version: f6cc84052e2a79dd540766c8a1b15ff399696371
