{
  "title": "Accelerate Bench: Evaluating Agents for Application-Level Program Optimization",
  "snapshot": "ICLR final manuscript, 26 September 2026",
  "metric_note": "Table 1 reports 384 selected agent-task outcomes: 12 agents and 32 tasks. Eight configurations use the median-reward attempt among the latest three eligible scored attempts; GPT-6 Astra, Gemini-3.8-Flash, Kimi K3 (max), and GLM-5.3 (max) use their sole attempt. All metrics for an agent-task pair use the same selected record. Scores and target rates are percentages. Per-task values retain published rounding; null indicates failed correctness, not missing data. Do not recompute Table 1 from rounded per-task values.",
  "models": [
    {
      "name": "GPT-6 Astra",
      "effort": "high",
      "score": 40.3,
      "gm": 2.11,
      "a125": 68.8,
      "a2": 43.8,
      "steps": null,
      "tokens": null
    },
    {
      "name": "Hy4-Preview",
      "effort": "high",
      "score": 36.5,
      "gm": 2.13,
      "a125": 56.2,
      "a2": 37.5,
      "steps": 538.0,
      "tokens": 49.48
    },
    {
      "name": "DeepSeek-V4.1-Flash (0910)",
      "effort": "high",
      "score": 29.9,
      "gm": 1.78,
      "a125": 50.0,
      "a2": 31.2,
      "steps": 676.4,
      "tokens": 63.35,
      "model_id": "deepseek-v4.1-flash-expires-on-0910"
    },
    {
      "name": "Gemini-3.8-Flash",
      "effort": "high",
      "score": 29.9,
      "gm": 1.71,
      "a125": 50.0,
      "a2": 25.0,
      "steps": null,
      "tokens": null
    },
    {
      "name": "GLM-5.3",
      "effort": "max",
      "score": 28.1,
      "gm": 1.77,
      "a125": 40.6,
      "a2": 21.9,
      "steps": 967.1,
      "tokens": 92.89
    },
    {
      "name": "GLM-5.3-Flash",
      "effort": "high",
      "score": 27.3,
      "gm": 1.65,
      "a125": 43.8,
      "a2": 21.9,
      "steps": 773.8,
      "tokens": 74.98
    },
    {
      "name": "Grok 4.6",
      "effort": "high",
      "score": 26.2,
      "gm": 1.66,
      "a125": 43.8,
      "a2": 21.9,
      "steps": null,
      "tokens": null
    },
    {
      "name": "DeepSeek-V4-Pro",
      "effort": "high",
      "score": 20.6,
      "gm": 1.46,
      "a125": 43.8,
      "a2": 12.5,
      "steps": 471.7,
      "tokens": 46.17
    },
    {
      "name": "GLM-5.2",
      "effort": "high",
      "score": 19.9,
      "gm": 1.46,
      "a125": 34.4,
      "a2": 15.6,
      "steps": 529.8,
      "tokens": 45.61
    },
    {
      "name": "DeepSeek-V4-Flash",
      "effort": "high",
      "score": 19.1,
      "gm": 1.44,
      "a125": 34.4,
      "a2": 12.5,
      "steps": 404.0,
      "tokens": 38.7
    },
    {
      "name": "Kimi K3",
      "effort": "max",
      "score": 19.0,
      "gm": 1.34,
      "a125": 34.4,
      "a2": 15.6,
      "steps": 433.0,
      "tokens": 37.37
    },
    {
      "name": "Hy3",
      "effort": "high",
      "score": 9.8,
      "gm": 1.25,
      "a125": 12.5,
      "a2": 6.2,
      "steps": 254.3,
      "tokens": 20.92
    }
  ],
  "tasks": [
    {
      "id": "gromacs_cpu",
      "name": "GROMACS",
      "area": "Molecular simulation & electronic structure",
      "area_id": 0,
      "diagram": "assets/tasks/gromacs_cpu.svg",
      "version": "12.0.3",
      "cpus": 4,
      "memory_gib": 6,
      "agent_hours": 8.0,
      "verify_minutes": 390.0,
      "rewards_percent": [
        65.0,
        63.0,
        0.0,
        58.0,
        52.0,
        53.0,
        53.0,
        52.0,
        53.0,
        53.0,
        53.0,
        0.0
      ]
    },
    {
      "id": "lammps_cpu",
      "name": "LAMMPS",
      "area": "Molecular simulation & electronic structure",
      "area_id": 0,
      "diagram": "assets/tasks/lammps_cpu.svg",
      "version": "4.1.3",
      "cpus": 4,
      "memory_gib": 8,
      "agent_hours": 8.0,
      "verify_minutes": 30.0,
      "rewards_percent": [
        54.0,
        47.0,
        56.0,
        1.0,
        3.0,
        7.0,
        11.0,
        0.0,
        1.0,
        0.0,
        7.0,
        3.0
      ]
    },
    {
      "id": "gpaw_cpu",
      "name": "GPAW",
      "area": "Molecular simulation & electronic structure",
      "area_id": 0,
      "diagram": "assets/tasks/gpaw_cpu.svg",
      "version": "1.0.2",
      "cpus": 4,
      "memory_gib": 6,
      "agent_hours": 8.0,
      "verify_minutes": 233.3,
      "rewards_percent": [
        31.0,
        5.0,
        4.0,
        23.0,
        null,
        null,
        null,
        12.0,
        null,
        12.0,
        null,
        0.0
      ]
    },
    {
      "id": "qe_cpu",
      "name": "Quantum ESPRESSO",
      "area": "Molecular simulation & electronic structure",
      "area_id": 0,
      "diagram": "assets/tasks/qe_cpu.svg",
      "version": "1.2.2",
      "cpus": 4,
      "memory_gib": 6,
      "agent_hours": 8.0,
      "verify_minutes": 233.3,
      "rewards_percent": [
        9.0,
        null,
        0.0,
        4.0,
        12.0,
        4.0,
        null,
        1.0,
        1.0,
        1.0,
        0.0,
        null
      ]
    },
    {
      "id": "siesta_cpu",
      "name": "SIESTA",
      "area": "Molecular simulation & electronic structure",
      "area_id": 0,
      "diagram": "assets/tasks/siesta_cpu.svg",
      "version": "1.2.2",
      "cpus": 4,
      "memory_gib": 6,
      "agent_hours": 8.0,
      "verify_minutes": 233.3,
      "rewards_percent": [
        61.0,
        55.0,
        null,
        39.0,
        2.0,
        49.8,
        null,
        null,
        null,
        51.0,
        49.7,
        null
      ]
    },
    {
      "id": "fluidity_cpu",
      "name": "Fluidity",
      "area": "Fluids, atmosphere & ocean",
      "area_id": 1,
      "diagram": "assets/tasks/fluidity_cpu.svg",
      "version": "1.0.2",
      "cpus": 4,
      "memory_gib": 8,
      "agent_hours": 8.0,
      "verify_minutes": 233.3,
      "rewards_percent": [
        0.0,
        0.0,
        null,
        null,
        null,
        0.0,
        null,
        null,
        0.0,
        null,
        0.0,
        null
      ]
    },
    {
      "id": "icon_cpu",
      "name": "ICON",
      "area": "Fluids, atmosphere & ocean",
      "area_id": 1,
      "diagram": "assets/tasks/icon_cpu.svg",
      "version": "1.0.2",
      "cpus": 4,
      "memory_gib": 6,
      "agent_hours": 8.0,
      "verify_minutes": 200.0,
      "rewards_percent": [
        0.0,
        2.0,
        null,
        0.0,
        0.0,
        2.0,
        null,
        null,
        0.0,
        0.0,
        0.0,
        0.0
      ]
    },
    {
      "id": "mpas_a_cpu",
      "name": "MPAS-A",
      "area": "Fluids, atmosphere & ocean",
      "area_id": 1,
      "diagram": "assets/tasks/mpas_a_cpu.svg",
      "version": "1.0.2",
      "cpus": 4,
      "memory_gib": 6,
      "agent_hours": 8.0,
      "verify_minutes": 233.3,
      "rewards_percent": [
        23.0,
        25.0,
        4.0,
        24.0,
        19.0,
        14.0,
        1.0,
        null,
        0.0,
        2.0,
        24.0,
        null
      ]
    },
    {
      "id": "nemo_cpu",
      "name": "NEMO",
      "area": "Fluids, atmosphere & ocean",
      "area_id": 1,
      "diagram": "assets/tasks/nemo_cpu.svg",
      "version": "1.0.2",
      "cpus": 4,
      "memory_gib": 8,
      "agent_hours": 8.0,
      "verify_minutes": 233.3,
      "rewards_percent": [
        0.0,
        3.0,
        1.0,
        null,
        null,
        3.0,
        2.0,
        null,
        3.0,
        1.0,
        1.0,
        3.0
      ]
    },
    {
      "id": "openfoam_cpu",
      "name": "OpenFOAM",
      "area": "Fluids, atmosphere & ocean",
      "area_id": 1,
      "diagram": "assets/tasks/openfoam_cpu.svg",
      "version": "1.0.3",
      "cpus": 4,
      "memory_gib": 8,
      "agent_hours": 8.0,
      "verify_minutes": 40.0,
      "rewards_percent": [
        23.0,
        12.0,
        7.0,
        12.0,
        0.0,
        2.0,
        2.0,
        25.0,
        0.0,
        13.0,
        null,
        0.0
      ]
    },
    {
      "id": "palabos_cpu",
      "name": "Palabos",
      "area": "Fluids, atmosphere & ocean",
      "area_id": 1,
      "diagram": "assets/tasks/palabos_cpu.svg",
      "version": "2.0.2",
      "cpus": 4,
      "memory_gib": 6,
      "agent_hours": 8.0,
      "verify_minutes": 120.0,
      "rewards_percent": [
        88.0,
        91.0,
        0.0,
        62.0,
        null,
        16.0,
        72.0,
        23.0,
        null,
        1.0,
        11.0,
        0.0
      ]
    },
    {
      "id": "su2_cpu",
      "name": "SU2",
      "area": "Fluids, atmosphere & ocean",
      "area_id": 1,
      "diagram": "assets/tasks/su2_cpu.svg",
      "version": "2.0.2",
      "cpus": 4,
      "memory_gib": 8,
      "agent_hours": 8.0,
      "verify_minutes": 60.0,
      "rewards_percent": [
        56.0,
        64.0,
        61.0,
        45.0,
        44.0,
        29.0,
        25.0,
        54.0,
        32.0,
        13.0,
        31.0,
        8.0
      ]
    },
    {
      "id": "openmc_cpu",
      "name": "OpenMC",
      "area": "Transport, plasma & astronomy",
      "area_id": 2,
      "diagram": "assets/tasks/openmc_cpu.svg",
      "version": "3.0.4",
      "cpus": 4,
      "memory_gib": 6,
      "agent_hours": 8.0,
      "verify_minutes": 120.0,
      "rewards_percent": [
        47.0,
        30.0,
        21.0,
        26.0,
        37.0,
        3.0,
        4.0,
        3.0,
        9.0,
        17.0,
        15.0,
        3.0
      ]
    },
    {
      "id": "shengbte_cpu",
      "name": "ShengBTE",
      "area": "Transport, plasma & astronomy",
      "area_id": 2,
      "diagram": "assets/tasks/shengbte_cpu.svg",
      "version": "1.0.2",
      "cpus": 4,
      "memory_gib": 6,
      "agent_hours": 8.0,
      "verify_minutes": 233.3,
      "rewards_percent": [
        54.0,
        null,
        55.0,
        null,
        43.0,
        38.0,
        37.0,
        36.0,
        36.0,
        35.0,
        15.0,
        null
      ]
    },
    {
      "id": "vpic_cpu",
      "name": "VPIC",
      "area": "Transport, plasma & astronomy",
      "area_id": 2,
      "diagram": "assets/tasks/vpic_cpu.svg",
      "version": "1.2.2",
      "cpus": 4,
      "memory_gib": 6,
      "agent_hours": 8.0,
      "verify_minutes": 150.0,
      "rewards_percent": [
        0.0,
        0.0,
        null,
        5.0,
        null,
        3.0,
        2.0,
        0.0,
        null,
        null,
        null,
        0.0
      ]
    },
    {
      "id": "presto_cpu",
      "name": "PRESTO",
      "area": "Transport, plasma & astronomy",
      "area_id": 2,
      "diagram": "assets/tasks/presto_cpu.svg",
      "version": "2.0.2",
      "cpus": 4,
      "memory_gib": 6,
      "agent_hours": 8.0,
      "verify_minutes": 120.0,
      "rewards_percent": [
        27.0,
        0.0,
        0.0,
        1.0,
        null,
        null,
        0.0,
        0.0,
        null,
        2.0,
        null,
        null
      ]
    },
    {
      "id": "abyss_cpu",
      "name": "ABySS",
      "area": "Bioinformatics & phylogenetics",
      "area_id": 3,
      "diagram": "assets/tasks/abyss_cpu.svg",
      "version": "1.0.3",
      "cpus": 4,
      "memory_gib": 8,
      "agent_hours": 8.0,
      "verify_minutes": 30.0,
      "rewards_percent": [
        null,
        42.0,
        49.0,
        null,
        49.9,
        45.0,
        43.0,
        36.0,
        10.0,
        17.0,
        37.0,
        9.0
      ]
    },
    {
      "id": "mhm2_cpu",
      "name": "MHM2",
      "area": "Bioinformatics & phylogenetics",
      "area_id": 3,
      "diagram": "assets/tasks/mhm2_cpu.svg",
      "version": "1.0.2",
      "cpus": 4,
      "memory_gib": 6,
      "agent_hours": 8.0,
      "verify_minutes": 150.0,
      "rewards_percent": [
        37.0,
        2.0,
        null,
        33.0,
        null,
        27.0,
        2.0,
        0.0,
        null,
        0.0,
        0.0,
        3.0
      ]
    },
    {
      "id": "mrbayes_cpu",
      "name": "MrBayes",
      "area": "Bioinformatics & phylogenetics",
      "area_id": 3,
      "diagram": "assets/tasks/mrbayes_cpu.svg",
      "version": "2.0.2",
      "cpus": 4,
      "memory_gib": 6,
      "agent_hours": 8.0,
      "verify_minutes": 120.0,
      "rewards_percent": [
        29.0,
        32.0,
        23.0,
        12.0,
        11.0,
        19.0,
        9.0,
        14.0,
        12.0,
        7.0,
        17.0,
        8.0
      ]
    },
    {
      "id": "wtdbg2_cpu",
      "name": "wtdbg2",
      "area": "Bioinformatics & phylogenetics",
      "area_id": 3,
      "diagram": "assets/tasks/wtdbg2_cpu.svg",
      "version": "2.0.3",
      "cpus": 4,
      "memory_gib": 8,
      "agent_hours": 8.0,
      "verify_minutes": 30.0,
      "rewards_percent": [
        58.0,
        58.0,
        39.0,
        16.0,
        13.0,
        0.0,
        34.0,
        22.0,
        0.0,
        24.0,
        0.0,
        0.0
      ]
    },
    {
      "id": "hpcg_cpu",
      "name": "HPCG-derived",
      "area": "Numerics, tensors & graphs",
      "area_id": 4,
      "diagram": "assets/tasks/hpcg_cpu.svg",
      "version": "12.0.3",
      "cpus": 4,
      "memory_gib": 6,
      "agent_hours": 8.0,
      "verify_minutes": 120.0,
      "rewards_percent": [
        97.0,
        99.0,
        98.0,
        92.0,
        99.0,
        97.0,
        99.0,
        97.0,
        98.0,
        98.0,
        null,
        97.0
      ]
    },
    {
      "id": "hpl_cpu",
      "name": "HPL-derived",
      "area": "Numerics, tensors & graphs",
      "area_id": 4,
      "diagram": "assets/tasks/hpl_cpu.svg",
      "version": "11.0.2",
      "cpus": 4,
      "memory_gib": 6,
      "agent_hours": 8.0,
      "verify_minutes": 120.0,
      "rewards_percent": [
        45.0,
        34.0,
        32.0,
        34.0,
        47.0,
        34.0,
        68.0,
        32.0,
        48.0,
        22.0,
        36.0,
        7.0
      ]
    },
    {
      "id": "ipcc22_sparsify_cpu",
      "name": "Graph sparsification",
      "area": "Numerics, tensors & graphs",
      "area_id": 4,
      "diagram": "assets/tasks/ipcc22_sparsify_cpu.svg",
      "version": "3.0.2",
      "cpus": 4,
      "memory_gib": 4,
      "agent_hours": 8.0,
      "verify_minutes": 120.0,
      "rewards_percent": [
        63.0,
        79.0,
        79.0,
        16.0,
        79.0,
        78.0,
        58.0,
        32.0,
        61.0,
        34.0,
        32.0,
        14.0
      ]
    },
    {
      "id": "minivite_cpu",
      "name": "miniVite",
      "area": "Numerics, tensors & graphs",
      "area_id": 4,
      "diagram": "assets/tasks/minivite_cpu.svg",
      "version": "11.0.2",
      "cpus": 4,
      "memory_gib": 6,
      "agent_hours": 8.0,
      "verify_minutes": 120.0,
      "rewards_percent": [
        10.0,
        18.0,
        19.0,
        8.0,
        14.0,
        12.0,
        19.95,
        4.0,
        7.0,
        3.0,
        6.0,
        2.0
      ]
    },
    {
      "id": "pac2020_nufft_cpu",
      "name": "NUFFT imaging",
      "area": "Numerics, tensors & graphs",
      "area_id": 4,
      "diagram": "assets/tasks/pac2020_nufft_cpu.svg",
      "version": "1.0.2",
      "cpus": 4,
      "memory_gib": 6,
      "agent_hours": 8.0,
      "verify_minutes": 200.0,
      "rewards_percent": [
        80.0,
        52.0,
        75.0,
        76.0,
        null,
        44.0,
        47.0,
        38.0,
        32.0,
        33.0,
        54.0,
        33.0
      ]
    },
    {
      "id": "pac_tensor_cpu",
      "name": "Tucker reconstruction",
      "area": "Numerics, tensors & graphs",
      "area_id": 4,
      "diagram": "assets/tasks/pac_tensor_cpu.svg",
      "version": "3.1.6",
      "cpus": 8,
      "memory_gib": 16,
      "agent_hours": 8.0,
      "verify_minutes": 120.0,
      "rewards_percent": [
        68.0,
        82.0,
        63.0,
        74.0,
        70.0,
        58.0,
        41.0,
        36.0,
        33.0,
        30.0,
        null,
        31.0
      ]
    },
    {
      "id": "quest_cpu",
      "name": "QuEST",
      "area": "Quantum circuits",
      "area_id": 5,
      "diagram": "assets/tasks/quest_cpu.svg",
      "version": "3.0.5",
      "cpus": 4,
      "memory_gib": 8,
      "agent_hours": 8.0,
      "verify_minutes": 60.0,
      "rewards_percent": [
        12.0,
        10.0,
        11.0,
        9.0,
        12.0,
        0.0,
        16.0,
        0.0,
        7.0,
        9.0,
        0.0,
        2.0
      ]
    },
    {
      "id": "asc26_qibotn_cpu",
      "name": "QiboTN",
      "area": "Quantum circuits",
      "area_id": 5,
      "diagram": "assets/tasks/asc26_qibotn_cpu.svg",
      "version": "3.2.7",
      "cpus": 8,
      "memory_gib": 32,
      "agent_hours": 8.0,
      "verify_minutes": 833.3,
      "rewards_percent": [
        null,
        0.0,
        29.0,
        null,
        31.0,
        null,
        34.0,
        2.0,
        0.0,
        1.0,
        0.0,
        0.0
      ]
    },
    {
      "id": "pac_mqa_cpu",
      "name": "Paged LM generation",
      "area": "Machine learning",
      "area_id": 6,
      "diagram": "assets/tasks/pac_mqa_cpu.svg",
      "version": "1.0.2",
      "cpus": 4,
      "memory_gib": 6,
      "agent_hours": 8.0,
      "verify_minutes": 200.0,
      "rewards_percent": [
        94.0,
        94.0,
        94.0,
        93.0,
        94.0,
        91.0,
        93.0,
        93.0,
        91.0,
        90.0,
        85.0,
        87.0
      ]
    },
    {
      "id": "pac_thundersvm_cpu",
      "name": "ThunderSVM",
      "area": "Machine learning",
      "area_id": 6,
      "diagram": "assets/tasks/pac_thundersvm_cpu.svg",
      "version": "4.3.4",
      "cpus": 4,
      "memory_gib": 8,
      "agent_hours": 8.0,
      "verify_minutes": 60.0,
      "rewards_percent": [
        77.0,
        85.0,
        53.0,
        70.0,
        83.0,
        65.0,
        8.0,
        48.0,
        43.0,
        40.0,
        52.0,
        null
      ]
    },
    {
      "id": "datalife_cpu",
      "name": "DataLife",
      "area": "Scientific I/O",
      "area_id": 7,
      "diagram": "assets/tasks/datalife_cpu.svg",
      "version": "1.0.2",
      "cpus": 4,
      "memory_gib": 6,
      "agent_hours": 8.0,
      "verify_minutes": 233.3,
      "rewards_percent": [
        78.0,
        84.0,
        83.0,
        87.0,
        83.0,
        79.0,
        59.0,
        null,
        62.0,
        1.0,
        82.0,
        1.0
      ]
    },
    {
      "id": "io500_cpu",
      "name": "IO500-derived",
      "area": "Scientific I/O",
      "area_id": 7,
      "diagram": "assets/tasks/io500_cpu.svg",
      "version": "12.0.2",
      "cpus": 4,
      "memory_gib": 6,
      "agent_hours": 8.0,
      "verify_minutes": 120.0,
      "rewards_percent": [
        4.0,
        2.0,
        3.0,
        37.0,
        0.0,
        1.0,
        1.0,
        0.0,
        null,
        null,
        0.0,
        1.0
      ]
    }
  ],
  "areas": [
    "Molecular simulation & electronic structure",
    "Fluids, atmosphere & ocean",
    "Transport, plasma & astronomy",
    "Bioinformatics & phylogenetics",
    "Numerics, tensors & graphs",
    "Quantum circuits",
    "Machine learning",
    "Scientific I/O"
  ],
  "authors": [
    {
      "name": "Bowen Liu",
      "affiliations": [
        1
      ],
      "role": "First author",
      "email": "bliu690@connect.hkust-gz.edu.cn"
    },
    {
      "name": "Yuhan Li",
      "affiliations": [
        1
      ],
      "role": "Project Lead"
    },
    {
      "name": "Nuo Chen",
      "affiliations": [
        1
      ],
      "role": "Corresponding author"
    },
    {
      "name": "Weile Luo",
      "affiliations": [
        1
      ],
      "role": ""
    },
    {
      "name": "Miao Peng",
      "affiliations": [
        1,
        3
      ],
      "role": ""
    },
    {
      "name": "Qintong Zhang",
      "affiliations": [
        1,
        5
      ],
      "role": ""
    },
    {
      "name": "Qifan Zhang",
      "affiliations": [
        1,
        3
      ],
      "role": ""
    },
    {
      "name": "Jia Li",
      "affiliations": [
        2,
        3
      ],
      "role": ""
    },
    {
      "name": "Ziqi Gao",
      "affiliations": [
        4
      ],
      "role": ""
    },
    {
      "name": "Lydell Huang",
      "affiliations": [
        1
      ],
      "role": ""
    }
  ],
  "institutions": [
    "Tencent Hy Team",
    "The Hong Kong University of Science and Technology",
    "The Hong Kong University of Science and Technology (Guangzhou)",
    "Tsinghua University",
    "Peking University"
  ],
  "sources": {
    "leaderboard": "Paper Table 1",
    "tasks": "Paper Appendix B, resource allocation table",
    "per_task": "Paper Appendix F, per-task reward table",
    "analysis": "Separate 480-run analysis cohort; paper Sections 5 and Appendices G–L"
  },
  "selection": {
    "snapshot_id": "median-main-table-20260921-12configs",
    "aggregation": "median_reward_attempt",
    "ties": "Sort by reward, original completion time, and trial ID; select the middle record for triplets.",
    "three_attempt_models": [
      "DeepSeek-V4-Flash",
      "DeepSeek-V4-Pro",
      "DeepSeek-V4.1-Flash (0910)",
      "GLM-5.2",
      "Grok 4.6",
      "Hy3",
      "Hy4-Preview",
      "GLM-5.3-Flash"
    ],
    "single_attempt_models": [
      "GPT-6 Astra",
      "Gemini-3.8-Flash",
      "Kimi K3",
      "GLM-5.3"
    ],
    "manifest_sha256": "6d1be766c4becc8ea6aba81accee32d3467eb39609e8a53d457945eeae3f5576"
  }
}