| CoTMode | 1.5B | 2.9B | 7.2B | 13.3B | |||||||||||
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| A | B | Δ | A | B | Δ | A | B | Δ | A | B | Δ | ||||
| agieval | 5151 | avg@1 | NoCoT | +0.2% | -0.4% | +0.7% | -0.5% | ||||||||
| aime24 | 30 | avg@128 | CoT | — | — | — | — | — | — | — | — | ||||
| aime25 | 30 | avg@128 | CoT | — | — | — | — | — | — | — | — | ||||
| aimo_progress_prize_1 | 10 | avg@512 | CoT | — | — | — | — | — | — | — | — | ||||
| arc:challenge | 1172 | avg@1 | NoCoT | -0.7% | -1.0% | +0.7% | -0.3% | ||||||||
| arc:easy | 2376 | avg@1 | NoCoT | -0.8% | -0.6% | +0.1% | -0.4% | ||||||||
| arithmetic | 20000 | avg@1 | FakeCoT | -0.9% | +1.6% | +0.5% | 0.0% | ||||||||
| asdiv | 2305 | avg@1 | FakeCoT | +1.2% | +0.6% | +0.8% | +0.3% | ||||||||
| bfcl_v4 | 3641 | avg@1 | FakeCoT | — | — | — | — | — | — | — | — | — | |||
| bigbench_hard | 13054 | avg@1 | NoCoT | — | — | — | — | — | — | — | — | — | |||
| BigCodeBench | 1140 | avg@4 | FakeCoT | — | — | — | — | — | — | — | — | ||||
| ceval_zho_mcf | 1343 | avg@1 | NoCoT | +0.6% | -0.2% | +0.5% | -0.6% | ||||||||
| commonsenseqa | 1221 | avg@1 | NoCoT | -0.2% | -0.5% | +0.7% | 0.0% | ||||||||
| gaia | 165 | avg@1 | FakeCoT | — | — | — | — | — | — | — | — | ||||
| gpqa:diamond | 198 | avg@1 | CoT | +2.0% | -1.6% | +0.4% | +2.1% | ||||||||
| gpqa:main | 448 | avg@1 | CoT | +0.7% | -0.7% | +0.9% | -1.6% | ||||||||
| gsm8k | 1319 | avg@1 | CoT | +1.1% | +1.8% | +0.6% | -2.3% | ||||||||
| gsm_plus | 9233 | avg@1 | CoT | -0.3% | +0.4% | +0.4% | -1.7% | ||||||||
| hellaswag | 10042 | avg@1 | NoCoT | -1.9% | +0.1% | +0.3% | -0.2% | ||||||||
| HumanEval | 164 | avg@16 | FakeCoT | -2.1% | -1.8% | +0.1% | +0.3% | ||||||||
| HumanEval | 164 | avg@16 | CoT | -10.4% | -6.1% | -10.8% | -8.8% | ||||||||
| HumanEval+ | 164 | avg@16 | FakeCoT | -1.3% | -1.4% | +0.8% | +1.9% | ||||||||
| HumanEval+ | 164 | avg@16 | CoT | -10.1% | -5.2% | — | — | -9.1% | |||||||
| ifbench_multiturn | 1387 | avg@1 | CoT | +3.4% | +1.9% | -2.7% | +2.9% | ||||||||
| ifbench_test | 300 | avg@1 | CoT | +3.0% | +3.7% | +3.0% | +3.0% | ||||||||
| ifeval | 541 | avg@1 | CoT | +7.2% | +0.4% | -0.4% | +3.9% | ||||||||
| lcb:codegeneration | 268 | avg@1 | CoT | 0.0% | +4.9% | -0.4% | 0.0% | ||||||||
| math | 5000 | avg@4 | CoT | +2.5% | +0.8% | +1.0% | -0.4% | ||||||||
| math_500 | 500 | avg@8 | CoT | — | — | — | — | — | — | — | — | ||||
| mathqa | 2985 | avg@1 | NoCoT | -0.3% | -0.3% | +0.7% | -0.3% | ||||||||
| MBPP | 500 | avg@8 | FakeCoT | +5.0% | -2.7% | -1.7% | -1.4% | ||||||||
| MBPP | 500 | avg@8 | CoT | -3.6% | — | — | — | — | — | — | |||||
| MBPP+ | 378 | avg@8 | FakeCoT | +5.2% | +2.1% | -3.0% | -0.5% | ||||||||
| MBPP+ | 378 | avg@8 | CoT | — | — | — | — | — | — | — | — | ||||
| mc_taco | 9442 | avg@1 | CoT | — | — | — | — | — | — | — | — | ||||
| med_mcqa | 4183 | avg@1 | NoCoT | -1.4% | +0.5% | 0.0% | +0.3% | ||||||||
| med_qa | 2545 | avg@1 | NoCoT | -0.4% | +0.2% | +0.2% | +0.4% | ||||||||
| mmlu | 14042 | avg@1 | NoCoT | +0.8% | +0.2% | +0.7% | +0.2% | ||||||||
| mmlu_pro | 12032 | avg@1 | NoCoT | +1.7% | +2.5% | -2.7% | -0.1% | ||||||||
| mmlu_redux_2 | 5700 | avg@1 | NoCoT | -0.2% | +0.4% | +0.4% | -0.3% | ||||||||
| MultiPL-E HumanEval | 2864 | avg@1 | FakeCoT | — | — | — | — | — | — | — | — | — | — | ||
| mutual | 886 | avg@4 | CoT | — | — | — | — | — | — | — | — | ||||
| olympiad_bench | 2673 | avg@1 | CoT | -0.5% | +0.4% | +0.5% | -0.6% | ||||||||
| openbookqa | 1000 | avg@1 | NoCoT | -0.2% | 0.0% | +0.6% | -0.3% | ||||||||
| race | 1045 | avg@4 | CoT | — | — | — | — | — | — | — | — | ||||
| tau2_bench | 278 | avg@1 | FakeCoT | — | — | — | — | — | — | — | — | ||||
| tau3_bench | 375 | avg@1 | FakeCoT | — | — | — | — | — | — | — | — | ||||
| tau_bench | 165 | avg@1 | FakeCoT | — | — | — | — | — | — | — | — | ||||
| truthfulqa:mc | 817 | avg@1 | NoCoT | -0.1% | -0.5% | +0.4% | -0.5% | ||||||||
| winogender | 960 | avg@2 | CoT | — | — | — | — | — | — | — | — | ||||
| winogrande | 1267 | avg@1 | NoCoT | +0.4% | -1.0% | -0.1% | +0.6% | ||||||||
| wmdp | 3668 | avg@1 | CoT | — | — | — | — | — | — | — | — | ||||