{
 "credit": "Findra (https://findra.dev/)",
 "kind": "Published by others, not tested by us",
 "license": "CC BY 4.0",
 "license_url": "https://creativecommons.org/licenses/by/4.0/",
 "note": "The numbers were published by the sources named in each row; Findra compiled them. Credit the original source with the figure.",
 "rows": [
  {
   "benchmark": "MMLU-Redux",
   "model_id": "qwen3-4b",
   "notes": "Maker (Qwen team) tech report Table 17; post-trained hybrid model, thinking mode; MMLU-Redux is a cleaned MMLU re-label, not plain MMLU",
   "retrieved": "2026-10-22",
   "score": 83.7,
   "setting": "thinking mode",
   "source_url": "https://arxiv.org/html/2505.09388",
   "unit": "percent"
  },
  {
   "benchmark": "GPQA-Diamond",
   "model_id": "qwen3-4b",
   "notes": "Maker (Qwen team) tech report Table 17; post-trained hybrid model, thinking mode",
   "retrieved": "2026-10-22",
   "score": 55.9,
   "setting": "thinking mode",
   "source_url": "https://arxiv.org/html/2505.09388",
   "unit": "percent"
  },
  {
   "benchmark": "IFEval strict prompt",
   "model_id": "qwen3-4b",
   "notes": "Maker (Qwen team) tech report Table 17; post-trained hybrid model, thinking mode",
   "retrieved": "2026-10-22",
   "score": 81.9,
   "setting": "thinking mode",
   "source_url": "https://arxiv.org/html/2505.09388",
   "unit": "percent"
  },
  {
   "benchmark": "LiveCodeBench v5",
   "model_id": "qwen3-4b",
   "notes": "Maker (Qwen team) tech report Table 17; post-trained hybrid model, thinking mode",
   "retrieved": "2026-10-22",
   "score": 54.2,
   "setting": "thinking mode",
   "source_url": "https://arxiv.org/html/2505.09388",
   "unit": "percent"
  },
  {
   "benchmark": "MATH-500",
   "model_id": "qwen3-4b",
   "notes": "Maker (Qwen team) tech report Table 17; post-trained hybrid model, thinking mode",
   "retrieved": "2026-10-22",
   "score": 97.0,
   "setting": "thinking mode",
   "source_url": "https://arxiv.org/html/2505.09388",
   "unit": "percent"
  },
  {
   "benchmark": "AIME'24",
   "model_id": "qwen3-4b",
   "notes": "Maker (Qwen team) tech report Table 17; post-trained hybrid model, thinking mode",
   "retrieved": "2026-10-22",
   "score": 73.8,
   "setting": "thinking mode",
   "source_url": "https://arxiv.org/html/2505.09388",
   "unit": "percent"
  },
  {
   "benchmark": "MMLU-Redux",
   "model_id": "qwen3-8b",
   "notes": "Maker (Qwen team) tech report Table 17; post-trained hybrid model, thinking mode; MMLU-Redux is a cleaned MMLU re-label, not plain MMLU",
   "retrieved": "2026-10-22",
   "score": 87.5,
   "setting": "thinking mode",
   "source_url": "https://arxiv.org/html/2505.09388",
   "unit": "percent"
  },
  {
   "benchmark": "GPQA-Diamond",
   "model_id": "qwen3-8b",
   "notes": "Maker (Qwen team) tech report Table 17; post-trained hybrid model, thinking mode",
   "retrieved": "2026-10-22",
   "score": 62.0,
   "setting": "thinking mode",
   "source_url": "https://arxiv.org/html/2505.09388",
   "unit": "percent"
  },
  {
   "benchmark": "IFEval strict prompt",
   "model_id": "qwen3-8b",
   "notes": "Maker (Qwen team) tech report Table 17; post-trained hybrid model, thinking mode",
   "retrieved": "2026-10-22",
   "score": 85.0,
   "setting": "thinking mode",
   "source_url": "https://arxiv.org/html/2505.09388",
   "unit": "percent"
  },
  {
   "benchmark": "LiveCodeBench v5",
   "model_id": "qwen3-8b",
   "notes": "Maker (Qwen team) tech report Table 17; post-trained hybrid model, thinking mode",
   "retrieved": "2026-10-22",
   "score": 57.5,
   "setting": "thinking mode",
   "source_url": "https://arxiv.org/html/2505.09388",
   "unit": "percent"
  },
  {
   "benchmark": "MATH-500",
   "model_id": "qwen3-8b",
   "notes": "Maker (Qwen team) tech report Table 17; post-trained hybrid model, thinking mode",
   "retrieved": "2026-10-22",
   "score": 97.4,
   "setting": "thinking mode",
   "source_url": "https://arxiv.org/html/2505.09388",
   "unit": "percent"
  },
  {
   "benchmark": "AIME'24",
   "model_id": "qwen3-8b",
   "notes": "Maker (Qwen team) tech report Table 17; post-trained hybrid model, thinking mode",
   "retrieved": "2026-10-22",
   "score": 76.0,
   "setting": "thinking mode",
   "source_url": "https://arxiv.org/html/2505.09388",
   "unit": "percent"
  },
  {
   "benchmark": "MMLU-Redux",
   "model_id": "qwen3-4b",
   "notes": "Maker (Qwen team) tech report Table 18; post-trained hybrid model, non-thinking mode; MMLU-Redux is a cleaned MMLU re-label, not plain MMLU",
   "retrieved": "2026-10-22",
   "score": 77.3,
   "setting": "non-thinking mode",
   "source_url": "https://arxiv.org/html/2505.09388",
   "unit": "percent"
  },
  {
   "benchmark": "GPQA-Diamond",
   "model_id": "qwen3-4b",
   "notes": "Maker (Qwen team) tech report Table 18; post-trained hybrid model, non-thinking mode",
   "retrieved": "2026-10-22",
   "score": 41.7,
   "setting": "non-thinking mode",
   "source_url": "https://arxiv.org/html/2505.09388",
   "unit": "percent"
  },
  {
   "benchmark": "IFEval strict prompt",
   "model_id": "qwen3-4b",
   "notes": "Maker (Qwen team) tech report Table 18; post-trained hybrid model, non-thinking mode",
   "retrieved": "2026-10-22",
   "score": 81.2,
   "setting": "non-thinking mode",
   "source_url": "https://arxiv.org/html/2505.09388",
   "unit": "percent"
  },
  {
   "benchmark": "LiveCodeBench v5",
   "model_id": "qwen3-4b",
   "notes": "Maker (Qwen team) tech report Table 18; post-trained hybrid model, non-thinking mode",
   "retrieved": "2026-10-22",
   "score": 21.3,
   "setting": "non-thinking mode",
   "source_url": "https://arxiv.org/html/2505.09388",
   "unit": "percent"
  },
  {
   "benchmark": "MATH-500",
   "model_id": "qwen3-4b",
   "notes": "Maker (Qwen team) tech report Table 18; post-trained hybrid model, non-thinking mode",
   "retrieved": "2026-10-22",
   "score": 84.8,
   "setting": "non-thinking mode",
   "source_url": "https://arxiv.org/html/2505.09388",
   "unit": "percent"
  },
  {
   "benchmark": "AIME'24",
   "model_id": "qwen3-4b",
   "notes": "Maker (Qwen team) tech report Table 18; post-trained hybrid model, non-thinking mode",
   "retrieved": "2026-10-22",
   "score": 25.0,
   "setting": "non-thinking mode",
   "source_url": "https://arxiv.org/html/2505.09388",
   "unit": "percent"
  },
  {
   "benchmark": "MMLU-Redux",
   "model_id": "qwen3-8b",
   "notes": "Maker (Qwen team) tech report Table 18; post-trained hybrid model, non-thinking mode; MMLU-Redux is a cleaned MMLU re-label, not plain MMLU",
   "retrieved": "2026-10-22",
   "score": 79.5,
   "setting": "non-thinking mode",
   "source_url": "https://arxiv.org/html/2505.09388",
   "unit": "percent"
  },
  {
   "benchmark": "GPQA-Diamond",
   "model_id": "qwen3-8b",
   "notes": "Maker (Qwen team) tech report Table 18; post-trained hybrid model, non-thinking mode",
   "retrieved": "2026-10-22",
   "score": 39.3,
   "setting": "non-thinking mode",
   "source_url": "https://arxiv.org/html/2505.09388",
   "unit": "percent"
  },
  {
   "benchmark": "IFEval strict prompt",
   "model_id": "qwen3-8b",
   "notes": "Maker (Qwen team) tech report Table 18; post-trained hybrid model, non-thinking mode",
   "retrieved": "2026-10-22",
   "score": 83.0,
   "setting": "non-thinking mode",
   "source_url": "https://arxiv.org/html/2505.09388",
   "unit": "percent"
  },
  {
   "benchmark": "LiveCodeBench v5",
   "model_id": "qwen3-8b",
   "notes": "Maker (Qwen team) tech report Table 18; post-trained hybrid model, non-thinking mode",
   "retrieved": "2026-10-22",
   "score": 22.8,
   "setting": "non-thinking mode",
   "source_url": "https://arxiv.org/html/2505.09388",
   "unit": "percent"
  },
  {
   "benchmark": "MATH-500",
   "model_id": "qwen3-8b",
   "notes": "Maker (Qwen team) tech report Table 18; post-trained hybrid model, non-thinking mode",
   "retrieved": "2026-10-22",
   "score": 87.4,
   "setting": "non-thinking mode",
   "source_url": "https://arxiv.org/html/2505.09388",
   "unit": "percent"
  },
  {
   "benchmark": "AIME'24",
   "model_id": "qwen3-8b",
   "notes": "Maker (Qwen team) tech report Table 18; post-trained hybrid model, non-thinking mode",
   "retrieved": "2026-10-22",
   "score": 29.1,
   "setting": "non-thinking mode",
   "source_url": "https://arxiv.org/html/2505.09388",
   "unit": "percent"
  },
  {
   "benchmark": "MMLU-Redux",
   "model_id": "qwen3-14b",
   "notes": "Maker (Qwen team) tech report Table 15; post-trained hybrid model, thinking mode; MMLU-Redux is a cleaned MMLU re-label, not plain MMLU",
   "retrieved": "2026-10-22",
   "score": 88.6,
   "setting": "thinking mode",
   "source_url": "https://arxiv.org/html/2505.09388",
   "unit": "percent"
  },
  {
   "benchmark": "GPQA-Diamond",
   "model_id": "qwen3-14b",
   "notes": "Maker (Qwen team) tech report Table 15; post-trained hybrid model, thinking mode",
   "retrieved": "2026-10-22",
   "score": 64.0,
   "setting": "thinking mode",
   "source_url": "https://arxiv.org/html/2505.09388",
   "unit": "percent"
  },
  {
   "benchmark": "IFEval strict prompt",
   "model_id": "qwen3-14b",
   "notes": "Maker (Qwen team) tech report Table 15; post-trained hybrid model, thinking mode",
   "retrieved": "2026-10-22",
   "score": 85.4,
   "setting": "thinking mode",
   "source_url": "https://arxiv.org/html/2505.09388",
   "unit": "percent"
  },
  {
   "benchmark": "LiveCodeBench v5",
   "model_id": "qwen3-14b",
   "notes": "Maker (Qwen team) tech report Table 15; post-trained hybrid model, thinking mode",
   "retrieved": "2026-10-22",
   "score": 63.5,
   "setting": "thinking mode",
   "source_url": "https://arxiv.org/html/2505.09388",
   "unit": "percent"
  },
  {
   "benchmark": "MATH-500",
   "model_id": "qwen3-14b",
   "notes": "Maker (Qwen team) tech report Table 15; post-trained hybrid model, thinking mode",
   "retrieved": "2026-10-22",
   "score": 96.8,
   "setting": "thinking mode",
   "source_url": "https://arxiv.org/html/2505.09388",
   "unit": "percent"
  },
  {
   "benchmark": "AIME'24",
   "model_id": "qwen3-14b",
   "notes": "Maker (Qwen team) tech report Table 15; post-trained hybrid model, thinking mode",
   "retrieved": "2026-10-22",
   "score": 79.3,
   "setting": "thinking mode",
   "source_url": "https://arxiv.org/html/2505.09388",
   "unit": "percent"
  },
  {
   "benchmark": "MMLU-Redux",
   "model_id": "qwen3-30b",
   "notes": "Maker (Qwen team) tech report Table 15; post-trained hybrid model, thinking mode; MMLU-Redux is a cleaned MMLU re-label, not plain MMLU",
   "retrieved": "2026-10-22",
   "score": 89.5,
   "setting": "thinking mode",
   "source_url": "https://arxiv.org/html/2505.09388",
   "unit": "percent"
  },
  {
   "benchmark": "GPQA-Diamond",
   "model_id": "qwen3-30b",
   "notes": "Maker (Qwen team) tech report Table 15; post-trained hybrid model, thinking mode",
   "retrieved": "2026-10-22",
   "score": 65.8,
   "setting": "thinking mode",
   "source_url": "https://arxiv.org/html/2505.09388",
   "unit": "percent"
  },
  {
   "benchmark": "IFEval strict prompt",
   "model_id": "qwen3-30b",
   "notes": "Maker (Qwen team) tech report Table 15; post-trained hybrid model, thinking mode",
   "retrieved": "2026-10-22",
   "score": 86.5,
   "setting": "thinking mode",
   "source_url": "https://arxiv.org/html/2505.09388",
   "unit": "percent"
  },
  {
   "benchmark": "LiveCodeBench v5",
   "model_id": "qwen3-30b",
   "notes": "Maker (Qwen team) tech report Table 15; post-trained hybrid model, thinking mode",
   "retrieved": "2026-10-22",
   "score": 62.6,
   "setting": "thinking mode",
   "source_url": "https://arxiv.org/html/2505.09388",
   "unit": "percent"
  },
  {
   "benchmark": "MATH-500",
   "model_id": "qwen3-30b",
   "notes": "Maker (Qwen team) tech report Table 15; post-trained hybrid model, thinking mode",
   "retrieved": "2026-10-22",
   "score": 98.0,
   "setting": "thinking mode",
   "source_url": "https://arxiv.org/html/2505.09388",
   "unit": "percent"
  },
  {
   "benchmark": "AIME'24",
   "model_id": "qwen3-30b",
   "notes": "Maker (Qwen team) tech report Table 15; post-trained hybrid model, thinking mode",
   "retrieved": "2026-10-22",
   "score": 80.4,
   "setting": "thinking mode",
   "source_url": "https://arxiv.org/html/2505.09388",
   "unit": "percent"
  },
  {
   "benchmark": "MMLU-Redux",
   "model_id": "qwen3-14b",
   "notes": "Maker (Qwen team) tech report Table 16; post-trained hybrid model, non-thinking mode; MMLU-Redux is a cleaned MMLU re-label, not plain MMLU",
   "retrieved": "2026-10-22",
   "score": 82.0,
   "setting": "non-thinking mode",
   "source_url": "https://arxiv.org/html/2505.09388",
   "unit": "percent"
  },
  {
   "benchmark": "GPQA-Diamond",
   "model_id": "qwen3-14b",
   "notes": "Maker (Qwen team) tech report Table 16; post-trained hybrid model, non-thinking mode",
   "retrieved": "2026-10-22",
   "score": 54.8,
   "setting": "non-thinking mode",
   "source_url": "https://arxiv.org/html/2505.09388",
   "unit": "percent"
  },
  {
   "benchmark": "IFEval strict prompt",
   "model_id": "qwen3-14b",
   "notes": "Maker (Qwen team) tech report Table 16; post-trained hybrid model, non-thinking mode",
   "retrieved": "2026-10-22",
   "score": 84.8,
   "setting": "non-thinking mode",
   "source_url": "https://arxiv.org/html/2505.09388",
   "unit": "percent"
  },
  {
   "benchmark": "LiveCodeBench v5",
   "model_id": "qwen3-14b",
   "notes": "Maker (Qwen team) tech report Table 16; post-trained hybrid model, non-thinking mode",
   "retrieved": "2026-10-22",
   "score": 29.0,
   "setting": "non-thinking mode",
   "source_url": "https://arxiv.org/html/2505.09388",
   "unit": "percent"
  },
  {
   "benchmark": "MATH-500",
   "model_id": "qwen3-14b",
   "notes": "Maker (Qwen team) tech report Table 16; post-trained hybrid model, non-thinking mode",
   "retrieved": "2026-10-22",
   "score": 90.0,
   "setting": "non-thinking mode",
   "source_url": "https://arxiv.org/html/2505.09388",
   "unit": "percent"
  },
  {
   "benchmark": "AIME'24",
   "model_id": "qwen3-14b",
   "notes": "Maker (Qwen team) tech report Table 16; post-trained hybrid model, non-thinking mode",
   "retrieved": "2026-10-22",
   "score": 31.7,
   "setting": "non-thinking mode",
   "source_url": "https://arxiv.org/html/2505.09388",
   "unit": "percent"
  },
  {
   "benchmark": "MMLU-Redux",
   "model_id": "qwen3-30b",
   "notes": "Maker (Qwen team) tech report Table 16; post-trained hybrid model, non-thinking mode; MMLU-Redux is a cleaned MMLU re-label, not plain MMLU",
   "retrieved": "2026-10-22",
   "score": 84.1,
   "setting": "non-thinking mode",
   "source_url": "https://arxiv.org/html/2505.09388",
   "unit": "percent"
  },
  {
   "benchmark": "GPQA-Diamond",
   "model_id": "qwen3-30b",
   "notes": "Maker (Qwen team) tech report Table 16; post-trained hybrid model, non-thinking mode",
   "retrieved": "2026-10-22",
   "score": 54.8,
   "setting": "non-thinking mode",
   "source_url": "https://arxiv.org/html/2505.09388",
   "unit": "percent"
  },
  {
   "benchmark": "IFEval strict prompt",
   "model_id": "qwen3-30b",
   "notes": "Maker (Qwen team) tech report Table 16; post-trained hybrid model, non-thinking mode",
   "retrieved": "2026-10-22",
   "score": 83.7,
   "setting": "non-thinking mode",
   "source_url": "https://arxiv.org/html/2505.09388",
   "unit": "percent"
  },
  {
   "benchmark": "LiveCodeBench v5",
   "model_id": "qwen3-30b",
   "notes": "Maker (Qwen team) tech report Table 16; post-trained hybrid model, non-thinking mode",
   "retrieved": "2026-10-22",
   "score": 29.8,
   "setting": "non-thinking mode",
   "source_url": "https://arxiv.org/html/2505.09388",
   "unit": "percent"
  },
  {
   "benchmark": "MATH-500",
   "model_id": "qwen3-30b",
   "notes": "Maker (Qwen team) tech report Table 16; post-trained hybrid model, non-thinking mode",
   "retrieved": "2026-10-22",
   "score": 89.8,
   "setting": "non-thinking mode",
   "source_url": "https://arxiv.org/html/2505.09388",
   "unit": "percent"
  },
  {
   "benchmark": "AIME'24",
   "model_id": "qwen3-30b",
   "notes": "Maker (Qwen team) tech report Table 16; post-trained hybrid model, non-thinking mode",
   "retrieved": "2026-10-22",
   "score": 32.8,
   "setting": "non-thinking mode",
   "source_url": "https://arxiv.org/html/2505.09388",
   "unit": "percent"
  },
  {
   "benchmark": "MMLU-Redux",
   "model_id": "qwen3-32b",
   "notes": "Maker (Qwen team) tech report Table 13; post-trained hybrid model, thinking mode; MMLU-Redux is a cleaned MMLU re-label, not plain MMLU",
   "retrieved": "2026-10-22",
   "score": 90.9,
   "setting": "thinking mode",
   "source_url": "https://arxiv.org/html/2505.09388",
   "unit": "percent"
  },
  {
   "benchmark": "GPQA-Diamond",
   "model_id": "qwen3-32b",
   "notes": "Maker (Qwen team) tech report Table 13; post-trained hybrid model, thinking mode",
   "retrieved": "2026-10-22",
   "score": 68.4,
   "setting": "thinking mode",
   "source_url": "https://arxiv.org/html/2505.09388",
   "unit": "percent"
  },
  {
   "benchmark": "IFEval strict prompt",
   "model_id": "qwen3-32b",
   "notes": "Maker (Qwen team) tech report Table 13; post-trained hybrid model, thinking mode",
   "retrieved": "2026-10-22",
   "score": 85.0,
   "setting": "thinking mode",
   "source_url": "https://arxiv.org/html/2505.09388",
   "unit": "percent"
  },
  {
   "benchmark": "LiveCodeBench v5",
   "model_id": "qwen3-32b",
   "notes": "Maker (Qwen team) tech report Table 13; post-trained hybrid model, thinking mode",
   "retrieved": "2026-10-22",
   "score": 65.7,
   "setting": "thinking mode",
   "source_url": "https://arxiv.org/html/2505.09388",
   "unit": "percent"
  },
  {
   "benchmark": "MATH-500",
   "model_id": "qwen3-32b",
   "notes": "Maker (Qwen team) tech report Table 13; post-trained hybrid model, thinking mode",
   "retrieved": "2026-10-22",
   "score": 97.2,
   "setting": "thinking mode",
   "source_url": "https://arxiv.org/html/2505.09388",
   "unit": "percent"
  },
  {
   "benchmark": "AIME'24",
   "model_id": "qwen3-32b",
   "notes": "Maker (Qwen team) tech report Table 13; post-trained hybrid model, thinking mode",
   "retrieved": "2026-10-22",
   "score": 81.4,
   "setting": "thinking mode",
   "source_url": "https://arxiv.org/html/2505.09388",
   "unit": "percent"
  },
  {
   "benchmark": "MMLU-Redux",
   "model_id": "qwen3-32b",
   "notes": "Maker (Qwen team) tech report Table 14; post-trained hybrid model, non-thinking mode; MMLU-Redux is a cleaned MMLU re-label, not plain MMLU",
   "retrieved": "2026-10-22",
   "score": 85.7,
   "setting": "non-thinking mode",
   "source_url": "https://arxiv.org/html/2505.09388",
   "unit": "percent"
  },
  {
   "benchmark": "GPQA-Diamond",
   "model_id": "qwen3-32b",
   "notes": "Maker (Qwen team) tech report Table 14; post-trained hybrid model, non-thinking mode",
   "retrieved": "2026-10-22",
   "score": 54.6,
   "setting": "non-thinking mode",
   "source_url": "https://arxiv.org/html/2505.09388",
   "unit": "percent"
  },
  {
   "benchmark": "IFEval strict prompt",
   "model_id": "qwen3-32b",
   "notes": "Maker (Qwen team) tech report Table 14; post-trained hybrid model, non-thinking mode",
   "retrieved": "2026-10-22",
   "score": 83.2,
   "setting": "non-thinking mode",
   "source_url": "https://arxiv.org/html/2505.09388",
   "unit": "percent"
  },
  {
   "benchmark": "LiveCodeBench v5",
   "model_id": "qwen3-32b",
   "notes": "Maker (Qwen team) tech report Table 14; post-trained hybrid model, non-thinking mode",
   "retrieved": "2026-10-22",
   "score": 31.3,
   "setting": "non-thinking mode",
   "source_url": "https://arxiv.org/html/2505.09388",
   "unit": "percent"
  },
  {
   "benchmark": "MATH-500",
   "model_id": "qwen3-32b",
   "notes": "Maker (Qwen team) tech report Table 14; post-trained hybrid model, non-thinking mode",
   "retrieved": "2026-10-22",
   "score": 88.6,
   "setting": "non-thinking mode",
   "source_url": "https://arxiv.org/html/2505.09388",
   "unit": "percent"
  },
  {
   "benchmark": "AIME'24",
   "model_id": "qwen3-32b",
   "notes": "Maker (Qwen team) tech report Table 14; post-trained hybrid model, non-thinking mode",
   "retrieved": "2026-10-22",
   "score": 31.0,
   "setting": "non-thinking mode",
   "source_url": "https://arxiv.org/html/2505.09388",
   "unit": "percent"
  },
  {
   "benchmark": "MMLU",
   "model_id": "qwen3-32b",
   "notes": "Maker (Qwen team) tech report Table 4; score is for the BASE model Qwen3-32B-Base, not the chat model",
   "retrieved": "2026-10-22",
   "score": 83.61,
   "setting": "",
   "source_url": "https://arxiv.org/html/2505.09388",
   "unit": "percent"
  },
  {
   "benchmark": "MMLU-Pro",
   "model_id": "qwen3-32b",
   "notes": "Maker (Qwen team) tech report Table 4; score is for the BASE model Qwen3-32B-Base, not the chat model",
   "retrieved": "2026-10-22",
   "score": 65.54,
   "setting": "",
   "source_url": "https://arxiv.org/html/2505.09388",
   "unit": "percent"
  },
  {
   "benchmark": "MMLU",
   "model_id": "qwen3-14b",
   "notes": "Maker (Qwen team) tech report Table 5; score is for the BASE model Qwen3-14B-Base, not the chat model",
   "retrieved": "2026-10-22",
   "score": 81.05,
   "setting": "",
   "source_url": "https://arxiv.org/html/2505.09388",
   "unit": "percent"
  },
  {
   "benchmark": "MMLU",
   "model_id": "qwen3-30b",
   "notes": "Maker (Qwen team) tech report Table 5; score is for the BASE model Qwen3-30B-A3B-Base, not the chat model",
   "retrieved": "2026-10-22",
   "score": 81.38,
   "setting": "",
   "source_url": "https://arxiv.org/html/2505.09388",
   "unit": "percent"
  },
  {
   "benchmark": "MMLU-Pro",
   "model_id": "qwen3-14b",
   "notes": "Maker (Qwen team) tech report Table 5; score is for the BASE model Qwen3-14B-Base, not the chat model",
   "retrieved": "2026-10-22",
   "score": 61.03,
   "setting": "",
   "source_url": "https://arxiv.org/html/2505.09388",
   "unit": "percent"
  },
  {
   "benchmark": "MMLU-Pro",
   "model_id": "qwen3-30b",
   "notes": "Maker (Qwen team) tech report Table 5; score is for the BASE model Qwen3-30B-A3B-Base, not the chat model",
   "retrieved": "2026-10-22",
   "score": 61.49,
   "setting": "",
   "source_url": "https://arxiv.org/html/2505.09388",
   "unit": "percent"
  },
  {
   "benchmark": "MMLU",
   "model_id": "qwen3-8b",
   "notes": "Maker (Qwen team) tech report Table 6; score is for the BASE model Qwen3-8B-Base, not the chat model",
   "retrieved": "2026-10-22",
   "score": 76.89,
   "setting": "",
   "source_url": "https://arxiv.org/html/2505.09388",
   "unit": "percent"
  },
  {
   "benchmark": "MMLU-Pro",
   "model_id": "qwen3-8b",
   "notes": "Maker (Qwen team) tech report Table 6; score is for the BASE model Qwen3-8B-Base, not the chat model",
   "retrieved": "2026-10-22",
   "score": 56.73,
   "setting": "",
   "source_url": "https://arxiv.org/html/2505.09388",
   "unit": "percent"
  },
  {
   "benchmark": "MMLU",
   "model_id": "qwen3-4b",
   "notes": "Maker (Qwen team) tech report Table 7; score is for the BASE model Qwen3-4B-Base, not the chat model",
   "retrieved": "2026-10-22",
   "score": 72.99,
   "setting": "",
   "source_url": "https://arxiv.org/html/2505.09388",
   "unit": "percent"
  },
  {
   "benchmark": "MMLU-Pro",
   "model_id": "qwen3-4b",
   "notes": "Maker (Qwen team) tech report Table 7; score is for the BASE model Qwen3-4B-Base, not the chat model",
   "retrieved": "2026-10-22",
   "score": 50.58,
   "setting": "",
   "source_url": "https://arxiv.org/html/2505.09388",
   "unit": "percent"
  },
  {
   "benchmark": "MMLU-Pro",
   "model_id": "gemma3-4b",
   "notes": "Maker (Google) Gemma 3 tech report Table 6, instruction-tuned (IT) model; Table 6 caption says zero-shot",
   "retrieved": "2026-10-22",
   "score": 43.6,
   "setting": "0-shot",
   "source_url": "https://arxiv.org/html/2503.19786",
   "unit": "percent"
  },
  {
   "benchmark": "GPQA Diamond",
   "model_id": "gemma3-4b",
   "notes": "Maker (Google) Gemma 3 tech report Table 6, instruction-tuned (IT) model; Table 6 caption says zero-shot",
   "retrieved": "2026-10-22",
   "score": 30.8,
   "setting": "0-shot",
   "source_url": "https://arxiv.org/html/2503.19786",
   "unit": "percent"
  },
  {
   "benchmark": "LiveCodeBench",
   "model_id": "gemma3-4b",
   "notes": "Maker (Google) Gemma 3 tech report Table 6, instruction-tuned (IT) model; Table 6 caption says zero-shot",
   "retrieved": "2026-10-22",
   "score": 12.6,
   "setting": "0-shot",
   "source_url": "https://arxiv.org/html/2503.19786",
   "unit": "percent"
  },
  {
   "benchmark": "MATH",
   "model_id": "gemma3-4b",
   "notes": "Maker (Google) Gemma 3 tech report Table 6, instruction-tuned (IT) model; Table 6 caption says zero-shot",
   "retrieved": "2026-10-22",
   "score": 75.6,
   "setting": "0-shot",
   "source_url": "https://arxiv.org/html/2503.19786",
   "unit": "percent"
  },
  {
   "benchmark": "MMLU",
   "model_id": "gemma3-4b",
   "notes": "Maker (Google) Gemma 3 tech report appendix Table 18, instruction-tuned (IT) model; 0-shot per the report's IT details table",
   "retrieved": "2026-10-22",
   "score": 58.1,
   "setting": "0-shot",
   "source_url": "https://arxiv.org/html/2503.19786",
   "unit": "percent"
  },
  {
   "benchmark": "HumanEval",
   "model_id": "gemma3-4b",
   "notes": "Maker (Google) Gemma 3 tech report appendix Table 18, instruction-tuned (IT) model; 0-shot per the report's IT details table",
   "retrieved": "2026-10-22",
   "score": 71.3,
   "setting": "0-shot",
   "source_url": "https://arxiv.org/html/2503.19786",
   "unit": "percent"
  },
  {
   "benchmark": "IFEval",
   "model_id": "gemma3-4b",
   "notes": "Maker (Google) Gemma 3 tech report appendix Table 18, instruction-tuned (IT) model; 0-shot per the report's IT details table",
   "retrieved": "2026-10-22",
   "score": 90.2,
   "setting": "0-shot",
   "source_url": "https://arxiv.org/html/2503.19786",
   "unit": "percent"
  },
  {
   "benchmark": "MMLU-Pro",
   "model_id": "gemma3-12b",
   "notes": "Maker (Google) Gemma 3 tech report Table 6, instruction-tuned (IT) model; Table 6 caption says zero-shot",
   "retrieved": "2026-10-22",
   "score": 60.6,
   "setting": "0-shot",
   "source_url": "https://arxiv.org/html/2503.19786",
   "unit": "percent"
  },
  {
   "benchmark": "GPQA Diamond",
   "model_id": "gemma3-12b",
   "notes": "Maker (Google) Gemma 3 tech report Table 6, instruction-tuned (IT) model; Table 6 caption says zero-shot",
   "retrieved": "2026-10-22",
   "score": 40.9,
   "setting": "0-shot",
   "source_url": "https://arxiv.org/html/2503.19786",
   "unit": "percent"
  },
  {
   "benchmark": "LiveCodeBench",
   "model_id": "gemma3-12b",
   "notes": "Maker (Google) Gemma 3 tech report Table 6, instruction-tuned (IT) model; Table 6 caption says zero-shot",
   "retrieved": "2026-10-22",
   "score": 24.6,
   "setting": "0-shot",
   "source_url": "https://arxiv.org/html/2503.19786",
   "unit": "percent"
  },
  {
   "benchmark": "MATH",
   "model_id": "gemma3-12b",
   "notes": "Maker (Google) Gemma 3 tech report Table 6, instruction-tuned (IT) model; Table 6 caption says zero-shot",
   "retrieved": "2026-10-22",
   "score": 83.8,
   "setting": "0-shot",
   "source_url": "https://arxiv.org/html/2503.19786",
   "unit": "percent"
  },
  {
   "benchmark": "MMLU",
   "model_id": "gemma3-12b",
   "notes": "Maker (Google) Gemma 3 tech report appendix Table 18, instruction-tuned (IT) model; 0-shot per the report's IT details table",
   "retrieved": "2026-10-22",
   "score": 71.9,
   "setting": "0-shot",
   "source_url": "https://arxiv.org/html/2503.19786",
   "unit": "percent"
  },
  {
   "benchmark": "HumanEval",
   "model_id": "gemma3-12b",
   "notes": "Maker (Google) Gemma 3 tech report appendix Table 18, instruction-tuned (IT) model; 0-shot per the report's IT details table",
   "retrieved": "2026-10-22",
   "score": 85.4,
   "setting": "0-shot",
   "source_url": "https://arxiv.org/html/2503.19786",
   "unit": "percent"
  },
  {
   "benchmark": "IFEval",
   "model_id": "gemma3-12b",
   "notes": "Maker (Google) Gemma 3 tech report appendix Table 18, instruction-tuned (IT) model; 0-shot per the report's IT details table",
   "retrieved": "2026-10-22",
   "score": 88.9,
   "setting": "0-shot",
   "source_url": "https://arxiv.org/html/2503.19786",
   "unit": "percent"
  },
  {
   "benchmark": "MMLU-Pro",
   "model_id": "gemma3-27b",
   "notes": "Maker (Google) Gemma 3 tech report Table 6, instruction-tuned (IT) model; Table 6 caption says zero-shot",
   "retrieved": "2026-10-22",
   "score": 67.5,
   "setting": "0-shot",
   "source_url": "https://arxiv.org/html/2503.19786",
   "unit": "percent"
  },
  {
   "benchmark": "GPQA Diamond",
   "model_id": "gemma3-27b",
   "notes": "Maker (Google) Gemma 3 tech report Table 6, instruction-tuned (IT) model; Table 6 caption says zero-shot",
   "retrieved": "2026-10-22",
   "score": 42.4,
   "setting": "0-shot",
   "source_url": "https://arxiv.org/html/2503.19786",
   "unit": "percent"
  },
  {
   "benchmark": "LiveCodeBench",
   "model_id": "gemma3-27b",
   "notes": "Maker (Google) Gemma 3 tech report Table 6, instruction-tuned (IT) model; Table 6 caption says zero-shot",
   "retrieved": "2026-10-22",
   "score": 29.7,
   "setting": "0-shot",
   "source_url": "https://arxiv.org/html/2503.19786",
   "unit": "percent"
  },
  {
   "benchmark": "MATH",
   "model_id": "gemma3-27b",
   "notes": "Maker (Google) Gemma 3 tech report Table 6, instruction-tuned (IT) model; Table 6 caption says zero-shot",
   "retrieved": "2026-10-22",
   "score": 89.0,
   "setting": "0-shot",
   "source_url": "https://arxiv.org/html/2503.19786",
   "unit": "percent"
  },
  {
   "benchmark": "MMLU",
   "model_id": "gemma3-27b",
   "notes": "Maker (Google) Gemma 3 tech report appendix Table 18, instruction-tuned (IT) model; 0-shot per the report's IT details table",
   "retrieved": "2026-10-22",
   "score": 76.9,
   "setting": "0-shot",
   "source_url": "https://arxiv.org/html/2503.19786",
   "unit": "percent"
  },
  {
   "benchmark": "HumanEval",
   "model_id": "gemma3-27b",
   "notes": "Maker (Google) Gemma 3 tech report appendix Table 18, instruction-tuned (IT) model; 0-shot per the report's IT details table",
   "retrieved": "2026-10-22",
   "score": 87.8,
   "setting": "0-shot",
   "source_url": "https://arxiv.org/html/2503.19786",
   "unit": "percent"
  },
  {
   "benchmark": "IFEval",
   "model_id": "gemma3-27b",
   "notes": "Maker (Google) Gemma 3 tech report appendix Table 18, instruction-tuned (IT) model; 0-shot per the report's IT details table",
   "retrieved": "2026-10-22",
   "score": 90.4,
   "setting": "0-shot",
   "source_url": "https://arxiv.org/html/2503.19786",
   "unit": "percent"
  },
  {
   "benchmark": "Chatbot Arena Elo",
   "model_id": "gemma3-27b",
   "notes": "Maker (Google) Gemma 3 tech report Table 5; preliminary result received 8 Mar 2025 from LMArena, 95% CI +8/-9",
   "retrieved": "2026-10-22",
   "score": 1338.0,
   "setting": "",
   "source_url": "https://arxiv.org/html/2503.19786",
   "unit": "elo"
  },
  {
   "benchmark": "MMLU",
   "model_id": "llama-3-1-8b",
   "notes": "Maker (Meta) model card, instruction-tuned model (Llama 3.1 Instruct); also on the unsloth HF mirror",
   "retrieved": "2026-10-22",
   "score": 69.4,
   "setting": "5-shot, macro_avg/acc",
   "source_url": "https://github.com/meta-llama/llama-models/blob/main/models/llama3_1/MODEL_CARD.md",
   "unit": "percent"
  },
  {
   "benchmark": "MMLU (CoT)",
   "model_id": "llama-3-1-8b",
   "notes": "Maker (Meta) model card, instruction-tuned model (Llama 3.1 Instruct)",
   "retrieved": "2026-10-22",
   "score": 73.0,
   "setting": "0-shot, macro_avg/acc",
   "source_url": "https://github.com/meta-llama/llama-models/blob/main/models/llama3_1/MODEL_CARD.md",
   "unit": "percent"
  },
  {
   "benchmark": "MMLU-Pro (CoT)",
   "model_id": "llama-3-1-8b",
   "notes": "Maker (Meta) model card, instruction-tuned model (Llama 3.1 Instruct)",
   "retrieved": "2026-10-22",
   "score": 48.3,
   "setting": "5-shot, macro_avg/acc",
   "source_url": "https://github.com/meta-llama/llama-models/blob/main/models/llama3_1/MODEL_CARD.md",
   "unit": "percent"
  },
  {
   "benchmark": "IFEval",
   "model_id": "llama-3-1-8b",
   "notes": "Maker (Meta) model card, instruction-tuned model (Llama 3.1 Instruct); card prints no shot count or metric for IFEval",
   "retrieved": "2026-10-22",
   "score": 80.4,
   "setting": "",
   "source_url": "https://github.com/meta-llama/llama-models/blob/main/models/llama3_1/MODEL_CARD.md",
   "unit": "percent"
  },
  {
   "benchmark": "GPQA",
   "model_id": "llama-3-1-8b",
   "notes": "Maker (Meta) model card, instruction-tuned model (Llama 3.1 Instruct); plain GPQA as printed, not named Diamond",
   "retrieved": "2026-10-22",
   "score": 30.4,
   "setting": "0-shot, em",
   "source_url": "https://github.com/meta-llama/llama-models/blob/main/models/llama3_1/MODEL_CARD.md",
   "unit": "percent"
  },
  {
   "benchmark": "HumanEval",
   "model_id": "llama-3-1-8b",
   "notes": "Maker (Meta) model card, instruction-tuned model (Llama 3.1 Instruct)",
   "retrieved": "2026-10-22",
   "score": 72.6,
   "setting": "0-shot, pass@1",
   "source_url": "https://github.com/meta-llama/llama-models/blob/main/models/llama3_1/MODEL_CARD.md",
   "unit": "percent"
  },
  {
   "benchmark": "MATH (CoT)",
   "model_id": "llama-3-1-8b",
   "notes": "Maker (Meta) model card, instruction-tuned model (Llama 3.1 Instruct)",
   "retrieved": "2026-10-22",
   "score": 51.9,
   "setting": "0-shot, final_em",
   "source_url": "https://github.com/meta-llama/llama-models/blob/main/models/llama3_1/MODEL_CARD.md",
   "unit": "percent"
  },
  {
   "benchmark": "MMLU",
   "model_id": "llama-3-1-70b",
   "notes": "Maker (Meta) model card, instruction-tuned model (Llama 3.1 Instruct); also on the unsloth HF mirror",
   "retrieved": "2026-10-22",
   "score": 83.6,
   "setting": "5-shot, macro_avg/acc",
   "source_url": "https://github.com/meta-llama/llama-models/blob/main/models/llama3_1/MODEL_CARD.md",
   "unit": "percent"
  },
  {
   "benchmark": "MMLU (CoT)",
   "model_id": "llama-3-1-70b",
   "notes": "Maker (Meta) model card, instruction-tuned model (Llama 3.1 Instruct)",
   "retrieved": "2026-10-22",
   "score": 86.0,
   "setting": "0-shot, macro_avg/acc",
   "source_url": "https://github.com/meta-llama/llama-models/blob/main/models/llama3_1/MODEL_CARD.md",
   "unit": "percent"
  },
  {
   "benchmark": "MMLU-Pro (CoT)",
   "model_id": "llama-3-1-70b",
   "notes": "Maker (Meta) model card, instruction-tuned model (Llama 3.1 Instruct)",
   "retrieved": "2026-10-22",
   "score": 66.4,
   "setting": "5-shot, macro_avg/acc",
   "source_url": "https://github.com/meta-llama/llama-models/blob/main/models/llama3_1/MODEL_CARD.md",
   "unit": "percent"
  },
  {
   "benchmark": "IFEval",
   "model_id": "llama-3-1-70b",
   "notes": "Maker (Meta) model card, instruction-tuned model (Llama 3.1 Instruct); card prints no shot count or metric for IFEval",
   "retrieved": "2026-10-22",
   "score": 87.5,
   "setting": "",
   "source_url": "https://github.com/meta-llama/llama-models/blob/main/models/llama3_1/MODEL_CARD.md",
   "unit": "percent"
  },
  {
   "benchmark": "GPQA",
   "model_id": "llama-3-1-70b",
   "notes": "Maker (Meta) model card, instruction-tuned model (Llama 3.1 Instruct); plain GPQA as printed, not named Diamond",
   "retrieved": "2026-10-22",
   "score": 46.7,
   "setting": "0-shot, em",
   "source_url": "https://github.com/meta-llama/llama-models/blob/main/models/llama3_1/MODEL_CARD.md",
   "unit": "percent"
  },
  {
   "benchmark": "HumanEval",
   "model_id": "llama-3-1-70b",
   "notes": "Maker (Meta) model card, instruction-tuned model (Llama 3.1 Instruct)",
   "retrieved": "2026-10-22",
   "score": 80.5,
   "setting": "0-shot, pass@1",
   "source_url": "https://github.com/meta-llama/llama-models/blob/main/models/llama3_1/MODEL_CARD.md",
   "unit": "percent"
  },
  {
   "benchmark": "MATH (CoT)",
   "model_id": "llama-3-1-70b",
   "notes": "Maker (Meta) model card, instruction-tuned model (Llama 3.1 Instruct)",
   "retrieved": "2026-10-22",
   "score": 68.0,
   "setting": "0-shot, final_em",
   "source_url": "https://github.com/meta-llama/llama-models/blob/main/models/llama3_1/MODEL_CARD.md",
   "unit": "percent"
  },
  {
   "benchmark": "MMLU",
   "model_id": "llama-3-2-3b",
   "notes": "Maker (Meta) model card, Llama 3.2 3B Instruct bf16 column",
   "retrieved": "2026-10-22",
   "score": 63.4,
   "setting": "5-shot, macro_avg/acc",
   "source_url": "https://github.com/meta-llama/llama-models/blob/main/models/llama3_2/MODEL_CARD.md",
   "unit": "percent"
  },
  {
   "benchmark": "IFEval",
   "model_id": "llama-3-2-3b",
   "notes": "Maker (Meta) model card, Llama 3.2 3B Instruct bf16 column; average of four IFEval accuracies, not strict-only",
   "retrieved": "2026-10-22",
   "score": 77.4,
   "setting": "0-shot, avg of prompt/instruction acc loose/strict",
   "source_url": "https://github.com/meta-llama/llama-models/blob/main/models/llama3_2/MODEL_CARD.md",
   "unit": "percent"
  },
  {
   "benchmark": "MATH (CoT)",
   "model_id": "llama-3-2-3b",
   "notes": "Maker (Meta) model card, Llama 3.2 3B Instruct bf16 column",
   "retrieved": "2026-10-22",
   "score": 48.0,
   "setting": "0-shot, final_em",
   "source_url": "https://github.com/meta-llama/llama-models/blob/main/models/llama3_2/MODEL_CARD.md",
   "unit": "percent"
  },
  {
   "benchmark": "GPQA",
   "model_id": "llama-3-2-3b",
   "notes": "Maker (Meta) model card, Llama 3.2 3B Instruct bf16 column; plain GPQA as printed, not named Diamond",
   "retrieved": "2026-10-22",
   "score": 32.8,
   "setting": "0-shot, acc",
   "source_url": "https://github.com/meta-llama/llama-models/blob/main/models/llama3_2/MODEL_CARD.md",
   "unit": "percent"
  },
  {
   "benchmark": "IFEval",
   "model_id": "mistral-7b",
   "notes": "Hugging Face Open LLM Leaderboard v2 (not the maker), row mistralai/Mistral-7B-Instruct-v0.3 bfloat16 chat model, submitted 2024-06-12",
   "retrieved": "2026-10-22",
   "score": 54.65,
   "setting": "0-shot, strict prompt+instruction accuracy",
   "source_url": "https://huggingface.co/datasets/open-llm-leaderboard/contents",
   "unit": "percent"
  },
  {
   "benchmark": "MMLU-PRO (raw accuracy)",
   "model_id": "mistral-7b",
   "notes": "Hugging Face Open LLM Leaderboard v2 (not the maker), row mistralai/Mistral-7B-Instruct-v0.3 bfloat16 chat model, submitted 2024-06-12; the leaderboard's normalized MMLU-PRO column prints 23.06",
   "retrieved": "2026-10-22",
   "score": 30.75,
   "setting": "5-shot",
   "source_url": "https://huggingface.co/datasets/open-llm-leaderboard/contents",
   "unit": "percent"
  },
  {
   "benchmark": "GPQA (raw accuracy)",
   "model_id": "mistral-7b",
   "notes": "Hugging Face Open LLM Leaderboard v2 (not the maker), row mistralai/Mistral-7B-Instruct-v0.3 bfloat16 chat model, submitted 2024-06-12; plain GPQA, 4-way multiple choice so chance is 25; normalized GPQA column prints 3.91",
   "retrieved": "2026-10-22",
   "score": 27.94,
   "setting": "0-shot",
   "source_url": "https://huggingface.co/datasets/open-llm-leaderboard/contents",
   "unit": "percent"
  },
  {
   "benchmark": "MATH Lvl 5",
   "model_id": "mistral-7b",
   "notes": "Hugging Face Open LLM Leaderboard v2 (not the maker), row mistralai/Mistral-7B-Instruct-v0.3 bfloat16 chat model, submitted 2024-06-12; level 5 problems only",
   "retrieved": "2026-10-22",
   "score": 3.85,
   "setting": "4-shot, exact match",
   "source_url": "https://huggingface.co/datasets/open-llm-leaderboard/contents",
   "unit": "percent"
  },
  {
   "benchmark": "MMLU",
   "model_id": "phi4-14b",
   "notes": "Maker (Microsoft) model card Model Quality table, OpenAI simple-evals",
   "retrieved": "2026-10-22",
   "score": 84.8,
   "setting": "simple-evals",
   "source_url": "https://huggingface.co/microsoft/phi-4",
   "unit": "percent"
  },
  {
   "benchmark": "GPQA",
   "model_id": "phi4-14b",
   "notes": "Maker (Microsoft) model card Model Quality table, OpenAI simple-evals; plain GPQA as printed, not named Diamond",
   "retrieved": "2026-10-22",
   "score": 56.1,
   "setting": "simple-evals",
   "source_url": "https://huggingface.co/microsoft/phi-4",
   "unit": "percent"
  },
  {
   "benchmark": "MATH",
   "model_id": "phi4-14b",
   "notes": "Maker (Microsoft) model card Model Quality table, OpenAI simple-evals",
   "retrieved": "2026-10-22",
   "score": 80.4,
   "setting": "simple-evals",
   "source_url": "https://huggingface.co/microsoft/phi-4",
   "unit": "percent"
  },
  {
   "benchmark": "HumanEval",
   "model_id": "phi4-14b",
   "notes": "Maker (Microsoft) model card Model Quality table, OpenAI simple-evals",
   "retrieved": "2026-10-22",
   "score": 82.6,
   "setting": "simple-evals",
   "source_url": "https://huggingface.co/microsoft/phi-4",
   "unit": "percent"
  },
  {
   "benchmark": "MMLU-Pro",
   "model_id": "phi4-14b",
   "notes": "Maker (Microsoft) phi-4 tech report Table 1; in the second benchmark group (not simple-evals); setting not stated in the table",
   "retrieved": "2026-10-22",
   "score": 70.4,
   "setting": "",
   "source_url": "https://arxiv.org/html/2412.08905",
   "unit": "percent"
  },
  {
   "benchmark": "IFEval",
   "model_id": "phi4-14b",
   "notes": "Maker (Microsoft) phi-4 tech report Table 1; in the second benchmark group (not simple-evals); setting not stated in the table",
   "retrieved": "2026-10-22",
   "score": 63.0,
   "setting": "",
   "source_url": "https://arxiv.org/html/2412.08905",
   "unit": "percent"
  },
  {
   "benchmark": "MMLU",
   "model_id": "gpt-oss-120b",
   "notes": "Maker (OpenAI) gpt-oss model card Table 3; MXFP4 weights as released",
   "retrieved": "2026-10-22",
   "score": 85.9,
   "setting": "low reasoning",
   "source_url": "https://arxiv.org/html/2508.10925",
   "unit": "percent"
  },
  {
   "benchmark": "MMLU",
   "model_id": "gpt-oss-120b",
   "notes": "Maker (OpenAI) gpt-oss model card Table 3; MXFP4 weights as released",
   "retrieved": "2026-10-22",
   "score": 88.0,
   "setting": "medium reasoning",
   "source_url": "https://arxiv.org/html/2508.10925",
   "unit": "percent"
  },
  {
   "benchmark": "MMLU",
   "model_id": "gpt-oss-120b",
   "notes": "Maker (OpenAI) gpt-oss model card Table 3; MXFP4 weights as released",
   "retrieved": "2026-10-22",
   "score": 90.0,
   "setting": "high reasoning",
   "source_url": "https://arxiv.org/html/2508.10925",
   "unit": "percent"
  },
  {
   "benchmark": "GPQA Diamond",
   "model_id": "gpt-oss-120b",
   "notes": "Maker (OpenAI) gpt-oss model card Table 3; MXFP4 weights as released",
   "retrieved": "2026-10-22",
   "score": 67.1,
   "setting": "low reasoning, no tools",
   "source_url": "https://arxiv.org/html/2508.10925",
   "unit": "percent"
  },
  {
   "benchmark": "GPQA Diamond",
   "model_id": "gpt-oss-120b",
   "notes": "Maker (OpenAI) gpt-oss model card Table 3; MXFP4 weights as released",
   "retrieved": "2026-10-22",
   "score": 73.1,
   "setting": "medium reasoning, no tools",
   "source_url": "https://arxiv.org/html/2508.10925",
   "unit": "percent"
  },
  {
   "benchmark": "GPQA Diamond",
   "model_id": "gpt-oss-120b",
   "notes": "Maker (OpenAI) gpt-oss model card Table 3; MXFP4 weights as released",
   "retrieved": "2026-10-22",
   "score": 80.1,
   "setting": "high reasoning, no tools",
   "source_url": "https://arxiv.org/html/2508.10925",
   "unit": "percent"
  },
  {
   "benchmark": "AIME 2024",
   "model_id": "gpt-oss-120b",
   "notes": "Maker (OpenAI) gpt-oss model card Table 3; MXFP4 weights as released",
   "retrieved": "2026-10-22",
   "score": 56.3,
   "setting": "low reasoning, no tools",
   "source_url": "https://arxiv.org/html/2508.10925",
   "unit": "percent"
  },
  {
   "benchmark": "AIME 2024",
   "model_id": "gpt-oss-120b",
   "notes": "Maker (OpenAI) gpt-oss model card Table 3; MXFP4 weights as released",
   "retrieved": "2026-10-22",
   "score": 80.4,
   "setting": "medium reasoning, no tools",
   "source_url": "https://arxiv.org/html/2508.10925",
   "unit": "percent"
  },
  {
   "benchmark": "AIME 2024",
   "model_id": "gpt-oss-120b",
   "notes": "Maker (OpenAI) gpt-oss model card Table 3; MXFP4 weights as released",
   "retrieved": "2026-10-22",
   "score": 95.8,
   "setting": "high reasoning, no tools",
   "source_url": "https://arxiv.org/html/2508.10925",
   "unit": "percent"
  },
  {
   "benchmark": "AIME 2025",
   "model_id": "gpt-oss-120b",
   "notes": "Maker (OpenAI) gpt-oss model card Table 3; MXFP4 weights as released",
   "retrieved": "2026-10-22",
   "score": 50.4,
   "setting": "low reasoning, no tools",
   "source_url": "https://arxiv.org/html/2508.10925",
   "unit": "percent"
  },
  {
   "benchmark": "AIME 2025",
   "model_id": "gpt-oss-120b",
   "notes": "Maker (OpenAI) gpt-oss model card Table 3; MXFP4 weights as released",
   "retrieved": "2026-10-22",
   "score": 80.0,
   "setting": "medium reasoning, no tools",
   "source_url": "https://arxiv.org/html/2508.10925",
   "unit": "percent"
  },
  {
   "benchmark": "AIME 2025",
   "model_id": "gpt-oss-120b",
   "notes": "Maker (OpenAI) gpt-oss model card Table 3; MXFP4 weights as released",
   "retrieved": "2026-10-22",
   "score": 92.5,
   "setting": "high reasoning, no tools",
   "source_url": "https://arxiv.org/html/2508.10925",
   "unit": "percent"
  },
  {
   "benchmark": "Codeforces Elo",
   "model_id": "gpt-oss-120b",
   "notes": "Maker (OpenAI) gpt-oss model card Table 3; MXFP4 weights as released",
   "retrieved": "2026-10-22",
   "score": 1595.0,
   "setting": "low reasoning, no tools",
   "source_url": "https://arxiv.org/html/2508.10925",
   "unit": "elo"
  },
  {
   "benchmark": "Codeforces Elo",
   "model_id": "gpt-oss-120b",
   "notes": "Maker (OpenAI) gpt-oss model card Table 3; MXFP4 weights as released",
   "retrieved": "2026-10-22",
   "score": 2205.0,
   "setting": "medium reasoning, no tools",
   "source_url": "https://arxiv.org/html/2508.10925",
   "unit": "elo"
  },
  {
   "benchmark": "Codeforces Elo",
   "model_id": "gpt-oss-120b",
   "notes": "Maker (OpenAI) gpt-oss model card Table 3; MXFP4 weights as released",
   "retrieved": "2026-10-22",
   "score": 2463.0,
   "setting": "high reasoning, no tools",
   "source_url": "https://arxiv.org/html/2508.10925",
   "unit": "elo"
  },
  {
   "benchmark": "MMLU",
   "model_id": "gpt-oss-20b",
   "notes": "Maker (OpenAI) gpt-oss model card Table 3; MXFP4 weights as released",
   "retrieved": "2026-10-22",
   "score": 80.4,
   "setting": "low reasoning",
   "source_url": "https://arxiv.org/html/2508.10925",
   "unit": "percent"
  },
  {
   "benchmark": "MMLU",
   "model_id": "gpt-oss-20b",
   "notes": "Maker (OpenAI) gpt-oss model card Table 3; MXFP4 weights as released",
   "retrieved": "2026-10-22",
   "score": 84.0,
   "setting": "medium reasoning",
   "source_url": "https://arxiv.org/html/2508.10925",
   "unit": "percent"
  },
  {
   "benchmark": "MMLU",
   "model_id": "gpt-oss-20b",
   "notes": "Maker (OpenAI) gpt-oss model card Table 3; MXFP4 weights as released",
   "retrieved": "2026-10-22",
   "score": 85.3,
   "setting": "high reasoning",
   "source_url": "https://arxiv.org/html/2508.10925",
   "unit": "percent"
  },
  {
   "benchmark": "GPQA Diamond",
   "model_id": "gpt-oss-20b",
   "notes": "Maker (OpenAI) gpt-oss model card Table 3; MXFP4 weights as released",
   "retrieved": "2026-10-22",
   "score": 56.8,
   "setting": "low reasoning, no tools",
   "source_url": "https://arxiv.org/html/2508.10925",
   "unit": "percent"
  },
  {
   "benchmark": "GPQA Diamond",
   "model_id": "gpt-oss-20b",
   "notes": "Maker (OpenAI) gpt-oss model card Table 3; MXFP4 weights as released",
   "retrieved": "2026-10-22",
   "score": 66.0,
   "setting": "medium reasoning, no tools",
   "source_url": "https://arxiv.org/html/2508.10925",
   "unit": "percent"
  },
  {
   "benchmark": "GPQA Diamond",
   "model_id": "gpt-oss-20b",
   "notes": "Maker (OpenAI) gpt-oss model card Table 3; MXFP4 weights as released",
   "retrieved": "2026-10-22",
   "score": 71.5,
   "setting": "high reasoning, no tools",
   "source_url": "https://arxiv.org/html/2508.10925",
   "unit": "percent"
  },
  {
   "benchmark": "AIME 2024",
   "model_id": "gpt-oss-20b",
   "notes": "Maker (OpenAI) gpt-oss model card Table 3; MXFP4 weights as released",
   "retrieved": "2026-10-22",
   "score": 42.1,
   "setting": "low reasoning, no tools",
   "source_url": "https://arxiv.org/html/2508.10925",
   "unit": "percent"
  },
  {
   "benchmark": "AIME 2024",
   "model_id": "gpt-oss-20b",
   "notes": "Maker (OpenAI) gpt-oss model card Table 3; MXFP4 weights as released",
   "retrieved": "2026-10-22",
   "score": 80.0,
   "setting": "medium reasoning, no tools",
   "source_url": "https://arxiv.org/html/2508.10925",
   "unit": "percent"
  },
  {
   "benchmark": "AIME 2024",
   "model_id": "gpt-oss-20b",
   "notes": "Maker (OpenAI) gpt-oss model card Table 3; MXFP4 weights as released",
   "retrieved": "2026-10-22",
   "score": 92.1,
   "setting": "high reasoning, no tools",
   "source_url": "https://arxiv.org/html/2508.10925",
   "unit": "percent"
  },
  {
   "benchmark": "AIME 2025",
   "model_id": "gpt-oss-20b",
   "notes": "Maker (OpenAI) gpt-oss model card Table 3; MXFP4 weights as released",
   "retrieved": "2026-10-22",
   "score": 37.1,
   "setting": "low reasoning, no tools",
   "source_url": "https://arxiv.org/html/2508.10925",
   "unit": "percent"
  },
  {
   "benchmark": "AIME 2025",
   "model_id": "gpt-oss-20b",
   "notes": "Maker (OpenAI) gpt-oss model card Table 3; MXFP4 weights as released",
   "retrieved": "2026-10-22",
   "score": 72.1,
   "setting": "medium reasoning, no tools",
   "source_url": "https://arxiv.org/html/2508.10925",
   "unit": "percent"
  },
  {
   "benchmark": "AIME 2025",
   "model_id": "gpt-oss-20b",
   "notes": "Maker (OpenAI) gpt-oss model card Table 3; MXFP4 weights as released",
   "retrieved": "2026-10-22",
   "score": 91.7,
   "setting": "high reasoning, no tools",
   "source_url": "https://arxiv.org/html/2508.10925",
   "unit": "percent"
  },
  {
   "benchmark": "Codeforces Elo",
   "model_id": "gpt-oss-20b",
   "notes": "Maker (OpenAI) gpt-oss model card Table 3; MXFP4 weights as released",
   "retrieved": "2026-10-22",
   "score": 1366.0,
   "setting": "low reasoning, no tools",
   "source_url": "https://arxiv.org/html/2508.10925",
   "unit": "elo"
  },
  {
   "benchmark": "Codeforces Elo",
   "model_id": "gpt-oss-20b",
   "notes": "Maker (OpenAI) gpt-oss model card Table 3; MXFP4 weights as released",
   "retrieved": "2026-10-22",
   "score": 1998.0,
   "setting": "medium reasoning, no tools",
   "source_url": "https://arxiv.org/html/2508.10925",
   "unit": "elo"
  },
  {
   "benchmark": "Codeforces Elo",
   "model_id": "gpt-oss-20b",
   "notes": "Maker (OpenAI) gpt-oss model card Table 3; MXFP4 weights as released",
   "retrieved": "2026-10-22",
   "score": 2230.0,
   "setting": "high reasoning, no tools",
   "source_url": "https://arxiv.org/html/2508.10925",
   "unit": "elo"
  },
  {
   "benchmark": "AIME 2024",
   "model_id": "deepseek-r1-14b",
   "notes": "Maker (DeepSeek) model card distilled-model table; pass@1 from 64 samples, temperature 0.6, top-p 0.95, max 32768 tokens",
   "retrieved": "2026-10-22",
   "score": 69.7,
   "setting": "pass@1",
   "source_url": "https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Qwen-14B",
   "unit": "percent"
  },
  {
   "benchmark": "MATH-500",
   "model_id": "deepseek-r1-14b",
   "notes": "Maker (DeepSeek) model card distilled-model table; pass@1 from 64 samples, temperature 0.6, top-p 0.95, max 32768 tokens",
   "retrieved": "2026-10-22",
   "score": 93.9,
   "setting": "pass@1",
   "source_url": "https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Qwen-14B",
   "unit": "percent"
  },
  {
   "benchmark": "GPQA Diamond",
   "model_id": "deepseek-r1-14b",
   "notes": "Maker (DeepSeek) model card distilled-model table; pass@1 from 64 samples, temperature 0.6, top-p 0.95, max 32768 tokens",
   "retrieved": "2026-10-22",
   "score": 59.1,
   "setting": "pass@1",
   "source_url": "https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Qwen-14B",
   "unit": "percent"
  },
  {
   "benchmark": "LiveCodeBench",
   "model_id": "deepseek-r1-14b",
   "notes": "Maker (DeepSeek) model card distilled-model table; pass@1 from 64 samples, temperature 0.6, top-p 0.95, max 32768 tokens",
   "retrieved": "2026-10-22",
   "score": 53.1,
   "setting": "pass@1",
   "source_url": "https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Qwen-14B",
   "unit": "percent"
  },
  {
   "benchmark": "CodeForces rating",
   "model_id": "deepseek-r1-14b",
   "notes": "Maker (DeepSeek) model card distilled-model table; pass@1 from 64 samples, temperature 0.6, top-p 0.95, max 32768 tokens",
   "retrieved": "2026-10-22",
   "score": 1481.0,
   "setting": "",
   "source_url": "https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Qwen-14B",
   "unit": "elo"
  },
  {
   "benchmark": "MMLU-Redux",
   "model_id": "deepseek-r1-14b",
   "notes": "NOT the maker: measured by the Qwen team and printed in the Qwen3 tech report Table 17 as a comparison column; DeepSeek publishes no MMLU for this model",
   "retrieved": "2026-10-22",
   "score": 84.1,
   "setting": "thinking mode",
   "source_url": "https://arxiv.org/html/2505.09388",
   "unit": "percent"
  },
  {
   "benchmark": "IFEval strict prompt",
   "model_id": "deepseek-r1-14b",
   "notes": "NOT the maker: measured by the Qwen team and printed in the Qwen3 tech report Table 17 as a comparison column; DeepSeek publishes no IFEval for this model",
   "retrieved": "2026-10-22",
   "score": 72.6,
   "setting": "thinking mode",
   "source_url": "https://arxiv.org/html/2505.09388",
   "unit": "percent"
  }
 ]
}