{
  "generatedBy": "bench.mjs",
  "startedAt": "2026-10-03T21:30:37.075Z",
  "finishedAt": "2026-10-03T21:30:40.994Z",
  "machine": {
    "cpu": "Apple M1",
    "os": "Darwin 26.3",
    "arch": "arm64",
    "node": "v22.18.0",
    "python": "Python 3.13.5"
  },
  "metricCount": 58,
  "metrics": [
    {
      "metric": "ollama_version",
      "value": "0.35.1",
      "unit": "",
      "note": "official ollama-darwin.tgz"
    },
    {
      "metric": "suite",
      "value": "5 Python tasks: parse log line, fix off-by-one, parse duration with edge cases, refactor invoice (400 randomized equivalence checks), merge intervals",
      "unit": "",
      "note": "tasks/ folder; tests written before any model ran"
    },
    {
      "metric": "sampling",
      "value": "temperature 0, seed 42, n=1 per task per configuration, default 4k context",
      "unit": ""
    },
    {
      "metric": "t4_test_change",
      "value": "t4 originally required exact float equality with the original function; that flagged 1-cent rounding differences from reordered arithmetic (e.g. total *= 0.9). Default test now allows < 0.011; strict results are reported separately",
      "unit": ""
    },
    {
      "metric": "qwen3.5:4b_think=false_passed",
      "value": 4,
      "unit": "of 5",
      "note": "tasks whose extracted code passed all assertions (t4 with 1-cent tolerance)",
      "n": 5
    },
    {
      "metric": "qwen3.5:4b_think=false_passed_strict",
      "value": 3,
      "unit": "of 5",
      "note": "same, with t4 requiring exact float equality",
      "n": 5
    },
    {
      "metric": "qwen3.5:4b_think=false_t1_parse_log",
      "value": "PASS",
      "unit": "",
      "note": "456 tokens",
      "n": 1
    },
    {
      "metric": "qwen3.5:4b_think=false_t2_fix_off_by_one",
      "value": "PASS",
      "unit": "",
      "note": "89 tokens",
      "n": 1
    },
    {
      "metric": "qwen3.5:4b_think=false_t3_parse_duration",
      "value": "FAIL",
      "unit": "",
      "note": "AssertionError: expected ValueError for '5s1m'",
      "n": 1
    },
    {
      "metric": "qwen3.5:4b_think=false_t4_refactor_invoice",
      "value": "PASS",
      "unit": "",
      "note": "191 tokens",
      "n": 1
    },
    {
      "metric": "qwen3.5:4b_think=false_t5_merge_intervals",
      "value": "PASS",
      "unit": "",
      "note": "180 tokens",
      "n": 1
    },
    {
      "metric": "qwen3.5:4b_think=false_eval_tps_median",
      "value": 3.48,
      "unit": "tok/s",
      "note": "per-task eval rate; min 2.968, max 3.589",
      "n": 5
    },
    {
      "metric": "qwen3.5:4b_think=false_eval_tokens_total",
      "value": 1177,
      "unit": "tokens",
      "note": "sum over the 5 tasks",
      "n": 5
    },
    {
      "metric": "qwen3.5:4b_think=false_wall_total_s",
      "value": 397.8,
      "unit": "s",
      "note": "sum of request wall times over the 5 tasks",
      "n": 5
    },
    {
      "metric": "qwen3.5:2b_think=false_passed",
      "value": 2,
      "unit": "of 5",
      "note": "tasks whose extracted code passed all assertions (t4 with 1-cent tolerance)",
      "n": 5
    },
    {
      "metric": "qwen3.5:2b_think=false_passed_strict",
      "value": 2,
      "unit": "of 5",
      "note": "same, with t4 requiring exact float equality",
      "n": 5
    },
    {
      "metric": "qwen3.5:2b_think=false_t1_parse_log",
      "value": "FAIL",
      "unit": "",
      "note": "IndexError: no such group",
      "n": 1
    },
    {
      "metric": "qwen3.5:2b_think=false_t2_fix_off_by_one",
      "value": "PASS",
      "unit": "",
      "note": "90 tokens",
      "n": 1
    },
    {
      "metric": "qwen3.5:2b_think=false_t3_parse_duration",
      "value": "FAIL",
      "unit": "",
      "note": "AssertionError",
      "n": 1
    },
    {
      "metric": "qwen3.5:2b_think=false_t4_refactor_invoice",
      "value": "FAIL",
      "unit": "",
      "note": "AssertionError: ([{'type': 'book', 'price': 45.37, 'qty': 4}, {'type': 'food', 'price': 39.96, 'qty': 4}, {'type': 'toy', 'price': 62.29, 'qty': 3}, {'type': 'toy', 'price': 29.25, 'qty': 1}], 0.05, '",
      "n": 1
    },
    {
      "metric": "qwen3.5:2b_think=false_t5_merge_intervals",
      "value": "PASS",
      "unit": "",
      "note": "170 tokens",
      "n": 1
    },
    {
      "metric": "qwen3.5:2b_think=false_eval_tps_median",
      "value": 26.81,
      "unit": "tok/s",
      "note": "per-task eval rate; min 26.452, max 27.175",
      "n": 5
    },
    {
      "metric": "qwen3.5:2b_think=false_eval_tokens_total",
      "value": 1483,
      "unit": "tokens",
      "note": "sum over the 5 tasks",
      "n": 5
    },
    {
      "metric": "qwen3.5:2b_think=false_wall_total_s",
      "value": 64.9,
      "unit": "s",
      "note": "sum of request wall times over the 5 tasks",
      "n": 5
    },
    {
      "metric": "qwen3.5:4b_think=false_session2_passed",
      "value": 4,
      "unit": "of 5",
      "note": "tasks whose extracted code passed all assertions (t4 with 1-cent tolerance)",
      "n": 5
    },
    {
      "metric": "qwen3.5:4b_think=false_session2_passed_strict",
      "value": 3,
      "unit": "of 5",
      "note": "same, with t4 requiring exact float equality",
      "n": 5
    },
    {
      "metric": "qwen3.5:4b_think=false_session2_t1_parse_log",
      "value": "PASS",
      "unit": "",
      "note": "456 tokens",
      "n": 1
    },
    {
      "metric": "qwen3.5:4b_think=false_session2_t2_fix_off_by_one",
      "value": "PASS",
      "unit": "",
      "note": "89 tokens",
      "n": 1
    },
    {
      "metric": "qwen3.5:4b_think=false_session2_t3_parse_duration",
      "value": "FAIL",
      "unit": "",
      "note": "AssertionError: expected ValueError for '5s1m'",
      "n": 1
    },
    {
      "metric": "qwen3.5:4b_think=false_session2_t4_refactor_invoice",
      "value": "PASS",
      "unit": "",
      "note": "191 tokens",
      "n": 1
    },
    {
      "metric": "qwen3.5:4b_think=false_session2_t5_merge_intervals",
      "value": "PASS",
      "unit": "",
      "note": "180 tokens",
      "n": 1
    },
    {
      "metric": "qwen3.5:4b_think=false_session2_eval_tps_median",
      "value": 18.35,
      "unit": "tok/s",
      "note": "per-task eval rate; min 17.501, max 18.529",
      "n": 5
    },
    {
      "metric": "qwen3.5:4b_think=false_session2_eval_tokens_total",
      "value": 1177,
      "unit": "tokens",
      "note": "sum over the 5 tasks",
      "n": 5
    },
    {
      "metric": "qwen3.5:4b_think=false_session2_wall_total_s",
      "value": 79.9,
      "unit": "s",
      "note": "sum of request wall times over the 5 tasks",
      "n": 5
    },
    {
      "metric": "qwen3.5:4b_think=default_passed",
      "value": 2,
      "unit": "of 5",
      "note": "tasks whose extracted code passed all assertions (t4 with 1-cent tolerance)",
      "n": 5
    },
    {
      "metric": "qwen3.5:4b_think=default_passed_strict",
      "value": 2,
      "unit": "of 5",
      "note": "same, with t4 requiring exact float equality",
      "n": 5
    },
    {
      "metric": "qwen3.5:4b_think=default_t1_parse_log",
      "value": "PASS",
      "unit": "",
      "note": "1165 tokens",
      "n": 1
    },
    {
      "metric": "qwen3.5:4b_think=default_t2_fix_off_by_one",
      "value": "PASS",
      "unit": "",
      "note": "3316 tokens",
      "n": 1
    },
    {
      "metric": "qwen3.5:4b_think=default_t3_parse_duration",
      "value": "FAIL",
      "unit": "",
      "note": "AttributeError: module 'solution' has no attribute 'parse_duration'",
      "n": 1
    },
    {
      "metric": "qwen3.5:4b_think=default_t4_refactor_invoice",
      "value": "FAIL",
      "unit": "",
      "note": "AttributeError: module 'solution' has no attribute 'invoice_total'",
      "n": 1
    },
    {
      "metric": "qwen3.5:4b_think=default_t5_merge_intervals",
      "value": "FAIL",
      "unit": "",
      "note": "AttributeError: module 'solution' has no attribute 'merge_intervals'",
      "n": 1
    },
    {
      "metric": "qwen3.5:4b_think=default_eval_tps_median",
      "value": 13.7,
      "unit": "tok/s",
      "note": "per-task eval rate; min 12.092, max 17.102",
      "n": 5
    },
    {
      "metric": "qwen3.5:4b_think=default_eval_tokens_total",
      "value": 15883,
      "unit": "tokens",
      "note": "sum over the 5 tasks",
      "n": 5
    },
    {
      "metric": "qwen3.5:4b_think=default_wall_total_s",
      "value": 1180.6,
      "unit": "s",
      "note": "sum of request wall times over the 5 tasks",
      "n": 5
    },
    {
      "metric": "qwen3.5:4b_think=default_hit_token_cap",
      "value": 3,
      "unit": "tasks",
      "note": "done_reason=length (answer cut off at num_predict)",
      "n": 5
    },
    {
      "metric": "qwen2.5-coder:7b_think=omit_passed",
      "value": 2,
      "unit": "of 5",
      "note": "tasks whose extracted code passed all assertions (t4 with 1-cent tolerance)",
      "n": 5
    },
    {
      "metric": "qwen2.5-coder:7b_think=omit_passed_strict",
      "value": 1,
      "unit": "of 5",
      "note": "same, with t4 requiring exact float equality",
      "n": 5
    },
    {
      "metric": "qwen2.5-coder:7b_think=omit_t1_parse_log",
      "value": "FAIL",
      "unit": "",
      "note": "ValueError: not enough values to unpack (expected 2, got 1)",
      "n": 1
    },
    {
      "metric": "qwen2.5-coder:7b_think=omit_t2_fix_off_by_one",
      "value": "PASS",
      "unit": "",
      "note": "80 tokens",
      "n": 1
    },
    {
      "metric": "qwen2.5-coder:7b_think=omit_t3_parse_duration",
      "value": "FAIL",
      "unit": "",
      "note": "ValueError: invalid literal for int() with base 10: '45s'",
      "n": 1
    },
    {
      "metric": "qwen2.5-coder:7b_think=omit_t4_refactor_invoice",
      "value": "PASS",
      "unit": "",
      "note": "171 tokens",
      "n": 1
    },
    {
      "metric": "qwen2.5-coder:7b_think=omit_t5_merge_intervals",
      "value": "FAIL",
      "unit": "",
      "note": "AssertionError: input was modified",
      "n": 1
    },
    {
      "metric": "qwen2.5-coder:7b_think=omit_eval_tps_median",
      "value": 11.29,
      "unit": "tok/s",
      "note": "per-task eval rate; min 10.841, max 11.853",
      "n": 5
    },
    {
      "metric": "qwen2.5-coder:7b_think=omit_eval_tokens_total",
      "value": 762,
      "unit": "tokens",
      "note": "sum over the 5 tasks",
      "n": 5
    },
    {
      "metric": "qwen2.5-coder:7b_think=omit_wall_total_s",
      "value": 92.1,
      "unit": "s",
      "note": "sum of request wall times over the 5 tasks",
      "n": 5
    },
    {
      "metric": "qwen2.5-coder:7b_ollama_ps",
      "value": "5.2 GB, 12%/88% CPU/GPU, context 4096",
      "unit": "",
      "note": "`ollama ps` after the suite (SIZE, PROCESSOR, CONTEXT)"
    },
    {
      "metric": "qwen3.5:4b_think=false_session_note",
      "value": "first qwen3.5:4b run happened in a slower session (3.48 tok/s median); the session-2 repeat produced identical tokens and pass/fail at 18.35 tok/s",
      "unit": ""
    },
    {
      "metric": "qwen3.5:9b_eval_tps",
      "value": "0.079, 0.086",
      "unit": "tok/s",
      "note": "two 10-token requests on this machine (58%/42% CPU/GPU split, heavy swap); the 5-task suite was not run on 9b at this measured rate",
      "n": 2
    }
  ],
  "log": [
    ""
  ]
}