{
 "schema": "nuvec-parity-record",
 "schema_version": 1,
 "created": "2026-10-31T13:14:24+00:00",
 "environment": {
  "machine": "Mac16,7",
  "chip": "Apple M4 Pro",
  "cpu_cores": {
   "performance": 10,
   "efficiency": 4
  },
  "gpu_cores": 20,
  "memory_bytes": 51539607552,
  "macos": {
   "version": "26.6.2",
   "build": "25G83"
  },
  "power": {
   "source": "ac",
   "low_power_mode": false
  },
  "swap_used_bytes_at_start": 16320624066,
  "nuvec": {
   "version": "0.5.0",
   "commit": "a31e0eec1d80efd3f634af8532acc800d2a53300",
   "dirty": false
  },
  "rust": "rustc 1.99.0 (b940084d7 2026-09-28)",
  "mlx": "0.32.4",
  "mlx_lm": "0.32.2",
  "python": "3.13.12"
 },
 "settings": {
  "tier": "full",
  "passes": 2,
  "runs_per_pass": 2,
  "max_tokens": 128,
  "prompt_tokens": [
   2048,
   32768
  ],
  "legacy_repeats": [
   1,
   40,
   160
  ],
  "extra_prompt_tokens": {
   "mlx-community/Qwen3.6-35B-A3B-4bit": [
    32768,
    65536,
    261952
   ]
  },
  "concurrency": {
   "levels": [
    1,
    4,
    8
   ],
   "prompt_tokens": 512,
   "max_tokens": 128,
   "eos": "ignored"
  },
  "load": {
   "samples": 3,
   "first_token_prompt_tokens": 16,
   "cold_method": "sudo -n purge"
  },
  "memory": {
   "method": "peak_with_mapped-vs-get_peak_memory+ru_maxrss",
   "note": "nuvec counts mapped files whole; both engines copy unaligned (mlx-community) checkpoints once"
  },
  "gates": {
   "gate": 0.98,
   "goal": 1.0
  }
 },
 "sections": [
  "decode",
  "ttft",
  "memory",
  "load",
  "concurrency"
 ],
 "models": [
  {
   "id": "mlx-community/Qwen3-0.6B-4bit",
   "kind": "dense",
   "snapshot": "73e3e38d981303bc594367cd910ea6eb48349da8",
   "decode": [
    {
     "repeats": 1,
     "prompt_tokens": {
      "nuvec": 25,
      "mlx_lm": 25
     },
     "tok_s": {
      "nuvec": {
       "median": 441.145,
       "samples": [
        438.17,
        438.85,
        443.67,
        443.44
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 400.3561452019685,
       "samples": [
        399.9584294884603,
        400.1766133610635,
        400.53567704287343,
        400.8833638490799
       ],
       "unit": "tok/s"
      },
      "ratio": 1.1019,
      "gate_pass": true,
      "goal_pass": true
     }
    },
    {
     "repeats": 40,
     "prompt_tokens": {
      "nuvec": 961,
      "mlx_lm": 961
     },
     "tok_s": {
      "nuvec": {
       "median": 351.93,
       "samples": [
        351.04,
        352.82,
        354.88,
        342.18
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 319.45981067014156,
       "samples": [
        325.47981634056833,
        315.8060620515409,
        314.7011268631312,
        323.1135592887423
       ],
       "unit": "tok/s"
      },
      "ratio": 1.1016,
      "gate_pass": true,
      "goal_pass": true
     }
    },
    {
     "repeats": 160,
     "prompt_tokens": {
      "nuvec": 3841,
      "mlx_lm": 3841
     },
     "tok_s": {
      "nuvec": {
       "median": 234.20999999999998,
       "samples": [
        233.66,
        234.1,
        234.32,
        235.0
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 220.64988738575653,
       "samples": [
        221.0234967377267,
        220.59658481465962,
        219.7597152074993,
        220.70318995685344
       ],
       "unit": "tok/s"
      },
      "ratio": 1.0615,
      "gate_pass": true,
      "goal_pass": true
     }
    }
   ],
   "ttft": {
    "legacy": [
     {
      "repeats": 1,
      "prompt_tokens": {
       "nuvec": 25,
       "mlx_lm": 25
      },
      "prefill_tok_s": {
       "nuvec": {
        "median": 2032.94,
        "samples": [
         2035.17,
         2030.71,
         2026.35,
         2035.4
        ],
        "unit": "tok/s"
       },
       "mlx_lm": {
        "median": 1669.558257186709,
        "samples": [
         1659.5123992427295,
         1666.7871184044716,
         1672.329395968946,
         1713.1501404927435
        ],
        "unit": "tok/s"
       },
       "ratio": 1.2177,
       "gate_pass": true,
       "goal_pass": true
      }
     },
     {
      "repeats": 40,
      "prompt_tokens": {
       "nuvec": 961,
       "mlx_lm": 961
      },
      "prefill_tok_s": {
       "nuvec": {
        "median": 6304.855,
        "samples": [
         6247.08,
         6350.32,
         6259.39,
         6355.13
        ],
        "unit": "tok/s"
       },
       "mlx_lm": {
        "median": 6005.2184243844895,
        "samples": [
         5982.520891572746,
         5992.268054870892,
         6030.650576788858,
         6018.168793898088
        ],
        "unit": "tok/s"
       },
       "ratio": 1.0499,
       "gate_pass": true,
       "goal_pass": true
      }
     },
     {
      "repeats": 160,
      "prompt_tokens": {
       "nuvec": 3841,
       "mlx_lm": 3841
      },
      "prefill_tok_s": {
       "nuvec": {
        "median": 4922.4,
        "samples": [
         4863.43,
         4929.19,
         4915.61,
         4932.32
        ],
        "unit": "tok/s"
       },
       "mlx_lm": {
        "median": 4800.915933669456,
        "samples": [
         4799.585647611485,
         4802.669290855627,
         4795.827214289081,
         4802.246219727426
        ],
        "unit": "tok/s"
       },
       "ratio": 1.0253,
       "gate_pass": true,
       "goal_pass": true
      }
     }
    ],
    "lengths": [
     {
      "prompt_tokens": 2048,
      "ttft_s": {
       "nuvec": {
        "median": 0.3623,
        "samples": [
         0.3623,
         0.3641,
         0.3623,
         0.3608
        ],
        "unit": "s"
       },
       "mlx_lm": {
        "median": 0.36724331250297837,
        "samples": [
         0.368129292037338,
         0.3668104999815114,
         0.36762837501009926,
         0.3668582499958575
        ],
        "unit": "s"
       },
       "ratio": 1.0136,
       "gate_pass": true,
       "goal_pass": true
      },
      "prefill_tok_s": {
       "nuvec": {
        "median": 5652.585,
        "samples": [
         5653.01,
         5625.29,
         5652.16,
         5676.94
        ],
        "unit": "tok/s"
       },
       "mlx_lm": {
        "median": 5576.690389839541,
        "samples": [
         5563.26281091557,
         5583.264383389316,
         5570.843110093824,
         5582.537669585257
        ],
        "unit": "tok/s"
       },
       "ratio": 1.0136,
       "gate_pass": true,
       "goal_pass": true
      }
     },
     {
      "prompt_tokens": 32768,
      "ttft_s": {
       "nuvec": {
        "median": 22.997549999999997,
        "samples": [
         23.045,
         23.0036,
         22.9837,
         22.9915
        ],
        "unit": "s"
       },
       "mlx_lm": {
        "median": 23.22463816648815,
        "samples": [
         23.217144459020346,
         23.226737540971953,
         23.232797875010874,
         23.222538792004343
        ],
        "unit": "s"
       },
       "ratio": 1.0099,
       "gate_pass": true,
       "goal_pass": true
      },
      "prefill_tok_s": {
       "nuvec": {
        "median": 1424.845,
        "samples": [
         1421.91,
         1424.47,
         1425.71,
         1425.22
        ],
        "unit": "tok/s"
       },
       "mlx_lm": {
        "median": 1410.915426662442,
        "samples": [
         1411.3708108177336,
         1410.78788797597,
         1410.419880390091,
         1411.042965348914
        ],
        "unit": "tok/s"
       },
       "ratio": 1.0099,
       "gate_pass": true,
       "goal_pass": true
      }
     }
    ]
   },
   "memory": {
    "load": {
     "peak_bytes": {
      "nuvec": {
       "median": 335462400.0,
       "samples": [
        335462400,
        335462400
       ],
       "unit": "bytes"
      },
      "mlx_lm": {
       "median": 335380488.0,
       "samples": [
        335380488,
        335380488
       ],
       "unit": "bytes"
      },
      "ratio": 0.9998
     },
     "os_peak_rss_bytes": {
      "nuvec": {
       "median": 97034240.0,
       "samples": [
        97026048,
        97042432
       ],
       "unit": "bytes"
      },
      "mlx_lm": {
       "median": 698048512.0,
       "samples": [
        697614336,
        698482688
       ],
       "unit": "bytes"
      },
      "ratio": 7.1938
     },
     "nuvec_raw": {
      "peak_bytes": 0,
      "active_bytes": 0,
      "mapped_bytes": 335462400
     }
    },
    "decode": {
     "peak_bytes": {
      "nuvec": {
       "median": 370245888.0,
       "samples": [
        370262272,
        370229504
       ],
       "unit": "bytes"
      },
      "mlx_lm": {
       "median": 459276552.0,
       "samples": [
        454918408,
        463634696
       ],
       "unit": "bytes"
      },
      "ratio": 1.2405
     },
     "os_peak_rss_bytes": {
      "nuvec": {
       "median": 198156288.0,
       "samples": [
        197967872,
        198344704
       ],
       "unit": "bytes"
      },
      "mlx_lm": {
       "median": 697909248.0,
       "samples": [
        697827328,
        697991168
       ],
       "unit": "bytes"
      },
      "ratio": 3.522
     },
     "nuvec_raw": {
      "peak_bytes": 34767104,
      "active_bytes": 0,
      "mapped_bytes": 335462400
     }
    },
    "prefill_2048": {
     "peak_bytes": {
      "nuvec": {
       "median": 2034165760.0,
       "samples": [
        2034165760,
        2034165760
       ],
       "unit": "bytes"
      },
      "mlx_lm": {
       "median": 1459462292.0,
       "samples": [
        1459462292,
        1459462292
       ],
       "unit": "bytes"
      },
      "ratio": 0.7175
     },
     "os_peak_rss_bytes": {
      "nuvec": {
       "median": 194568192.0,
       "samples": [
        194396160,
        194740224
       ],
       "unit": "bytes"
      },
      "mlx_lm": {
       "median": 699064320.0,
       "samples": [
        699990016,
        698138624
       ],
       "unit": "bytes"
      },
      "ratio": 3.5929
     },
     "nuvec_raw": {
      "peak_bytes": 1698703360,
      "active_bytes": 0,
      "mapped_bytes": 335462400
     }
    },
    "prefill_32768": {
     "peak_bytes": {
      "nuvec": {
       "median": 4273924096.0,
       "samples": [
        4273924096,
        4273924096
       ],
       "unit": "bytes"
      },
      "mlx_lm": {
       "median": 4739530808.0,
       "samples": [
        4739530808,
        4739530808
       ],
       "unit": "bytes"
      },
      "ratio": 1.1089
     },
     "os_peak_rss_bytes": {
      "nuvec": {
       "median": 194428928.0,
       "samples": [
        194281472,
        194576384
       ],
       "unit": "bytes"
      },
      "mlx_lm": {
       "median": 698777600.0,
       "samples": [
        699858944,
        697696256
       ],
       "unit": "bytes"
      },
      "ratio": 3.594
     },
     "nuvec_raw": {
      "peak_bytes": 3938461696,
      "active_bytes": 0,
      "mapped_bytes": 335462400
     }
    }
   },
   "load": {
    "warm": {
     "model_ready_s": {
      "nuvec": {
       "median": 0.0625,
       "samples": [
        0.0625,
        0.0625,
        0.0637
       ],
       "unit": "s"
      },
      "mlx_lm": {
       "median": 0.3007255839766003,
       "samples": [
        0.303773750027176,
        0.3007255839766003,
        0.2996969160158187
       ],
       "unit": "s"
      },
      "ratio": 4.8116,
      "gate_pass": true,
      "goal_pass": true
     },
     "first_token_s": {
      "nuvec": {
       "median": 0.1262,
       "samples": [
        0.1228,
        0.1267,
        0.1262
       ],
       "unit": "s"
      },
      "mlx_lm": {
       "median": 0.3422044990584254,
       "samples": [
        0.3456928749801591,
        0.34103062498616055,
        0.3422044990584254
       ],
       "unit": "s"
      },
      "ratio": 2.7116,
      "gate_pass": true,
      "goal_pass": true
     }
    },
    "cold": {
     "model_ready_s": {
      "nuvec": null,
      "mlx_lm": null,
      "ratio": null,
      "reason": "purge unavailable (needs passwordless sudo)"
     },
     "first_token_s": {
      "nuvec": null,
      "mlx_lm": null,
      "ratio": null,
      "reason": "purge unavailable (needs passwordless sudo)"
     }
    }
   },
   "concurrency": [
    {
     "level": 1,
     "total_tokens": 128,
     "wall_s": {
      "nuvec": {
       "median": 0.351,
       "samples": [
        0.351
       ],
       "unit": "s"
      },
      "mlx_lm": {
       "median": 0.3711368329823017,
       "samples": [
        0.3711368329823017
       ],
       "unit": "s"
      },
      "ratio": 1.0574,
      "gate_pass": true,
      "goal_pass": true
     },
     "aggregate_tok_s": {
      "nuvec": {
       "median": 364.6723646723647,
       "samples": [
        364.6723646723647
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 344.88627542420153,
       "samples": [
        344.88627542420153
       ],
       "unit": "tok/s"
      },
      "ratio": 1.0574,
      "gate_pass": true,
      "goal_pass": true
     },
     "decode_aggregate_tok_s": {
      "nuvec": {
       "median": 414.8970924534466,
       "samples": [
        414.8970924534466
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 399.30111863360185,
       "samples": [
        399.30111863360185
       ],
       "unit": "tok/s"
      },
      "ratio": 1.0391,
      "gate_pass": true,
      "goal_pass": true
     },
     "nuvec_matches_solo": true
    },
    {
     "level": 4,
     "total_tokens": 512,
     "wall_s": {
      "nuvec": {
       "median": 0.666,
       "samples": [
        0.666
       ],
       "unit": "s"
      },
      "mlx_lm": {
       "median": 0.6525763329700567,
       "samples": [
        0.6525763329700567
       ],
       "unit": "s"
      },
      "ratio": 0.9798,
      "gate_pass": false,
      "goal_pass": false
     },
     "aggregate_tok_s": {
      "nuvec": {
       "median": 768.7687687687687,
       "samples": [
        768.7687687687687
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 784.5825448032192,
       "samples": [
        784.5825448032192
       ],
       "unit": "tok/s"
      },
      "ratio": 0.9798,
      "gate_pass": false,
      "goal_pass": false
     },
     "decode_aggregate_tok_s": {
      "nuvec": {
       "median": 967.6190476190476,
       "samples": [
        967.6190476190476
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 996.2119921793435,
       "samples": [
        996.2119921793435
       ],
       "unit": "tok/s"
      },
      "ratio": 0.9713,
      "gate_pass": false,
      "goal_pass": false
     },
     "nuvec_matches_solo": false
    },
    {
     "level": 8,
     "total_tokens": 1024,
     "wall_s": {
      "nuvec": {
       "median": 1.0981,
       "samples": [
        1.0981
       ],
       "unit": "s"
      },
      "mlx_lm": {
       "median": 1.0875222089816816,
       "samples": [
        1.0875222089816816
       ],
       "unit": "s"
      },
      "ratio": 0.9904,
      "gate_pass": true,
      "goal_pass": false
     },
     "aggregate_tok_s": {
      "nuvec": {
       "median": 932.5198069392586,
       "samples": [
        932.5198069392586
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 941.5899662029324,
       "samples": [
        941.5899662029324
       ],
       "unit": "tok/s"
      },
      "ratio": 0.9904,
      "gate_pass": true,
      "goal_pass": false
     },
     "decode_aggregate_tok_s": {
      "nuvec": {
       "median": 1217.6414189837008,
       "samples": [
        1217.6414189837008
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 1219.161779739398,
       "samples": [
        1219.161779739398
       ],
       "unit": "tok/s"
      },
      "ratio": 0.9988,
      "gate_pass": true,
      "goal_pass": false
     },
     "nuvec_matches_solo": false
    }
   ],
   "status": "ok"
  },
  {
   "id": "mlx-community/Llama-3.2-3B-Instruct-4bit",
   "kind": "dense",
   "snapshot": "7f0dc925e0d0afb0322d96f9255cfddf2ba5636e",
   "decode": [
    {
     "repeats": 1,
     "prompt_tokens": {
      "nuvec": 26,
      "mlx_lm": 26
     },
     "tok_s": {
      "nuvec": {
       "median": 117.995,
       "samples": [
        118.7,
        117.48,
        117.47,
        118.51
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 118.65020525362385,
       "samples": [
        118.53996564017311,
        118.47608499293572,
        118.87304137357879,
        118.76044486707458
       ],
       "unit": "tok/s"
      },
      "ratio": 0.9945,
      "gate_pass": true,
      "goal_pass": false
     }
    },
    {
     "repeats": 40,
     "prompt_tokens": {
      "nuvec": 962,
      "mlx_lm": 962
     },
     "tok_s": {
      "nuvec": {
       "median": 109.685,
       "samples": [
        109.91,
        109.8,
        109.57,
        109.3
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 109.84598629533897,
       "samples": [
        109.84378332001177,
        110.1571944323582,
        109.833218900953,
        109.84818927066617
       ],
       "unit": "tok/s"
      },
      "ratio": 0.9985,
      "gate_pass": true,
      "goal_pass": false
     }
    },
    {
     "repeats": 160,
     "prompt_tokens": {
      "nuvec": 3842,
      "mlx_lm": 3842
     },
     "tok_s": {
      "nuvec": {
       "median": 95.14,
       "samples": [
        95.0,
        95.16,
        96.28,
        95.12
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 95.2502916584483,
       "samples": [
        95.41907965188425,
        95.08150366501235,
        96.03914764951041,
        94.61543795773748
       ],
       "unit": "tok/s"
      },
      "ratio": 0.9988,
      "gate_pass": true,
      "goal_pass": false
     }
    }
   ],
   "ttft": {
    "legacy": [
     {
      "repeats": 1,
      "prompt_tokens": {
       "nuvec": 26,
       "mlx_lm": 26
      },
      "prefill_tok_s": {
       "nuvec": {
        "median": 594.46,
        "samples": [
         603.98,
         596.4,
         592.52,
         572.92
        ],
        "unit": "tok/s"
       },
       "mlx_lm": {
        "median": 524.3179841215838,
        "samples": [
         541.8063510842615,
         501.6294949589973,
         547.347299168334,
         506.8296171589061
        ],
        "unit": "tok/s"
       },
       "ratio": 1.1338,
       "gate_pass": true,
       "goal_pass": true
      }
     },
     {
      "repeats": 40,
      "prompt_tokens": {
       "nuvec": 962,
       "mlx_lm": 962
      },
      "prefill_tok_s": {
       "nuvec": {
        "median": 1128.185,
        "samples": [
         1126.21,
         1130.16,
         1126.06,
         1130.19
        ],
        "unit": "tok/s"
       },
       "mlx_lm": {
        "median": 1117.4936028106226,
        "samples": [
         1118.232773738867,
         1120.491844695677,
         1115.9805837733147,
         1116.754431882378
        ],
        "unit": "tok/s"
       },
       "ratio": 1.0096,
       "gate_pass": true,
       "goal_pass": true
      }
     },
     {
      "repeats": 160,
      "prompt_tokens": {
       "nuvec": 3842,
       "mlx_lm": 3842
      },
      "prefill_tok_s": {
       "nuvec": {
        "median": 1075.47,
        "samples": [
         1074.4,
         1076.54,
         1074.35,
         1076.69
        ],
        "unit": "tok/s"
       },
       "mlx_lm": {
        "median": 1074.022727174719,
        "samples": [
         1074.8436347887591,
         1073.1644107797877,
         1074.8720393422832,
         1073.201819560679
        ],
        "unit": "tok/s"
       },
       "ratio": 1.0013,
       "gate_pass": true,
       "goal_pass": true
      }
     }
    ],
    "lengths": [
     {
      "prompt_tokens": 2048,
      "ttft_s": {
       "nuvec": {
        "median": 1.80955,
        "samples": [
         1.8181,
         1.8073,
         1.8114,
         1.8077
        ],
        "unit": "s"
       },
       "mlx_lm": {
        "median": 1.8054569789965171,
        "samples": [
         1.806790209026076,
         1.8039151669945568,
         1.806389041012153,
         1.8045249169808812
        ],
        "unit": "s"
       },
       "ratio": 0.9977,
       "gate_pass": true,
       "goal_pass": false
      },
      "prefill_tok_s": {
       "nuvec": {
        "median": 1131.78,
        "samples": [
         1126.45,
         1133.16,
         1130.61,
         1132.95
        ],
        "unit": "tok/s"
       },
       "mlx_lm": {
        "median": 1134.339156036387,
        "samples": [
         1133.5018253745932,
         1135.308376730434,
         1133.7535566826002,
         1134.9247553901737
        ],
        "unit": "tok/s"
       },
       "ratio": 0.9977,
       "gate_pass": true,
       "goal_pass": false
      }
     },
     {
      "prompt_tokens": 32768,
      "ttft_s": {
       "nuvec": {
        "median": 54.35355,
        "samples": [
         53.9257,
         53.9009,
         54.7826,
         54.7814
        ],
        "unit": "s"
       },
       "mlx_lm": {
        "median": 54.81749943748582,
        "samples": [
         54.4893907920341,
         54.88545612501912,
         54.91261208301876,
         54.749542749952525
        ],
        "unit": "s"
       },
       "ratio": 1.0085,
       "gate_pass": true,
       "goal_pass": true
      },
      "prefill_tok_s": {
       "nuvec": {
        "median": 602.905,
        "samples": [
         607.65,
         607.93,
         598.15,
         598.16
        ],
        "unit": "tok/s"
       },
       "mlx_lm": {
        "median": 597.7662369715894,
        "samples": [
         601.3647707140527,
         597.0251923453171,
         596.7299452165965,
         598.5072815978616
        ],
        "unit": "tok/s"
       },
       "ratio": 1.0086,
       "gate_pass": true,
       "goal_pass": true
      }
     }
    ]
   },
   "memory": {
    "load": {
     "peak_bytes": {
      "nuvec": {
       "median": 1807440896.0,
       "samples": [
        1807440896,
        1807440896
       ],
       "unit": "bytes"
      },
      "mlx_lm": {
       "median": 1807441224.0,
       "samples": [
        1807441224,
        1807441224
       ],
       "unit": "bytes"
      },
      "ratio": 1.0
     },
     "os_peak_rss_bytes": {
      "nuvec": {
       "median": 1925390336.0,
       "samples": [
        1925283840,
        1925496832
       ],
       "unit": "bytes"
      },
      "mlx_lm": {
       "median": 2260590592.0,
       "samples": [
        2259910656,
        2261270528
       ],
       "unit": "bytes"
      },
      "ratio": 1.1741
     },
     "nuvec_raw": {
      "peak_bytes": 1807440896,
      "active_bytes": 1807440896,
      "mapped_bytes": 0
     }
    },
    "decode": {
     "peak_bytes": {
      "nuvec": {
       "median": 1856252928.0,
       "samples": [
        1856252928,
        1856252928
       ],
       "unit": "bytes"
      },
      "mlx_lm": {
       "median": 1993438572.0,
       "samples": [
        1994995052,
        1991882092
       ],
       "unit": "bytes"
      },
      "ratio": 1.0739
     },
     "os_peak_rss_bytes": {
      "nuvec": {
       "median": 2042380288.0,
       "samples": [
        2042626048,
        2042134528
       ],
       "unit": "bytes"
      },
      "mlx_lm": {
       "median": 2254798848.0,
       "samples": [
        2259435520,
        2250162176
       ],
       "unit": "bytes"
      },
      "ratio": 1.104
     },
     "nuvec_raw": {
      "peak_bytes": 1856252928,
      "active_bytes": 1807440896,
      "mapped_bytes": 0
     }
    },
    "prefill_2048": {
     "peak_bytes": {
      "nuvec": {
       "median": 2323303680.0,
       "samples": [
        2323303680,
        2323303680
       ],
       "unit": "bytes"
      },
      "mlx_lm": {
       "median": 2809730112.0,
       "samples": [
        2809730112,
        2809730112
       ],
       "unit": "bytes"
      },
      "ratio": 1.2094
     },
     "os_peak_rss_bytes": {
      "nuvec": {
       "median": 2038374400.0,
       "samples": [
        2038398976,
        2038349824
       ],
       "unit": "bytes"
      },
      "mlx_lm": {
       "median": 2260959232.0,
       "samples": [
        2259091456,
        2262827008
       ],
       "unit": "bytes"
      },
      "ratio": 1.1092
     },
     "nuvec_raw": {
      "peak_bytes": 2323303680,
      "active_bytes": 1807440896,
      "mapped_bytes": 0
     }
    },
    "prefill_32768": {
     "peak_bytes": {
      "nuvec": {
       "median": 5741661440.0,
       "samples": [
        5741661440,
        5741661440
       ],
       "unit": "bytes"
      },
      "mlx_lm": {
       "median": 6186365996.0,
       "samples": [
        6186365996,
        6186365996
       ],
       "unit": "bytes"
      },
      "ratio": 1.0775
     },
     "os_peak_rss_bytes": {
      "nuvec": {
       "median": 2038497280.0,
       "samples": [
        2038431744,
        2038562816
       ],
       "unit": "bytes"
      },
      "mlx_lm": {
       "median": 2261106688.0,
       "samples": [
        2262286336,
        2259927040
       ],
       "unit": "bytes"
      },
      "ratio": 1.1092
     },
     "nuvec_raw": {
      "peak_bytes": 5741661440,
      "active_bytes": 1807440896,
      "mapped_bytes": 0
     }
    }
   },
   "load": {
    "warm": {
     "model_ready_s": {
      "nuvec": {
       "median": 0.1928,
       "samples": [
        0.1928,
        0.1984,
        0.1913
       ],
       "unit": "s"
      },
      "mlx_lm": {
       "median": 0.5859965419513173,
       "samples": [
        0.5859965419513173,
        0.584845083008986,
        0.5931792499613948
       ],
       "unit": "s"
      },
      "ratio": 3.0394,
      "gate_pass": true,
      "goal_pass": true
     },
     "first_token_s": {
      "nuvec": {
       "median": 0.2756,
       "samples": [
        0.2756,
        0.2811,
        0.274
       ],
       "unit": "s"
      },
      "mlx_lm": {
       "median": 0.6577345009427518,
       "samples": [
        0.6577345009427518,
        0.6567266240017489,
        0.665286915958859
       ],
       "unit": "s"
      },
      "ratio": 2.3866,
      "gate_pass": true,
      "goal_pass": true
     }
    },
    "cold": {
     "model_ready_s": {
      "nuvec": null,
      "mlx_lm": null,
      "ratio": null,
      "reason": "purge unavailable (needs passwordless sudo)"
     },
     "first_token_s": {
      "nuvec": null,
      "mlx_lm": null,
      "ratio": null,
      "reason": "purge unavailable (needs passwordless sudo)"
     }
    }
   },
   "concurrency": [
    {
     "level": 1,
     "total_tokens": 128,
     "wall_s": {
      "nuvec": {
       "median": 1.3043,
       "samples": [
        1.3043
       ],
       "unit": "s"
      },
      "mlx_lm": {
       "median": 1.3063696669996716,
       "samples": [
        1.3063696669996716
       ],
       "unit": "s"
      },
      "ratio": 1.0016,
      "gate_pass": true,
      "goal_pass": true
     },
     "aggregate_tok_s": {
      "nuvec": {
       "median": 98.1369316874952,
       "samples": [
        98.1369316874952
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 97.9814544331671,
       "samples": [
        97.9814544331671
       ],
       "unit": "tok/s"
      },
      "ratio": 1.0016,
      "gate_pass": true,
      "goal_pass": true
     },
     "decode_aggregate_tok_s": {
      "nuvec": {
       "median": 115.80195130847086,
       "samples": [
        115.80195130847086
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 117.6419186509911,
       "samples": [
        117.6419186509911
       ],
       "unit": "tok/s"
      },
      "ratio": 0.9844,
      "gate_pass": true,
      "goal_pass": false
     },
     "nuvec_matches_solo": true
    },
    {
     "level": 4,
     "total_tokens": 512,
     "wall_s": {
      "nuvec": {
       "median": 2.3106,
       "samples": [
        2.3106
       ],
       "unit": "s"
      },
      "mlx_lm": {
       "median": 2.300536915950943,
       "samples": [
        2.300536915950943
       ],
       "unit": "s"
      },
      "ratio": 0.9956,
      "gate_pass": true,
      "goal_pass": false
     },
     "aggregate_tok_s": {
      "nuvec": {
       "median": 221.58746645892842,
       "samples": [
        221.58746645892842
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 222.55674162410094,
       "samples": [
        222.55674162410094
       ],
       "unit": "tok/s"
      },
      "ratio": 0.9956,
      "gate_pass": true,
      "goal_pass": false
     },
     "decode_aggregate_tok_s": {
      "nuvec": {
       "median": 315.5475495372383,
       "samples": [
        315.5475495372383
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 319.64284749151767,
       "samples": [
        319.64284749151767
       ],
       "unit": "tok/s"
      },
      "ratio": 0.9872,
      "gate_pass": true,
      "goal_pass": false
     },
     "nuvec_matches_solo": true
    },
    {
     "level": 8,
     "total_tokens": 1024,
     "wall_s": {
      "nuvec": {
       "median": 4.311,
       "samples": [
        4.311
       ],
       "unit": "s"
      },
      "mlx_lm": {
       "median": 4.297167084005196,
       "samples": [
        4.297167084005196
       ],
       "unit": "s"
      },
      "ratio": 0.9968,
      "gate_pass": true,
      "goal_pass": false
     },
     "aggregate_tok_s": {
      "nuvec": {
       "median": 237.5318951519369,
       "samples": [
        237.5318951519369
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 238.29652884839092,
       "samples": [
        238.29652884839092
       ],
       "unit": "tok/s"
      },
      "ratio": 0.9968,
      "gate_pass": true,
      "goal_pass": false
     },
     "decode_aggregate_tok_s": {
      "nuvec": {
       "median": 346.63937222790855,
       "samples": [
        346.63937222790855
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 347.3680035814935,
       "samples": [
        347.3680035814935
       ],
       "unit": "tok/s"
      },
      "ratio": 0.9979,
      "gate_pass": true,
      "goal_pass": false
     },
     "nuvec_matches_solo": true
    }
   ],
   "status": "ok"
  },
  {
   "id": "mlx-community/Meta-Llama-3.1-8B-Instruct-4bit",
   "kind": "dense",
   "snapshot": "241a666dad6cb93c8ff213d39a7f34a36bf26db4",
   "decode": [
    {
     "repeats": 1,
     "prompt_tokens": {
      "nuvec": 25,
      "mlx_lm": 25
     },
     "tok_s": {
      "nuvec": {
       "median": 56.09,
       "samples": [
        55.64,
        55.71,
        56.59,
        56.47
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 55.35630256843134,
       "samples": [
        55.30541106842751,
        55.225639133685796,
        56.13670924249033,
        55.40719406843517
       ],
       "unit": "tok/s"
      },
      "ratio": 1.0133,
      "gate_pass": true,
      "goal_pass": true
     }
    },
    {
     "repeats": 40,
     "prompt_tokens": {
      "nuvec": 961,
      "mlx_lm": 961
     },
     "tok_s": {
      "nuvec": {
       "median": 53.365,
       "samples": [
        53.35,
        53.29,
        53.38,
        53.82
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 53.11183055656337,
       "samples": [
        53.116973918060424,
        53.13676912763363,
        53.106687195066314,
        53.1033137514393
       ],
       "unit": "tok/s"
      },
      "ratio": 1.0048,
      "gate_pass": true,
      "goal_pass": true
     }
    },
    {
     "repeats": 160,
     "prompt_tokens": {
      "nuvec": 3841,
      "mlx_lm": 3841
     },
     "tok_s": {
      "nuvec": {
       "median": 49.345,
       "samples": [
        49.22,
        49.32,
        49.37,
        49.39
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 49.26182391087178,
       "samples": [
        49.27672423433053,
        49.246923587413036,
        49.08368505172382,
        49.29637192253882
       ],
       "unit": "tok/s"
      },
      "ratio": 1.0017,
      "gate_pass": true,
      "goal_pass": true
     }
    }
   ],
   "ttft": {
    "legacy": [
     {
      "repeats": 1,
      "prompt_tokens": {
       "nuvec": 25,
       "mlx_lm": 25
      },
      "prefill_tok_s": {
       "nuvec": {
        "median": 265.815,
        "samples": [
         265.51,
         275.57,
         258.27,
         266.12
        ],
        "unit": "tok/s"
       },
       "mlx_lm": {
        "median": 234.03126804166482,
        "samples": [
         241.2418171388245,
         241.39895513069854,
         226.8207189445051,
         221.39812911970608
        ],
        "unit": "tok/s"
       },
       "ratio": 1.1358,
       "gate_pass": true,
       "goal_pass": true
      }
     },
     {
      "repeats": 40,
      "prompt_tokens": {
       "nuvec": 961,
       "mlx_lm": 961
      },
      "prefill_tok_s": {
       "nuvec": {
        "median": 482.825,
        "samples": [
         482.63,
         483.05,
         482.12,
         483.02
        ],
        "unit": "tok/s"
       },
       "mlx_lm": {
        "median": 479.84291394932507,
        "samples": [
         479.6675968437137,
         479.72538335421746,
         479.9962739093712,
         479.9604445444326
        ],
        "unit": "tok/s"
       },
       "ratio": 1.0062,
       "gate_pass": true,
       "goal_pass": true
      }
     },
     {
      "repeats": 160,
      "prompt_tokens": {
       "nuvec": 3841,
       "mlx_lm": 3841
      },
      "prefill_tok_s": {
       "nuvec": {
        "median": 461.525,
        "samples": [
         460.81,
         461.78,
         461.27,
         461.83
        ],
        "unit": "tok/s"
       },
       "mlx_lm": {
        "median": 461.10350164780016,
        "samples": [
         461.13433524018126,
         461.0726680554191,
         460.2737319527774,
         461.15455936641814
        ],
        "unit": "tok/s"
       },
       "ratio": 1.0009,
       "gate_pass": true,
       "goal_pass": true
      }
     }
    ],
    "lengths": [
     {
      "prompt_tokens": 2048,
      "ttft_s": {
       "nuvec": {
        "median": 4.384,
        "samples": [
         4.3841,
         4.3856,
         4.3838,
         4.3839
        ],
        "unit": "s"
       },
       "mlx_lm": {
        "median": 4.386385104473447,
        "samples": [
         4.387875082960818,
         4.385347959003411,
         4.38620520895347,
         4.386564999993425
        ],
        "unit": "s"
       },
       "ratio": 1.0005,
       "gate_pass": true,
       "goal_pass": true
      },
      "prefill_tok_s": {
       "nuvec": {
        "median": 467.155,
        "samples": [
         467.15,
         466.98,
         467.18,
         467.16
        ],
        "unit": "tok/s"
       },
       "mlx_lm": {
        "median": 466.899270051801,
        "samples": [
         466.7407255855756,
         467.00969208049264,
         466.9184186411206,
         466.88012146248144
        ],
        "unit": "tok/s"
       },
       "ratio": 1.0005,
       "gate_pass": true,
       "goal_pass": true
      }
     },
     {
      "prompt_tokens": 32768,
      "ttft_s": {
       "nuvec": {
        "median": 109.14685,
        "samples": [
         108.6796,
         108.703,
         109.5907,
         109.9632
        ],
        "unit": "s"
       },
       "mlx_lm": {
        "median": 109.6925437914906,
        "samples": [
         109.43077662499854,
         108.99807045800844,
         110.89987579203444,
         109.95431095798267
        ],
        "unit": "s"
       },
       "ratio": 1.005,
       "gate_pass": true,
       "goal_pass": true
      },
      "prefill_tok_s": {
       "nuvec": {
        "median": 300.225,
        "samples": [
         301.51,
         301.45,
         299.0,
         297.99
        ],
        "unit": "tok/s"
       },
       "mlx_lm": {
        "median": 298.72756592779507,
        "samples": [
         299.4404408943437,
         300.6291750148356,
         295.47373038945835,
         298.01469096124646
        ],
        "unit": "tok/s"
       },
       "ratio": 1.005,
       "gate_pass": true,
       "goal_pass": true
      }
     }
    ]
   },
   "memory": {
    "load": {
     "peak_bytes": {
      "nuvec": {
       "median": 4517421056.0,
       "samples": [
        4517421056,
        4517421056
       ],
       "unit": "bytes"
      },
      "mlx_lm": {
       "median": 4517406216.0,
       "samples": [
        4517406216,
        4517406216
       ],
       "unit": "bytes"
      },
      "ratio": 1.0
     },
     "os_peak_rss_bytes": {
      "nuvec": {
       "median": 4605460480.0,
       "samples": [
        4605247488,
        4605673472
       ],
       "unit": "bytes"
      },
      "mlx_lm": {
       "median": 4930412544.0,
       "samples": [
        4928503808,
        4932321280
       ],
       "unit": "bytes"
      },
      "ratio": 1.0706
     },
     "nuvec_raw": {
      "peak_bytes": 4517421056,
      "active_bytes": 4517421056,
      "mapped_bytes": 0
     }
    },
    "decode": {
     "peak_bytes": {
      "nuvec": {
       "median": 4583191552.0,
       "samples": [
        4583191552,
        4583191552
       ],
       "unit": "bytes"
      },
      "mlx_lm": {
       "median": 4662444376.0,
       "samples": [
        4662444376,
        4662444376
       ],
       "unit": "bytes"
      },
      "ratio": 1.0173
     },
     "os_peak_rss_bytes": {
      "nuvec": {
       "median": 4722663424.0,
       "samples": [
        4722671616,
        4722655232
       ],
       "unit": "bytes"
      },
      "mlx_lm": {
       "median": 4926029824.0,
       "samples": [
        4923359232,
        4928700416
       ],
       "unit": "bytes"
      },
      "ratio": 1.0431
     },
     "nuvec_raw": {
      "peak_bytes": 4583191552,
      "active_bytes": 4517421056,
      "mapped_bytes": 0
     }
    },
    "prefill_2048": {
     "peak_bytes": {
      "nuvec": {
       "median": 5175904000.0,
       "samples": [
        5175904000,
        5175904000
       ],
       "unit": "bytes"
      },
      "mlx_lm": {
       "median": 5536538676.0,
       "samples": [
        5536538676,
        5536538676
       ],
       "unit": "bytes"
      },
      "ratio": 1.0697
     },
     "os_peak_rss_bytes": {
      "nuvec": {
       "median": 4718559232.0,
       "samples": [
        4718542848,
        4718575616
       ],
       "unit": "bytes"
      },
      "mlx_lm": {
       "median": 4939169792.0,
       "samples": [
        4950228992,
        4928110592
       ],
       "unit": "bytes"
      },
      "ratio": 1.0468
     },
     "nuvec_raw": {
      "peak_bytes": 5175904000,
      "active_bytes": 4517421056,
      "mapped_bytes": 0
     }
    },
    "prefill_32768": {
     "peak_bytes": {
      "nuvec": {
       "median": 9089175808.0,
       "samples": [
        9089175808,
        9089175808
       ],
       "unit": "bytes"
      },
      "mlx_lm": {
       "median": 9533866024.0,
       "samples": [
        9533866024,
        9533866024
       ],
       "unit": "bytes"
      },
      "ratio": 1.0489
     },
     "os_peak_rss_bytes": {
      "nuvec": {
       "median": 4718788608.0,
       "samples": [
        4718854144,
        4718723072
       ],
       "unit": "bytes"
      },
      "mlx_lm": {
       "median": 4932009984.0,
       "samples": [
        4928700416,
        4935319552
       ],
       "unit": "bytes"
      },
      "ratio": 1.0452
     },
     "nuvec_raw": {
      "peak_bytes": 9089175808,
      "active_bytes": 4517421056,
      "mapped_bytes": 0
     }
    }
   },
   "load": {
    "warm": {
     "model_ready_s": {
      "nuvec": {
       "median": 0.305,
       "samples": [
        0.3137,
        0.305,
        0.3042
       ],
       "unit": "s"
      },
      "mlx_lm": {
       "median": 0.5797772499499843,
       "samples": [
        0.581799375009723,
        0.5797772499499843,
        0.5755162499845028
       ],
       "unit": "s"
      },
      "ratio": 1.9009,
      "gate_pass": true,
      "goal_pass": true
     },
     "first_token_s": {
      "nuvec": {
       "median": 0.4368,
       "samples": [
        0.4458,
        0.4368,
        0.4368
       ],
       "unit": "s"
      },
      "mlx_lm": {
       "median": 0.7269022919354029,
       "samples": [
        0.7282022500294261,
        0.7269022919354029,
        0.7219190000323579
       ],
       "unit": "s"
      },
      "ratio": 1.6642,
      "gate_pass": true,
      "goal_pass": true
     }
    },
    "cold": {
     "model_ready_s": {
      "nuvec": null,
      "mlx_lm": null,
      "ratio": null,
      "reason": "purge unavailable (needs passwordless sudo)"
     },
     "first_token_s": {
      "nuvec": null,
      "mlx_lm": null,
      "ratio": null,
      "reason": "purge unavailable (needs passwordless sudo)"
     }
    }
   },
   "concurrency": [
    {
     "level": 1,
     "total_tokens": 128,
     "wall_s": {
      "nuvec": {
       "median": 2.8,
       "samples": [
        2.8
       ],
       "unit": "s"
      },
      "mlx_lm": {
       "median": 2.8205434170085937,
       "samples": [
        2.8205434170085937
       ],
       "unit": "s"
      },
      "ratio": 1.0073,
      "gate_pass": true,
      "goal_pass": true
     },
     "aggregate_tok_s": {
      "nuvec": {
       "median": 45.714285714285715,
       "samples": [
        45.714285714285715
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 45.38132589207011,
       "samples": [
        45.38132589207011
       ],
       "unit": "tok/s"
      },
      "ratio": 1.0073,
      "gate_pass": true,
      "goal_pass": true
     },
     "decode_aggregate_tok_s": {
      "nuvec": {
       "median": 54.999783465419426,
       "samples": [
        54.999783465419426
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 54.95557677234104,
       "samples": [
        54.95557677234104
       ],
       "unit": "tok/s"
      },
      "ratio": 1.0008,
      "gate_pass": true,
      "goal_pass": true
     },
     "nuvec_matches_solo": true
    },
    {
     "level": 4,
     "total_tokens": 512,
     "wall_s": {
      "nuvec": {
       "median": 5.19,
       "samples": [
        5.19
       ],
       "unit": "s"
      },
      "mlx_lm": {
       "median": 5.228358583000954,
       "samples": [
        5.228358583000954
       ],
       "unit": "s"
      },
      "ratio": 1.0074,
      "gate_pass": true,
      "goal_pass": true
     },
     "aggregate_tok_s": {
      "nuvec": {
       "median": 98.65125240847783,
       "samples": [
        98.65125240847783
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 97.92748371633762,
       "samples": [
        97.92748371633762
       ],
       "unit": "tok/s"
      },
      "ratio": 1.0074,
      "gate_pass": true,
      "goal_pass": true
     },
     "decode_aggregate_tok_s": {
      "nuvec": {
       "median": 144.77058991165575,
       "samples": [
        144.77058991165575
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 144.0562119542208,
       "samples": [
        144.0562119542208
       ],
       "unit": "tok/s"
      },
      "ratio": 1.005,
      "gate_pass": true,
      "goal_pass": true
     },
     "nuvec_matches_solo": true
    },
    {
     "level": 8,
     "total_tokens": 1024,
     "wall_s": {
      "nuvec": {
       "median": 9.9304,
       "samples": [
        9.9304
       ],
       "unit": "s"
      },
      "mlx_lm": {
       "median": 9.96507954201661,
       "samples": [
        9.96507954201661
       ],
       "unit": "s"
      },
      "ratio": 1.0035,
      "gate_pass": true,
      "goal_pass": true
     },
     "aggregate_tok_s": {
      "nuvec": {
       "median": 103.11769918633689,
       "samples": [
        103.11769918633689
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 102.75883857047221,
       "samples": [
        102.75883857047221
       ],
       "unit": "tok/s"
      },
      "ratio": 1.0035,
      "gate_pass": true,
      "goal_pass": true
     },
     "decode_aggregate_tok_s": {
      "nuvec": {
       "median": 152.89691497366442,
       "samples": [
        152.89691497366442
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 152.47253784902856,
       "samples": [
        152.47253784902856
       ],
       "unit": "tok/s"
      },
      "ratio": 1.0028,
      "gate_pass": true,
      "goal_pass": true
     },
     "nuvec_matches_solo": true
    }
   ],
   "status": "ok"
  },
  {
   "id": "mlx-community/Qwen3-8B-4bit",
   "kind": "dense",
   "snapshot": "545dc4251c05440727734bcd94334791f6ab0192",
   "decode": [
    {
     "repeats": 1,
     "prompt_tokens": {
      "nuvec": 25,
      "mlx_lm": 25
     },
     "tok_s": {
      "nuvec": {
       "median": 54.765,
       "samples": [
        54.31,
        54.57,
        54.96,
        55.19
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 54.83623677211648,
       "samples": [
        54.91815203637082,
        54.26048600055222,
        54.91919203127383,
        54.754321507862144
       ],
       "unit": "tok/s"
      },
      "ratio": 0.9987,
      "gate_pass": true,
      "goal_pass": false
     }
    },
    {
     "repeats": 40,
     "prompt_tokens": {
      "nuvec": 961,
      "mlx_lm": 961
     },
     "tok_s": {
      "nuvec": {
       "median": 52.129999999999995,
       "samples": [
        52.05,
        52.45,
        52.14,
        52.12
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 51.77837288095046,
       "samples": [
        51.93037187307667,
        51.62637388882423,
        51.50790227481795,
        52.081141393494526
       ],
       "unit": "tok/s"
      },
      "ratio": 1.0068,
      "gate_pass": true,
      "goal_pass": true
     }
    },
    {
     "repeats": 160,
     "prompt_tokens": {
      "nuvec": 3841,
      "mlx_lm": 3841
     },
     "tok_s": {
      "nuvec": {
       "median": 47.91,
       "samples": [
        47.91,
        48.0,
        47.88,
        47.91
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 47.39469490858919,
       "samples": [
        47.42031245007466,
        47.354606675150535,
        47.37533420886809,
        47.414055608310285
       ],
       "unit": "tok/s"
      },
      "ratio": 1.0109,
      "gate_pass": true,
      "goal_pass": true
     }
    }
   ],
   "ttft": {
    "legacy": [
     {
      "repeats": 1,
      "prompt_tokens": {
       "nuvec": 25,
       "mlx_lm": 25
      },
      "prefill_tok_s": {
       "nuvec": {
        "median": 264.66999999999996,
        "samples": [
         262.09,
         270.01,
         267.25,
         252.23
        ],
        "unit": "tok/s"
       },
       "mlx_lm": {
        "median": 224.2976132278152,
        "samples": [
         224.72440355818617,
         222.06899456374117,
         223.8708228974442,
         229.32808027816426
        ],
        "unit": "tok/s"
       },
       "ratio": 1.18,
       "gate_pass": true,
       "goal_pass": true
      }
     },
     {
      "repeats": 40,
      "prompt_tokens": {
       "nuvec": 961,
       "mlx_lm": 961
      },
      "prefill_tok_s": {
       "nuvec": {
        "median": 480.925,
        "samples": [
         480.68,
         481.17,
         478.9,
         481.2
        ],
        "unit": "tok/s"
       },
       "mlx_lm": {
        "median": 477.0709769017615,
        "samples": [
         477.1658728839761,
         476.5481352339605,
         476.976080919547,
         477.9930162574437
        ],
        "unit": "tok/s"
       },
       "ratio": 1.0081,
       "gate_pass": true,
       "goal_pass": true
      }
     },
     {
      "repeats": 160,
      "prompt_tokens": {
       "nuvec": 3841,
       "mlx_lm": 3841
      },
      "prefill_tok_s": {
       "nuvec": {
        "median": 456.99,
        "samples": [
         456.55,
         457.27,
         456.75,
         457.23
        ],
        "unit": "tok/s"
       },
       "mlx_lm": {
        "median": 456.2689404037468,
        "samples": [
         456.1720224211901,
         456.32504407896914,
         456.2371409613491,
         456.3007398461445
        ],
        "unit": "tok/s"
       },
       "ratio": 1.0016,
       "gate_pass": true,
       "goal_pass": true
      }
     }
    ],
    "lengths": [
     {
      "prompt_tokens": 2048,
      "ttft_s": {
       "nuvec": {
        "median": 4.407299999999999,
        "samples": [
         4.4079,
         4.4071,
         4.4075,
         4.407
        ],
        "unit": "s"
       },
       "mlx_lm": {
        "median": 4.412655708496459,
        "samples": [
         4.419791415974032,
         4.412773041985929,
         4.4097213330096565,
         4.412538375006989
        ],
        "unit": "s"
       },
       "ratio": 1.0012,
       "gate_pass": true,
       "goal_pass": true
      },
      "prefill_tok_s": {
       "nuvec": {
        "median": 464.68,
        "samples": [
         464.62,
         464.7,
         464.66,
         464.72
        ],
        "unit": "tok/s"
       },
       "mlx_lm": {
        "median": 464.1195997921715,
        "samples": [
         463.3702831762848,
         464.10725874954943,
         464.4284401078537,
         464.1319408347936
        ],
        "unit": "tok/s"
       },
       "ratio": 1.0012,
       "gate_pass": true,
       "goal_pass": true
      }
     },
     {
      "prompt_tokens": 32768,
      "ttft_s": {
       "nuvec": {
        "median": 114.22895,
        "samples": [
         114.2078,
         114.2237,
         114.2446,
         114.2342
        ],
        "unit": "s"
       },
       "mlx_lm": {
        "median": 114.95514516651747,
        "samples": [
         115.27778995898552,
         114.66225358302472,
         115.23855970799923,
         114.6717306250357
        ],
        "unit": "s"
       },
       "ratio": 1.0064,
       "gate_pass": true,
       "goal_pass": true
      },
      "prefill_tok_s": {
       "nuvec": {
        "median": 286.865,
        "samples": [
         286.92,
         286.88,
         286.82,
         286.85
        ],
        "unit": "tok/s"
       },
       "mlx_lm": {
        "median": 285.05204469020623,
        "samples": [
         284.25250008400116,
         285.7784403851205,
         284.34926714660617,
         285.7548222338063
        ],
        "unit": "tok/s"
       },
       "ratio": 1.0064,
       "gate_pass": true,
       "goal_pass": true
      }
     }
    ]
   },
   "memory": {
    "load": {
     "peak_bytes": {
      "nuvec": {
       "median": 4607737856.0,
       "samples": [
        4607737856,
        4607737856
       ],
       "unit": "bytes"
      },
      "mlx_lm": {
       "median": 4607731720.0,
       "samples": [
        4607731720,
        4607731720
       ],
       "unit": "bytes"
      },
      "ratio": 1.0
     },
     "os_peak_rss_bytes": {
      "nuvec": {
       "median": 4692303872.0,
       "samples": [
        4692180992,
        4692426752
       ],
       "unit": "bytes"
      },
      "mlx_lm": {
       "median": 4973248512.0,
       "samples": [
        4973232128,
        4973264896
       ],
       "unit": "bytes"
      },
      "ratio": 1.0599
     },
     "nuvec_raw": {
      "peak_bytes": 4607737856,
      "active_bytes": 4607737856,
      "mapped_bytes": 0
     }
    },
    "decode": {
     "peak_bytes": {
      "nuvec": {
       "median": 4675469568.0,
       "samples": [
        4675469568,
        4675469568
       ],
       "unit": "bytes"
      },
      "mlx_lm": {
       "median": 4773456216.0,
       "samples": [
        4773456216,
        4773456216
       ],
       "unit": "bytes"
      },
      "ratio": 1.021
     },
     "os_peak_rss_bytes": {
      "nuvec": {
       "median": 4816109568.0,
       "samples": [
        4816306176,
        4815912960
       ],
       "unit": "bytes"
      },
      "mlx_lm": {
       "median": 4976148480.0,
       "samples": [
        4976623616,
        4975673344
       ],
       "unit": "bytes"
      },
      "ratio": 1.0332
     },
     "nuvec_raw": {
      "peak_bytes": 4675469568,
      "active_bytes": 4607737856,
      "mapped_bytes": 0
     }
    },
    "prefill_2048": {
     "peak_bytes": {
      "nuvec": {
       "median": 5257834496.0,
       "samples": [
        5257834496,
        5257834496
       ],
       "unit": "bytes"
      },
      "mlx_lm": {
       "median": 5610080308.0,
       "samples": [
        5610080308,
        5610080308
       ],
       "unit": "bytes"
      },
      "ratio": 1.067
     },
     "os_peak_rss_bytes": {
      "nuvec": {
       "median": 4811186176.0,
       "samples": [
        4811227136,
        4811145216
       ],
       "unit": "bytes"
      },
      "mlx_lm": {
       "median": 4974493696.0,
       "samples": [
        4973592576,
        4975394816
       ],
       "unit": "bytes"
      },
      "ratio": 1.0339
     },
     "nuvec_raw": {
      "peak_bytes": 5257834496,
      "active_bytes": 4607737856,
      "mapped_bytes": 0
     }
    },
    "prefill_32768": {
     "peak_bytes": {
      "nuvec": {
       "median": 9691197440.0,
       "samples": [
        9691197440,
        9691197440
       ],
       "unit": "bytes"
      },
      "mlx_lm": {
       "median": 10127501352.0,
       "samples": [
        10127501352,
        10127501352
       ],
       "unit": "bytes"
      },
      "ratio": 1.045
     },
     "os_peak_rss_bytes": {
      "nuvec": {
       "median": 4813627392.0,
       "samples": [
        4813815808,
        4813438976
       ],
       "unit": "bytes"
      },
      "mlx_lm": {
       "median": 4973043712.0,
       "samples": [
        4973182976,
        4972904448
       ],
       "unit": "bytes"
      },
      "ratio": 1.0331
     },
     "nuvec_raw": {
      "peak_bytes": 9691197440,
      "active_bytes": 4607737856,
      "mapped_bytes": 0
     }
    }
   },
   "load": {
    "warm": {
     "model_ready_s": {
      "nuvec": {
       "median": 0.3066,
       "samples": [
        0.3066,
        0.3066,
        0.3028
       ],
       "unit": "s"
      },
      "mlx_lm": {
       "median": 0.43580233299871907,
       "samples": [
        0.43580233299871907,
        0.4225023750332184,
        0.43922991602448747
       ],
       "unit": "s"
      },
      "ratio": 1.4214,
      "gate_pass": true,
      "goal_pass": true
     },
     "first_token_s": {
      "nuvec": {
       "median": 0.451,
       "samples": [
        0.4532,
        0.451,
        0.4435
       ],
       "unit": "s"
      },
      "mlx_lm": {
       "median": 0.592327042017132,
       "samples": [
        0.592327042017132,
        0.592222583014518,
        0.6036570410360582
       ],
       "unit": "s"
      },
      "ratio": 1.3134,
      "gate_pass": true,
      "goal_pass": true
     }
    },
    "cold": {
     "model_ready_s": {
      "nuvec": null,
      "mlx_lm": null,
      "ratio": null,
      "reason": "purge unavailable (needs passwordless sudo)"
     },
     "first_token_s": {
      "nuvec": null,
      "mlx_lm": null,
      "ratio": null,
      "reason": "purge unavailable (needs passwordless sudo)"
     }
    }
   },
   "concurrency": [
    {
     "level": 1,
     "total_tokens": 128,
     "wall_s": {
      "nuvec": {
       "median": 2.8731,
       "samples": [
        2.8731
       ],
       "unit": "s"
      },
      "mlx_lm": {
       "median": 2.861868667008821,
       "samples": [
        2.861868667008821
       ],
       "unit": "s"
      },
      "ratio": 0.9961,
      "gate_pass": true,
      "goal_pass": false
     },
     "aggregate_tok_s": {
      "nuvec": {
       "median": 44.55118165048206,
       "samples": [
        44.55118165048206
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 44.72602166394433,
       "samples": [
        44.72602166394433
       ],
       "unit": "tok/s"
      },
      "ratio": 0.9961,
      "gate_pass": true,
      "goal_pass": false
     },
     "decode_aggregate_tok_s": {
      "nuvec": {
       "median": 53.278516591853,
       "samples": [
        53.278516591853
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 54.137941549274366,
       "samples": [
        54.137941549274366
       ],
       "unit": "tok/s"
      },
      "ratio": 0.9841,
      "gate_pass": true,
      "goal_pass": false
     },
     "nuvec_matches_solo": true
    },
    {
     "level": 4,
     "total_tokens": 512,
     "wall_s": {
      "nuvec": {
       "median": 5.2951,
       "samples": [
        5.2951
       ],
       "unit": "s"
      },
      "mlx_lm": {
       "median": 5.347400666039903,
       "samples": [
        5.347400666039903
       ],
       "unit": "s"
      },
      "ratio": 1.0099,
      "gate_pass": true,
      "goal_pass": true
     },
     "aggregate_tok_s": {
      "nuvec": {
       "median": 96.69316915638987,
       "samples": [
        96.69316915638987
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 95.74745413254571,
       "samples": [
        95.74745413254571
       ],
       "unit": "tok/s"
      },
      "ratio": 1.0099,
      "gate_pass": true,
      "goal_pass": true
     },
     "decode_aggregate_tok_s": {
      "nuvec": {
       "median": 140.7202216066482,
       "samples": [
        140.7202216066482
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 139.57919233208636,
       "samples": [
        139.57919233208636
       ],
       "unit": "tok/s"
      },
      "ratio": 1.0082,
      "gate_pass": true,
      "goal_pass": true
     },
     "nuvec_matches_solo": true
    },
    {
     "level": 8,
     "total_tokens": 1024,
     "wall_s": {
      "nuvec": {
       "median": 10.0611,
       "samples": [
        10.0611
       ],
       "unit": "s"
      },
      "mlx_lm": {
       "median": 10.14591650001239,
       "samples": [
        10.14591650001239
       ],
       "unit": "s"
      },
      "ratio": 1.0084,
      "gate_pass": true,
      "goal_pass": true
     },
     "aggregate_tok_s": {
      "nuvec": {
       "median": 101.77813559153572,
       "samples": [
        101.77813559153572
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 100.92730410296097,
       "samples": [
        100.92730410296097
       ],
       "unit": "tok/s"
      },
      "ratio": 1.0084,
      "gate_pass": true,
      "goal_pass": true
     },
     "decode_aggregate_tok_s": {
      "nuvec": {
       "median": 150.19587552664646,
       "samples": [
        150.19587552664646
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 148.81556595441984,
       "samples": [
        148.81556595441984
       ],
       "unit": "tok/s"
      },
      "ratio": 1.0093,
      "gate_pass": true,
      "goal_pass": true
     },
     "nuvec_matches_solo": false
    }
   ],
   "status": "ok"
  },
  {
   "id": "mlx-community/Qwen3.5-9B-4bit",
   "kind": "hybrid",
   "snapshot": "8b2b98c00a6b4d291155e4890773ca8f769aee53",
   "decode": [
    {
     "repeats": 1,
     "prompt_tokens": {
      "nuvec": 25,
      "mlx_lm": 25
     },
     "tok_s": {
      "nuvec": {
       "median": 50.495,
       "samples": [
        50.46,
        50.48,
        50.51,
        50.51
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 50.53126445378399,
       "samples": [
        50.610521796923365,
        50.49153292668953,
        50.57099598087844,
        50.39512140175304
       ],
       "unit": "tok/s"
      },
      "ratio": 0.9993,
      "gate_pass": true,
      "goal_pass": false
     }
    },
    {
     "repeats": 40,
     "prompt_tokens": {
      "nuvec": 961,
      "mlx_lm": 961
     },
     "tok_s": {
      "nuvec": {
       "median": 49.92,
       "samples": [
        49.87,
        49.94,
        49.9,
        50.03
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 49.996312201778785,
       "samples": [
        49.996146950202764,
        49.99647745335481,
        50.0290194510897,
        49.78913858837147
       ],
       "unit": "tok/s"
      },
      "ratio": 0.9985,
      "gate_pass": true,
      "goal_pass": false
     }
    },
    {
     "repeats": 160,
     "prompt_tokens": {
      "nuvec": 3841,
      "mlx_lm": 3841
     },
     "tok_s": {
      "nuvec": {
       "median": 49.135,
       "samples": [
        48.74,
        49.17,
        49.11,
        49.16
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 48.90671075476131,
       "samples": [
        48.82058875740089,
        48.82222468408023,
        49.005616084816864,
        48.991196825442394
       ],
       "unit": "tok/s"
      },
      "ratio": 1.0047,
      "gate_pass": true,
      "goal_pass": true
     }
    }
   ],
   "ttft": {
    "legacy": [
     {
      "repeats": 1,
      "prompt_tokens": {
       "nuvec": 25,
       "mlx_lm": 25
      },
      "prefill_tok_s": {
       "nuvec": {
        "median": 248.12,
        "samples": [
         244.34,
         254.05,
         244.31,
         251.9
        ],
        "unit": "tok/s"
       },
       "mlx_lm": {
        "median": 206.16380352819436,
        "samples": [
         206.5014930878409,
         205.8261139685478,
         204.80961295119383,
         212.14531955981516
        ],
        "unit": "tok/s"
       },
       "ratio": 1.2035,
       "gate_pass": true,
       "goal_pass": true
      }
     },
     {
      "repeats": 40,
      "prompt_tokens": {
       "nuvec": 961,
       "mlx_lm": 961
      },
      "prefill_tok_s": {
       "nuvec": {
        "median": 475.46,
        "samples": [
         475.89,
         474.96,
         475.91,
         475.03
        ],
        "unit": "tok/s"
       },
       "mlx_lm": {
        "median": 475.5202810421739,
        "samples": [
         475.4562119242014,
         475.2730068740471,
         475.58435016014647,
         475.7065919889579
        ],
        "unit": "tok/s"
       },
       "ratio": 0.9999,
       "gate_pass": true,
       "goal_pass": false
      }
     },
     {
      "repeats": 160,
      "prompt_tokens": {
       "nuvec": 3841,
       "mlx_lm": 3841
      },
      "prefill_tok_s": {
       "nuvec": {
        "median": 474.21500000000003,
        "samples": [
         473.69,
         474.43,
         474.0,
         474.43
        ],
        "unit": "tok/s"
       },
       "mlx_lm": {
        "median": 474.97704072006417,
        "samples": [
         475.0124460313756,
         474.94163540875275,
         475.15245037525546,
         474.7624224764731
        ],
        "unit": "tok/s"
       },
       "ratio": 0.9984,
       "gate_pass": true,
       "goal_pass": false
      }
     }
    ],
    "lengths": [
     {
      "prompt_tokens": 2048,
      "ttft_s": {
       "nuvec": {
        "median": 4.30855,
        "samples": [
         4.3114,
         4.3057,
         4.3118,
         4.3027
        ],
        "unit": "s"
       },
       "mlx_lm": {
        "median": 4.312862729042536,
        "samples": [
         4.316060167038813,
         4.308444750029594,
         4.3164775829645805,
         4.309665291046258
        ],
        "unit": "s"
       },
       "ratio": 1.001,
       "gate_pass": true,
       "goal_pass": true
      },
      "prefill_tok_s": {
       "nuvec": {
        "median": 475.33,
        "samples": [
         475.02,
         475.64,
         474.97,
         475.98
        ],
        "unit": "tok/s"
       },
       "mlx_lm": {
        "median": 474.8588708512318,
        "samples": [
         474.5068235239879,
         475.34554086737046,
         474.46093733525714,
         475.2109181784757
        ],
        "unit": "tok/s"
       },
       "ratio": 1.001,
       "gate_pass": true,
       "goal_pass": true
      }
     },
     {
      "prompt_tokens": 32768,
      "ttft_s": {
       "nuvec": {
        "median": 78.8515,
        "samples": [
         78.8524,
         78.854,
         78.8506,
         78.849
        ],
        "unit": "s"
       },
       "mlx_lm": {
        "median": 78.1588280625001,
        "samples": [
         78.15650120796636,
         78.15708249999443,
         78.16057362500578,
         78.16202820796752
        ],
        "unit": "s"
       },
       "ratio": 0.9912,
       "gate_pass": true,
       "goal_pass": false
      },
      "prefill_tok_s": {
       "nuvec": {
        "median": 415.565,
        "samples": [
         415.56,
         415.55,
         415.57,
         415.58
        ],
        "unit": "tok/s"
       },
       "mlx_lm": {
        "median": 419.24886578571954,
        "samples": [
         419.26134734214554,
         419.2582290927036,
         419.2395024787355,
         419.2317004980145
        ],
        "unit": "tok/s"
       },
       "ratio": 0.9912,
       "gate_pass": true,
       "goal_pass": false
      }
     }
    ]
   },
   "memory": {
    "load": {
     "peak_bytes": {
      "nuvec": {
       "median": 5038063616.0,
       "samples": [
        5038063616,
        5038063616
       ],
       "unit": "bytes"
      },
      "mlx_lm": {
       "median": 5038041608.0,
       "samples": [
        5038041608,
        5038041608
       ],
       "unit": "bytes"
      },
      "ratio": 1.0
     },
     "os_peak_rss_bytes": {
      "nuvec": {
       "median": 5172232192.0,
       "samples": [
        5172412416,
        5172051968
       ],
       "unit": "bytes"
      },
      "mlx_lm": {
       "median": 5666373632.0,
       "samples": [
        5667356672,
        5665390592
       ],
       "unit": "bytes"
      },
      "ratio": 1.0955
     },
     "nuvec_raw": {
      "peak_bytes": 5038063616,
      "active_bytes": 5038063616,
      "mapped_bytes": 0
     }
    },
    "decode": {
     "peak_bytes": {
      "nuvec": {
       "median": 5153585152.0,
       "samples": [
        5153585152,
        5153585152
       ],
       "unit": "bytes"
      },
      "mlx_lm": {
       "median": 5226481789.0,
       "samples": [
        5226481789,
        5226481789
       ],
       "unit": "bytes"
      },
      "ratio": 1.0141
     },
     "os_peak_rss_bytes": {
      "nuvec": {
       "median": 5346869248.0,
       "samples": [
        5345558528,
        5348179968
       ],
       "unit": "bytes"
      },
      "mlx_lm": {
       "median": 5667381248.0,
       "samples": [
        5667373056,
        5667389440
       ],
       "unit": "bytes"
      },
      "ratio": 1.0599
     },
     "nuvec_raw": {
      "peak_bytes": 5153585152,
      "active_bytes": 5038065152,
      "mapped_bytes": 0
     }
    },
    "prefill_2048": {
     "peak_bytes": {
      "nuvec": {
       "median": 6624513792.0,
       "samples": [
        6624513792,
        6624513792
       ],
       "unit": "bytes"
      },
      "mlx_lm": {
       "median": 6769838664.0,
       "samples": [
        6769838664,
        6769838664
       ],
       "unit": "bytes"
      },
      "ratio": 1.0219
     },
     "os_peak_rss_bytes": {
      "nuvec": {
       "median": 5339013120.0,
       "samples": [
        5338234880,
        5339791360
       ],
       "unit": "bytes"
      },
      "mlx_lm": {
       "median": 5666775040.0,
       "samples": [
        5667405824,
        5666144256
       ],
       "unit": "bytes"
      },
      "ratio": 1.0614
     },
     "nuvec_raw": {
      "peak_bytes": 6624513792,
      "active_bytes": 5038065152,
      "mapped_bytes": 0
     }
    },
    "prefill_32768": {
     "peak_bytes": {
      "nuvec": {
       "median": 9626328320.0,
       "samples": [
        9626328320,
        9626328320
       ],
       "unit": "bytes"
      },
      "mlx_lm": {
       "median": 9791040054.0,
       "samples": [
        9791040054,
        9791040054
       ],
       "unit": "bytes"
      },
      "ratio": 1.0171
     },
     "os_peak_rss_bytes": {
      "nuvec": {
       "median": 5337399296.0,
       "samples": [
        5337759744,
        5337038848
       ],
       "unit": "bytes"
      },
      "mlx_lm": {
       "median": 5667028992.0,
       "samples": [
        5667356672,
        5666701312
       ],
       "unit": "bytes"
      },
      "ratio": 1.0618
     },
     "nuvec_raw": {
      "peak_bytes": 9626328320,
      "active_bytes": 5038065664,
      "mapped_bytes": 0
     }
    }
   },
   "load": {
    "warm": {
     "model_ready_s": {
      "nuvec": {
       "median": 0.382,
       "samples": [
        0.3775,
        0.382,
        0.3925
       ],
       "unit": "s"
      },
      "mlx_lm": {
       "median": 0.8040562079986557,
       "samples": [
        0.8040562079986557,
        0.7956642500357702,
        0.8159460419556126
       ],
       "unit": "s"
      },
      "ratio": 2.1049,
      "gate_pass": true,
      "goal_pass": true
     },
     "first_token_s": {
      "nuvec": {
       "median": 0.5294,
       "samples": [
        0.5237,
        0.5294,
        0.5396
       ],
       "unit": "s"
      },
      "mlx_lm": {
       "median": 0.967931249004323,
       "samples": [
        0.967931249004323,
        0.9590203340630978,
        0.9774604999693111
       ],
       "unit": "s"
      },
      "ratio": 1.8284,
      "gate_pass": true,
      "goal_pass": true
     }
    },
    "cold": {
     "model_ready_s": {
      "nuvec": null,
      "mlx_lm": null,
      "ratio": null,
      "reason": "purge unavailable (needs passwordless sudo)"
     },
     "first_token_s": {
      "nuvec": null,
      "mlx_lm": null,
      "ratio": null,
      "reason": "purge unavailable (needs passwordless sudo)"
     }
    }
   },
   "concurrency": [
    {
     "level": 1,
     "total_tokens": 128,
     "wall_s": {
      "nuvec": {
       "median": 3.0558,
       "samples": [
        3.0558
       ],
       "unit": "s"
      },
      "mlx_lm": {
       "median": 3.057254125014879,
       "samples": [
        3.057254125014879
       ],
       "unit": "s"
      },
      "ratio": 1.0005,
      "gate_pass": true,
      "goal_pass": true
     },
     "aggregate_tok_s": {
      "nuvec": {
       "median": 41.88755808626219,
       "samples": [
        41.88755808626219
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 41.86763506268131,
       "samples": [
        41.86763506268131
       ],
       "unit": "tok/s"
      },
      "ratio": 1.0005,
      "gate_pass": true,
      "goal_pass": true
     },
     "decode_aggregate_tok_s": {
      "nuvec": {
       "median": 49.71424097706098,
       "samples": [
        49.71424097706098
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 50.289672133956735,
       "samples": [
        50.289672133956735
       ],
       "unit": "tok/s"
      },
      "ratio": 0.9886,
      "gate_pass": true,
      "goal_pass": false
     },
     "nuvec_matches_solo": true
    },
    {
     "level": 4,
     "total_tokens": 512,
     "wall_s": {
      "nuvec": {
       "median": 5.6594,
       "samples": [
        5.6594
       ],
       "unit": "s"
      },
      "mlx_lm": {
       "median": 5.666273334005382,
       "samples": [
        5.666273334005382
       ],
       "unit": "s"
      },
      "ratio": 1.0012,
      "gate_pass": true,
      "goal_pass": true
     },
     "aggregate_tok_s": {
      "nuvec": {
       "median": 90.46895430611019,
       "samples": [
        90.46895430611019
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 90.35921315819702,
       "samples": [
        90.35921315819702
       ],
       "unit": "tok/s"
      },
      "ratio": 1.0012,
      "gate_pass": true,
      "goal_pass": true
     },
     "decode_aggregate_tok_s": {
      "nuvec": {
       "median": 128.99949212798373,
       "samples": [
        128.99949212798373
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 129.62153478136634,
       "samples": [
        129.62153478136634
       ],
       "unit": "tok/s"
      },
      "ratio": 0.9952,
      "gate_pass": true,
      "goal_pass": false
     },
     "nuvec_matches_solo": false
    },
    {
     "level": 8,
     "total_tokens": 1024,
     "wall_s": {
      "nuvec": {
       "median": 10.7114,
       "samples": [
        10.7114
       ],
       "unit": "s"
      },
      "mlx_lm": {
       "median": 10.783130125026219,
       "samples": [
        10.783130125026219
       ],
       "unit": "s"
      },
      "ratio": 1.0067,
      "gate_pass": true,
      "goal_pass": true
     },
     "aggregate_tok_s": {
      "nuvec": {
       "median": 95.59908135257763,
       "samples": [
        95.59908135257763
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 94.96314967241575,
       "samples": [
        94.96314967241575
       ],
       "unit": "tok/s"
      },
      "ratio": 1.0067,
      "gate_pass": true,
      "goal_pass": true
     },
     "decode_aggregate_tok_s": {
      "nuvec": {
       "median": 138.22753122364017,
       "samples": [
        138.22753122364017
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 137.85514256854827,
       "samples": [
        137.85514256854827
       ],
       "unit": "tok/s"
      },
      "ratio": 1.0027,
      "gate_pass": true,
      "goal_pass": true
     },
     "nuvec_matches_solo": false
    }
   ],
   "status": "ok"
  },
  {
   "id": "mlx-community/Qwen3.8-27B-4bit",
   "kind": "hybrid",
   "snapshot": "10c35caafbb80f7dc6a7a432cdd11af10a6d4818",
   "decode": [
    {
     "repeats": 1,
     "prompt_tokens": {
      "nuvec": 25,
      "mlx_lm": 25
     },
     "tok_s": {
      "nuvec": {
       "median": 15.565000000000001,
       "samples": [
        15.75,
        15.77,
        15.35,
        15.38
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 15.565636906566318,
       "samples": [
        15.616169634405287,
        15.608235740952445,
        15.523038072180192,
        15.505462195891461
       ],
       "unit": "tok/s"
      },
      "ratio": 1.0,
      "gate_pass": true,
      "goal_pass": false
     }
    },
    {
     "repeats": 40,
     "prompt_tokens": {
      "nuvec": 961,
      "mlx_lm": 961
     },
     "tok_s": {
      "nuvec": {
       "median": 15.41,
       "samples": [
        15.58,
        15.66,
        15.24,
        15.1
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 15.240045663576193,
       "samples": [
        15.094678249415745,
        14.332933765146084,
        15.385417892037761,
        15.385413077736644
       ],
       "unit": "tok/s"
      },
      "ratio": 1.0112,
      "gate_pass": true,
      "goal_pass": true
     }
    },
    {
     "repeats": 160,
     "prompt_tokens": {
      "nuvec": 3841,
      "mlx_lm": 3841
     },
     "tok_s": {
      "nuvec": {
       "median": 15.43,
       "samples": [
        15.41,
        15.45,
        15.47,
        15.38
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 14.65247932618013,
       "samples": [
        14.131397254657006,
        14.179423635468682,
        15.126531650687992,
        15.125535016891577
       ],
       "unit": "tok/s"
      },
      "ratio": 1.0531,
      "gate_pass": true,
      "goal_pass": true
     }
    }
   ],
   "ttft": {
    "legacy": [
     {
      "repeats": 1,
      "prompt_tokens": {
       "nuvec": 25,
       "mlx_lm": 25
      },
      "prefill_tok_s": {
       "nuvec": {
        "median": 75.995,
        "samples": [
         76.38,
         77.07,
         75.35,
         75.61
        ],
        "unit": "tok/s"
       },
       "mlx_lm": {
        "median": 65.8489014159889,
        "samples": [
         65.93368922394068,
         65.85804698531105,
         65.7765925508904,
         65.83975584666676
        ],
        "unit": "tok/s"
       },
       "ratio": 1.1541,
       "gate_pass": true,
       "goal_pass": true
      }
     },
     {
      "repeats": 40,
      "prompt_tokens": {
       "nuvec": 961,
       "mlx_lm": 961
      },
      "prefill_tok_s": {
       "nuvec": {
        "median": 132.92000000000002,
        "samples": [
         133.82,
         133.87,
         131.95,
         132.02
        ],
        "unit": "tok/s"
       },
       "mlx_lm": {
        "median": 133.10003338051814,
        "samples": [
         133.12210800844628,
         130.87262906546167,
         133.07795875258998,
         133.1253943482906
        ],
        "unit": "tok/s"
       },
       "ratio": 0.9986,
       "gate_pass": true,
       "goal_pass": false
      }
     },
     {
      "repeats": 160,
      "prompt_tokens": {
       "nuvec": 3841,
       "mlx_lm": 3841
      },
      "prefill_tok_s": {
       "nuvec": {
        "median": 133.33,
        "samples": [
         133.3,
         133.38,
         133.33,
         133.33
        ],
        "unit": "tok/s"
       },
       "mlx_lm": {
        "median": 132.02134308240744,
        "samples": [
         130.87569531824326,
         130.8882219653052,
         133.15526161609833,
         133.1544641995097
        ],
        "unit": "tok/s"
       },
       "ratio": 1.0099,
       "gate_pass": true,
       "goal_pass": true
      }
     }
    ],
    "lengths": [
     {
      "prompt_tokens": 2048,
      "ttft_s": {
       "nuvec": {
        "median": 15.2881,
        "samples": [
         15.292,
         15.2858,
         15.2892,
         15.287
        ],
        "unit": "s"
       },
       "mlx_lm": {
        "median": 15.411326437490061,
        "samples": [
         15.412225459003821,
         15.40926862496417,
         15.416021917015314,
         15.4104274159763
        ],
        "unit": "s"
       },
       "ratio": 1.0081,
       "gate_pass": true,
       "goal_pass": true
      },
      "prefill_tok_s": {
       "nuvec": {
        "median": 133.95999999999998,
        "samples": [
         133.93,
         133.98,
         133.95,
         133.97
        ],
        "unit": "tok/s"
       },
       "mlx_lm": {
        "median": 132.88927564257466,
        "samples": [
         132.8815235312794,
         132.90702173119928,
         132.8487991924516,
         132.8970277538699
        ],
        "unit": "tok/s"
       },
       "ratio": 1.0081,
       "gate_pass": true,
       "goal_pass": true
      }
     },
     {
      "prompt_tokens": 32768,
      "ttft_s": {
       "nuvec": {
        "median": 280.0256,
        "samples": [
         276.3761,
         279.7799,
         280.2713,
         280.2899
        ],
        "unit": "s"
       },
       "mlx_lm": {
        "median": 279.7072128544969,
        "samples": [
         279.67833962501027,
         279.70403004199034,
         279.71039566700347,
         279.72097920801025
        ],
        "unit": "s"
       },
       "ratio": 0.9989,
       "gate_pass": true,
       "goal_pass": false
      },
      "prefill_tok_s": {
       "nuvec": {
        "median": 117.02000000000001,
        "samples": [
         118.56,
         117.12,
         116.92,
         116.91
        ],
        "unit": "tok/s"
       },
       "mlx_lm": {
        "median": 117.15107261566673,
        "samples": [
         117.16316695792382,
         117.15240568782913,
         117.14973954350434,
         117.14530705840471
        ],
        "unit": "tok/s"
       },
       "ratio": 0.9989,
       "gate_pass": true,
       "goal_pass": false
      }
     }
    ]
   },
   "memory": {
    "load": {
     "peak_bytes": {
      "nuvec": {
       "median": 15132852224.0,
       "samples": [
        15132852224,
        15132852224
       ],
       "unit": "bytes"
      },
      "mlx_lm": {
       "median": 15133588488.0,
       "samples": [
        15133588488,
        15133588488
       ],
       "unit": "bytes"
      },
      "ratio": 1.0
     },
     "os_peak_rss_bytes": {
      "nuvec": {
       "median": 15266529280.0,
       "samples": [
        15265021952,
        15268036608
       ],
       "unit": "bytes"
      },
      "mlx_lm": {
       "median": 15768428544.0,
       "samples": [
        15768109056,
        15768748032
       ],
       "unit": "bytes"
      },
      "ratio": 1.0329
     },
     "nuvec_raw": {
      "peak_bytes": 15132852224,
      "active_bytes": 15132852224,
      "mapped_bytes": 0
     }
    },
    "decode": {
     "peak_bytes": {
      "nuvec": {
       "median": 15438841856.0,
       "samples": [
        15438841856,
        15438841856
       ],
       "unit": "bytes"
      },
      "mlx_lm": {
       "median": 15414862838.0,
       "samples": [
        15414862838,
        15414862838
       ],
       "unit": "bytes"
      },
      "ratio": 0.9984
     },
     "os_peak_rss_bytes": {
      "nuvec": {
       "median": 15635152896.0,
       "samples": [
        15635021824,
        15635283968
       ],
       "unit": "bytes"
      },
      "mlx_lm": {
       "median": 15768895488.0,
       "samples": [
        15768027136,
        15769763840
       ],
       "unit": "bytes"
      },
      "ratio": 1.0086
     },
     "nuvec_raw": {
      "peak_bytes": 15438841856,
      "active_bytes": 15132853760,
      "mapped_bytes": 0
     }
    },
    "prefill_2048": {
     "peak_bytes": {
      "nuvec": {
       "median": 18994081536.0,
       "samples": [
        18994081536,
        18994081536
       ],
       "unit": "bytes"
      },
      "mlx_lm": {
       "median": 18129036340.0,
       "samples": [
        18129036340,
        18129036340
       ],
       "unit": "bytes"
      },
      "ratio": 0.9545
     },
     "os_peak_rss_bytes": {
      "nuvec": {
       "median": 15619162112.0,
       "samples": [
        15619342336,
        15618981888
       ],
       "unit": "bytes"
      },
      "mlx_lm": {
       "median": 15768805376.0,
       "samples": [
        15768027136,
        15769583616
       ],
       "unit": "bytes"
      },
      "ratio": 1.0096
     },
     "nuvec_raw": {
      "peak_bytes": 18994081536,
      "active_bytes": 15132853760,
      "mapped_bytes": 0
     }
    },
    "prefill_32768": {
     "peak_bytes": {
      "nuvec": {
       "median": 23987764480.0,
       "samples": [
        23987764480,
        23987764480
       ],
       "unit": "bytes"
      },
      "mlx_lm": {
       "median": 23166827586.0,
       "samples": [
        23166770242,
        23166884930
       ],
       "unit": "bytes"
      },
      "ratio": 0.9658
     },
     "os_peak_rss_bytes": {
      "nuvec": {
       "median": 15621742592.0,
       "samples": [
        15621488640,
        15621996544
       ],
       "unit": "bytes"
      },
      "mlx_lm": {
       "median": 15768936448.0,
       "samples": [
        15768092672,
        15769780224
       ],
       "unit": "bytes"
      },
      "ratio": 1.0094
     },
     "nuvec_raw": {
      "peak_bytes": 23987764480,
      "active_bytes": 15132854272,
      "mapped_bytes": 0
     }
    }
   },
   "load": {
    "warm": {
     "model_ready_s": {
      "nuvec": {
       "median": 0.8882,
       "samples": [
        0.8882,
        0.9036,
        0.8842
       ],
       "unit": "s"
      },
      "mlx_lm": {
       "median": 1.0855010419618338,
       "samples": [
        1.0816592919873074,
        1.0922939580050297,
        1.0855010419618338
       ],
       "unit": "s"
      },
      "ratio": 1.2221,
      "gate_pass": true,
      "goal_pass": true
     },
     "first_token_s": {
      "nuvec": {
       "median": 1.2625,
       "samples": [
        1.2625,
        1.278,
        1.2575
       ],
       "unit": "s"
      },
      "mlx_lm": {
       "median": 1.5740464169648476,
       "samples": [
        1.5714819999993779,
        1.5821057909633964,
        1.5740464169648476
       ],
       "unit": "s"
      },
      "ratio": 1.2468,
      "gate_pass": true,
      "goal_pass": true
     }
    },
    "cold": {
     "model_ready_s": {
      "nuvec": null,
      "mlx_lm": null,
      "ratio": null,
      "reason": "purge unavailable (needs passwordless sudo)"
     },
     "first_token_s": {
      "nuvec": null,
      "mlx_lm": null,
      "ratio": null,
      "reason": "purge unavailable (needs passwordless sudo)"
     }
    }
   },
   "concurrency": [
    {
     "level": 1,
     "total_tokens": 128,
     "wall_s": {
      "nuvec": {
       "median": 9.9399,
       "samples": [
        9.9399
       ],
       "unit": "s"
      },
      "mlx_lm": {
       "median": 10.033648292010184,
       "samples": [
        10.033648292010184
       ],
       "unit": "s"
      },
      "ratio": 1.0094,
      "gate_pass": true,
      "goal_pass": true
     },
     "aggregate_tok_s": {
      "nuvec": {
       "median": 12.877393132727693,
       "samples": [
        12.877393132727693
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 12.757074622789666,
       "samples": [
        12.757074622789666
       ],
       "unit": "tok/s"
      },
      "ratio": 1.0094,
      "gate_pass": true,
      "goal_pass": true
     },
     "decode_aggregate_tok_s": {
      "nuvec": {
       "median": 15.480630927131331,
       "samples": [
        15.480630927131331
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 15.457852146969282,
       "samples": [
        15.457852146969282
       ],
       "unit": "tok/s"
      },
      "ratio": 1.0015,
      "gate_pass": true,
      "goal_pass": true
     },
     "nuvec_matches_solo": true
    },
    {
     "level": 4,
     "total_tokens": 512,
     "wall_s": {
      "nuvec": {
       "median": 17.8845,
       "samples": [
        17.8845
       ],
       "unit": "s"
      },
      "mlx_lm": {
       "median": 17.910904166987166,
       "samples": [
        17.910904166987166
       ],
       "unit": "s"
      },
      "ratio": 1.0015,
      "gate_pass": true,
      "goal_pass": true
     },
     "aggregate_tok_s": {
      "nuvec": {
       "median": 28.628141686935617,
       "samples": [
        28.628141686935617
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 28.58593822101415,
       "samples": [
        28.58593822101415
       ],
       "unit": "tok/s"
      },
      "ratio": 1.0015,
      "gate_pass": true,
      "goal_pass": true
     },
     "decode_aggregate_tok_s": {
      "nuvec": {
       "median": 42.96164742695251,
       "samples": [
        42.96164742695251
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 43.22211769583798,
       "samples": [
        43.22211769583798
       ],
       "unit": "tok/s"
      },
      "ratio": 0.994,
      "gate_pass": true,
      "goal_pass": false
     },
     "nuvec_matches_solo": false
    },
    {
     "level": 8,
     "total_tokens": 1024,
     "wall_s": {
      "nuvec": {
       "median": 34.1668,
       "samples": [
        34.1668
       ],
       "unit": "s"
      },
      "mlx_lm": {
       "median": 34.52082270901883,
       "samples": [
        34.52082270901883
       ],
       "unit": "s"
      },
      "ratio": 1.0104,
      "gate_pass": true,
      "goal_pass": true
     },
     "aggregate_tok_s": {
      "nuvec": {
       "median": 29.97061474882049,
       "samples": [
        29.97061474882049
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 29.66325596094418,
       "samples": [
        29.66325596094418
       ],
       "unit": "tok/s"
      },
      "ratio": 1.0104,
      "gate_pass": true,
      "goal_pass": true
     },
     "decode_aggregate_tok_s": {
      "nuvec": {
       "median": 45.5783197929237,
       "samples": [
        45.5783197929237
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 45.215557605276715,
       "samples": [
        45.215557605276715
       ],
       "unit": "tok/s"
      },
      "ratio": 1.008,
      "gate_pass": true,
      "goal_pass": true
     },
     "nuvec_matches_solo": false
    }
   ],
   "status": "ok"
  },
  {
   "id": "mlx-community/Qwen3-30B-A3B-4bit",
   "kind": "moe",
   "snapshot": "d388dead1515f5e085ef7a0431dd8fadf0886c57",
   "decode": [
    {
     "repeats": 1,
     "prompt_tokens": {
      "nuvec": 25,
      "mlx_lm": 25
     },
     "tok_s": {
      "nuvec": {
       "median": 94.735,
       "samples": [
        95.26,
        95.11,
        94.33,
        94.36
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 94.92056299458443,
       "samples": [
        94.88551511094158,
        94.84153819454295,
        94.95561087822729,
        95.0319788200638
       ],
       "unit": "tok/s"
      },
      "ratio": 0.998,
      "gate_pass": true,
      "goal_pass": false
     }
    },
    {
     "repeats": 40,
     "prompt_tokens": {
      "nuvec": 961,
      "mlx_lm": 961
     },
     "tok_s": {
      "nuvec": {
       "median": 86.06,
       "samples": [
        86.06,
        86.3,
        85.89,
        86.06
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 86.26261965366854,
       "samples": [
        85.97405020399482,
        86.23990040789036,
        86.2853388994467,
        86.39248128805595
       ],
       "unit": "tok/s"
      },
      "ratio": 0.9977,
      "gate_pass": true,
      "goal_pass": false
     }
    },
    {
     "repeats": 160,
     "prompt_tokens": {
      "nuvec": 3841,
      "mlx_lm": 3841
     },
     "tok_s": {
      "nuvec": {
       "median": 76.96000000000001,
       "samples": [
        77.04,
        76.98,
        76.89,
        76.94
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 77.17953855119316,
       "samples": [
        77.22864306605655,
        77.41806679837903,
        77.13043403632979,
        77.0629703925446
       ],
       "unit": "tok/s"
      },
      "ratio": 0.9972,
      "gate_pass": true,
      "goal_pass": false
     }
    }
   ],
   "ttft": {
    "legacy": [
     {
      "repeats": 1,
      "prompt_tokens": {
       "nuvec": 25,
       "mlx_lm": 25
      },
      "prefill_tok_s": {
       "nuvec": {
        "median": 257.315,
        "samples": [
         257.01,
         259.28,
         257.62,
         255.29
        ],
        "unit": "tok/s"
       },
       "mlx_lm": {
        "median": 234.818025499619,
        "samples": [
         235.020019007174,
         235.1285141385976,
         234.61603199206402,
         234.4104355277541
        ],
        "unit": "tok/s"
       },
       "ratio": 1.0958,
       "gate_pass": true,
       "goal_pass": true
      }
     },
     {
      "repeats": 40,
      "prompt_tokens": {
       "nuvec": 961,
       "mlx_lm": 961
      },
      "prefill_tok_s": {
       "nuvec": {
        "median": 970.075,
        "samples": [
         968.48,
         971.59,
         968.56,
         971.93
        ],
        "unit": "tok/s"
       },
       "mlx_lm": {
        "median": 967.5302621654732,
        "samples": [
         963.1366782560652,
         968.1701070788437,
         966.8904172521028,
         969.9539279376369
        ],
        "unit": "tok/s"
       },
       "ratio": 1.0026,
       "gate_pass": true,
       "goal_pass": true
      }
     },
     {
      "repeats": 160,
      "prompt_tokens": {
       "nuvec": 3841,
       "mlx_lm": 3841
      },
      "prefill_tok_s": {
       "nuvec": {
        "median": 871.7750000000001,
        "samples": [
         871.48,
         872.23,
         871.02,
         872.07
        ],
        "unit": "tok/s"
       },
       "mlx_lm": {
        "median": 873.7763488120229,
        "samples": [
         873.1458850481001,
         873.7286640873665,
         874.2253595807352,
         873.8240335366793
        ],
        "unit": "tok/s"
       },
       "ratio": 0.9977,
       "gate_pass": true,
       "goal_pass": false
      }
     }
    ],
    "lengths": [
     {
      "prompt_tokens": 2048,
      "ttft_s": {
       "nuvec": {
        "median": 2.1996,
        "samples": [
         2.2003,
         2.1975,
         2.2046,
         2.1989
        ],
        "unit": "s"
       },
       "mlx_lm": {
        "median": 2.1853344369737897,
        "samples": [
         2.18471512501128,
         2.1852919579832815,
         2.189557541976683,
         2.185376915964298
        ],
        "unit": "s"
       },
       "ratio": 0.9935,
       "gate_pass": true,
       "goal_pass": false
      },
      "prefill_tok_s": {
       "nuvec": {
        "median": 931.065,
        "samples": [
         930.78,
         931.97,
         928.97,
         931.35
        ],
        "unit": "tok/s"
       },
       "mlx_lm": {
        "median": 937.1563299985593,
        "samples": [
         937.4219899674223,
         937.1745466404486,
         935.348791131158,
         937.1381133566699
        ],
        "unit": "tok/s"
       },
       "ratio": 0.9935,
       "gate_pass": true,
       "goal_pass": false
      }
     },
     {
      "prompt_tokens": 32768,
      "ttft_s": {
       "nuvec": {
        "median": 93.33315,
        "samples": [
         93.376,
         93.3318,
         93.3205,
         93.3345
        ],
        "unit": "s"
       },
       "mlx_lm": {
        "median": 93.68019887548871,
        "samples": [
         93.67623233399354,
         93.68416541698389,
         93.72662183397915,
         93.66424766601995
        ],
        "unit": "s"
       },
       "ratio": 1.0037,
       "gate_pass": true,
       "goal_pass": true
      },
      "prefill_tok_s": {
       "nuvec": {
        "median": 351.085,
        "samples": [
         350.93,
         351.09,
         351.13,
         351.08
        ],
        "unit": "tok/s"
       },
       "mlx_lm": {
        "median": 349.78576531736746,
        "samples": [
         349.8005757017305,
         349.7709549330044,
         349.6125151938471,
         349.84533390842324
        ],
        "unit": "tok/s"
       },
       "ratio": 1.0037,
       "gate_pass": true,
       "goal_pass": true
      }
     }
    ]
   },
   "memory": {
    "load": {
     "peak_bytes": {
      "nuvec": {
       "median": 17174708224.0,
       "samples": [
        17174708224,
        17174708224
       ],
       "unit": "bytes"
      },
      "mlx_lm": {
       "median": 17174654984.0,
       "samples": [
        17174654984,
        17174654984
       ],
       "unit": "bytes"
      },
      "ratio": 1.0
     },
     "os_peak_rss_bytes": {
      "nuvec": {
       "median": 11895234560.0,
       "samples": [
        11895177216,
        11895291904
       ],
       "unit": "bytes"
      },
      "mlx_lm": {
       "median": 17544568832.0,
       "samples": [
        17545854976,
        17543282688
       ],
       "unit": "bytes"
      },
      "ratio": 1.4749
     },
     "nuvec_raw": {
      "peak_bytes": 11810013184,
      "active_bytes": 11810013184,
      "mapped_bytes": 5364695040
     }
    },
    "decode": {
     "peak_bytes": {
      "nuvec": {
       "median": 17223415296.0,
       "samples": [
        17223415296,
        17223415296
       ],
       "unit": "bytes"
      },
      "mlx_lm": {
       "median": 17252849792.0,
       "samples": [
        17252849792,
        17252849792
       ],
       "unit": "bytes"
      },
      "ratio": 1.0017
     },
     "os_peak_rss_bytes": {
      "nuvec": {
       "median": 12017360896.0,
       "samples": [
        12017090560,
        12017631232
       ],
       "unit": "bytes"
      },
      "mlx_lm": {
       "median": 17542791168.0,
       "samples": [
        17543380992,
        17542201344
       ],
       "unit": "bytes"
      },
      "ratio": 1.4598
     },
     "nuvec_raw": {
      "peak_bytes": 11858720256,
      "active_bytes": 11810013184,
      "mapped_bytes": 5364695040
     }
    },
    "prefill_2048": {
     "peak_bytes": {
      "nuvec": {
       "median": 17669956608.0,
       "samples": [
        17669956608,
        17669956608
       ],
       "unit": "bytes"
      },
      "mlx_lm": {
       "median": 18141952760.0,
       "samples": [
        18141952760,
        18141952760
       ],
       "unit": "bytes"
      },
      "ratio": 1.0267
     },
     "os_peak_rss_bytes": {
      "nuvec": {
       "median": 12009742336.0,
       "samples": [
        12009783296,
        12009701376
       ],
       "unit": "bytes"
      },
      "mlx_lm": {
       "median": 17544790016.0,
       "samples": [
        17545986048,
        17543593984
       ],
       "unit": "bytes"
      },
      "ratio": 1.4609
     },
     "nuvec_raw": {
      "peak_bytes": 12305261568,
      "active_bytes": 11810013184,
      "mapped_bytes": 5364695040
     }
    },
    "prefill_32768": {
     "peak_bytes": {
      "nuvec": {
       "median": 20693842944.0,
       "samples": [
        20693842944,
        20693842944
       ],
       "unit": "bytes"
      },
      "mlx_lm": {
       "median": 21067602568.0,
       "samples": [
        21067602568,
        21067602568
       ],
       "unit": "bytes"
      },
      "ratio": 1.0181
     },
     "os_peak_rss_bytes": {
      "nuvec": {
       "median": 12012888064.0,
       "samples": [
        12012044288,
        12013731840
       ],
       "unit": "bytes"
      },
      "mlx_lm": {
       "median": 17544552448.0,
       "samples": [
        17543610368,
        17545494528
       ],
       "unit": "bytes"
      },
      "ratio": 1.4605
     },
     "nuvec_raw": {
      "peak_bytes": 15329147904,
      "active_bytes": 11810013184,
      "mapped_bytes": 5364695040
     }
    }
   },
   "load": {
    "warm": {
     "model_ready_s": {
      "nuvec": {
       "median": 0.6751,
       "samples": [
        0.6751,
        0.6681,
        0.6833
       ],
       "unit": "s"
      },
      "mlx_lm": {
       "median": 0.7072105840197764,
       "samples": [
        0.7072105840197764,
        0.6980346670025028,
        0.7465110839693807
       ],
       "unit": "s"
      },
      "ratio": 1.0476,
      "gate_pass": true,
      "goal_pass": true
     },
     "first_token_s": {
      "nuvec": {
       "median": 0.7916,
       "samples": [
        0.7916,
        0.784,
        0.7991
       ],
       "unit": "s"
      },
      "mlx_lm": {
       "median": 0.9474821669864468,
       "samples": [
        0.9317624180112034,
        0.9474821669864468,
        0.9835478760069236
       ],
       "unit": "s"
      },
      "ratio": 1.1969,
      "gate_pass": true,
      "goal_pass": true
     }
    },
    "cold": {
     "model_ready_s": {
      "nuvec": null,
      "mlx_lm": null,
      "ratio": null,
      "reason": "purge unavailable (needs passwordless sudo)"
     },
     "first_token_s": {
      "nuvec": null,
      "mlx_lm": null,
      "ratio": null,
      "reason": "purge unavailable (needs passwordless sudo)"
     }
    }
   },
   "concurrency": [
    {
     "level": 1,
     "total_tokens": 128,
     "wall_s": {
      "nuvec": {
       "median": 1.6532,
       "samples": [
        1.6532
       ],
       "unit": "s"
      },
      "mlx_lm": {
       "median": 1.65451220900286,
       "samples": [
        1.65451220900286
       ],
       "unit": "s"
      },
      "ratio": 1.0008,
      "gate_pass": true,
      "goal_pass": true
     },
     "aggregate_tok_s": {
      "nuvec": {
       "median": 77.42559883861601,
       "samples": [
        77.42559883861601
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 77.36419187691757,
       "samples": [
        77.36419187691757
       ],
       "unit": "tok/s"
      },
      "ratio": 1.0008,
      "gate_pass": true,
      "goal_pass": true
     },
     "decode_aggregate_tok_s": {
      "nuvec": {
       "median": 92.01564990581075,
       "samples": [
        92.01564990581075
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 93.14275006767207,
       "samples": [
        93.14275006767207
       ],
       "unit": "tok/s"
      },
      "ratio": 0.9879,
      "gate_pass": true,
      "goal_pass": false
     },
     "nuvec_matches_solo": true
    },
    {
     "level": 4,
     "total_tokens": 512,
     "wall_s": {
      "nuvec": {
       "median": 3.9702,
       "samples": [
        3.9702
       ],
       "unit": "s"
      },
      "mlx_lm": {
       "median": 3.9040901249973103,
       "samples": [
        3.9040901249973103
       ],
       "unit": "s"
      },
      "ratio": 0.9833,
      "gate_pass": true,
      "goal_pass": false
     },
     "aggregate_tok_s": {
      "nuvec": {
       "median": 128.96075764445115,
       "samples": [
        128.96075764445115
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 131.14451347363624,
       "samples": [
        131.14451347363624
       ],
       "unit": "tok/s"
      },
      "ratio": 0.9833,
      "gate_pass": true,
      "goal_pass": false
     },
     "decode_aggregate_tok_s": {
      "nuvec": {
       "median": 161.64444585865655,
       "samples": [
        161.64444585865655
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 165.78627161430413,
       "samples": [
        165.78627161430413
       ],
       "unit": "tok/s"
      },
      "ratio": 0.975,
      "gate_pass": false,
      "goal_pass": false
     },
     "nuvec_matches_solo": true
    },
    {
     "level": 8,
     "total_tokens": 1024,
     "wall_s": {
      "nuvec": {
       "median": 6.2236,
       "samples": [
        6.2236
       ],
       "unit": "s"
      },
      "mlx_lm": {
       "median": 6.297245583962649,
       "samples": [
        6.297245583962649
       ],
       "unit": "s"
      },
      "ratio": 1.0118,
      "gate_pass": true,
      "goal_pass": true
     },
     "aggregate_tok_s": {
      "nuvec": {
       "median": 164.5349958223536,
       "samples": [
        164.5349958223536
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 162.61077741796288,
       "samples": [
        162.61077741796288
       ],
       "unit": "tok/s"
      },
      "ratio": 1.0118,
      "gate_pass": true,
      "goal_pass": true
     },
     "decode_aggregate_tok_s": {
      "nuvec": {
       "median": 218.0631868131868,
       "samples": [
        218.0631868131868
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 215.17510039796767,
       "samples": [
        215.17510039796767
       ],
       "unit": "tok/s"
      },
      "ratio": 1.0134,
      "gate_pass": true,
      "goal_pass": true
     },
     "nuvec_matches_solo": true
    }
   ],
   "status": "ok"
  },
  {
   "id": "mlx-community/Qwen3.6-35B-A3B-4bit",
   "kind": "hybrid-moe",
   "snapshot": "38740b847e4cb78f352aba30aa41c76e08e6eb46",
   "decode": [
    {
     "repeats": 1,
     "prompt_tokens": {
      "nuvec": 25,
      "mlx_lm": 25
     },
     "tok_s": {
      "nuvec": {
       "median": 92.78999999999999,
       "samples": [
        92.84,
        92.86,
        92.69,
        92.74
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 93.40298042268412,
       "samples": [
        93.18283205492934,
        93.43286797432278,
        93.37309287104546,
        93.47063169095797
       ],
       "unit": "tok/s"
      },
      "ratio": 0.9934,
      "gate_pass": true,
      "goal_pass": false
     }
    },
    {
     "repeats": 40,
     "prompt_tokens": {
      "nuvec": 961,
      "mlx_lm": 961
     },
     "tok_s": {
      "nuvec": {
       "median": 91.03,
       "samples": [
        91.07,
        91.08,
        90.93,
        90.99
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 91.44383974673855,
       "samples": [
        91.56352649760424,
        91.54319020631645,
        91.34448928716066,
        91.18207692859855
       ],
       "unit": "tok/s"
      },
      "ratio": 0.9955,
      "gate_pass": true,
      "goal_pass": false
     }
    },
    {
     "repeats": 160,
     "prompt_tokens": {
      "nuvec": 3841,
      "mlx_lm": 3841
     },
     "tok_s": {
      "nuvec": {
       "median": 88.89500000000001,
       "samples": [
        88.9,
        88.74,
        88.89,
        88.98
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 88.96107594663312,
       "samples": [
        89.18135899057376,
        89.06297857097579,
        88.83080041916377,
        88.85917332229045
       ],
       "unit": "tok/s"
      },
      "ratio": 0.9993,
      "gate_pass": true,
      "goal_pass": false
     }
    }
   ],
   "ttft": {
    "legacy": [
     {
      "repeats": 1,
      "prompt_tokens": {
       "nuvec": 25,
       "mlx_lm": 25
      },
      "prefill_tok_s": {
       "nuvec": {
        "median": 323.20500000000004,
        "samples": [
         321.04,
         323.79,
         322.62,
         325.1
        ],
        "unit": "tok/s"
       },
       "mlx_lm": {
        "median": 292.906525348694,
        "samples": [
         293.92990540291834,
         293.1902174085706,
         292.0465490058803,
         292.62283328881745
        ],
        "unit": "tok/s"
       },
       "ratio": 1.1034,
       "gate_pass": true,
       "goal_pass": true
      }
     },
     {
      "repeats": 40,
      "prompt_tokens": {
       "nuvec": 961,
       "mlx_lm": 961
      },
      "prefill_tok_s": {
       "nuvec": {
        "median": 1017.8,
        "samples": [
         1017.54,
         1019.59,
         1015.5,
         1018.06
        ],
        "unit": "tok/s"
       },
       "mlx_lm": {
        "median": 1025.6327872563688,
        "samples": [
         1024.1407527595568,
         1027.5126948839825,
         1024.9198533722115,
         1026.345721140526
        ],
        "unit": "tok/s"
       },
       "ratio": 0.9924,
       "gate_pass": true,
       "goal_pass": false
      }
     },
     {
      "repeats": 160,
      "prompt_tokens": {
       "nuvec": 3841,
       "mlx_lm": 3841
      },
      "prefill_tok_s": {
       "nuvec": {
        "median": 1038.455,
        "samples": [
         1037.88,
         1039.03,
         1037.08,
         1039.51
        ],
        "unit": "tok/s"
       },
       "mlx_lm": {
        "median": 1047.2122462645045,
        "samples": [
         1048.1510715725274,
         1047.864362543363,
         1046.3241447650628,
         1046.560129985646
        ],
        "unit": "tok/s"
       },
       "ratio": 0.9916,
       "gate_pass": true,
       "goal_pass": false
      }
     }
    ],
    "lengths": [
     {
      "prompt_tokens": 2048,
      "ttft_s": {
       "nuvec": {
        "median": 1.97145,
        "samples": [
         1.9731,
         1.9701,
         1.972,
         1.9709
        ],
        "unit": "s"
       },
       "mlx_lm": {
        "median": 1.9822296874772292,
        "samples": [
         1.9836977499653585,
         1.9807616249890998,
         1.9842310000094585,
         1.9790022500092164
        ],
        "unit": "s"
       },
       "ratio": 1.0055,
       "gate_pass": true,
       "goal_pass": true
      },
      "prefill_tok_s": {
       "nuvec": {
        "median": 1038.82,
        "samples": [
         1037.96,
         1039.54,
         1038.52,
         1039.12
        ],
        "unit": "tok/s"
       },
       "mlx_lm": {
        "median": 1033.1805321444515,
        "samples": [
         1032.4153465596078,
         1033.9457177292952,
         1032.1378911982715,
         1034.864917404951
        ],
        "unit": "tok/s"
       },
       "ratio": 1.0055,
       "gate_pass": true,
       "goal_pass": true
      }
     },
     {
      "prompt_tokens": 32768,
      "ttft_s": {
       "nuvec": {
        "median": 44.55075,
        "samples": [
         44.5592,
         44.5471,
         44.5541,
         44.5474
        ],
        "unit": "s"
       },
       "mlx_lm": {
        "median": 43.77219514545868,
        "samples": [
         43.77658812497975,
         43.76828533294611,
         43.775180124968756,
         43.76921016594861
        ],
        "unit": "s"
       },
       "ratio": 0.9825,
       "gate_pass": true,
       "goal_pass": false
      },
      "prefill_tok_s": {
       "nuvec": {
        "median": 735.5250000000001,
        "samples": [
         735.38,
         735.58,
         735.47,
         735.58
        ],
        "unit": "tok/s"
       },
       "mlx_lm": {
        "median": 748.6030810995884,
        "samples": [
         748.5279553182437,
         748.6699501872932,
         748.5520312298975,
         748.6541309692793
        ],
        "unit": "tok/s"
       },
       "ratio": 0.9825,
       "gate_pass": true,
       "goal_pass": false
      }
     },
     {
      "prompt_tokens": 65536,
      "ttft_s": {
       "nuvec": {
        "median": 117.54925,
        "samples": [
         117.5697,
         117.5288
        ],
        "unit": "s"
       },
       "mlx_lm": {
        "median": 115.57720593799604,
        "samples": [
         115.6495956669678,
         115.50481620902428
        ],
        "unit": "s"
       },
       "ratio": 0.9832,
       "gate_pass": true,
       "goal_pass": false
      },
      "prefill_tok_s": {
       "nuvec": {
        "median": 557.52,
        "samples": [
         557.42,
         557.62
        ],
        "unit": "tok/s"
       },
       "mlx_lm": {
        "median": 567.0324453458481,
        "samples": [
         566.67729465066,
         567.3875960410362
        ],
        "unit": "tok/s"
       },
       "ratio": 0.9832,
       "gate_pass": true,
       "goal_pass": false
      }
     },
     {
      "prompt_tokens": 261952,
      "ttft_s": {
       "nuvec": {
        "median": 1224.3465,
        "samples": [
         1224.3465
        ],
        "unit": "s"
       },
       "mlx_lm": null,
       "ratio": null,
       "reason": "mlx-lm: exit 1: RuntimeError: [METAL] Command buffer execution failed: Insufficient Memory (00000008:kIOGPUCommandBufferCallbackErrorOutOfMemory)."
      },
      "prefill_tok_s": {
       "nuvec": {
        "median": 213.95,
        "samples": [
         213.95
        ],
        "unit": "tok/s"
       },
       "mlx_lm": null,
       "ratio": null,
       "reason": "mlx-lm: exit 1: RuntimeError: [METAL] Command buffer execution failed: Insufficient Memory (00000008:kIOGPUCommandBufferCallbackErrorOutOfMemory)."
      }
     }
    ]
   },
   "memory": {
    "load": {
     "peak_bytes": {
      "nuvec": {
       "median": 19508838400.0,
       "samples": [
        19508838400,
        19508838400
       ],
       "unit": "bytes"
      },
      "mlx_lm": {
       "median": 19508787464.0,
       "samples": [
        19508787464,
        19508787464
       ],
       "unit": "bytes"
      },
      "ratio": 1.0
     },
     "os_peak_rss_bytes": {
      "nuvec": {
       "median": 19642458112.0,
       "samples": [
        19641319424,
        19643596800
       ],
       "unit": "bytes"
      },
      "mlx_lm": {
       "median": 20143931392.0,
       "samples": [
        20144046080,
        20143816704
       ],
       "unit": "bytes"
      },
      "ratio": 1.0255
     },
     "nuvec_raw": {
      "peak_bytes": 19508838400,
      "active_bytes": 19508838400,
      "mapped_bytes": 0
     }
    },
    "decode": {
     "peak_bytes": {
      "nuvec": {
       "median": 19615111680.0,
       "samples": [
        19615087104,
        19615136256
       ],
       "unit": "bytes"
      },
      "mlx_lm": {
       "median": 19654036909.0,
       "samples": [
        19654036909,
        19654036909
       ],
       "unit": "bytes"
      },
      "ratio": 1.002
     },
     "os_peak_rss_bytes": {
      "nuvec": {
       "median": 19831824384.0,
       "samples": [
        19833667584,
        19829981184
       ],
       "unit": "bytes"
      },
      "mlx_lm": {
       "median": 20144087040.0,
       "samples": [
        20144046080,
        20144128000
       ],
       "unit": "bytes"
      },
      "ratio": 1.0157
     },
     "nuvec_raw": {
      "peak_bytes": 19615136256,
      "active_bytes": 19508839936,
      "mapped_bytes": 0
     }
    },
    "prefill_2048": {
     "peak_bytes": {
      "nuvec": {
       "median": 21372346624.0,
       "samples": [
        21372346624,
        21372346624
       ],
       "unit": "bytes"
      },
      "mlx_lm": {
       "median": 21452473854.0,
       "samples": [
        21452465662,
        21452482046
       ],
       "unit": "bytes"
      },
      "ratio": 1.0037
     },
     "os_peak_rss_bytes": {
      "nuvec": {
       "median": 19840622592.0,
       "samples": [
        19841761280,
        19839483904
       ],
       "unit": "bytes"
      },
      "mlx_lm": {
       "median": 20014784512.0,
       "samples": [
        20088799232,
        19940769792
       ],
       "unit": "bytes"
      },
      "ratio": 1.0088
     },
     "nuvec_raw": {
      "peak_bytes": 21372346624,
      "active_bytes": 19508839936,
      "mapped_bytes": 0
     }
    },
    "prefill_32768": {
     "peak_bytes": {
      "nuvec": {
       "median": 24016349952.0,
       "samples": [
        24016349952,
        24016349952
       ],
       "unit": "bytes"
      },
      "mlx_lm": {
       "median": 23998842288.0,
       "samples": [
        23998817712,
        23998866864
       ],
       "unit": "bytes"
      },
      "ratio": 0.9993
     },
     "os_peak_rss_bytes": {
      "nuvec": {
       "median": 19690659840.0,
       "samples": [
        19838238720,
        19543080960
       ],
       "unit": "bytes"
      },
      "mlx_lm": {
       "median": 19391504384.0,
       "samples": [
        19586531328,
        19196477440
       ],
       "unit": "bytes"
      },
      "ratio": 0.9848
     },
     "nuvec_raw": {
      "peak_bytes": 24016349952,
      "active_bytes": 19508840448,
      "mapped_bytes": 0
     }
    },
    "prefill_65536": {
     "peak_bytes": {
      "nuvec": {
       "median": 26898963200.0,
       "samples": [
        26898963200,
        26898963200
       ],
       "unit": "bytes"
      },
      "mlx_lm": {
       "median": 26883597744.0,
       "samples": [
        26883597744,
        26883597744
       ],
       "unit": "bytes"
      },
      "ratio": 0.9994
     },
     "os_peak_rss_bytes": {
      "nuvec": {
       "median": 19332743168.0,
       "samples": [
        19309772800,
        19355713536
       ],
       "unit": "bytes"
      },
      "mlx_lm": {
       "median": 19449503744.0,
       "samples": [
        19534315520,
        19364691968
       ],
       "unit": "bytes"
      },
      "ratio": 1.006
     },
     "nuvec_raw": {
      "peak_bytes": 26898963200,
      "active_bytes": 19508840448,
      "mapped_bytes": 0
     }
    }
   },
   "load": {
    "warm": {
     "model_ready_s": {
      "nuvec": {
       "median": 1.6681,
       "samples": [
        1.6681,
        1.6722,
        1.6447
       ],
       "unit": "s"
      },
      "mlx_lm": {
       "median": 1.9231466659693979,
       "samples": [
        2.0426529169781134,
        1.9053026670007966,
        1.9231466659693979
       ],
       "unit": "s"
      },
      "ratio": 1.1529,
      "gate_pass": true,
      "goal_pass": true
     },
     "first_token_s": {
      "nuvec": {
       "median": 1.7857,
       "samples": [
        1.7857,
        1.7897,
        1.7613
       ],
       "unit": "s"
      },
      "mlx_lm": {
       "median": 2.603534249996301,
       "samples": [
        2.766178374993615,
        2.577910083986353,
        2.603534249996301
       ],
       "unit": "s"
      },
      "ratio": 1.458,
      "gate_pass": true,
      "goal_pass": true
     }
    },
    "cold": {
     "model_ready_s": {
      "nuvec": null,
      "mlx_lm": null,
      "ratio": null,
      "reason": "purge unavailable (needs passwordless sudo)"
     },
     "first_token_s": {
      "nuvec": null,
      "mlx_lm": null,
      "ratio": null,
      "reason": "purge unavailable (needs passwordless sudo)"
     }
    }
   },
   "concurrency": [
    {
     "level": 1,
     "total_tokens": 128,
     "wall_s": {
      "nuvec": {
       "median": 1.7006,
       "samples": [
        1.7006
       ],
       "unit": "s"
      },
      "mlx_lm": {
       "median": 1.6812032909947447,
       "samples": [
        1.6812032909947447
       ],
       "unit": "s"
      },
      "ratio": 0.9886,
      "gate_pass": true,
      "goal_pass": false
     },
     "aggregate_tok_s": {
      "nuvec": {
       "median": 75.26755262848407,
       "samples": [
        75.26755262848407
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 76.1359442285318,
       "samples": [
        76.1359442285318
       ],
       "unit": "tok/s"
      },
      "ratio": 0.9886,
      "gate_pass": true,
      "goal_pass": false
     },
     "decode_aggregate_tok_s": {
      "nuvec": {
       "median": 89.68293199632795,
       "samples": [
        89.68293199632795
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 92.09838738190501,
       "samples": [
        92.09838738190501
       ],
       "unit": "tok/s"
      },
      "ratio": 0.9738,
      "gate_pass": false,
      "goal_pass": false
     },
     "nuvec_matches_solo": true
    },
    {
     "level": 4,
     "total_tokens": 512,
     "wall_s": {
      "nuvec": {
       "median": 3.695,
       "samples": [
        3.695
       ],
       "unit": "s"
      },
      "mlx_lm": {
       "median": 3.611474667035509,
       "samples": [
        3.611474667035509
       ],
       "unit": "s"
      },
      "ratio": 0.9774,
      "gate_pass": false,
      "goal_pass": false
     },
     "aggregate_tok_s": {
      "nuvec": {
       "median": 138.5656292286874,
       "samples": [
        138.5656292286874
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 141.77034236828163,
       "samples": [
        141.77034236828163
       ],
       "unit": "tok/s"
      },
      "ratio": 0.9774,
      "gate_pass": false,
      "goal_pass": false
     },
     "decode_aggregate_tok_s": {
      "nuvec": {
       "median": 176.6833611574847,
       "samples": [
        176.6833611574847
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 183.25128359493874,
       "samples": [
        183.25128359493874
       ],
       "unit": "tok/s"
      },
      "ratio": 0.9642,
      "gate_pass": false,
      "goal_pass": false
     },
     "nuvec_matches_solo": false
    },
    {
     "level": 8,
     "total_tokens": 1024,
     "wall_s": {
      "nuvec": {
       "median": 6.1434,
       "samples": [
        6.1434
       ],
       "unit": "s"
      },
      "mlx_lm": {
       "median": 6.069529250031337,
       "samples": [
        6.069529250031337
       ],
       "unit": "s"
      },
      "ratio": 0.988,
      "gate_pass": true,
      "goal_pass": false
     },
     "aggregate_tok_s": {
      "nuvec": {
       "median": 166.68294429794577,
       "samples": [
        166.68294429794577
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 168.71160147958972,
       "samples": [
        168.71160147958972
       ],
       "unit": "tok/s"
      },
      "ratio": 0.988,
      "gate_pass": true,
      "goal_pass": false
     },
     "decode_aggregate_tok_s": {
      "nuvec": {
       "median": 220.31399080579408,
       "samples": [
        220.31399080579408
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 225.05705132661046,
       "samples": [
        225.05705132661046
       ],
       "unit": "tok/s"
      },
      "ratio": 0.9789,
      "gate_pass": false,
      "goal_pass": false
     },
     "nuvec_matches_solo": false
    }
   ],
   "status": "partial",
   "reason": "ttft-261952 mlx-lm: exit 1: RuntimeError: [METAL] Command buffer execution failed: Insufficient Memory (00000008:kIOGPUCommandBufferCallbackErrorOutOfMemory)."
  },
  {
   "id": "mlx-community/gpt-oss-20b-MXFP4-Q8",
   "kind": "moe",
   "snapshot": "773a7da77e569019bb0fd17a554b263738d669a3",
   "decode": [
    {
     "repeats": 1,
     "prompt_tokens": {
      "nuvec": 24,
      "mlx_lm": 24
     },
     "tok_s": {
      "nuvec": {
       "median": 74.53999999999999,
       "samples": [
        74.6,
        74.7,
        74.42,
        74.48
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 75.32558273163343,
       "samples": [
        75.27767532374664,
        75.15982003478894,
        75.3867671217307,
        75.37349013952023
       ],
       "unit": "tok/s"
      },
      "ratio": 0.9896,
      "gate_pass": true,
      "goal_pass": false
     }
    },
    {
     "repeats": 40,
     "prompt_tokens": {
      "nuvec": 921,
      "mlx_lm": 921
     },
     "tok_s": {
      "nuvec": {
       "median": 72.575,
       "samples": [
        72.84,
        72.39,
        72.61,
        72.54
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 73.66650290377129,
       "samples": [
        73.57972837831825,
        73.55757645522861,
        73.86838863243761,
        73.75327742922433
       ],
       "unit": "tok/s"
      },
      "ratio": 0.9852,
      "gate_pass": true,
      "goal_pass": false
     }
    },
    {
     "repeats": 160,
     "prompt_tokens": {
      "nuvec": 3681,
      "mlx_lm": 3681
     },
     "tok_s": {
      "nuvec": {
       "median": 69.83,
       "samples": [
        69.81,
        69.84,
        69.82,
        69.92
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 71.04549578762739,
       "samples": [
        70.94585463693157,
        70.97900284071105,
        71.11198873454373,
        71.18184207361773
       ],
       "unit": "tok/s"
      },
      "ratio": 0.9829,
      "gate_pass": true,
      "goal_pass": false
     }
    }
   ],
   "ttft": {
    "legacy": [
     {
      "repeats": 1,
      "prompt_tokens": {
       "nuvec": 24,
       "mlx_lm": 24
      },
      "prefill_tok_s": {
       "nuvec": {
        "median": 225.215,
        "samples": [
         224.56,
         225.87,
         224.21,
         226.33
        ],
        "unit": "tok/s"
       },
       "mlx_lm": {
        "median": 208.42354436983948,
        "samples": [
         210.32045442455177,
         209.59745334715035,
         206.66190490060458,
         207.24963539252857
        ],
        "unit": "tok/s"
       },
       "ratio": 1.0806,
       "gate_pass": true,
       "goal_pass": true
      }
     },
     {
      "repeats": 40,
      "prompt_tokens": {
       "nuvec": 921,
       "mlx_lm": 921
      },
      "prefill_tok_s": {
       "nuvec": {
        "median": 934.46,
        "samples": [
         932.27,
         936.33,
         932.59,
         936.47
        ],
        "unit": "tok/s"
       },
       "mlx_lm": {
        "median": 920.1021688390194,
        "samples": [
         919.6757056140669,
         921.2312290296288,
         919.9807763280681,
         920.2235613499709
        ],
        "unit": "tok/s"
       },
       "ratio": 1.0156,
       "gate_pass": true,
       "goal_pass": true
      }
     },
     {
      "repeats": 160,
      "prompt_tokens": {
       "nuvec": 3681,
       "mlx_lm": 3681
      },
      "prefill_tok_s": {
       "nuvec": {
        "median": 941.61,
        "samples": [
         941.24,
         942.31,
         941.18,
         941.98
        ],
        "unit": "tok/s"
       },
       "mlx_lm": {
        "median": 906.4756405379511,
        "samples": [
         905.8000641497553,
         906.4068633305537,
         906.5444177453485,
         907.0766581219194
        ],
        "unit": "tok/s"
       },
       "ratio": 1.0388,
       "gate_pass": true,
       "goal_pass": true
      }
     }
    ],
    "lengths": [
     {
      "prompt_tokens": 2048,
      "ttft_s": {
       "nuvec": {
        "median": 2.15175,
        "samples": [
         2.1523,
         2.1512,
         2.1526,
         2.1509
        ],
        "unit": "s"
       },
       "mlx_lm": {
        "median": 2.2374914794927463,
        "samples": [
         2.2381450419779867,
         2.236837917007506,
         2.238560708006844,
         2.236316208029166
        ],
        "unit": "s"
       },
       "ratio": 1.0398,
       "gate_pass": true,
       "goal_pass": true
      },
      "prefill_tok_s": {
       "nuvec": {
        "median": 951.77,
        "samples": [
         951.53,
         952.01,
         951.39,
         952.18
        ],
        "unit": "tok/s"
       },
       "mlx_lm": {
        "median": 915.3108262114646,
        "samples": [
         915.0434675091727,
         915.5781849137564,
         914.8735581191746,
         915.7917796450055
        ],
        "unit": "tok/s"
       },
       "ratio": 1.0398,
       "gate_pass": true,
       "goal_pass": true
      }
     },
     {
      "prompt_tokens": 32768,
      "ttft_s": {
       "nuvec": {
        "median": 47.372600000000006,
        "samples": [
         47.4622,
         47.374,
         47.3639,
         47.3712
        ],
        "unit": "s"
       },
       "mlx_lm": {
        "median": 48.76352593750926,
        "samples": [
         48.75515283300774,
         48.771879874984734,
         48.76118274999317,
         48.76586912502535
        ],
        "unit": "s"
       },
       "ratio": 1.0294,
       "gate_pass": true,
       "goal_pass": true
      },
      "prefill_tok_s": {
       "nuvec": {
        "median": 691.71,
        "samples": [
         690.4,
         691.69,
         691.83,
         691.73
        ],
        "unit": "tok/s"
       },
       "mlx_lm": {
        "median": 671.9776604680656,
        "samples": [
         672.0930629062807,
         671.8625585889466,
         672.0099503739907,
         671.9453705621404
        ],
        "unit": "tok/s"
       },
       "ratio": 1.0294,
       "gate_pass": true,
       "goal_pass": true
      }
     }
    ]
   },
   "memory": {
    "load": {
     "peak_bytes": {
      "nuvec": {
       "median": 12076177408.0,
       "samples": [
        12076177408,
        12076177408
       ],
       "unit": "bytes"
      },
      "mlx_lm": {
       "median": 12079160296.0,
       "samples": [
        12079160296,
        12079160296
       ],
       "unit": "bytes"
      },
      "ratio": 1.0002
     },
     "os_peak_rss_bytes": {
      "nuvec": {
       "median": 12243607552.0,
       "samples": [
        12243632128,
        12243582976
       ],
       "unit": "bytes"
      },
      "mlx_lm": {
       "median": 12708012032.0,
       "samples": [
        12701122560,
        12714901504
       ],
       "unit": "bytes"
      },
      "ratio": 1.0379
     },
     "nuvec_raw": {
      "peak_bytes": 12076177408,
      "active_bytes": 12076177408,
      "mapped_bytes": 0
     }
    },
    "decode": {
     "peak_bytes": {
      "nuvec": {
       "median": 12118135296.0,
       "samples": [
        12118135296,
        12118135296
       ],
       "unit": "bytes"
      },
      "mlx_lm": {
       "median": 12128795581.0,
       "samples": [
        12128795566,
        12128795596
       ],
       "unit": "bytes"
      },
      "ratio": 1.0009
     },
     "os_peak_rss_bytes": {
      "nuvec": {
       "median": 12354060288.0,
       "samples": [
        12353912832,
        12354207744
       ],
       "unit": "bytes"
      },
      "mlx_lm": {
       "median": 12708233216.0,
       "samples": [
        12714737664,
        12701728768
       ],
       "unit": "bytes"
      },
      "ratio": 1.0287
     },
     "nuvec_raw": {
      "peak_bytes": 12118135296,
      "active_bytes": 12076178176,
      "mapped_bytes": 0
     }
    },
    "prefill_2048": {
     "peak_bytes": {
      "nuvec": {
       "median": 12509726976.0,
       "samples": [
        12509726976,
        12509726976
       ],
       "unit": "bytes"
      },
      "mlx_lm": {
       "median": 12954535666.0,
       "samples": [
        12954535666,
        12954535666
       ],
       "unit": "bytes"
      },
      "ratio": 1.0356
     },
     "os_peak_rss_bytes": {
      "nuvec": {
       "median": 12349505536.0,
       "samples": [
        12349489152,
        12349521920
       ],
       "unit": "bytes"
      },
      "mlx_lm": {
       "median": 12708274176.0,
       "samples": [
        12714672128,
        12701876224
       ],
       "unit": "bytes"
      },
      "ratio": 1.0291
     },
     "nuvec_raw": {
      "peak_bytes": 12509726976,
      "active_bytes": 12076178176,
      "mapped_bytes": 0
     }
    },
    "prefill_32768": {
     "peak_bytes": {
      "nuvec": {
       "median": 13354019584.0,
       "samples": [
        13354019584,
        13354019584
       ],
       "unit": "bytes"
      },
      "mlx_lm": {
       "median": 13736994534.0,
       "samples": [
        13736994534,
        13736994534
       ],
       "unit": "bytes"
      },
      "ratio": 1.0287
     },
     "os_peak_rss_bytes": {
      "nuvec": {
       "median": 12349685760.0,
       "samples": [
        12349702144,
        12349669376
       ],
       "unit": "bytes"
      },
      "mlx_lm": {
       "median": 12713222144.0,
       "samples": [
        12714672128,
        12711772160
       ],
       "unit": "bytes"
      },
      "ratio": 1.0294
     },
     "nuvec_raw": {
      "peak_bytes": 13354019584,
      "active_bytes": 12076178432,
      "mapped_bytes": 0
     }
    }
   },
   "load": {
    "warm": {
     "model_ready_s": {
      "nuvec": {
       "median": 0.787,
       "samples": [
        0.793,
        0.772,
        0.787
       ],
       "unit": "s"
      },
      "mlx_lm": {
       "median": 1.1495915420236997,
       "samples": [
        1.1495915420236997,
        1.1509335410082713,
        1.1398809580132365
       ],
       "unit": "s"
      },
      "ratio": 1.4607,
      "gate_pass": true,
      "goal_pass": true
     },
     "first_token_s": {
      "nuvec": {
       "median": 0.9332,
       "samples": [
        0.9369,
        0.9146,
        0.9332
       ],
       "unit": "s"
      },
      "mlx_lm": {
       "median": 1.3724054159829393,
       "samples": [
        1.3765907510532998,
        1.3724054159829393,
        1.3666775419842452
       ],
       "unit": "s"
      },
      "ratio": 1.4706,
      "gate_pass": true,
      "goal_pass": true
     }
    },
    "cold": {
     "model_ready_s": {
      "nuvec": null,
      "mlx_lm": null,
      "ratio": null,
      "reason": "purge unavailable (needs passwordless sudo)"
     },
     "first_token_s": {
      "nuvec": null,
      "mlx_lm": null,
      "ratio": null,
      "reason": "purge unavailable (needs passwordless sudo)"
     }
    }
   },
   "concurrency": [
    {
     "level": 1,
     "total_tokens": 128,
     "wall_s": {
      "nuvec": {
       "median": 1.9733,
       "samples": [
        1.9733
       ],
       "unit": "s"
      },
      "mlx_lm": {
       "median": 1.9748381250537932,
       "samples": [
        1.9748381250537932
       ],
       "unit": "s"
      },
      "ratio": 1.0008,
      "gate_pass": true,
      "goal_pass": true
     },
     "aggregate_tok_s": {
      "nuvec": {
       "median": 64.86596057365834,
       "samples": [
        64.86596057365834
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 64.81543898516409,
       "samples": [
        64.81543898516409
       ],
       "unit": "tok/s"
      },
      "ratio": 1.0008,
      "gate_pass": true,
      "goal_pass": true
     },
     "decode_aggregate_tok_s": {
      "nuvec": {
       "median": 73.95760540414628,
       "samples": [
        73.95760540414628
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 74.62056251993428,
       "samples": [
        74.62056251993428
       ],
       "unit": "tok/s"
      },
      "ratio": 0.9911,
      "gate_pass": true,
      "goal_pass": false
     },
     "nuvec_matches_solo": true
    },
    {
     "level": 4,
     "total_tokens": 512,
     "wall_s": {
      "nuvec": {
       "median": 4.4779,
       "samples": [
        4.4779
       ],
       "unit": "s"
      },
      "mlx_lm": {
       "median": 4.457289458019659,
       "samples": [
        4.457289458019659
       ],
       "unit": "s"
      },
      "ratio": 0.9954,
      "gate_pass": true,
      "goal_pass": false
     },
     "aggregate_tok_s": {
      "nuvec": {
       "median": 114.33931083766944,
       "samples": [
        114.33931083766944
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 114.86801672231488,
       "samples": [
        114.86801672231488
       ],
       "unit": "tok/s"
      },
      "ratio": 0.9954,
      "gate_pass": true,
      "goal_pass": false
     },
     "decode_aggregate_tok_s": {
      "nuvec": {
       "median": 139.63332508727083,
       "samples": [
        139.63332508727083
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 140.6577892631148,
       "samples": [
        140.6577892631148
       ],
       "unit": "tok/s"
      },
      "ratio": 0.9927,
      "gate_pass": true,
      "goal_pass": false
     },
     "nuvec_matches_solo": false
    },
    {
     "level": 8,
     "total_tokens": 1024,
     "wall_s": {
      "nuvec": {
       "median": 8.1335,
       "samples": [
        8.1335
       ],
       "unit": "s"
      },
      "mlx_lm": {
       "median": 8.103042374947108,
       "samples": [
        8.103042374947108
       ],
       "unit": "s"
      },
      "ratio": 0.9963,
      "gate_pass": true,
      "goal_pass": false
     },
     "aggregate_tok_s": {
      "nuvec": {
       "median": 125.89905944550317,
       "samples": [
        125.89905944550317
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 126.37228742206646,
       "samples": [
        126.37228742206646
       ],
       "unit": "tok/s"
      },
      "ratio": 0.9963,
      "gate_pass": true,
      "goal_pass": false
     },
     "decode_aggregate_tok_s": {
      "nuvec": {
       "median": 155.75177826833456,
       "samples": [
        155.75177826833456
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 156.49037676546808,
       "samples": [
        156.49037676546808
       ],
       "unit": "tok/s"
      },
      "ratio": 0.9953,
      "gate_pass": true,
      "goal_pass": false
     },
     "nuvec_matches_solo": false
    }
   ],
   "status": "ok"
  },
  {
   "id": "lmstudio-community/gpt-oss-20b-MLX-8bit",
   "kind": "moe",
   "snapshot": "d23249e3b54df6248c79b67c155bab2f836742dd",
   "decode": [
    {
     "repeats": 1,
     "prompt_tokens": {
      "nuvec": 24,
      "mlx_lm": 24
     },
     "tok_s": {
      "nuvec": {
       "median": 53.495,
       "samples": [
        53.56,
        53.48,
        53.51,
        53.4
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 52.81326378188551,
       "samples": [
        52.89254914005191,
        52.855655097860236,
        52.4101039053057,
        52.77087246591079
       ],
       "unit": "tok/s"
      },
      "ratio": 1.0129,
      "gate_pass": true,
      "goal_pass": true
     }
    },
    {
     "repeats": 40,
     "prompt_tokens": {
      "nuvec": 921,
      "mlx_lm": 921
     },
     "tok_s": {
      "nuvec": {
       "median": 52.5,
       "samples": [
        52.49,
        52.51,
        52.48,
        52.54
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 52.08005439216869,
       "samples": [
        52.12010585899178,
        52.10770184754256,
        52.04651221976955,
        52.052406936794824
       ],
       "unit": "tok/s"
      },
      "ratio": 1.0081,
      "gate_pass": true,
      "goal_pass": true
     }
    },
    {
     "repeats": 160,
     "prompt_tokens": {
      "nuvec": 3681,
      "mlx_lm": 3681
     },
     "tok_s": {
      "nuvec": {
       "median": 51.114999999999995,
       "samples": [
        51.11,
        51.11,
        51.12,
        51.14
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 50.887985764222776,
       "samples": [
        50.97068978935764,
        50.823083845323396,
        50.88899126850511,
        50.886980259940444
       ],
       "unit": "tok/s"
      },
      "ratio": 1.0045,
      "gate_pass": true,
      "goal_pass": true
     }
    }
   ],
   "ttft": {
    "legacy": [
     {
      "repeats": 1,
      "prompt_tokens": {
       "nuvec": 24,
       "mlx_lm": 24
      },
      "prefill_tok_s": {
       "nuvec": {
        "median": 161.18,
        "samples": [
         160.47,
         160.94,
         162.28,
         161.42
        ],
        "unit": "tok/s"
       },
       "mlx_lm": {
        "median": 145.72879040982195,
        "samples": [
         145.81123063688048,
         146.0513270643174,
         145.601506779147,
         145.6463501827634
        ],
        "unit": "tok/s"
       },
       "ratio": 1.106,
       "gate_pass": true,
       "goal_pass": true
      }
     },
     {
      "repeats": 40,
      "prompt_tokens": {
       "nuvec": 921,
       "mlx_lm": 921
      },
      "prefill_tok_s": {
       "nuvec": {
        "median": 886.665,
        "samples": [
         884.37,
         888.13,
         885.2,
         888.38
        ],
        "unit": "tok/s"
       },
       "mlx_lm": {
        "median": 881.1432008973267,
        "samples": [
         878.8482060219886,
         882.818468509115,
         879.4679332855384,
         883.9943742444482
        ],
        "unit": "tok/s"
       },
       "ratio": 1.0063,
       "gate_pass": true,
       "goal_pass": true
      }
     },
     {
      "repeats": 160,
      "prompt_tokens": {
       "nuvec": 3681,
       "mlx_lm": 3681
      },
      "prefill_tok_s": {
       "nuvec": {
        "median": 902.165,
        "samples": [
         901.56,
         902.6,
         901.88,
         902.45
        ],
        "unit": "tok/s"
       },
       "mlx_lm": {
        "median": 883.3232247779881,
        "samples": [
         882.4380004738014,
         883.3233395545753,
         883.3703285103346,
         883.3231100014009
        ],
        "unit": "tok/s"
       },
       "ratio": 1.0213,
       "gate_pass": true,
       "goal_pass": true
      }
     }
    ],
    "lengths": [
     {
      "prompt_tokens": 2048,
      "ttft_s": {
       "nuvec": {
        "median": 2.25115,
        "samples": [
         2.2573,
         2.2505,
         2.2518,
         2.2494
        ],
        "unit": "s"
       },
       "mlx_lm": {
        "median": 2.3032964790181722,
        "samples": [
         2.305031333002262,
         2.300920000008773,
         2.305987208033912,
         2.3015616250340827
        ],
        "unit": "s"
       },
       "ratio": 1.0232,
       "gate_pass": true,
       "goal_pass": true
      },
      "prefill_tok_s": {
       "nuvec": {
        "median": 909.755,
        "samples": [
         907.26,
         910.02,
         909.49,
         910.45
        ],
        "unit": "tok/s"
       },
       "mlx_lm": {
        "median": 889.1608963601027,
        "samples": [
         888.4911760971669,
         890.0787511048586,
         888.1228798082223,
         889.8306166230384
        ],
        "unit": "tok/s"
       },
       "ratio": 1.0232,
       "gate_pass": true,
       "goal_pass": true
      }
     },
     {
      "prompt_tokens": 32768,
      "ttft_s": {
       "nuvec": {
        "median": 49.549350000000004,
        "samples": [
         48.8249,
         49.6218,
         49.5445,
         49.5542
        ],
        "unit": "s"
       },
       "mlx_lm": {
        "median": 50.375510312995175,
        "samples": [
         50.52366883400828,
         50.37178833398502,
         50.37923229200533,
         50.3694613330299
        ],
        "unit": "s"
       },
       "ratio": 1.0167,
       "gate_pass": true,
       "goal_pass": true
      },
      "prefill_tok_s": {
       "nuvec": {
        "median": 661.325,
        "samples": [
         671.13,
         660.36,
         661.39,
         661.26
        ],
        "unit": "tok/s"
       },
       "mlx_lm": {
        "median": 650.4748036354038,
        "samples": [
         648.5673102572342,
         650.5228637652311,
         650.4267435055764,
         650.5529170412688
        ],
        "unit": "tok/s"
       },
       "ratio": 1.0167,
       "gate_pass": true,
       "goal_pass": true
      }
     }
    ]
   },
   "memory": {
    "load": {
     "peak_bytes": {
      "nuvec": {
       "median": 22228588544.0,
       "samples": [
        22228588544,
        22228588544
       ],
       "unit": "bytes"
      },
      "mlx_lm": {
       "median": 22232390632.0,
       "samples": [
        22232390632,
        22232390632
       ],
       "unit": "bytes"
      },
      "ratio": 1.0002
     },
     "os_peak_rss_bytes": {
      "nuvec": {
       "median": 6647275520.0,
       "samples": [
        6647267328,
        6647283712
       ],
       "unit": "bytes"
      },
      "mlx_lm": {
       "median": 19464454144.0,
       "samples": [
        19440205824,
        19488702464
       ],
       "unit": "bytes"
      },
      "ratio": 2.9282
     },
     "nuvec_raw": {
      "peak_bytes": 6480140288,
      "active_bytes": 6480140288,
      "mapped_bytes": 15748448256
     }
    },
    "decode": {
     "peak_bytes": {
      "nuvec": {
       "median": 22271085696.0,
       "samples": [
        22270546432,
        22271624960
       ],
       "unit": "bytes"
      },
      "mlx_lm": {
       "median": 22261606984.0,
       "samples": [
        22261606984,
        22261606984
       ],
       "unit": "bytes"
      },
      "ratio": 0.9996
     },
     "os_peak_rss_bytes": {
      "nuvec": {
       "median": 6758711296.0,
       "samples": [
        6758531072,
        6758891520
       ],
       "unit": "bytes"
      },
      "mlx_lm": {
       "median": 19327696896.0,
       "samples": [
        19118669824,
        19536723968
       ],
       "unit": "bytes"
      },
      "ratio": 2.8597
     },
     "nuvec_raw": {
      "peak_bytes": 6523176704,
      "active_bytes": 6480141056,
      "mapped_bytes": 15748448256
     }
    },
    "prefill_2048": {
     "peak_bytes": {
      "nuvec": {
       "median": 22662138112.0,
       "samples": [
        22662138112,
        22662138112
       ],
       "unit": "bytes"
      },
      "mlx_lm": {
       "median": 23107766002.0,
       "samples": [
        23107766002,
        23107766002
       ],
       "unit": "bytes"
      },
      "ratio": 1.0197
     },
     "os_peak_rss_bytes": {
      "nuvec": {
       "median": 6753402880.0,
       "samples": [
        6753107968,
        6753697792
       ],
       "unit": "bytes"
      },
      "mlx_lm": {
       "median": 18767626240.0,
       "samples": [
        18474795008,
        19060457472
       ],
       "unit": "bytes"
      },
      "ratio": 2.779
     },
     "nuvec_raw": {
      "peak_bytes": 6913689856,
      "active_bytes": 6480141056,
      "mapped_bytes": 15748448256
     }
    },
    "prefill_32768": {
     "peak_bytes": {
      "nuvec": {
       "median": 23506430720.0,
       "samples": [
        23506430720,
        23506430720
       ],
       "unit": "bytes"
      },
      "mlx_lm": {
       "median": 23890224870.0,
       "samples": [
        23890224870,
        23890224870
       ],
       "unit": "bytes"
      },
      "ratio": 1.0163
     },
     "os_peak_rss_bytes": {
      "nuvec": {
       "median": 6753263616.0,
       "samples": [
        6753157120,
        6753370112
       ],
       "unit": "bytes"
      },
      "mlx_lm": {
       "median": 18590687232.0,
       "samples": [
        19064356864,
        18117017600
       ],
       "unit": "bytes"
      },
      "ratio": 2.7528
     },
     "nuvec_raw": {
      "peak_bytes": 7757982464,
      "active_bytes": 6480141312,
      "mapped_bytes": 15748448256
     }
    }
   },
   "load": {
    "warm": {
     "model_ready_s": {
      "nuvec": {
       "median": 0.6226,
       "samples": [
        0.6226,
        0.6207,
        0.623
       ],
       "unit": "s"
      },
      "mlx_lm": {
       "median": 2.3036967089865357,
       "samples": [
        2.3036967089865357,
        2.288584667025134,
        2.3529685829998925
       ],
       "unit": "s"
      },
      "ratio": 3.7001,
      "gate_pass": true,
      "goal_pass": true
     },
     "first_token_s": {
      "nuvec": {
       "median": 0.8558,
       "samples": [
        0.8558,
        0.8522,
        0.8572
       ],
       "unit": "s"
      },
      "mlx_lm": {
       "median": 3.084101209009532,
       "samples": [
        3.084101209009532,
        3.035204333020374,
        3.154850749007892
       ],
       "unit": "s"
      },
      "ratio": 3.6038,
      "gate_pass": true,
      "goal_pass": true
     }
    },
    "cold": {
     "model_ready_s": {
      "nuvec": null,
      "mlx_lm": null,
      "ratio": null,
      "reason": "purge unavailable (needs passwordless sudo)"
     },
     "first_token_s": {
      "nuvec": null,
      "mlx_lm": null,
      "ratio": null,
      "reason": "purge unavailable (needs passwordless sudo)"
     }
    }
   },
   "concurrency": [
    {
     "level": 1,
     "total_tokens": 128,
     "wall_s": {
      "nuvec": {
       "median": 2.6542,
       "samples": [
        2.6542
       ],
       "unit": "s"
      },
      "mlx_lm": {
       "median": 2.731764625001233,
       "samples": [
        2.731764625001233
       ],
       "unit": "s"
      },
      "ratio": 1.0292,
      "gate_pass": true,
      "goal_pass": true
     },
     "aggregate_tok_s": {
      "nuvec": {
       "median": 48.225453997438024,
       "samples": [
        48.225453997438024
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 46.85615987136601,
       "samples": [
        46.85615987136601
       ],
       "unit": "tok/s"
      },
      "ratio": 1.0292,
      "gate_pass": true,
      "goal_pass": true
     },
     "decode_aggregate_tok_s": {
      "nuvec": {
       "median": 53.49620893007582,
       "samples": [
        53.49620893007582
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 52.31172246416136,
       "samples": [
        52.31172246416136
       ],
       "unit": "tok/s"
      },
      "ratio": 1.0226,
      "gate_pass": true,
      "goal_pass": true
     },
     "nuvec_matches_solo": true
    },
    {
     "level": 4,
     "total_tokens": 512,
     "wall_s": {
      "nuvec": {
       "median": 7.234,
       "samples": [
        7.234
       ],
       "unit": "s"
      },
      "mlx_lm": {
       "median": 7.824309165996965,
       "samples": [
        7.824309165996965
       ],
       "unit": "s"
      },
      "ratio": 1.0816,
      "gate_pass": true,
      "goal_pass": true
     },
     "aggregate_tok_s": {
      "nuvec": {
       "median": 70.77688692286425,
       "samples": [
        70.77688692286425
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 65.43708704981387,
       "samples": [
        65.43708704981387
       ],
       "unit": "tok/s"
      },
      "ratio": 1.0816,
      "gate_pass": true,
      "goal_pass": true
     },
     "decode_aggregate_tok_s": {
      "nuvec": {
       "median": 80.23501911109707,
       "samples": [
        80.23501911109707
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 73.81414060273367,
       "samples": [
        73.81414060273367
       ],
       "unit": "tok/s"
      },
      "ratio": 1.087,
      "gate_pass": true,
      "goal_pass": true
     },
     "nuvec_matches_solo": false
    },
    {
     "level": 8,
     "total_tokens": 1024,
     "wall_s": {
      "nuvec": {
       "median": 14.669,
       "samples": [
        14.669
       ],
       "unit": "s"
      },
      "mlx_lm": {
       "median": 14.799305166990962,
       "samples": [
        14.799305166990962
       ],
       "unit": "s"
      },
      "ratio": 1.0089,
      "gate_pass": true,
      "goal_pass": true
     },
     "aggregate_tok_s": {
      "nuvec": {
       "median": 69.80707614697661,
       "samples": [
        69.80707614697661
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 69.19243764794957,
       "samples": [
        69.19243764794957
       ],
       "unit": "tok/s"
      },
      "ratio": 1.0089,
      "gate_pass": true,
      "goal_pass": true
     },
     "decode_aggregate_tok_s": {
      "nuvec": {
       "median": 78.79267290183488,
       "samples": [
        78.79267290183488
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 78.13276708117567,
       "samples": [
        78.13276708117567
       ],
       "unit": "tok/s"
      },
      "ratio": 1.0084,
      "gate_pass": true,
      "goal_pass": true
     },
     "nuvec_matches_solo": false
    }
   ],
   "status": "ok"
  },
  {
   "id": "mlx-community/gemma-4-e4b-it-4bit",
   "kind": "dense",
   "snapshot": "475b9088d29754a3379866cf5aeb6b41acd313c2",
   "decode": [
    {
     "repeats": 1,
     "prompt_tokens": {
      "nuvec": 24,
      "mlx_lm": 24
     },
     "tok_s": {
      "nuvec": {
       "median": 70.57499999999999,
       "samples": [
        69.43,
        69.55,
        71.6,
        71.81
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 71.83970840363907,
       "samples": [
        71.76786770585237,
        71.78934011796953,
        71.89007668930859,
        71.90997327067424
       ],
       "unit": "tok/s"
      },
      "ratio": 0.9824,
      "gate_pass": true,
      "goal_pass": false
     }
    },
    {
     "repeats": 40,
     "prompt_tokens": {
      "nuvec": 921,
      "mlx_lm": 921
     },
     "tok_s": {
      "nuvec": {
       "median": 69.225,
       "samples": [
        69.11,
        69.06,
        69.37,
        69.34
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 69.13050343178236,
       "samples": [
        69.10449394102652,
        69.15651292253818,
        69.33253760203709,
        68.57472850145277
       ],
       "unit": "tok/s"
      },
      "ratio": 1.0014,
      "gate_pass": true,
      "goal_pass": true
     }
    },
    {
     "repeats": 160,
     "prompt_tokens": {
      "nuvec": 3681,
      "mlx_lm": 3681
     },
     "tok_s": {
      "nuvec": {
       "median": 66.57499999999999,
       "samples": [
        66.03,
        66.79,
        66.58,
        66.57
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 66.86507283180973,
       "samples": [
        66.7647065200796,
        67.00787774485462,
        66.68270436543118,
        66.96543914353984
       ],
       "unit": "tok/s"
      },
      "ratio": 0.9957,
      "gate_pass": true,
      "goal_pass": false
     }
    }
   ],
   "ttft": {
    "legacy": [
     {
      "repeats": 1,
      "prompt_tokens": {
       "nuvec": 24,
       "mlx_lm": 24
      },
      "prefill_tok_s": {
       "nuvec": {
        "median": 534.265,
        "samples": [
         535.9,
         532.6,
         532.63,
         545.73
        ],
        "unit": "tok/s"
       },
       "mlx_lm": {
        "median": 423.18843678439714,
        "samples": [
         424.4819991916867,
         424.8169793864234,
         419.79110759312334,
         421.8948743771075
        ],
        "unit": "tok/s"
       },
       "ratio": 1.2625,
       "gate_pass": true,
       "goal_pass": true
      }
     },
     {
      "repeats": 40,
      "prompt_tokens": {
       "nuvec": 921,
       "mlx_lm": 921
      },
      "prefill_tok_s": {
       "nuvec": {
        "median": 1331.0900000000001,
        "samples": [
         1323.44,
         1332.92,
         1329.26,
         1334.3
        ],
        "unit": "tok/s"
       },
       "mlx_lm": {
        "median": 1296.9399854447283,
        "samples": [
         1298.020641847664,
         1295.8448193873262,
         1298.3375072277818,
         1295.859329041793
        ],
        "unit": "tok/s"
       },
       "ratio": 1.0263,
       "gate_pass": true,
       "goal_pass": true
      }
     },
     {
      "repeats": 160,
      "prompt_tokens": {
       "nuvec": 3681,
       "mlx_lm": 3681
      },
      "prefill_tok_s": {
       "nuvec": {
        "median": 1385.26,
        "samples": [
         1383.24,
         1386.86,
         1384.23,
         1386.29
        ],
        "unit": "tok/s"
       },
       "mlx_lm": {
        "median": 1291.6074509883674,
        "samples": [
         1291.1973645741189,
         1291.2258614403847,
         1292.3141117634013,
         1291.9890405363499
        ],
        "unit": "tok/s"
       },
       "ratio": 1.0725,
       "gate_pass": true,
       "goal_pass": true
      }
     }
    ],
    "lengths": [
     {
      "prompt_tokens": 2048,
      "ttft_s": {
       "nuvec": {
        "median": 1.4816500000000001,
        "samples": [
         1.587,
         1.4802,
         1.4831,
         1.4799
        ],
        "unit": "s"
       },
       "mlx_lm": {
        "median": 1.5824297919753008,
        "samples": [
         1.5838549169711769,
         1.5801257920102216,
         1.5834025419899262,
         1.5814570419606753
        ],
        "unit": "s"
       },
       "ratio": 1.068,
       "gate_pass": true,
       "goal_pass": true
      },
      "prefill_tok_s": {
       "nuvec": {
        "median": 1382.23,
        "samples": [
         1290.52,
         1383.6,
         1380.86,
         1383.84
        ],
        "unit": "tok/s"
       },
       "mlx_lm": {
        "median": 1294.2127254451584,
        "samples": [
         1293.047726818573,
         1296.0993424419414,
         1293.417148002172,
         1295.008302888145
        ],
        "unit": "tok/s"
       },
       "ratio": 1.068,
       "gate_pass": true,
       "goal_pass": true
      }
     },
     {
      "prompt_tokens": 32768,
      "ttft_s": {
       "nuvec": {
        "median": 27.17255,
        "samples": [
         27.1895,
         27.1715,
         27.1736,
         27.1675
        ],
        "unit": "s"
       },
       "mlx_lm": {
        "median": 29.104294875025516,
        "samples": [
         29.104984750039876,
         29.103605000011157,
         29.106851375021506,
         29.101058207976166
        ],
        "unit": "s"
       },
       "ratio": 1.0711,
       "gate_pass": true,
       "goal_pass": true
      },
      "prefill_tok_s": {
       "nuvec": {
        "median": 1205.9250000000002,
        "samples": [
         1205.17,
         1205.97,
         1205.88,
         1206.15
        ],
        "unit": "tok/s"
       },
       "mlx_lm": {
        "median": 1125.8819414494487,
        "samples": [
         1125.8552540542082,
         1125.908628844689,
         1125.7830528560835,
         1126.0071632384413
        ],
        "unit": "tok/s"
       },
       "ratio": 1.0711,
       "gate_pass": true,
       "goal_pass": true
      }
     }
    ]
   },
   "memory": {
    "load": {
     "peak_bytes": {
      "nuvec": {
       "median": 4198784000.0,
       "samples": [
        4198784000,
        4198784000
       ],
       "unit": "bytes"
      },
      "mlx_lm": {
       "median": 4201486540.0,
       "samples": [
        4201486540,
        4201486540
       ],
       "unit": "bytes"
      },
      "ratio": 1.0006
     },
     "os_peak_rss_bytes": {
      "nuvec": {
       "median": 4405952512.0,
       "samples": [
        4406099968,
        4405805056
       ],
       "unit": "bytes"
      },
      "mlx_lm": {
       "median": 5391253504.0,
       "samples": [
        5390598144,
        5391908864
       ],
       "unit": "bytes"
      },
      "ratio": 1.2236
     },
     "nuvec_raw": {
      "peak_bytes": 4198784000,
      "active_bytes": 4198784000,
      "mapped_bytes": 0
     }
    },
    "decode": {
     "peak_bytes": {
      "nuvec": {
       "median": 4237652224.0,
       "samples": [
        4237652224,
        4237652224
       ],
       "unit": "bytes"
      },
      "mlx_lm": {
       "median": 4290298783.5,
       "samples": [
        4290249536,
        4290348031
       ],
       "unit": "bytes"
      },
      "ratio": 1.0124
     },
     "os_peak_rss_bytes": {
      "nuvec": {
       "median": 4525727744.0,
       "samples": [
        4526456832,
        4524998656
       ],
       "unit": "bytes"
      },
      "mlx_lm": {
       "median": 5390589952.0,
       "samples": [
        5390385152,
        5390794752
       ],
       "unit": "bytes"
      },
      "ratio": 1.1911
     },
     "nuvec_raw": {
      "peak_bytes": 4237652224,
      "active_bytes": 4198791680,
      "mapped_bytes": 0
     }
    },
    "prefill_2048": {
     "peak_bytes": {
      "nuvec": {
       "median": 4573096704.0,
       "samples": [
        4573096704,
        4573096704
       ],
       "unit": "bytes"
      },
      "mlx_lm": {
       "median": 5079964442.0,
       "samples": [
        5079964442,
        5079964442
       ],
       "unit": "bytes"
      },
      "ratio": 1.1108
     },
     "os_peak_rss_bytes": {
      "nuvec": {
       "median": 4517519360.0,
       "samples": [
        4516184064,
        4518854656
       ],
       "unit": "bytes"
      },
      "mlx_lm": {
       "median": 5391015936.0,
       "samples": [
        5390843904,
        5391187968
       ],
       "unit": "bytes"
      },
      "ratio": 1.1934
     },
     "nuvec_raw": {
      "peak_bytes": 4573096704,
      "active_bytes": 4198791680,
      "mapped_bytes": 0
     }
    },
    "prefill_32768": {
     "peak_bytes": {
      "nuvec": {
       "median": 5169290240.0,
       "samples": [
        5169290240,
        5169290240
       ],
       "unit": "bytes"
      },
      "mlx_lm": {
       "median": 5719751458.0,
       "samples": [
        5719751458,
        5719751458
       ],
       "unit": "bytes"
      },
      "ratio": 1.1065
     },
     "os_peak_rss_bytes": {
      "nuvec": {
       "median": 4518461440.0,
       "samples": [
        4518346752,
        4518576128
       ],
       "unit": "bytes"
      },
      "mlx_lm": {
       "median": 5391892480.0,
       "samples": [
        5392793600,
        5390991360
       ],
       "unit": "bytes"
      },
      "ratio": 1.1933
     },
     "nuvec_raw": {
      "peak_bytes": 5169290240,
      "active_bytes": 4198794240,
      "mapped_bytes": 0
     }
    }
   },
   "load": {
    "warm": {
     "model_ready_s": {
      "nuvec": {
       "median": 0.4261,
       "samples": [
        0.4261,
        0.4233,
        0.4272
       ],
       "unit": "s"
      },
      "mlx_lm": {
       "median": 1.1987828750279732,
       "samples": [
        1.1815470000146888,
        1.1987828750279732,
        1.2080584589857608
       ],
       "unit": "s"
      },
      "ratio": 2.8134,
      "gate_pass": true,
      "goal_pass": true
     },
     "first_token_s": {
      "nuvec": {
       "median": 0.5105,
       "samples": [
        0.5105,
        0.5102,
        0.5116
       ],
       "unit": "s"
      },
      "mlx_lm": {
       "median": 1.3017422500415705,
       "samples": [
        1.2812969170045108,
        1.3017422500415705,
        1.3107286260346882
       ],
       "unit": "s"
      },
      "ratio": 2.5499,
      "gate_pass": true,
      "goal_pass": true
     }
    },
    "cold": {
     "model_ready_s": {
      "nuvec": null,
      "mlx_lm": null,
      "ratio": null,
      "reason": "purge unavailable (needs passwordless sudo)"
     },
     "first_token_s": {
      "nuvec": null,
      "mlx_lm": null,
      "ratio": null,
      "reason": "purge unavailable (needs passwordless sudo)"
     }
    }
   },
   "concurrency": [
    {
     "level": 1,
     "total_tokens": 128,
     "wall_s": {
      "nuvec": {
       "median": 1.9553,
       "samples": [
        1.9553
       ],
       "unit": "s"
      },
      "mlx_lm": {
       "median": 1.984897957998328,
       "samples": [
        1.984897957998328
       ],
       "unit": "s"
      },
      "ratio": 1.0151,
      "gate_pass": true,
      "goal_pass": true
     },
     "aggregate_tok_s": {
      "nuvec": {
       "median": 65.46310029151536,
       "samples": [
        65.46310029151536
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 64.48694225524909,
       "samples": [
        64.48694225524909
       ],
       "unit": "tok/s"
      },
      "ratio": 1.0151,
      "gate_pass": true,
      "goal_pass": true
     },
     "decode_aggregate_tok_s": {
      "nuvec": {
       "median": 70.8626269389577,
       "samples": [
        70.8626269389577
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 70.573234778886,
       "samples": [
        70.573234778886
       ],
       "unit": "tok/s"
      },
      "ratio": 1.0041,
      "gate_pass": true,
      "goal_pass": true
     },
     "nuvec_matches_solo": true
    },
    {
     "level": 4,
     "total_tokens": 512,
     "wall_s": {
      "nuvec": {
       "median": 3.1341,
       "samples": [
        3.1341
       ],
       "unit": "s"
      },
      "mlx_lm": {
       "median": 3.1849353339639492,
       "samples": [
        3.1849353339639492
       ],
       "unit": "s"
      },
      "ratio": 1.0162,
      "gate_pass": true,
      "goal_pass": true
     },
     "aggregate_tok_s": {
      "nuvec": {
       "median": 163.36428320730033,
       "samples": [
        163.36428320730033
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 160.75679607685103,
       "samples": [
        160.75679607685103
       ],
       "unit": "tok/s"
      },
      "ratio": 1.0162,
      "gate_pass": true,
      "goal_pass": true
     },
     "decode_aggregate_tok_s": {
      "nuvec": {
       "median": 198.40649898453367,
       "samples": [
        198.40649898453367
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 196.3647650499326,
       "samples": [
        196.3647650499326
       ],
       "unit": "tok/s"
      },
      "ratio": 1.0104,
      "gate_pass": true,
      "goal_pass": true
     },
     "nuvec_matches_solo": false
    },
    {
     "level": 8,
     "total_tokens": 1024,
     "wall_s": {
      "nuvec": {
       "median": 5.5831,
       "samples": [
        5.5831
       ],
       "unit": "s"
      },
      "mlx_lm": {
       "median": 5.653411625011358,
       "samples": [
        5.653411625011358
       ],
       "unit": "s"
      },
      "ratio": 1.0126,
      "gate_pass": true,
      "goal_pass": true
     },
     "aggregate_tok_s": {
      "nuvec": {
       "median": 183.41064999731333,
       "samples": [
        183.41064999731333
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 181.12956705110653,
       "samples": [
        181.12956705110653
       ],
       "unit": "tok/s"
      },
      "ratio": 1.0126,
      "gate_pass": true,
      "goal_pass": true
     },
     "decode_aggregate_tok_s": {
      "nuvec": {
       "median": 227.171093819873,
       "samples": [
        227.171093819873
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 224.9749691979948,
       "samples": [
        224.9749691979948
       ],
       "unit": "tok/s"
      },
      "ratio": 1.0098,
      "gate_pass": true,
      "goal_pass": true
     },
     "nuvec_matches_solo": false
    }
   ],
   "status": "ok"
  },
  {
   "id": "mlx-community/gemma-4-26b-a4b-it-4bit",
   "kind": "moe",
   "snapshot": "0d77464eeb233a2da68ebf9d7dc4edaac7db956d",
   "decode": [
    {
     "repeats": 1,
     "prompt_tokens": {
      "nuvec": 24,
      "mlx_lm": 24
     },
     "tok_s": {
      "nuvec": {
       "median": 80.34,
       "samples": [
        80.38,
        80.33,
        80.35,
        80.3
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 79.6719034056454,
       "samples": [
        79.95332699928638,
        79.57679346950995,
        79.54512504699704,
        79.76701334178082
       ],
       "unit": "tok/s"
      },
      "ratio": 1.0084,
      "gate_pass": true,
      "goal_pass": true
     }
    },
    {
     "repeats": 40,
     "prompt_tokens": {
      "nuvec": 921,
      "mlx_lm": 921
     },
     "tok_s": {
      "nuvec": {
       "median": 73.285,
       "samples": [
        73.3,
        73.27,
        73.23,
        73.35
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 72.50816871725789,
       "samples": [
        72.53785216653667,
        72.59724726366476,
        72.46239510936589,
        72.4784852679791
       ],
       "unit": "tok/s"
      },
      "ratio": 1.0107,
      "gate_pass": true,
      "goal_pass": true
     }
    },
    {
     "repeats": 160,
     "prompt_tokens": {
      "nuvec": 3681,
      "mlx_lm": 3681
     },
     "tok_s": {
      "nuvec": {
       "median": 59.13,
       "samples": [
        58.97,
        59.2,
        59.13,
        59.13
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 58.734714362632246,
       "samples": [
        58.738962336919776,
        58.72908977062103,
        58.73447668162896,
        58.73495204363553
       ],
       "unit": "tok/s"
      },
      "ratio": 1.0067,
      "gate_pass": true,
      "goal_pass": true
     }
    }
   ],
   "ttft": {
    "legacy": [
     {
      "repeats": 1,
      "prompt_tokens": {
       "nuvec": 24,
       "mlx_lm": 24
      },
      "prefill_tok_s": {
       "nuvec": {
        "median": 247.985,
        "samples": [
         249.44,
         246.35,
         249.19,
         246.78
        ],
        "unit": "tok/s"
       },
       "mlx_lm": {
        "median": 224.73056177104624,
        "samples": [
         225.66765747319138,
         222.74152336975783,
         224.36669007100636,
         225.09443347108612
        ],
        "unit": "tok/s"
       },
       "ratio": 1.1035,
       "gate_pass": true,
       "goal_pass": true
      }
     },
     {
      "repeats": 40,
      "prompt_tokens": {
       "nuvec": 921,
       "mlx_lm": 921
      },
      "prefill_tok_s": {
       "nuvec": {
        "median": 861.4300000000001,
        "samples": [
         859.09,
         863.85,
         858.31,
         863.77
        ],
        "unit": "tok/s"
       },
       "mlx_lm": {
        "median": 855.1988471431889,
        "samples": [
         854.4480393478248,
         855.910568954527,
         855.5772185143322,
         854.8204757720455
        ],
        "unit": "tok/s"
       },
       "ratio": 1.0073,
       "gate_pass": true,
       "goal_pass": true
      }
     },
     {
      "repeats": 160,
      "prompt_tokens": {
       "nuvec": 3681,
       "mlx_lm": 3681
      },
      "prefill_tok_s": {
       "nuvec": {
        "median": 880.3,
        "samples": [
         879.16,
         881.44,
         879.06,
         881.51
        ],
        "unit": "tok/s"
       },
       "mlx_lm": {
        "median": 814.2286452969415,
        "samples": [
         814.0883732072274,
         814.494273305258,
         813.2324024549639,
         814.3689173866558
        ],
        "unit": "tok/s"
       },
       "ratio": 1.0811,
       "gate_pass": true,
       "goal_pass": true
      }
     }
    ],
    "lengths": [
     {
      "prompt_tokens": 2048,
      "ttft_s": {
       "nuvec": {
        "median": 2.30225,
        "samples": [
         2.303,
         2.3006,
         2.302,
         2.3025
        ],
        "unit": "s"
       },
       "mlx_lm": {
        "median": 2.451851854508277,
        "samples": [
         2.4527401250088587,
         2.4509635840076953,
         2.453074791992549,
         2.4498112080036663
        ],
        "unit": "s"
       },
       "ratio": 1.065,
       "gate_pass": true,
       "goal_pass": true
      },
      "prefill_tok_s": {
       "nuvec": {
        "median": 889.575,
        "samples": [
         889.28,
         890.19,
         889.67,
         889.48
        ],
        "unit": "tok/s"
       },
       "mlx_lm": {
        "median": 835.2871177905836,
        "samples": [
         834.984505336701,
         835.5897302444661,
         834.8705904464003,
         835.9827864731261
        ],
        "unit": "tok/s"
       },
       "ratio": 1.065,
       "gate_pass": true,
       "goal_pass": true
      }
     },
     {
      "prompt_tokens": 32768,
      "ttft_s": {
       "nuvec": {
        "median": 46.6922,
        "samples": [
         46.7259,
         46.6406,
         46.6585,
         47.0197
        ],
        "unit": "s"
       },
       "mlx_lm": {
        "median": 50.21465829201043,
        "samples": [
         50.57435595802963,
         50.570417709008325,
         49.85889887501253,
         49.852184666960966
        ],
        "unit": "s"
       },
       "ratio": 1.0754,
       "gate_pass": true,
       "goal_pass": true
      },
      "prefill_tok_s": {
       "nuvec": {
        "median": 701.785,
        "samples": [
         701.28,
         702.56,
         702.29,
         696.9
        ],
        "unit": "tok/s"
       },
       "mlx_lm": {
        "median": 652.5912144799036,
        "samples": [
         647.9172968053875,
         647.9677543609234,
         657.2146745988836,
         657.3031897981526
        ],
        "unit": "tok/s"
       },
       "ratio": 1.0754,
       "gate_pass": true,
       "goal_pass": true
      }
     }
    ]
   },
   "memory": {
    "load": {
     "peak_bytes": {
      "nuvec": {
       "median": 14200083456.0,
       "samples": [
        14200083456,
        14200083456
       ],
       "unit": "bytes"
      },
      "mlx_lm": {
       "median": 14202810516.0,
       "samples": [
        14202810516,
        14202810516
       ],
       "unit": "bytes"
      },
      "ratio": 1.0002
     },
     "os_peak_rss_bytes": {
      "nuvec": {
       "median": 14406418432.0,
       "samples": [
        14406549504,
        14406287360
       ],
       "unit": "bytes"
      },
      "mlx_lm": {
       "median": 15406276608.0,
       "samples": [
        15403401216,
        15409152000
       ],
       "unit": "bytes"
      },
      "ratio": 1.0694
     },
     "nuvec_raw": {
      "peak_bytes": 14200083456,
      "active_bytes": 14200083456,
      "mapped_bytes": 0
     }
    },
    "decode": {
     "peak_bytes": {
      "nuvec": {
       "median": 14333798656.0,
       "samples": [
        14333795584,
        14333801728
       ],
       "unit": "bytes"
      },
      "mlx_lm": {
       "median": 14278988806.0,
       "samples": [
        14278948486,
        14279029126
       ],
       "unit": "bytes"
      },
      "ratio": 0.9962
     },
     "os_peak_rss_bytes": {
      "nuvec": {
       "median": 14610948096.0,
       "samples": [
        14609711104,
        14612185088
       ],
       "unit": "bytes"
      },
      "mlx_lm": {
       "median": 15398313984.0,
       "samples": [
        15398338560,
        15398289408
       ],
       "unit": "bytes"
      },
      "ratio": 1.0539
     },
     "nuvec_raw": {
      "peak_bytes": 14333801728,
      "active_bytes": 14200091136,
      "mapped_bytes": 0
     }
    },
    "prefill_2048": {
     "peak_bytes": {
      "nuvec": {
       "median": 15180475648.0,
       "samples": [
        15180475648,
        15180475648
       ],
       "unit": "bytes"
      },
      "mlx_lm": {
       "median": 15486410502.0,
       "samples": [
        15486410502,
        15486410502
       ],
       "unit": "bytes"
      },
      "ratio": 1.0202
     },
     "os_peak_rss_bytes": {
      "nuvec": {
       "median": 14604173312.0,
       "samples": [
        14604288000,
        14604058624
       ],
       "unit": "bytes"
      },
      "mlx_lm": {
       "median": 15403458560.0,
       "samples": [
        15409954816,
        15396962304
       ],
       "unit": "bytes"
      },
      "ratio": 1.0547
     },
     "nuvec_raw": {
      "peak_bytes": 15180475648,
      "active_bytes": 14200091136,
      "mapped_bytes": 0
     }
    },
    "prefill_32768": {
     "peak_bytes": {
      "nuvec": {
       "median": 16090401024.0,
       "samples": [
        16090401024,
        16090401024
       ],
       "unit": "bytes"
      },
      "mlx_lm": {
       "median": 16640862422.0,
       "samples": [
        16640862422,
        16640862422
       ],
       "unit": "bytes"
      },
      "ratio": 1.0342
     },
     "os_peak_rss_bytes": {
      "nuvec": {
       "median": 14604394496.0,
       "samples": [
        14604402688,
        14604386304
       ],
       "unit": "bytes"
      },
      "mlx_lm": {
       "median": 15397961728.0,
       "samples": [
        15399043072,
        15396880384
       ],
       "unit": "bytes"
      },
      "ratio": 1.0543
     },
     "nuvec_raw": {
      "peak_bytes": 16090401024,
      "active_bytes": 14200093696,
      "mapped_bytes": 0
     }
    }
   },
   "load": {
    "warm": {
     "model_ready_s": {
      "nuvec": {
       "median": 0.9556,
       "samples": [
        0.9696,
        0.95,
        0.9556
       ],
       "unit": "s"
      },
      "mlx_lm": {
       "median": 1.39875612501055,
       "samples": [
        1.4614195420290343,
        1.397902917000465,
        1.39875612501055
       ],
       "unit": "s"
      },
      "ratio": 1.4637,
      "gate_pass": true,
      "goal_pass": true
     },
     "first_token_s": {
      "nuvec": {
       "median": 1.0895,
       "samples": [
        1.1113,
        1.0819,
        1.0895
       ],
       "unit": "s"
      },
      "mlx_lm": {
       "median": 1.6226967090042308,
       "samples": [
        1.6714693340472877,
        1.6131097080069594,
        1.6226967090042308
       ],
       "unit": "s"
      },
      "ratio": 1.4894,
      "gate_pass": true,
      "goal_pass": true
     }
    },
    "cold": {
     "model_ready_s": {
      "nuvec": null,
      "mlx_lm": null,
      "ratio": null,
      "reason": "purge unavailable (needs passwordless sudo)"
     },
     "first_token_s": {
      "nuvec": null,
      "mlx_lm": null,
      "ratio": null,
      "reason": "purge unavailable (needs passwordless sudo)"
     }
    }
   },
   "concurrency": [
    {
     "level": 1,
     "total_tokens": 128,
     "wall_s": {
      "nuvec": {
       "median": 1.8713,
       "samples": [
        1.8713
       ],
       "unit": "s"
      },
      "mlx_lm": {
       "median": 1.8997167920460925,
       "samples": [
        1.8997167920460925
       ],
       "unit": "s"
      },
      "ratio": 1.0152,
      "gate_pass": true,
      "goal_pass": true
     },
     "aggregate_tok_s": {
      "nuvec": {
       "median": 68.40164591460483,
       "samples": [
        68.40164591460483
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 67.37846427210734,
       "samples": [
        67.37846427210734
       ],
       "unit": "tok/s"
      },
      "ratio": 1.0152,
      "gate_pass": true,
      "goal_pass": true
     },
     "decode_aggregate_tok_s": {
      "nuvec": {
       "median": 79.45445445445445,
       "samples": [
        79.45445445445445
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 78.21534550691,
       "samples": [
        78.21534550691
       ],
       "unit": "tok/s"
      },
      "ratio": 1.0158,
      "gate_pass": true,
      "goal_pass": true
     },
     "nuvec_matches_solo": true
    },
    {
     "level": 4,
     "total_tokens": 512,
     "wall_s": {
      "nuvec": {
       "median": 4.3187,
       "samples": [
        4.3187
       ],
       "unit": "s"
      },
      "mlx_lm": {
       "median": 4.29511770803947,
       "samples": [
        4.29511770803947
       ],
       "unit": "s"
      },
      "ratio": 0.9945,
      "gate_pass": true,
      "goal_pass": false
     },
     "aggregate_tok_s": {
      "nuvec": {
       "median": 118.55419454928567,
       "samples": [
        118.55419454928567
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 119.20511492424389,
       "samples": [
        119.20511492424389
       ],
       "unit": "tok/s"
      },
      "ratio": 0.9945,
      "gate_pass": true,
      "goal_pass": false
     },
     "decode_aggregate_tok_s": {
      "nuvec": {
       "median": 149.1616995037731,
       "samples": [
        149.1616995037731
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 148.9192689726508,
       "samples": [
        148.9192689726508
       ],
       "unit": "tok/s"
      },
      "ratio": 1.0016,
      "gate_pass": true,
      "goal_pass": true
     },
     "nuvec_matches_solo": false
    },
    {
     "level": 8,
     "total_tokens": 1024,
     "wall_s": {
      "nuvec": {
       "median": 7.2442,
       "samples": [
        7.2442
       ],
       "unit": "s"
      },
      "mlx_lm": {
       "median": 7.231314250035211,
       "samples": [
        7.231314250035211
       ],
       "unit": "s"
      },
      "ratio": 0.9982,
      "gate_pass": true,
      "goal_pass": false
     },
     "aggregate_tok_s": {
      "nuvec": {
       "median": 141.35446288064935,
       "samples": [
        141.35446288064935
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 141.60634769744848,
       "samples": [
        141.60634769744848
       ],
       "unit": "tok/s"
      },
      "ratio": 0.9982,
      "gate_pass": true,
      "goal_pass": false
     },
     "decode_aggregate_tok_s": {
      "nuvec": {
       "median": 186.6274797942689,
       "samples": [
        186.6274797942689
       ],
       "unit": "tok/s"
      },
      "mlx_lm": {
       "median": 182.73415258590367,
       "samples": [
        182.73415258590367
       ],
       "unit": "tok/s"
      },
      "ratio": 1.0213,
      "gate_pass": true,
      "goal_pass": true
     },
     "nuvec_matches_solo": false
    }
   ],
   "status": "ok"
  }
 ],
 "summary": {
  "below_gate": {
   "concurrency_4": [
    {
     "model": "mlx-community/Qwen3-0.6B-4bit",
     "ratio": 0.9798
    },
    {
     "model": "mlx-community/Qwen3.6-35B-A3B-4bit",
     "ratio": 0.9774
    }
   ]
  },
  "below_goal": {
   "concurrency_4": [
    {
     "model": "mlx-community/Qwen3-0.6B-4bit",
     "ratio": 0.9798
    },
    {
     "model": "mlx-community/Llama-3.2-3B-Instruct-4bit",
     "ratio": 0.9956
    },
    {
     "model": "mlx-community/Qwen3-30B-A3B-4bit",
     "ratio": 0.9833
    },
    {
     "model": "mlx-community/Qwen3.6-35B-A3B-4bit",
     "ratio": 0.9774
    },
    {
     "model": "mlx-community/gpt-oss-20b-MXFP4-Q8",
     "ratio": 0.9954
    },
    {
     "model": "mlx-community/gemma-4-26b-a4b-it-4bit",
     "ratio": 0.9945
    }
   ],
   "concurrency_8": [
    {
     "model": "mlx-community/Qwen3-0.6B-4bit",
     "ratio": 0.9904
    },
    {
     "model": "mlx-community/Llama-3.2-3B-Instruct-4bit",
     "ratio": 0.9968
    },
    {
     "model": "mlx-community/Qwen3.6-35B-A3B-4bit",
     "ratio": 0.988
    },
    {
     "model": "mlx-community/gpt-oss-20b-MXFP4-Q8",
     "ratio": 0.9963
    },
    {
     "model": "mlx-community/gemma-4-26b-a4b-it-4bit",
     "ratio": 0.9982
    }
   ],
   "decode": [
    {
     "model": "mlx-community/Llama-3.2-3B-Instruct-4bit",
     "ratio": 0.9945
    },
    {
     "model": "mlx-community/Llama-3.2-3B-Instruct-4bit",
     "ratio": 0.9985
    },
    {
     "model": "mlx-community/Llama-3.2-3B-Instruct-4bit",
     "ratio": 0.9988
    },
    {
     "model": "mlx-community/Qwen3-8B-4bit",
     "ratio": 0.9987
    },
    {
     "model": "mlx-community/Qwen3.5-9B-4bit",
     "ratio": 0.9993
    },
    {
     "model": "mlx-community/Qwen3.5-9B-4bit",
     "ratio": 0.9985
    },
    {
     "model": "mlx-community/Qwen3.8-27B-4bit",
     "ratio": 1.0
    },
    {
     "model": "mlx-community/Qwen3-30B-A3B-4bit",
     "ratio": 0.998
    },
    {
     "model": "mlx-community/Qwen3-30B-A3B-4bit",
     "ratio": 0.9977
    },
    {
     "model": "mlx-community/Qwen3-30B-A3B-4bit",
     "ratio": 0.9972
    },
    {
     "model": "mlx-community/Qwen3.6-35B-A3B-4bit",
     "ratio": 0.9934
    },
    {
     "model": "mlx-community/Qwen3.6-35B-A3B-4bit",
     "ratio": 0.9955
    },
    {
     "model": "mlx-community/Qwen3.6-35B-A3B-4bit",
     "ratio": 0.9993
    },
    {
     "model": "mlx-community/gpt-oss-20b-MXFP4-Q8",
     "ratio": 0.9896
    },
    {
     "model": "mlx-community/gpt-oss-20b-MXFP4-Q8",
     "ratio": 0.9852
    },
    {
     "model": "mlx-community/gpt-oss-20b-MXFP4-Q8",
     "ratio": 0.9829
    },
    {
     "model": "mlx-community/gemma-4-e4b-it-4bit",
     "ratio": 0.9824
    },
    {
     "model": "mlx-community/gemma-4-e4b-it-4bit",
     "ratio": 0.9957
    }
   ],
   "ttft_2048": [
    {
     "model": "mlx-community/Llama-3.2-3B-Instruct-4bit",
     "ratio": 0.9977
    },
    {
     "model": "mlx-community/Qwen3-30B-A3B-4bit",
     "ratio": 0.9935
    }
   ],
   "concurrency_1": [
    {
     "model": "mlx-community/Qwen3-8B-4bit",
     "ratio": 0.9961
    },
    {
     "model": "mlx-community/Qwen3.6-35B-A3B-4bit",
     "ratio": 0.9886
    }
   ],
   "prefill_legacy": [
    {
     "model": "mlx-community/Qwen3.5-9B-4bit",
     "ratio": 0.9999
    },
    {
     "model": "mlx-community/Qwen3.5-9B-4bit",
     "ratio": 0.9984
    },
    {
     "model": "mlx-community/Qwen3.8-27B-4bit",
     "ratio": 0.9986
    },
    {
     "model": "mlx-community/Qwen3-30B-A3B-4bit",
     "ratio": 0.9977
    },
    {
     "model": "mlx-community/Qwen3.6-35B-A3B-4bit",
     "ratio": 0.9924
    },
    {
     "model": "mlx-community/Qwen3.6-35B-A3B-4bit",
     "ratio": 0.9916
    }
   ],
   "ttft_32768": [
    {
     "model": "mlx-community/Qwen3.5-9B-4bit",
     "ratio": 0.9912
    },
    {
     "model": "mlx-community/Qwen3.8-27B-4bit",
     "ratio": 0.9989
    },
    {
     "model": "mlx-community/Qwen3.6-35B-A3B-4bit",
     "ratio": 0.9825
    }
   ],
   "ttft_65536": [
    {
     "model": "mlx-community/Qwen3.6-35B-A3B-4bit",
     "ratio": 0.9832
    }
   ]
  },
  "correctness_flags": [],
  "decode_geomean": {
   "value": 1.0088,
   "n": 36
  },
  "memory_geomean": {
   "value": 1.0201,
   "n": 49
  },
  "load_warm_geomean": {
   "value": 1.7968,
   "n": 12
  },
  "ttft_geomean": {
   "2048": {
    "value": 1.0178,
    "n": 12
   },
   "32768": {
    "value": 1.0162,
    "n": 12
   },
   "65536": {
    "value": 0.9832,
    "n": 1
   },
   "legacy_prefill": {
    "value": 1.0149,
    "n": 24
   }
  },
  "concurrency_geomean": {
   "1": {
    "value": 1.01,
    "n": 12
   },
   "4": {
    "value": 1.0033,
    "n": 12
   },
   "8": {
    "value": 1.0026,
    "n": 12
   }
  }
 }
}