{
 "title": "JevBench v1.2.3 cost correction: the v1.1-tier cost of every row, each decision counted once",
 "date": "2026-09-20",
 "what_was_wrong": [
  "The v1.1 and v1.1.3 aggregations built their cost average from a row list that contained the 242-decision standard+judge run twice (once as the standard tier, once as the judge tier) and the 72 easy decisions once: 556 rows instead of 314. The standard and judge tiers were therefore over-weighted in the price, which made the affected rows look 1.5-3.3 % more expensive than they are.",
  "Rows without their own token counts were priced at the input tokens of the gemini-3.1-flash-lite run measured on the 242 standard+judge decisions only (452 per decision) and that figure was applied to all 314 v1.1 decisions, which excludes the shorter easy tier. Over all 314 decisions the same run averages 383.41 input tokens, which is the figure used from v1.2.3 on. This made the affected rows look 4-11 % more expensive.",
  "A metered row's price left out the requests whose answer came back unparseable. Those requests returned HTTP 200 with generated tokens and were billed, and JevBench already counts them as wrong answers, so from v1.2.3 they are priced too. Only DeepSeek V4.1 Flash had any (9 of its 314 v1.1 decisions); its price rises by 2.6 %.",
  "No tariff was wrong. The hard-tier costs, and classifier.dev's flat plan price, were already correct."
 ],
 "rule": "usd_per_1000_v11_tiers = 1000 x (mean input tokens per decision x $/M in + output tokens charged x $/M out) / 1e6, over all 314 v1.1 decisions (72 easy + 242 standard+judge), each decision counted exactly once and priced exactly once. A metered row uses the provider's own tariff and its own measured token counts, including the requests whose answer could not be parsed; an estimated row uses the reference tariff for its weights or size class and, when the run reports no usage, the input tokens of the gemini-3.1-flash-lite run on the same prompts over the same 314 decisions. usd_per_1000 = (v11 x 314 + hard x 220) / 534.",
 "n_v11_decisions": 314,
 "n_hard_decisions": 220,
 "gemini_fallback_input_tokens_per_decision": 383.4076433121019,
 "systems": {
  "classifier-dev-fast": {
   "display": "classifier.dev (fast tier)",
   "kind": "estimate",
   "n_decisions": 314,
   "usd_per_1000_v11_tiers_old": 0.003333333333333333,
   "usd_per_1000_hard": 0.003333333333333333,
   "usd_per_1000_old": 0.003333333333333333,
   "method": "flat plan price per decision, no token dependency",
   "usd_per_1000_v11_tiers": 0.003333333333333333,
   "usd_per_1000": 0.003333333333333333,
   "delta_pct": 0.0,
   "unchanged": true
  },
  "jev-1.13.0": {
   "display": "Jev 1.13.0 (TypeSafe AI)",
   "kind": "measured",
   "n_decisions": 314,
   "usd_per_1000_v11_tiers_old": 0.025888392086330935,
   "usd_per_1000_hard": 0.06163175454545452,
   "usd_per_1000_old": 0.04061412193840432,
   "method": "public tariff x measured tokens, over every decision the provider billed (including the answers that failed to parse: those requests were charged too)",
   "price_in_per_m": 0.042,
   "price_out_per_m": 0.0,
   "mean_input_tokens": 588.0509554140127,
   "mean_output_tokens": 47.32165605095541,
   "n_priced_decisions": 314,
   "usd_per_1000_v11_tiers": 0.024698140127388538,
   "usd_per_1000": 0.03991423595505617,
   "delta_pct": -1.7232576994021993,
   "unchanged": false
  },
  "semif-qwen3.5-4b": {
   "display": "SemIf, formerly OpenJev (Qwen3.5-4B, TheoLeeCJ)",
   "kind": "estimate",
   "n_decisions": 314,
   "usd_per_1000_v11_tiers_old": 0.012921852517985612,
   "usd_per_1000_hard": 0.03732368181818181,
   "usd_per_1000_old": 0.022975040619190038,
   "method": "tariff x measured input tokens, over all 314 decisions",
   "price_in_per_m": 0.03,
   "price_out_per_m": 0.15,
   "mean_input_tokens": 395.6592356687898,
   "output_tokens_charged": 1,
   "tokens_from": null,
   "usd_per_1000_v11_tiers": 0.012019777070063693,
   "usd_per_1000": 0.02244460674157303,
   "delta_pct": -2.3087396727993554,
   "unchanged": false
  },
  "djev": {
   "display": "djev (Maisa, diffusion-gemma)",
   "kind": "announced",
   "n_decisions": 314,
   "usd_per_1000_v11_tiers_old": 0.01388441082802548,
   "usd_per_1000_hard": 0.04317393181818183,
   "usd_per_1000_old": 0.025951254681647943,
   "method": "public tariff x measured tokens, over every decision the provider billed (including the answers that failed to parse: those requests were charged too)",
   "price_in_per_m": 0.035,
   "price_out_per_m": 0.0,
   "mean_input_tokens": 396.69745222929936,
   "mean_output_tokens": 8.0,
   "n_priced_decisions": 314,
   "usd_per_1000_v11_tiers": 0.013884410828025478,
   "usd_per_1000": 0.025951254681647943,
   "delta_pct": 0.0,
   "unchanged": true
  },
  "laya": {
   "display": "Laya (Convai Innovations, ModernBERT-large 421M)",
   "kind": "estimate",
   "n_decisions": 314,
   "usd_per_1000_v11_tiers_old": 0.002048248407643312,
   "usd_per_1000_hard": 0.004074727272727273,
   "usd_per_1000_old": 0.0028831273408239703,
   "method": "tariff x measured input tokens, over all 314 decisions",
   "price_in_per_m": 0.01,
   "price_out_per_m": 0.0,
   "mean_input_tokens": 204.8248407643312,
   "output_tokens_charged": 0,
   "tokens_from": null,
   "usd_per_1000_v11_tiers": 0.002048248407643312,
   "usd_per_1000": 0.0028831273408239703,
   "delta_pct": 0.0,
   "unchanged": true
  },
  "open-alternative-jev": {
   "display": "open-alternative-jev (Qwen3.5-4B, IkerMoel)",
   "kind": "estimate",
   "n_decisions": 314,
   "usd_per_1000_v11_tiers_old": 0.011652229299363059,
   "usd_per_1000_hard": 0.03718513636363636,
   "usd_per_1000_old": 0.022171404494382024,
   "method": "tariff x input tokens of the gemini-3.1-flash-lite run on the same prompts, over all 314 decisions",
   "price_in_per_m": 0.03,
   "price_out_per_m": 0.15,
   "mean_input_tokens": 383.4076433121019,
   "output_tokens_charged": 1,
   "tokens_from": "gemini-3.1-flash-lite",
   "usd_per_1000_v11_tiers": 0.011652229299363059,
   "usd_per_1000": 0.022171404494382024,
   "delta_pct": 0.0,
   "unchanged": true
  },
  "system-one-open": {
   "display": "system-one-open (Gemma 4 E2B LoRA on an L4)",
   "kind": "estimate",
   "n_decisions": 314,
   "usd_per_1000_v11_tiers_old": 0.009236694214876034,
   "usd_per_1000_hard": 0.024890090909090907,
   "usd_per_1000_old": 0.015685659145076917,
   "method": "tariff x input tokens of the gemini-3.1-flash-lite run on the same prompts, over all 314 decisions",
   "price_in_per_m": 0.02,
   "price_out_per_m": 0.1,
   "mean_input_tokens": 383.4076433121019,
   "output_tokens_charged": 2,
   "tokens_from": "gemini-3.1-flash-lite",
   "usd_per_1000_v11_tiers": 0.00786815286624204,
   "usd_per_1000": 0.014880936329588014,
   "delta_pct": -5.130309208213748,
   "unchanged": false
  },
  "openjev-razorback16": {
   "display": "OpenJev (DiffusionGemma 26B-A4B NVFP4, razorback16)",
   "kind": "estimate",
   "n_decisions": 314,
   "usd_per_1000_v11_tiers_old": 0.03718122302158273,
   "usd_per_1000_hard": 0.11002254545454546,
   "usd_per_1000_old": 0.0671907566081966,
   "method": "tariff x measured input tokens, over all 314 decisions",
   "price_in_per_m": 0.09,
   "price_out_per_m": 0.3,
   "mean_input_tokens": 379.81847133757964,
   "output_tokens_charged": 1,
   "tokens_from": null,
   "usd_per_1000_v11_tiers": 0.034483662420382165,
   "usd_per_1000": 0.06560455056179774,
   "delta_pct": -2.3607503866169424,
   "unchanged": false
  },
  "jeff": {
   "display": "jeff (Logan Markewich, GLiFormer 400M)",
   "kind": "estimate",
   "n_decisions": 314,
   "usd_per_1000_v11_tiers_old": 0.0027223885350318475,
   "usd_per_1000_hard": 0.010767181818181818,
   "usd_per_1000_old": 0.006036722846441948,
   "method": "tariff x measured input tokens, over all 314 decisions",
   "price_in_per_m": 0.01,
   "price_out_per_m": 0.0,
   "mean_input_tokens": 272.2388535031847,
   "output_tokens_charged": 0,
   "tokens_from": null,
   "usd_per_1000_v11_tiers": 0.0027223885350318475,
   "usd_per_1000": 0.006036722846441948,
   "delta_pct": 0.0,
   "unchanged": true
  },
  "openjev-sglang": {
   "display": "openjev-sglang (Qwen3.6-35B-A3B on SGLang)",
   "kind": "estimate",
   "n_decisions": 314,
   "usd_per_1000_v11_tiers_old": 0.06850989208633093,
   "usd_per_1000_hard": 0.22896636363636363,
   "usd_per_1000_old": 0.134615554522674,
   "method": "tariff x measured input tokens, over all 314 decisions",
   "price_in_per_m": 0.1,
   "price_out_per_m": 0.9,
   "mean_input_tokens": 610.296178343949,
   "output_tokens_charged": 2,
   "tokens_from": null,
   "usd_per_1000_v11_tiers": 0.0628296178343949,
   "usd_per_1000": 0.13127546816479402,
   "delta_pct": -2.481203877013638,
   "unchanged": false
  },
  "openjev-verdict": {
   "display": "openJev Verdict (heman10x, ModernBERT-base 151M)",
   "kind": "estimate",
   "n_decisions": 314,
   "usd_per_1000_v11_tiers_old": 0.0022600000000000003,
   "usd_per_1000_hard": 0.006175,
   "usd_per_1000_old": 0.003872921348314607,
   "method": "tariff x input tokens of the gemini-3.1-flash-lite run on the same prompts, over all 314 decisions",
   "price_in_per_m": 0.005,
   "price_out_per_m": 0.0,
   "mean_input_tokens": 383.4076433121019,
   "output_tokens_charged": 0,
   "tokens_from": "gemini-3.1-flash-lite",
   "usd_per_1000_v11_tiers": 0.0019170382165605096,
   "usd_per_1000": 0.00367125468164794,
   "delta_pct": -5.20709429729129,
   "unchanged": false
  },
  "gpt-5.6-luna": {
   "display": "GPT-5.6 Luna (low reasoning effort)",
   "kind": "measured",
   "n_decisions": 314,
   "usd_per_1000_v11_tiers_old": 0.16415539568345325,
   "usd_per_1000_hard": 0.3659363636363635,
   "usd_per_1000_old": 0.24728613154420276,
   "method": "public tariff x measured tokens, over every decision the provider billed (including the answers that failed to parse: those requests were charged too)",
   "price_in_per_m": 0.2,
   "price_out_per_m": 1.2,
   "mean_input_tokens": 431.4299363057325,
   "mean_output_tokens": 57.27388535031847,
   "n_priced_decisions": 314,
   "usd_per_1000_v11_tiers": 0.15501464968152867,
   "usd_per_1000": 0.24191123595505612,
   "delta_pct": -2.173553185365702,
   "unchanged": false
  },
  "open-jev-deberta-v3-large": {
   "display": "open-jev-deberta-v3-large (local CPU)",
   "kind": "estimate",
   "n_decisions": 314,
   "usd_per_1000_v11_tiers_old": 0.004518347107438017,
   "usd_per_1000_hard": 0.012345045454545454,
   "usd_per_1000_old": 0.007742829572538459,
   "method": "tariff x input tokens of the gemini-3.1-flash-lite run on the same prompts, over all 314 decisions",
   "price_in_per_m": 0.01,
   "price_out_per_m": 0.0,
   "mean_input_tokens": 383.4076433121019,
   "output_tokens_charged": 0,
   "tokens_from": "gemini-3.1-flash-lite",
   "usd_per_1000_v11_tiers": 0.003834076433121019,
   "usd_per_1000": 0.007340468164794008,
   "delta_pct": -5.196568050154547,
   "unchanged": false
  },
  "nimble-9b": {
   "display": "Bespoke Nimble 9B (Bespoke Labs)",
   "kind": "estimate",
   "n_decisions": 314,
   "usd_per_1000_v11_tiers_old": 0.09917246376811595,
   "usd_per_1000_hard": 0.12181333333333334,
   "usd_per_1000_old": 0.10850016283992836,
   "method": "tariff x measured input tokens, over all 314 decisions",
   "price_in_per_m": 0.1,
   "price_out_per_m": 0.15,
   "mean_input_tokens": 929.7051282051282,
   "output_tokens_charged": 1,
   "tokens_from": null,
   "usd_per_1000_v11_tiers": 0.09312051282051283,
   "usd_per_1000": 0.10494152501680593,
   "delta_pct": -3.2798456057365897,
   "unchanged": false
  },
  "gemini-3.1-flash-lite": {
   "display": "Gemini 3.1 Flash-Lite",
   "kind": "measured",
   "n_decisions": 314,
   "usd_per_1000_v11_tiers_old": 0.18556834532374103,
   "usd_per_1000_hard": 0.38614204545454534,
   "usd_per_1000_old": 0.26820170492819223,
   "method": "public tariff x measured tokens, over every decision the provider billed (including the answers that failed to parse: those requests were charged too)",
   "price_in_per_m": 0.25,
   "price_out_per_m": 1.5,
   "mean_input_tokens": 383.4076433121019,
   "mean_output_tokens": 54.80573248407644,
   "n_priced_decisions": 314,
   "usd_per_1000_v11_tiers": 0.17806050955414016,
   "usd_per_1000": 0.26378698501872655,
   "delta_pct": -1.6460446851550288,
   "unchanged": false
  },
  "deepseek-flash": {
   "display": "DeepSeek V4.1 Flash (thinking default)",
   "kind": "measured",
   "n_decisions": 314,
   "usd_per_1000_v11_tiers_old": 0.2252337662337665,
   "usd_per_1000_hard": 1.083477954545455,
   "usd_per_1000_old": 0.5788175142273461,
   "method": "public tariff x measured tokens, over every decision the provider billed (including the answers that failed to parse: those requests were charged too)",
   "price_in_per_m": 0.15,
   "price_out_per_m": 0.6,
   "mean_input_tokens": 411.4171974522293,
   "mean_output_tokens": 314.6687898089172,
   "n_priced_decisions": 314,
   "usd_per_1000_v11_tiers": 0.2505138535031847,
   "usd_per_1000": 0.593682584269663,
   "delta_pct": 2.5681790334489274,
   "unchanged": false
  },
  "system-one-sg": {
   "display": "system-one (Qwen3-8B, Sean Goedecke)",
   "kind": "estimate",
   "n_decisions": 314,
   "usd_per_1000_v11_tiers_old": 0.0522559082733813,
   "usd_per_1000_hard": 0.14759526363636366,
   "usd_per_1000_old": 0.09153429437798077,
   "method": "tariff x measured input tokens, over all 314 decisions",
   "price_in_per_m": 0.117,
   "price_out_per_m": 0.455,
   "mean_input_tokens": 412.31847133757964,
   "output_tokens_charged": 1,
   "tokens_from": null,
   "usd_per_1000_v11_tiers": 0.04869626114649681,
   "usd_per_1000": 0.08944116853932584,
   "delta_pct": -2.286712158408745,
   "unchanged": false
  },
  "gliner2": {
   "display": "GLiNER2 (Fastino, gliner2.5-base)",
   "kind": "estimate",
   "n_decisions": 314,
   "usd_per_1000_v11_tiers_old": 0.0022600000000000003,
   "usd_per_1000_hard": 0.006175,
   "usd_per_1000_old": 0.003872921348314607,
   "method": "tariff x input tokens of the gemini-3.1-flash-lite run on the same prompts, over all 314 decisions",
   "price_in_per_m": 0.005,
   "price_out_per_m": 0.0,
   "mean_input_tokens": 383.4076433121019,
   "output_tokens_charged": 0,
   "tokens_from": "gemini-3.1-flash-lite",
   "usd_per_1000_v11_tiers": 0.0019170382165605096,
   "usd_per_1000": 0.00367125468164794,
   "delta_pct": -5.20709429729129,
   "unchanged": false
  },
  "qwen3.8-27b": {
   "display": "Qwen3.8 27B (Chutes TEE)",
   "kind": "estimate",
   "n_decisions": 295,
   "usd_per_1000_v11_tiers_old": 1.096335976699029,
   "usd_per_1000_hard": 5.0156564166666655,
   "usd_per_1000_old": 2.7110372815546095,
   "method": "tariff x measured input tokens, over all 314 decisions",
   "price_in_per_m": 0.214,
   "price_out_per_m": 2.55,
   "mean_input_tokens": 415.91808873720134,
   "output_tokens_charged": 367.0546075085324,
   "tokens_from": null,
   "usd_per_1000_v11_tiers": 1.0249957201365187,
   "usd_per_1000": 2.669088141927965,
   "delta_pct": -1.5473464681603164,
   "unchanged": false
  },
  "needle-3-tools": {
   "display": "Needle 3, options as tools (post-hoc adapter mode)",
   "kind": "estimate",
   "n_decisions": 314,
   "usd_per_1000_v11_tiers_old": 0.016219537190082647,
   "usd_per_1000_hard": null,
   "usd_per_1000_old": 0.016219537190082647,
   "method": "tariff x input tokens of the gemini-3.1-flash-lite run on the same prompts, over all 314 decisions",
   "price_in_per_m": 0.027,
   "price_out_per_m": 0.201,
   "mean_input_tokens": 383.4076433121019,
   "output_tokens_charged": 20,
   "tokens_from": "gemini-3.1-flash-lite",
   "usd_per_1000_v11_tiers": 0.014372006369426754,
   "usd_per_1000": 0.014372006369426754,
   "delta_pct": -11.3907739721794,
   "unchanged": false
  },
  "needle-3": {
   "display": "Needle 3 (Cactus, 2-bit, local CPU)",
   "kind": "estimate",
   "n_decisions": 314,
   "usd_per_1000_v11_tiers_old": 0.016219537190082647,
   "usd_per_1000_hard": 0.03735162272727273,
   "usd_per_1000_old": 0.02492563984585384,
   "method": "tariff x input tokens of the gemini-3.1-flash-lite run on the same prompts, over all 314 decisions",
   "price_in_per_m": 0.027,
   "price_out_per_m": 0.201,
   "mean_input_tokens": 383.4076433121019,
   "output_tokens_charged": 20,
   "tokens_from": "gemini-3.1-flash-lite",
   "usd_per_1000_v11_tiers": 0.014372006369426754,
   "usd_per_1000": 0.023839264044943822,
   "delta_pct": -4.358467054921875,
   "unchanged": false
  }
 }
}