{ "schema": "jevbench-combinations/v1", "benchmark": "JevBench v1.2.6", "bar": { "reference": "classifier-dev-fast", "score": 84.83602134050506, "listing": "unranked honorable mention only" }, "source_hashes": { "jev-1.13.0": [ { "sha256": "c1685865c276ad99478251d15d99098fb4f31298222f01455d6b5043e4ac4830", "rows": 72 }, { "sha256": "218d38ac39cf98fd9225d97c9b64665f8a520827922138e285a1c324dcd00251", "rows": 242 }, { "sha256": "eb01d37bea76535afb6ad1debe338f572fff68415a50411e7409513fc1d9fa51", "rows": 220 } ], "semif-qwen3.5-4b": [ { "sha256": "30ccd0e88cc5bdaa93d940626bc5cf42441927f9be2749ab231b33b1628e0901", "rows": 72 }, { "sha256": "d35aee6825d091808409d3520a91fb694329a41b3d6529c55564826e1c154929", "rows": 242 }, { "sha256": "31224e5b00ece05e6f42d227db70edf04c90539d7e472752106e697954ff198c", "rows": 220 } ], "djev": [ { "sha256": "f267e29321a40f81c65306fa1a2774faa271768d6cbb5535b86ccf4cc89ea369", "rows": 72 }, { "sha256": "325d22cf206f33f123500fa45a5dbdb252f8a2156501805df12d0cbe304c2d64", "rows": 242 }, { "sha256": "9f19551bed7a517f3b5ccdba4a4316d9da54338d09e0329a964ee905213a90e7", "rows": 220 } ], "open-alternative-jev": [ { "sha256": "0108217fd1be5890cc03b21599c87c6557dcd4ccaf4df33c4977243a2be51113", "rows": 72 }, { "sha256": "a342e08f01ecb9209b58b1cb47478b5f91df90b647b9a8fb40c79453fb3546ca", "rows": 242 }, { "sha256": "4e6e3bee4ee21911bb3a74bf91143cf42e71129625f2dcf264ca350056a2aead", "rows": 220 } ], "system-one-open": [ { "sha256": "cdf70c12d824308bba4dd0746e3dc6274f9a6faa087213f0d3307984017a275e", "rows": 72 }, { "sha256": "109ce4d250c9850736e5a5325b51800eeb93e830a6ef71aa24ccdc787e1df578", "rows": 242 }, { "sha256": "88cf5b8ddf6b947c1510ee1175c3252289f24501ad42084c92248cb0dd0ce847", "rows": 220 } ], "openjev-razorback16": [ { "sha256": "d89ebe9738c36786b0736715d1dd6329d5be3e33ceee5c2a217a4a04654281bf", "rows": 72 }, { "sha256": "de21565d2b89d4fbb9b882ff3b7780c973a6407d1dfda5ee233a00104d652ae2", "rows": 242 }, { "sha256": "292745cf9274a9515a6514aee9f0e56eead1a47564e9b53dc3bde79c5a8d4649", "rows": 220 } ], "openjev-sglang": [ { "sha256": "84dadac61cfe19c02f0ee58df97133440795d11caa38a4b2abfbefdaefe86ac3", "rows": 72 }, { "sha256": "46a10d1ac0992ca283ce3e64b55c54fae9acc51ff98297a4e53220f390cba490", "rows": 242 }, { "sha256": "73719ee5b77a960d311efc9994cec9a4c0386b70b31977a1c357a99aec6b94e9", "rows": 220 } ], "classifier-dev-fast": [ { "sha256": "c625d44c3a2bfaa32ad4a82f4f5c03a191c0d4d8e19581a16f339e148c57d7fc", "rows": 72 }, { "sha256": "5dbf296c8b5ad29a8f199948ecc418bd239d409b04054560380cfbfcfba5b4a2", "rows": 242 }, { "sha256": "aa497b68373c4bf97feeecf32a268067de609a9736089ad4254565352550a638", "rows": 220 } ], "laya": [ { "sha256": "00633c2dc709aec4c391a65815c0c6595443d2d12d987f2c3f4d1a80521f1d85", "rows": 72 }, { "sha256": "132d19149fbaf831feee275d99f304794e1de7013797b5c79dd2cc5052b972e6", "rows": 242 }, { "sha256": "6ccc56379d4639d6b7477dcae668c7c3c47fa6239dcccaf1747f591bff884ae7", "rows": 220 } ], "openjev-verdict-1.4": [ { "sha256": "7335208f87813167a77e9f4052080fb0402d629a8f4b40b3f22de1cbf67b2fd9", "rows": 72 }, { "sha256": "f424fb5ae5a658b3f02c77306cc43a2cc3988cc8fc60e65728fab8d86188b00f", "rows": 242 }, { "sha256": "06954876258017a70c35bfe2239d5bbb6f6c475d0ecbdabbf393c0cfc2c8fae5", "rows": 220 } ], "simplejev-qwen3.8-27b": [ { "sha256": "068e81eba916525cf91ce95e81f3187726f02645fb520a029f97f443d4cd0e6c", "rows": 72 }, { "sha256": "2276119bc264f5165b09ce52b2fb6185d08204d293da638d2f664e06df2591d0", "rows": 242 }, { "sha256": "1daac4643a3d75b2eeddfb2c8ecf41dc850c09c5a70cb0c8286117f442c35f16", "rows": 220 } ], "simplejev-qwen3.6-35b-a3b": [ { "sha256": "47e0e4eb09be8fe0be871dd6502751688753bd49bba807134a86605497ab2621", "rows": 72 }, { "sha256": "ce0531404344a845912494ddd23a2c1d0afe22ef19daa4a119f607afb1cde342", "rows": 242 }, { "sha256": "bcce85d1f253a569e8407ce33f031624bffa99cb0ce077979bc8967dda203b25", "rows": 220 } ] }, "complete_systems": [ "classifier-dev-fast", "djev", "jev-1.13.0", "laya", "open-alternative-jev", "openjev-razorback16", "openjev-sglang", "openjev-verdict-1.4", "semif-qwen3.5-4b", "simplejev-qwen3.6-35b-a3b", "simplejev-qwen3.8-27b", "system-one-open" ], "committees": [ { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "djev" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9794520547945206, "hard": 0.6954545454545454 }, "accuracy": 0.8389513108614233, "ece_hard": 0.0395606580896503, "calibration": 87.12001913325832, "p50_s": 0.6695001281797885, "p95_s": 1.1348857231438156, "cost_usd_per_1000": 0.08831009737827715, "axes": { "intelligence": 85.62162723121628, "calibration": 87.12001913325832, "speed": 81.19297202879132, "cost": 41.61968909031034 }, "jevbench_score": 70.85649692925469 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "djev" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9794520547945206, "hard": 0.6863636363636364 }, "accuracy": 0.8352059925093633, "ece_hard": 0.0537356197745687, "calibration": 85.46734967278327, "p50_s": 0.6695001281797885, "p95_s": 1.1348857231438156, "cost_usd_per_1000": 0.08831009737827715, "axes": { "intelligence": 85.34889995848901, "calibration": 85.46734967278327, "speed": 81.19297202879132, "cost": 41.61968909031034 }, "jevbench_score": 70.46182034949827 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "open-alternative-jev" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9657534246575342, "hard": 0.6409090909090909 }, "accuracy": 0.8052434456928839, "ece_hard": 0.0480709888821369, "calibration": 85.52189781990481, "p50_s": 0.6719580851495266, "p95_s": 1.1348857231438156, "cost_usd_per_1000": 0.08453024719101122, "axes": { "intelligence": 82.43503528435035, "calibration": 85.52189781990481, "speed": 81.17705683541841, "cost": 42.18963583057602 }, "jevbench_score": 70.09823112385502 }, { "members": [ "jev-1.13.0", "djev", "open-alternative-jev" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9726027397260274, "hard": 0.6545454545454545 }, "accuracy": 0.8183520599250936, "ece_hard": 0.06802569324224442, "calibration": 84.09283904935688, "p50_s": 0.6722430549561977, "p95_s": 1.1239915058016774, "cost_usd_per_1000": 0.08803689513108615, "axes": { "intelligence": 83.91090701535909, "calibration": 84.09283904935688, "speed": 81.21710646636012, "cost": 41.660058473978935 }, "jevbench_score": 69.90145321400585 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "open-alternative-jev" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.958904109589041, "hard": 0.6318181818181818 }, "accuracy": 0.799625468164794, "ece_hard": 0.05838299133363121, "calibration": 83.92096747261274, "p50_s": 0.6719580851495266, "p95_s": 1.1348857231438156, "cost_usd_per_1000": 0.08453024719101122, "axes": { "intelligence": 81.97052718970528, "calibration": 83.92096747261274, "speed": 81.17705683541841, "cost": 42.18963583057602 }, "jevbench_score": 69.6693601694214 }, { "members": [ "jev-1.13.0", "open-alternative-jev", "system-one-open" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8020833333333334, "judge": 0.9794520547945206, "hard": 0.6545454545454545 }, "accuracy": 0.8164794007490637, "ece_hard": 0.05702825880694417, "calibration": 84.80332744606399, "p50_s": 1.3041210994124413, "p95_s": 2.030777859687805, "cost_usd_per_1000": 0.07696657677902621, "axes": { "intelligence": 83.51935450394355, "calibration": 84.80332744606399, "speed": 75.77019658655011, "cost": 43.41093487016646 }, "jevbench_score": 69.47433424319723 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "system-one-open" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9794520547945206, "hard": 0.6181818181818182 }, "accuracy": 0.8052434456928839, "ece_hard": 0.05408130583506763, "calibration": 84.80341831257005, "p50_s": 1.3041210994124413, "p95_s": 2.030777859687805, "cost_usd_per_1000": 0.07723977902621722, "axes": { "intelligence": 83.0117787463678, "calibration": 84.80341831257005, "speed": 75.77019658655011, "cost": 43.36476932066527 }, "jevbench_score": 69.35010663721552 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "system-one-open" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9794520547945206, "hard": 0.6227272727272727 }, "accuracy": 0.8071161048689138, "ece_hard": 0.06253866512741826, "calibration": 84.55366801966257, "p50_s": 1.3041210994124413, "p95_s": 2.030777859687805, "cost_usd_per_1000": 0.07723977902621722, "axes": { "intelligence": 83.14814238273142, "calibration": 84.55366801966257, "speed": 75.77019658655011, "cost": 43.36476932066527 }, "jevbench_score": 69.32743215730079 }, { "members": [ "jev-1.13.0", "djev", "open-alternative-jev" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9726027397260274, "hard": 0.6363636363636364 }, "accuracy": 0.8108614232209738, "ece_hard": 0.09127237326247599, "calibration": 81.40005719540585, "p50_s": 0.6722430549561977, "p95_s": 1.1239915058016774, "cost_usd_per_1000": 0.08803689513108615, "axes": { "intelligence": 83.36545246990454, "calibration": 81.40005719540585, "speed": 81.21710646636012, "cost": 41.660058473978935 }, "jevbench_score": 69.22206525414448 }, { "members": [ "jev-1.13.0", "djev", "system-one-open" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9657534246575342, "hard": 0.6727272727272727 }, "accuracy": 0.8258426966292135, "ece_hard": 0.06030659620752806, "calibration": 83.5727539191667, "p50_s": 1.3041210994124413, "p95_s": 2.030777859687805, "cost_usd_per_1000": 0.08074642696629214, "axes": { "intelligence": 84.55624740556247, "calibration": 83.5727539191667, "speed": 75.77019658655011, "cost": 42.78630058377141 }, "jevbench_score": 69.18363736474558 }, { "members": [ "semif-qwen3.5-4b", "djev", "system-one-open" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9726027397260274, "hard": 0.6227272727272727 }, "accuracy": 0.8052434456928839, "ece_hard": 0.08469404597810404, "calibration": 78.65943513389516, "p50_s": 1.3041210994124413, "p95_s": 2.030777859687805, "cost_usd_per_1000": 0.06327679775280898, "axes": { "intelligence": 82.95636156081362, "calibration": 78.65943513389516, "speed": 75.77019658655011, "cost": 45.9626652185008 }, "jevbench_score": 69.04401437308921 }, { "members": [ "semif-qwen3.5-4b", "djev", "system-one-open" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9726027397260274, "hard": 0.6227272727272727 }, "accuracy": 0.8052434456928839, "ece_hard": 0.08782905137769649, "calibration": 78.47230678648533, "p50_s": 1.3041210994124413, "p95_s": 2.030777859687805, "cost_usd_per_1000": 0.06327679775280898, "axes": { "intelligence": 82.95636156081362, "calibration": 78.47230678648533, "speed": 75.77019658655011, "cost": 45.9626652185008 }, "jevbench_score": 69.00291429878109 }, { "members": [ "jev-1.13.0", "djev", "system-one-open" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9657534246575342, "hard": 0.65 }, "accuracy": 0.8164794007490637, "ece_hard": 0.06354726142176578, "calibration": 82.71902454747364, "p50_s": 1.3041210994124413, "p95_s": 2.030777859687805, "cost_usd_per_1000": 0.08074642696629214, "axes": { "intelligence": 83.8744292237443, "calibration": 82.71902454747364, "speed": 75.77019658655011, "cost": 42.78630058377141 }, "jevbench_score": 68.86674136252734 }, { "members": [ "jev-1.13.0", "open-alternative-jev", "system-one-open" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9452054794520548, "hard": 0.6136363636363636 }, "accuracy": 0.7883895131086143, "ece_hard": 0.06172769759265953, "calibration": 83.55895817803227, "p50_s": 1.3041210994124413, "p95_s": 2.030777859687805, "cost_usd_per_1000": 0.07696657677902621, "axes": { "intelligence": 81.04151100041511, "calibration": 83.55895817803227, "speed": 75.77019658655011, "cost": 43.41093487016646 }, "jevbench_score": 68.69885910846105 }, { "members": [ "semif-qwen3.5-4b", "djev", "open-alternative-jev" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9452054794520548, "hard": 0.5818181818181818 }, "accuracy": 0.7790262172284644, "ece_hard": 0.14595699122316214, "calibration": 73.2437689465666, "p50_s": 0.5735304296016693, "p95_s": 1.1444025911390774, "cost_usd_per_1000": 0.070567265917603, "axes": { "intelligence": 80.67029887920299, "calibration": 73.2437689465666, "speed": 81.82864702484957, "cost": 44.541901260836084 }, "jevbench_score": 68.12234945447156 }, { "members": [ "semif-qwen3.5-4b", "djev", "open-alternative-jev" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9452054794520548, "hard": 0.5863636363636363 }, "accuracy": 0.7808988764044944, "ece_hard": 0.15140514694337298, "calibration": 72.70728170647902, "p50_s": 0.5735304296016693, "p95_s": 1.1444025911390774, "cost_usd_per_1000": 0.070567265917603, "axes": { "intelligence": 80.80666251556663, "calibration": 72.70728170647902, "speed": 81.82864702484957, "cost": 44.541901260836084 }, "jevbench_score": 68.02597886660847 }, { "members": [ "djev", "open-alternative-jev", "system-one-open" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.958904109589041, "hard": 0.6090909090909091 }, "accuracy": 0.795880149812734, "ece_hard": 0.12384089814940558, "calibration": 74.6812491550473, "p50_s": 1.3041210994124413, "p95_s": 2.030777859687805, "cost_usd_per_1000": 0.06300359550561799, "axes": { "intelligence": 82.16370900788709, "calibration": 74.6812491550473, "speed": 75.77019658655011, "cost": 46.0190399622533 }, "jevbench_score": 68.01142812981439 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "openjev-razorback16" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6590909090909091 }, "accuracy": 0.8220973782771536, "ece_hard": 0.06273831106273409, "calibration": 83.81495216876573, "p50_s": 0.6811514683067799, "p95_s": 1.1347835324704645, "cost_usd_per_1000": 0.12796339325842696, "axes": { "intelligence": 84.33893731838938, "calibration": 83.81495216876573, "speed": 81.11843278006634, "cost": 36.78742756204445 }, "jevbench_score": 67.77072775937233 }, { "members": [ "djev", "open-alternative-jev", "system-one-open" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.958904109589041, "hard": 0.6 }, "accuracy": 0.7902621722846442, "ece_hard": 0.13044528187715135, "calibration": 73.93673322175258, "p50_s": 1.3041210994124413, "p95_s": 2.030777859687805, "cost_usd_per_1000": 0.06300359550561799, "axes": { "intelligence": 81.59931506849315, "calibration": 73.93673322175258, "speed": 75.77019658655011, "cost": 46.0190399622533 }, "jevbench_score": 67.72448090459815 }, { "members": [ "semif-qwen3.5-4b", "open-alternative-jev", "system-one-open" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8020833333333334, "judge": 0.910958904109589, "hard": 0.5590909090909091 }, "accuracy": 0.7584269662921348, "ece_hard": 0.12196744331030088, "calibration": 74.07586998644128, "p50_s": 1.3041210994124413, "p95_s": 2.030777859687805, "cost_usd_per_1000": 0.05949694756554307, "axes": { "intelligence": 78.7379099211291, "calibration": 74.07586998644128, "speed": 75.77019658655011, "cost": 46.765159442990324 }, "jevbench_score": 67.4249090244371 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "openjev-razorback16" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6454545454545455 }, "accuracy": 0.8164794007490637, "ece_hard": 0.07289534352950582, "calibration": 82.4196727683094, "p50_s": 0.6811514683067799, "p95_s": 1.1347835324704645, "cost_usd_per_1000": 0.12796339325842696, "axes": { "intelligence": 83.92984640929846, "calibration": 82.4196727683094, "speed": 81.11843278006634, "cost": 36.78742756204445 }, "jevbench_score": 67.40491593747868 }, { "members": [ "semif-qwen3.5-4b", "open-alternative-jev", "system-one-open" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.910958904109589, "hard": 0.5590909090909091 }, "accuracy": 0.7565543071161048, "ece_hard": 0.12281363784726532, "calibration": 74.01281472102082, "p50_s": 1.3041210994124413, "p95_s": 2.030777859687805, "cost_usd_per_1000": 0.05949694756554307, "axes": { "intelligence": 78.44624325446243, "calibration": 74.01281472102082, "speed": 75.77019658655011, "cost": 46.765159442990324 }, "jevbench_score": 67.34804232190076 }, { "members": [ "jev-1.13.0", "system-one-open", "openjev-razorback16" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.65 }, "accuracy": 0.8108614232209738, "ece_hard": 0.051315080207149665, "calibration": 84.97617581855849, "p50_s": 1.3041210994124413, "p95_s": 2.030777859687805, "cost_usd_per_1000": 0.12039972284644193, "axes": { "intelligence": 83.29908675799086, "calibration": 84.97617581855849, "speed": 75.77019658655011, "cost": 37.5812353839736 }, "jevbench_score": 67.00415309556654 }, { "members": [ "jev-1.13.0", "open-alternative-jev", "openjev-razorback16" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9178082191780822, "hard": 0.6454545454545455 }, "accuracy": 0.799625468164794, "ece_hard": 0.07750377426845723, "calibration": 81.94568488667103, "p50_s": 0.6816155403852463, "p95_s": 1.1239915058016774, "cost_usd_per_1000": 0.12769019101123594, "axes": { "intelligence": 82.10393316728934, "calibration": 81.94568488667103, "speed": 81.15697487293535, "cost": 36.815273901199916 }, "jevbench_score": 66.95931227998263 }, { "members": [ "semif-qwen3.5-4b", "djev", "openjev-razorback16" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9452054794520548, "hard": 0.6545454545454545 }, "accuracy": 0.8089887640449438, "ece_hard": 0.10275654501306085, "calibration": 77.73473529293796, "p50_s": 0.6500487640500069, "p95_s": 1.1367489621043205, "cost_usd_per_1000": 0.11400041198501872, "axes": { "intelligence": 82.85211706102118, "calibration": 77.73473529293796, "speed": 81.31389496755949, "cost": 38.2928073750382 }, "jevbench_score": 66.91911308955365 }, { "members": [ "jev-1.13.0", "open-alternative-jev", "openjev-razorback16" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9383561643835616, "hard": 0.6454545454545455 }, "accuracy": 0.8052434456928839, "ece_hard": 0.09411918378944127, "calibration": 80.73467650270085, "p50_s": 0.6816155403852463, "p95_s": 1.1239915058016774, "cost_usd_per_1000": 0.12769019101123594, "axes": { "intelligence": 82.67927563304276, "calibration": 80.73467650270085, "speed": 81.15697487293535, "cost": 36.815273901199916 }, "jevbench_score": 66.82710763321784 }, { "members": [ "jev-1.13.0", "system-one-open", "openjev-razorback16" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6590909090909091 }, "accuracy": 0.8146067415730337, "ece_hard": 0.0663522908559217, "calibration": 83.74390862329432, "p50_s": 1.3041210994124413, "p95_s": 2.030777859687805, "cost_usd_per_1000": 0.12039972284644193, "axes": { "intelligence": 83.57181403071814, "calibration": 83.74390862329432, "speed": 75.77019658655011, "cost": 37.5812353839736 }, "jevbench_score": 66.81448582907534 }, { "members": [ "semif-qwen3.5-4b", "djev", "openjev-razorback16" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9452054794520548, "hard": 0.6545454545454545 }, "accuracy": 0.8089887640449438, "ece_hard": 0.1079061420007763, "calibration": 77.12880060780253, "p50_s": 0.6500487640500069, "p95_s": 1.1367489621043205, "cost_usd_per_1000": 0.11400041198501872, "axes": { "intelligence": 82.85211706102118, "calibration": 77.12880060780253, "speed": 81.31389496755949, "cost": 38.2928073750382 }, "jevbench_score": 66.78832316470144 }, { "members": [ "jev-1.13.0", "djev", "openjev-razorback16" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.952054794520548, "hard": 0.6772727272727272 }, "accuracy": 0.8202247191011236, "ece_hard": 0.1021994573880076, "calibration": 79.29158999564825, "p50_s": 0.6792524389922618, "p95_s": 1.086352695524692, "cost_usd_per_1000": 0.13147004119850186, "axes": { "intelligence": 83.72571606475717, "calibration": 79.29158999564825, "speed": 81.31997947189848, "cost": 36.43519602918106 }, "jevbench_score": 66.59645195807249 }, { "members": [ "djev", "open-alternative-jev", "openjev-razorback16" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9246575342465754, "hard": 0.6454545454545455 }, "accuracy": 0.799625468164794, "ece_hard": 0.11883559528702836, "calibration": 76.4373777854438, "p50_s": 0.6516780346632004, "p95_s": 1.131779689341783, "cost_usd_per_1000": 0.11372720973782771, "axes": { "intelligence": 82.00404732254047, "calibration": 76.4373777854438, "speed": 81.3220502108525, "cost": 38.324068480148846 }, "jevbench_score": 66.48217949543644 }, { "members": [ "djev", "open-alternative-jev", "openjev-razorback16" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9246575342465754, "hard": 0.65 }, "accuracy": 0.8014981273408239, "ece_hard": 0.1221469050775356, "calibration": 76.09146151226726, "p50_s": 0.6516780346632004, "p95_s": 1.131779689341783, "cost_usd_per_1000": 0.11372720973782771, "axes": { "intelligence": 82.14041095890411, "calibration": 76.09146151226726, "speed": 81.3220502108525, "cost": 38.324068480148846 }, "jevbench_score": 66.43442507273606 }, { "members": [ "jev-1.13.0", "djev", "open-alternative-jev" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9452054794520548, "hard": 0.6636363636363637 }, "accuracy": 0.8146067415730337, "ece_hard": 0.15151515151515113, "calibration": 68.80398484848489, "p50_s": 0.6722430549561977, "p95_s": 1.1239915058016774, "cost_usd_per_1000": 0.08803689513108615, "axes": { "intelligence": 83.41651100041511, "calibration": 68.80398484848489, "speed": 81.21710646636012, "cost": 41.660058473978935 }, "jevbench_score": 66.38321014771127 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "djev" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6863636363636364 }, "accuracy": 0.8333333333333334, "ece_hard": 0.165151515151515, "calibration": 67.0250984848485, "p50_s": 0.6695001281797885, "p95_s": 1.1348857231438156, "cost_usd_per_1000": 0.08831009737827715, "axes": { "intelligence": 85.1571191365712, "calibration": 67.0250984848485, "speed": 81.19297202879132, "cost": 41.61968909031034 }, "jevbench_score": 66.27029792583583 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "djev", "open-alternative-jev", "system-one-open" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9726027397260274, "hard": 0.6181818181818182 }, "accuracy": 0.8033707865168539, "ece_hard": 0.07997185533817011, "calibration": 82.23858307210071, "p50_s": 1.3041210994124413, "p95_s": 2.030777859687805, "cost_usd_per_1000": 0.12536243820224718, "axes": { "intelligence": 82.81999792444998, "calibration": 82.23858307210071, "speed": 75.77019658655011, "cost": 37.05497709298896 }, "jevbench_score": 66.12854228532142 }, { "members": [ "djev", "system-one-open", "openjev-razorback16" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9452054794520548, "hard": 0.6454545454545455 }, "accuracy": 0.8052434456928839, "ece_hard": 0.09507663204841531, "calibration": 77.95905035481783, "p50_s": 1.3041210994124413, "p95_s": 2.030777859687805, "cost_usd_per_1000": 0.1064367415730337, "axes": { "intelligence": 82.5793897882939, "calibration": 77.95905035481783, "speed": 75.77019658655011, "cost": 39.1872528788934 }, "jevbench_score": 66.12190552930377 }, { "members": [ "semif-qwen3.5-4b", "open-alternative-jev", "openjev-razorback16" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9041095890410958, "hard": 0.6045454545454545 }, "accuracy": 0.7771535580524345, "ece_hard": 0.10972915919758466, "calibration": 75.63749564948898, "p50_s": 0.6506742924451828, "p95_s": 1.1367489621043205, "cost_usd_per_1000": 0.1102205617977528, "axes": { "intelligence": 80.20143212951434, "calibration": 75.63749564948898, "speed": 81.30971785161026, "cost": 38.73212139075841 }, "jevbench_score": 66.11244986355935 }, { "members": [ "jev-1.13.0", "djev", "openjev-razorback16" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9452054794520548, "hard": 0.6636363636363637 }, "accuracy": 0.8127340823970037, "ece_hard": 0.12287562744024, "calibration": 76.70951258378736, "p50_s": 0.6792524389922618, "p95_s": 1.086352695524692, "cost_usd_per_1000": 0.13147004119850186, "axes": { "intelligence": 83.12484433374846, "calibration": 76.70951258378736, "speed": 81.31997947189848, "cost": 36.43519602918106 }, "jevbench_score": 65.9287147963788 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "djev", "open-alternative-jev", "system-one-open" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9726027397260274, "hard": 0.6136363636363636 }, "accuracy": 0.8014981273408239, "ece_hard": 0.08368507577193746, "calibration": 81.30630100390826, "p50_s": 1.3041210994124413, "p95_s": 2.030777859687805, "cost_usd_per_1000": 0.12536243820224718, "axes": { "intelligence": 82.68363428808635, "calibration": 81.30630100390826, "speed": 75.77019658655011, "cost": 37.05497709298896 }, "jevbench_score": 65.91316728238024 }, { "members": [ "semif-qwen3.5-4b", "open-alternative-jev", "openjev-razorback16" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9041095890410958, "hard": 0.6 }, "accuracy": 0.7752808988764045, "ece_hard": 0.12037264970642486, "calibration": 74.58644617108854, "p50_s": 0.6506742924451828, "p95_s": 1.1367489621043205, "cost_usd_per_1000": 0.1102205617977528, "axes": { "intelligence": 80.06506849315069, "calibration": 74.58644617108854, "speed": 81.30971785161026, "cost": 38.73212139075841 }, "jevbench_score": 65.85354871128916 }, { "members": [ "jev-1.13.0", "djev", "system-one-open" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9383561643835616, "hard": 0.6681818181818182 }, "accuracy": 0.8164794007490637, "ece_hard": 0.13333333333333297, "calibration": 69.29250000000003, "p50_s": 1.3041210994124413, "p95_s": 2.030777859687805, "cost_usd_per_1000": 0.08074642696629214, "axes": { "intelligence": 83.65276048152761, "calibration": 69.29250000000003, "speed": 75.77019658655011, "cost": 42.78630058377141 }, "jevbench_score": 65.84039304968722 }, { "members": [ "djev", "system-one-open", "openjev-razorback16" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9383561643835616, "hard": 0.6409090909090909 }, "accuracy": 0.8014981273408239, "ece_hard": 0.10778914199997307, "calibration": 76.63188399967542, "p50_s": 1.3041210994124413, "p95_s": 2.030777859687805, "cost_usd_per_1000": 0.1064367415730337, "axes": { "intelligence": 82.25124533001245, "calibration": 76.63188399967542, "speed": 75.77019658655011, "cost": 39.1872528788934 }, "jevbench_score": 65.77317469791706 }, { "members": [ "semif-qwen3.5-4b", "system-one-open", "openjev-razorback16" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.958904109589041, "hard": 0.5954545454545455 }, "accuracy": 0.7902621722846442, "ece_hard": 0.11936797425714693, "calibration": 74.99733234129859, "p50_s": 1.3041210994124413, "p95_s": 2.030777859687805, "cost_usd_per_1000": 0.1029300936329588, "axes": { "intelligence": 81.75461809879619, "calibration": 74.99733234129859, "speed": 75.77019658655011, "cost": 39.62372896471127 }, "jevbench_score": 65.50175964482776 }, { "members": [ "semif-qwen3.5-4b", "system-one-open", "openjev-razorback16" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.958904109589041, "hard": 0.6045454545454545 }, "accuracy": 0.7940074906367042, "ece_hard": 0.12175919400229429, "calibration": 74.71126949981735, "p50_s": 1.3041210994124413, "p95_s": 2.030777859687805, "cost_usd_per_1000": 0.1029300936329588, "axes": { "intelligence": 82.02734537152345, "calibration": 74.71126949981735, "speed": 75.77019658655011, "cost": 39.62372896471127 }, "jevbench_score": 65.49371608405538 }, { "members": [ "open-alternative-jev", "system-one-open", "openjev-razorback16" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.8972602739726028, "hard": 0.6045454545454545 }, "accuracy": 0.7752808988764045, "ece_hard": 0.11296949452200167, "calibration": 75.88399970052998, "p50_s": 1.3041210994124413, "p95_s": 2.030777859687805, "cost_usd_per_1000": 0.10265689138576778, "axes": { "intelligence": 80.00965130759651, "calibration": 75.88399970052998, "speed": 75.77019658655011, "cost": 39.658356730150174 }, "jevbench_score": 65.35539293327498 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "djev", "open-alternative-jev", "openjev-razorback16" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6409090909090909 }, "accuracy": 0.8146067415730337, "ece_hard": 0.08858612628899915, "calibration": 81.38585378440033, "p50_s": 0.6818747088313103, "p95_s": 1.1444025911390774, "cost_usd_per_1000": 0.1760860524344569, "axes": { "intelligence": 83.79348277293484, "calibration": 81.38585378440033, "speed": 81.07716585059153, "cost": 32.62825127850175 }, "jevbench_score": 65.17227489753022 }, { "members": [ "open-alternative-jev", "system-one-open", "openjev-razorback16" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.8972602739726028, "hard": 0.6 }, "accuracy": 0.7734082397003745, "ece_hard": 0.132539959538061, "calibration": 73.88599080811515, "p50_s": 1.3041210994124413, "p95_s": 2.030777859687805, "cost_usd_per_1000": 0.10265689138576778, "axes": { "intelligence": 79.87328767123287, "calibration": 73.88599080811515, "speed": 75.77019658655011, "cost": 39.658356730150174 }, "jevbench_score": 64.89320109437733 }, { "members": [ "jev-1.13.0", "open-alternative-jev", "system-one-open" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9041095890410958, "hard": 0.6272727272727273 }, "accuracy": 0.7827715355805244, "ece_hard": 0.1530303030303032, "calibration": 67.10780303030302, "p50_s": 1.3041210994124413, "p95_s": 2.030777859687805, "cost_usd_per_1000": 0.07696657677902621, "axes": { "intelligence": 80.29991697799919, "calibration": 67.10780303030302, "speed": 75.77019658655011, "cost": 43.41093487016646 }, "jevbench_score": 64.88531502470569 }, { "members": [ "jev-1.13.0", "djev", "open-alternative-jev", "system-one-open", "openjev-razorback16" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9657534246575342, "hard": 0.6454545454545455 }, "accuracy": 0.8146067415730337, "ece_hard": 0.06409691212443323, "calibration": 83.80431797811627, "p50_s": 1.3041210994124413, "p95_s": 2.030777859687805, "cost_usd_per_1000": 0.16852238202247188, "axes": { "intelligence": 83.73806558738066, "calibration": 83.80431797811627, "speed": 75.77019658655011, "cost": 33.20027232567968 }, "jevbench_score": 64.819749009673 }, { "members": [ "semif-qwen3.5-4b", "djev", "system-one-open" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9657534246575342, "hard": 0.6045454545454545 }, "accuracy": 0.795880149812734, "ece_hard": 0.19242424242424222, "calibration": 61.62915909090911, "p50_s": 1.3041210994124413, "p95_s": 2.030777859687805, "cost_usd_per_1000": 0.06327679775280898, "axes": { "intelligence": 82.21912619344127, "calibration": 61.62915909090911, "speed": 75.77019658655011, "cost": 45.9626652185008 }, "jevbench_score": 64.81353055787862 }, { "members": [ "djev", "open-alternative-jev", "system-one-open" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8020833333333334, "judge": 0.8972602739726028, "hard": 0.6045454545454545 }, "accuracy": 0.7734082397003745, "ece_hard": 0.16969696969696962, "calibration": 63.46438636363637, "p50_s": 1.3041210994124413, "p95_s": 2.030777859687805, "cost_usd_per_1000": 0.06300359550561799, "axes": { "intelligence": 79.71798464092986, "calibration": 63.46438636363637, "speed": 75.77019658655011, "cost": 46.0190399622533 }, "jevbench_score": 64.80829425932453 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "djev", "open-alternative-jev", "openjev-razorback16" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9657534246575342, "hard": 0.6409090909090909 }, "accuracy": 0.8127340823970037, "ece_hard": 0.10492886059426342, "calibration": 79.47647690391136, "p50_s": 0.6818747088313103, "p95_s": 1.1444025911390774, "cost_usd_per_1000": 0.1760860524344569, "axes": { "intelligence": 83.60170195101702, "calibration": 79.47647690391136, "speed": 81.07716585059153, "cost": 32.62825127850175 }, "jevbench_score": 64.7495151607313 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "djev", "system-one-open", "openjev-razorback16" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9657534246575342, "hard": 0.6454545454545455 }, "accuracy": 0.8146067415730337, "ece_hard": 0.06914263776544548, "calibration": 83.25934586503905, "p50_s": 1.3041210994124413, "p95_s": 2.030777859687805, "cost_usd_per_1000": 0.1687955842696629, "axes": { "intelligence": 83.73806558738066, "calibration": 83.25934586503905, "speed": 75.77019658655011, "cost": 33.179167564083926 }, "jevbench_score": 64.70382494570713 }, { "members": [ "semif-qwen3.5-4b", "djev", "simplejev-qwen3.8-27b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6863636363636364 }, "accuracy": 0.8333333333333334, "ece_hard": 0.042076559941121854, "calibration": 85.20254477398242, "p50_s": 2.3997448533773422, "p95_s": 4.381843151897192, "cost_usd_per_1000": 0.15239237827715355, "axes": { "intelligence": 85.1571191365712, "calibration": 85.20254477398242, "speed": 69.78178103134502, "cost": 34.5111025951973 }, "jevbench_score": 64.65373262622421 }, { "members": [ "semif-qwen3.5-4b", "djev", "simplejev-qwen3.8-27b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6863636363636364 }, "accuracy": 0.8333333333333334, "ece_hard": 0.044203795365580244, "calibration": 85.14753875356429, "p50_s": 2.3997448533773422, "p95_s": 4.381843151897192, "cost_usd_per_1000": 0.15239237827715355, "axes": { "intelligence": 85.1571191365712, "calibration": 85.14753875356429, "speed": 69.78178103134502, "cost": 34.5111025951973 }, "jevbench_score": 64.64329512740844 }, { "members": [ "jev-1.13.0", "system-one-open", "openjev-sglang" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9657534246575342, "hard": 0.6909090909090909 }, "accuracy": 0.8295880149812734, "ece_hard": 0.05183634315422017, "calibration": 85.67918102349884, "p50_s": 1.3879805654287338, "p95_s": 2.030777859687805, "cost_usd_per_1000": 0.18607064044943822, "axes": { "intelligence": 84.51836861768369, "calibration": 85.67918102349884, "speed": 75.49954194936167, "cost": 31.90966442617821 }, "jevbench_score": 64.62837578113631 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "open-alternative-jev", "system-one-open", "openjev-razorback16" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9726027397260274, "hard": 0.6136363636363636 }, "accuracy": 0.8014981273408239, "ece_hard": 0.06519816944656086, "calibration": 83.17941969562601, "p50_s": 1.3041210994124413, "p95_s": 2.030777859687805, "cost_usd_per_1000": 0.165015734082397, "axes": { "intelligence": 82.68363428808635, "calibration": 83.17941969562601, "speed": 75.77019658655011, "cost": 33.47423932824064 }, "jevbench_score": 64.62657548954687 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "open-alternative-jev", "system-one-open", "openjev-razorback16" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9726027397260274, "hard": 0.6136363636363636 }, "accuracy": 0.8014981273408239, "ece_hard": 0.06172784903046859, "calibration": 83.119697613554, "p50_s": 1.3041210994124413, "p95_s": 2.030777859687805, "cost_usd_per_1000": 0.165015734082397, "axes": { "intelligence": 82.68363428808635, "calibration": 83.119697613554, "speed": 75.77019658655011, "cost": 33.47423932824064 }, "jevbench_score": 64.61497203857165 }, { "members": [ "djev", "system-one-open", "openjev-sglang" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9657534246575342, "hard": 0.6818181818181818 }, "accuracy": 0.8258426966292135, "ece_hard": 0.054809159233216054, "calibration": 82.97122525803542, "p50_s": 1.3879805654287338, "p95_s": 2.030777859687805, "cost_usd_per_1000": 0.17210765917603, "axes": { "intelligence": 84.24564134495643, "calibration": 82.97122525803542, "speed": 75.49954194936167, "cost": 32.92599406497797 }, "jevbench_score": 64.56386773811899 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "djev", "system-one-open", "openjev-razorback16" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9657534246575342, "hard": 0.6454545454545455 }, "accuracy": 0.8146067415730337, "ece_hard": 0.07794984806600543, "calibration": 82.06517501228655, "p50_s": 1.3041210994124413, "p95_s": 2.030777859687805, "cost_usd_per_1000": 0.1687955842696629, "axes": { "intelligence": 83.73806558738066, "calibration": 82.06517501228655, "speed": 75.77019658655011, "cost": 33.179167564083926 }, "jevbench_score": 64.47055828068818 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "simplejev-qwen3.8-27b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9794520547945206, "hard": 0.7136363636363636 }, "accuracy": 0.846441947565543, "ece_hard": 0.05012008018157422, "calibration": 86.05740438275414, "p50_s": 2.3997448533773422, "p95_s": 4.381843151897192, "cost_usd_per_1000": 0.16635535955056177, "axes": { "intelligence": 86.16708177667083, "calibration": 86.05740438275414, "speed": 69.78178103134502, "cost": 33.36889608087208 }, "jevbench_score": 64.46194113950642 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "openjev-sglang" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9794520547945206, "hard": 0.6772727272727272 }, "accuracy": 0.8295880149812734, "ece_hard": 0.06479721254882308, "calibration": 84.50101910065642, "p50_s": 1.3671194911003113, "p95_s": 1.6307271040976046, "cost_usd_per_1000": 0.19363431086142321, "axes": { "intelligence": 84.78450601909506, "calibration": 84.50101910065642, "speed": 76.51812235281348, "cost": 31.390530573402955 }, "jevbench_score": 64.40733808789405 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "openjev-sglang" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9726027397260274, "hard": 0.6772727272727272 }, "accuracy": 0.8277153558052435, "ece_hard": 0.06556136227108918, "calibration": 84.61587084168687, "p50_s": 1.3671194911003113, "p95_s": 1.6307271040976046, "cost_usd_per_1000": 0.19363431086142321, "axes": { "intelligence": 84.59272519717726, "calibration": 84.61587084168687, "speed": 76.51812235281348, "cost": 31.390530573402955 }, "jevbench_score": 64.39274682928007 }, { "members": [ "jev-1.13.0", "open-alternative-jev", "simplejev-qwen3.8-27b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8020833333333334, "judge": 0.958904109589041, "hard": 0.7136363636363636 }, "accuracy": 0.8352059925093633, "ece_hard": 0.03798970972024887, "calibration": 87.03354896069351, "p50_s": 2.3997448533773422, "p95_s": 4.385688671469688, "cost_usd_per_1000": 0.16608215730337078, "axes": { "intelligence": 84.7167393109174, "calibration": 87.03354896069351, "speed": 69.7779713205367, "cost": 33.390310677437384 }, "jevbench_score": 64.37966031666548 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "system-one-open" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9452054794520548, "hard": 0.6136363636363636 }, "accuracy": 0.7940074906367042, "ece_hard": 0.1924242424242423, "calibration": 63.80207575757577, "p50_s": 1.3041210994124413, "p95_s": 2.030777859687805, "cost_usd_per_1000": 0.07723977902621722, "axes": { "intelligence": 81.91651100041511, "calibration": 63.80207575757577, "speed": 75.77019658655011, "cost": 43.36476932066527 }, "jevbench_score": 64.37398987446088 }, { "members": [ "djev", "system-one-open", "simplejev-qwen3.8-27b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6681818181818182 }, "accuracy": 0.8258426966292135, "ece_hard": 0.048953010084816645, "calibration": 82.71021858710488, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.14482870786516855, "axes": { "intelligence": 84.61166459111665, "calibration": 82.71021858710488, "speed": 69.75909261944061, "cost": 35.1743603199228 }, "jevbench_score": 64.37305343474661 }, { "members": [ "jev-1.13.0", "djev", "open-alternative-jev", "system-one-open", "openjev-razorback16" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9657534246575342, "hard": 0.6409090909090909 }, "accuracy": 0.8127340823970037, "ece_hard": 0.08278555968140468, "calibration": 81.54344256813457, "p50_s": 1.3041210994124413, "p95_s": 2.030777859687805, "cost_usd_per_1000": 0.16852238202247188, "axes": { "intelligence": 83.60170195101702, "calibration": 81.54344256813457, "speed": 75.77019658655011, "cost": 33.20027232567968 }, "jevbench_score": 64.35185277233495 }, { "members": [ "jev-1.13.0", "djev", "openjev-sglang" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9657534246575342, "hard": 0.7 }, "accuracy": 0.8333333333333334, "ece_hard": 0.06235577472311554, "calibration": 84.76296806419462, "p50_s": 1.3671194911003113, "p95_s": 1.6257096692919728, "cost_usd_per_1000": 0.19714095880149812, "axes": { "intelligence": 84.79109589041096, "calibration": 84.76296806419462, "speed": 76.53150536083959, "cost": 31.156694067130275 }, "jevbench_score": 64.34088161865014 }, { "members": [ "semif-qwen3.5-4b", "system-one-open", "simplejev-qwen3.8-27b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.958904109589041, "hard": 0.6454545454545455 }, "accuracy": 0.8108614232209738, "ece_hard": 0.06290625607246245, "calibration": 83.00780797687543, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.1413220599250936, "axes": { "intelligence": 83.25461809879619, "calibration": 83.00780797687543, "speed": 69.75909261944061, "cost": 35.49370122608096 }, "jevbench_score": 64.31612160016375 }, { "members": [ "djev", "system-one-open", "openjev-sglang" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9657534246575342, "hard": 0.6636363636363637 }, "accuracy": 0.8183520599250936, "ece_hard": 0.059212327163661205, "calibration": 82.19168391306076, "p50_s": 1.3879805654287338, "p95_s": 2.030777859687805, "cost_usd_per_1000": 0.17210765917603, "axes": { "intelligence": 83.70018679950188, "calibration": 82.19168391306076, "speed": 75.49954194936167, "cost": 32.92599406497797 }, "jevbench_score": 64.30716728722157 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "simplejev-qwen3.8-27b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9794520547945206, "hard": 0.7090909090909091 }, "accuracy": 0.8445692883895131, "ece_hard": 0.057621106692257416, "calibration": 85.33897028412622, "p50_s": 2.3997448533773422, "p95_s": 4.381843151897192, "cost_usd_per_1000": 0.16635535955056177, "axes": { "intelligence": 86.0307181403072, "calibration": 85.33897028412622, "speed": 69.78178103134502, "cost": 33.36889608087208 }, "jevbench_score": 64.3015155176426 }, { "members": [ "semif-qwen3.5-4b", "djev", "openjev-sglang" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9657534246575342, "hard": 0.6727272727272727 }, "accuracy": 0.8239700374531835, "ece_hard": 0.07135063855649967, "calibration": 81.8173425641703, "p50_s": 1.3671194911003113, "p95_s": 1.6307271040976046, "cost_usd_per_1000": 0.179671329588015, "axes": { "intelligence": 84.26458073889582, "calibration": 81.8173425641703, "speed": 76.51812235281348, "cost": 32.365636550689075 }, "jevbench_score": 64.28131059916525 }, { "members": [ "semif-qwen3.5-4b", "djev", "open-alternative-jev", "system-one-open", "openjev-razorback16" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9657534246575342, "hard": 0.6227272727272727 }, "accuracy": 0.8033707865168539, "ece_hard": 0.09446110815418166, "calibration": 78.56519595075869, "p50_s": 1.3041210994124413, "p95_s": 2.030777859687805, "cost_usd_per_1000": 0.15105275280898878, "axes": { "intelligence": 82.76458073889582, "calibration": 78.56519595075869, "speed": 75.77019658655011, "cost": 34.62614067004084 }, "jevbench_score": 64.26792898416241 }, { "members": [ "jev-1.13.0", "open-alternative-jev", "openjev-sglang" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.958904109589041, "hard": 0.6909090909090909 }, "accuracy": 0.8239700374531835, "ece_hard": 0.058616061411224905, "calibration": 84.72315137775772, "p50_s": 1.3671194911003113, "p95_s": 1.6307271040976046, "cost_usd_per_1000": 0.19336110861423222, "axes": { "intelligence": 83.74325446243255, "calibration": 84.72315137775772, "speed": 76.51812235281348, "cost": 31.408926178436772 }, "jevbench_score": 64.2602383912045 }, { "members": [ "semif-qwen3.5-4b", "system-one-open", "simplejev-qwen3.8-27b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.958904109589041, "hard": 0.6318181818181818 }, "accuracy": 0.8052434456928839, "ece_hard": 0.05834876435403151, "calibration": 83.04332437607292, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.1413220599250936, "axes": { "intelligence": 82.84552718970528, "calibration": 83.04332437607292, "speed": 69.75909261944061, "cost": 35.49370122608096 }, "jevbench_score": 64.24383768196333 }, { "members": [ "jev-1.13.0", "system-one-open", "simplejev-qwen3.8-27b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9726027397260274, "hard": 0.7318181818181818 }, "accuracy": 0.848314606741573, "ece_hard": 0.060733770574652506, "calibration": 83.52753913886548, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.15879168913857677, "axes": { "intelligence": 85.93742216687423, "calibration": 83.52753913886548, "speed": 69.75909261944061, "cost": 33.975166944366165 }, "jevbench_score": 64.22344647732811 }, { "members": [ "djev", "system-one-open", "simplejev-qwen3.8-27b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9657534246575342, "hard": 0.6909090909090909 }, "accuracy": 0.8333333333333334, "ece_hard": 0.06460188787443218, "calibration": 81.40482445413056, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.14482870786516855, "axes": { "intelligence": 85.10170195101703, "calibration": 81.40482445413056, "speed": 69.75909261944061, "cost": 35.1743603199228 }, "jevbench_score": 64.21017524282502 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "simplejev-qwen3.6-35b-a3b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9794520547945206, "hard": 0.6636363636363637 }, "accuracy": 0.8239700374531835, "ece_hard": 0.07000870057305038, "calibration": 82.7838131284737, "p50_s": 1.7256089895963669, "p95_s": 1.9284205481410026, "cost_usd_per_1000": 0.17791052808988764, "axes": { "intelligence": 84.37541511000416, "calibration": 82.7838131284737, "speed": 74.77855855005357, "cost": 32.493950536225114 }, "jevbench_score": 64.18525018830223 }, { "members": [ "jev-1.13.0", "system-one-open", "simplejev-qwen3.8-27b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9726027397260274, "hard": 0.7090909090909091 }, "accuracy": 0.8389513108614233, "ece_hard": 0.055764958179675056, "calibration": 83.7328115005626, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.15879168913857677, "axes": { "intelligence": 85.25560398505606, "calibration": 83.7328115005626, "speed": 69.75909261944061, "cost": 33.975166944366165 }, "jevbench_score": 64.13502345458214 }, { "members": [ "jev-1.13.0", "system-one-open", "openjev-sglang" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9657534246575342, "hard": 0.6818181818181818 }, "accuracy": 0.8258426966292135, "ece_hard": 0.07050453524811995, "calibration": 83.26408432438194, "p50_s": 1.3879805654287338, "p95_s": 2.030777859687805, "cost_usd_per_1000": 0.18607064044943822, "axes": { "intelligence": 84.24564134495643, "calibration": 83.26408432438194, "speed": 75.49954194936167, "cost": 31.90966442617821 }, "jevbench_score": 64.11622233869313 }, { "members": [ "jev-1.13.0", "open-alternative-jev", "openjev-sglang" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9657534246575342, "hard": 0.7045454545454546 }, "accuracy": 0.8314606741573034, "ece_hard": 0.07737344800444632, "calibration": 83.25154587474462, "p50_s": 1.3671194911003113, "p95_s": 1.6307271040976046, "cost_usd_per_1000": 0.19336110861423222, "axes": { "intelligence": 84.34412619344127, "calibration": 83.25154587474462, "speed": 76.51812235281348, "cost": 31.408926178436772 }, "jevbench_score": 64.09381696368014 }, { "members": [ "djev", "open-alternative-jev", "openjev-sglang" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9452054794520548, "hard": 0.6863636363636364 }, "accuracy": 0.8220973782771536, "ece_hard": 0.0758149788310863, "calibration": 81.20557126682206, "p50_s": 1.3671194911003113, "p95_s": 1.6307271040976046, "cost_usd_per_1000": 0.179398127340824, "axes": { "intelligence": 83.80666251556663, "calibration": 81.20557126682206, "speed": 76.51812235281348, "cost": 32.38546284039515 }, "jevbench_score": 64.08327465730912 }, { "members": [ "jev-1.13.0", "djev", "simplejev-qwen3.8-27b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9726027397260274, "hard": 0.7227272727272728 }, "accuracy": 0.846441947565543, "ece_hard": 0.047430927920759626, "calibration": 84.88709416983284, "p50_s": 2.3997448533773422, "p95_s": 4.381843151897192, "cost_usd_per_1000": 0.16986200749063668, "axes": { "intelligence": 85.95636156081362, "calibration": 84.88709416983284, "speed": 69.78178103134502, "cost": 33.097112427150094 }, "jevbench_score": 64.07121526670615 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "open-alternative-jev" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8020833333333334, "judge": 0.958904109589041, "hard": 0.6227272727272727 }, "accuracy": 0.797752808988764, "ece_hard": 0.20757575757575725, "calibration": 60.00809090909094, "p50_s": 0.6719580851495266, "p95_s": 1.1348857231438156, "cost_usd_per_1000": 0.08453024719101122, "axes": { "intelligence": 81.98946658364467, "calibration": 60.00809090909094, "speed": 81.17705683541841, "cost": 42.18963583057602 }, "jevbench_score": 64.0695593941902 }, { "members": [ "semif-qwen3.5-4b", "djev", "open-alternative-jev", "system-one-open", "openjev-razorback16" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9657534246575342, "hard": 0.6227272727272727 }, "accuracy": 0.8033707865168539, "ece_hard": 0.10424647337496586, "calibration": 77.59845779080636, "p50_s": 1.3041210994124413, "p95_s": 2.030777859687805, "cost_usd_per_1000": 0.15105275280898878, "axes": { "intelligence": 82.76458073889582, "calibration": 77.59845779080636, "speed": 75.77019658655011, "cost": 34.62614067004084 }, "jevbench_score": 64.06930723945564 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "simplejev-qwen3.6-35b-a3b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9794520547945206, "hard": 0.6636363636363637 }, "accuracy": 0.8239700374531835, "ece_hard": 0.07265121045195877, "calibration": 82.09035964260859, "p50_s": 1.7256089895963669, "p95_s": 1.9284205481410026, "cost_usd_per_1000": 0.17791052808988764, "axes": { "intelligence": 84.37541511000416, "calibration": 82.09035964260859, "speed": 74.77855855005357, "cost": 32.493950536225114 }, "jevbench_score": 64.05041106157728 }, { "members": [ "semif-qwen3.5-4b", "djev", "openjev-sglang" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9657534246575342, "hard": 0.6727272727272727 }, "accuracy": 0.8239700374531835, "ece_hard": 0.08395827902372231, "calibration": 80.56010091200501, "p50_s": 1.3671194911003113, "p95_s": 1.6307271040976046, "cost_usd_per_1000": 0.179671329588015, "axes": { "intelligence": 84.26458073889582, "calibration": 80.56010091200501, "speed": 76.51812235281348, "cost": 32.365636550689075 }, "jevbench_score": 64.03293091677823 }, { "members": [ "djev", "system-one-open", "simplejev-qwen3.6-35b-a3b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.952054794520548, "hard": 0.6590909090909091 }, "accuracy": 0.8146067415730337, "ece_hard": 0.07385398594247372, "calibration": 79.09637722402283, "p50_s": 1.7357386276125908, "p95_s": 2.0568130582571027, "cost_usd_per_1000": 0.1563838764044944, "axes": { "intelligence": 83.47192818596929, "calibration": 79.09637722402283, "speed": 74.47320850632062, "cost": 34.174240776685366 }, "jevbench_score": 64.02491262723176 }, { "members": [ "semif-qwen3.5-4b", "djev", "open-alternative-jev" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9315068493150684, "hard": 0.6090909090909091 }, "accuracy": 0.7865168539325843, "ece_hard": 0.2318181818181815, "calibration": 56.77509848484851, "p50_s": 0.5735304296016693, "p95_s": 1.1444025911390774, "cost_usd_per_1000": 0.070567265917603, "axes": { "intelligence": 81.1049190535492, "calibration": 56.77509848484851, "speed": 81.82864702484957, "cost": 44.541901260836084 }, "jevbench_score": 64.00586971872426 }, { "members": [ "jev-1.13.0", "djev", "openjev-sglang" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9657534246575342, "hard": 0.6954545454545454 }, "accuracy": 0.8314606741573034, "ece_hard": 0.08210883631557, "calibration": 83.06684232090845, "p50_s": 1.3671194911003113, "p95_s": 1.6257096692919728, "cost_usd_per_1000": 0.19714095880149812, "axes": { "intelligence": 84.65473225404733, "calibration": 83.06684232090845, "speed": 76.53150536083959, "cost": 31.156694067130275 }, "jevbench_score": 63.99081507529205 }, { "members": [ "jev-1.13.0", "open-alternative-jev", "simplejev-qwen3.6-35b-a3b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8020833333333334, "judge": 0.9726027397260274, "hard": 0.6545454545454545 }, "accuracy": 0.8146067415730337, "ece_hard": 0.06892714727699593, "calibration": 82.64102319830559, "p50_s": 1.7256089895963669, "p95_s": 1.9284205481410026, "cost_usd_per_1000": 0.17763732584269662, "axes": { "intelligence": 83.32757368202574, "calibration": 82.64102319830559, "speed": 74.77855855005357, "cost": 32.5139731999152 }, "jevbench_score": 63.967280666009245 }, { "members": [ "jev-1.13.0", "system-one-open", "simplejev-qwen3.6-35b-a3b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9726027397260274, "hard": 0.6636363636363637 }, "accuracy": 0.8220973782771536, "ece_hard": 0.08372862531894754, "calibration": 80.69308375967769, "p50_s": 1.7357386276125908, "p95_s": 2.0568130582571027, "cost_usd_per_1000": 0.1703468576779026, "axes": { "intelligence": 84.18363428808635, "calibration": 80.69308375967769, "speed": 74.47320850632062, "cost": 33.05997619849816 }, "jevbench_score": 63.95016159258793 }, { "members": [ "jev-1.13.0", "open-alternative-jev", "simplejev-qwen3.6-35b-a3b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8020833333333334, "judge": 0.9657534246575342, "hard": 0.6590909090909091 }, "accuracy": 0.8146067415730337, "ece_hard": 0.06345253870355722, "calibration": 82.56642864011525, "p50_s": 1.7256089895963669, "p95_s": 1.9284205481410026, "cost_usd_per_1000": 0.17763732584269662, "axes": { "intelligence": 83.27215649647157, "calibration": 82.56642864011525, "speed": 74.77855855005357, "cost": 32.5139731999152 }, "jevbench_score": 63.94220537729924 }, { "members": [ "jev-1.13.0", "system-one-open", "openjev-razorback16" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6636363636363637 }, "accuracy": 0.8164794007490637, "ece_hard": 0.13939393939393913, "calibration": 69.97177272727276, "p50_s": 1.3041210994124413, "p95_s": 2.030777859687805, "cost_usd_per_1000": 0.12039972284644193, "axes": { "intelligence": 83.70817766708178, "calibration": 69.97177272727276, "speed": 75.77019658655011, "cost": 37.5812353839736 }, "jevbench_score": 63.905767884195676 }, { "members": [ "jev-1.13.0", "open-alternative-jev", "simplejev-qwen3.8-27b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8020833333333334, "judge": 0.9452054794520548, "hard": 0.7045454545454546 }, "accuracy": 0.8277153558052435, "ece_hard": 0.056770422661774136, "calibration": 85.13828980148442, "p50_s": 2.3997448533773422, "p95_s": 4.385688671469688, "cost_usd_per_1000": 0.16608215730337078, "axes": { "intelligence": 84.0604503943545, "calibration": 85.13828980148442, "speed": 69.7779713205367, "cost": 33.390310677437384 }, "jevbench_score": 63.901912820605276 }, { "members": [ "djev", "open-alternative-jev", "simplejev-qwen3.8-27b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9452054794520548, "hard": 0.6818181818181818 }, "accuracy": 0.8220973782771536, "ece_hard": 0.06904517644326678, "calibration": 82.40826930239919, "p50_s": 2.3997448533773422, "p95_s": 4.385688671469688, "cost_usd_per_1000": 0.15211917602996256, "axes": { "intelligence": 83.96196554586966, "calibration": 82.40826930239919, "speed": 69.7779713205367, "cost": 34.53448106971932 }, "jevbench_score": 63.90078096156789 }, { "members": [ "jev-1.13.0", "djev", "simplejev-qwen3.8-27b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.7227272727272728 }, "accuracy": 0.848314606741573, "ece_hard": 0.05851752586427308, "calibration": 83.535860085239, "p50_s": 2.3997448533773422, "p95_s": 4.381843151897192, "cost_usd_per_1000": 0.16986200749063668, "axes": { "intelligence": 86.24802822748029, "calibration": 83.535860085239, "speed": 69.78178103134502, "cost": 33.097112427150094 }, "jevbench_score": 63.868772520935224 }, { "members": [ "djev", "open-alternative-jev", "openjev-sglang" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9452054794520548, "hard": 0.6590909090909091 }, "accuracy": 0.8127340823970037, "ece_hard": 0.08110032554640029, "calibration": 80.60230045884151, "p50_s": 1.3671194911003113, "p95_s": 1.6307271040976046, "cost_usd_per_1000": 0.179398127340824, "axes": { "intelligence": 83.28014736405149, "calibration": 80.60230045884151, "speed": 76.51812235281348, "cost": 32.38546284039515 }, "jevbench_score": 63.86322319233723 }, { "members": [ "open-alternative-jev", "system-one-open", "simplejev-qwen3.8-27b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.78125, "judge": 0.910958904109589, "hard": 0.6590909090909091 }, "accuracy": 0.795880149812734, "ece_hard": 0.05831441373413093, "calibration": 82.59334233833974, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.1410488576779026, "axes": { "intelligence": 81.15457658779577, "calibration": 82.59334233833974, "speed": 69.75909261944061, "cost": 35.51891280235006 }, "jevbench_score": 63.83805345952742 }, { "members": [ "djev", "system-one-open", "simplejev-qwen3.6-35b-a3b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.958904109589041, "hard": 0.65 }, "accuracy": 0.8127340823970037, "ece_hard": 0.08362995243468449, "calibration": 78.04349656616408, "p50_s": 1.7357386276125908, "p95_s": 2.0568130582571027, "cost_usd_per_1000": 0.1563838764044944, "axes": { "intelligence": 83.39098173515981, "calibration": 78.04349656616408, "speed": 74.47320850632062, "cost": 34.174240776685366 }, "jevbench_score": 63.79530032924639 }, { "members": [ "semif-qwen3.5-4b", "system-one-open", "openjev-sglang" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.958904109589041, "hard": 0.6227272727272727 }, "accuracy": 0.799625468164794, "ece_hard": 0.07322851125669572, "calibration": 80.27204326313304, "p50_s": 1.3879805654287338, "p95_s": 2.030777859687805, "cost_usd_per_1000": 0.16860101123595506, "axes": { "intelligence": 82.28113325031133, "calibration": 80.27204326313304, "speed": 75.49954194936167, "cost": 33.194194746698045 }, "jevbench_score": 63.78488240400505 }, { "members": [ "semif-qwen3.5-4b", "djev", "simplejev-qwen3.6-35b-a3b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.952054794520548, "hard": 0.65 }, "accuracy": 0.8108614232209738, "ece_hard": 0.0899039361215802, "calibration": 78.73808471397517, "p50_s": 1.7256089895963669, "p95_s": 1.9284205481410026, "cost_usd_per_1000": 0.16394754681647938, "axes": { "intelligence": 83.19920091324202, "calibration": 78.73808471397517, "speed": 74.77855855005357, "cost": 33.55885232172082 }, "jevbench_score": 63.67545364855542 }, { "members": [ "semif-qwen3.5-4b", "djev", "simplejev-qwen3.6-35b-a3b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.952054794520548, "hard": 0.65 }, "accuracy": 0.8108614232209738, "ece_hard": 0.09294233707953749, "calibration": 78.46372714080923, "p50_s": 1.7256089895963669, "p95_s": 1.9284205481410026, "cost_usd_per_1000": 0.16394754681647938, "axes": { "intelligence": 83.19920091324202, "calibration": 78.46372714080923, "speed": 74.77855855005357, "cost": 33.55885232172082 }, "jevbench_score": 63.61991281119475 }, { "members": [ "semif-qwen3.5-4b", "open-alternative-jev", "simplejev-qwen3.8-27b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9246575342465754, "hard": 0.6409090909090909 }, "accuracy": 0.7940074906367042, "ece_hard": 0.060780127500780975, "calibration": 82.82697138529979, "p50_s": 2.3997448533773422, "p95_s": 4.385688671469688, "cost_usd_per_1000": 0.14861252808988762, "axes": { "intelligence": 81.2843503528435, "calibration": 82.82697138529979, "speed": 69.7779713205367, "cost": 34.838337335506964 }, "jevbench_score": 63.604613784275365 }, { "members": [ "jev-1.13.0", "djev", "simplejev-qwen3.6-35b-a3b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.958904109589041, "hard": 0.6772727272727272 }, "accuracy": 0.8220973782771536, "ece_hard": 0.0957733333899133, "calibration": 80.38270188075279, "p50_s": 1.724735751748085, "p95_s": 1.9244587391614911, "cost_usd_per_1000": 0.18141717602996255, "axes": { "intelligence": 83.91749688667497, "calibration": 80.38270188075279, "speed": 74.78968829969378, "cost": 32.239647929208815 }, "jevbench_score": 63.50556491970039 }, { "members": [ "jev-1.13.0", "open-alternative-jev", "openjev-razorback16" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.910958904109589, "hard": 0.65 }, "accuracy": 0.799625468164794, "ece_hard": 0.16818181818181782, "calibration": 66.29356818181822, "p50_s": 0.6816155403852463, "p95_s": 1.1239915058016774, "cost_usd_per_1000": 0.12769019101123594, "axes": { "intelligence": 82.04851598173516, "calibration": 66.29356818181822, "speed": 81.15697487293535, "cost": 36.815273901199916 }, "jevbench_score": 63.492730301897396 }, { "members": [ "jev-1.13.0", "system-one-open", "simplejev-qwen3.6-35b-a3b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9657534246575342, "hard": 0.6363636363636364 }, "accuracy": 0.8089887640449438, "ece_hard": 0.09109738299259694, "calibration": 79.3400312496334, "p50_s": 1.7357386276125908, "p95_s": 2.0568130582571027, "cost_usd_per_1000": 0.1703468576779026, "axes": { "intelligence": 83.17367164798672, "calibration": 79.3400312496334, "speed": 74.47320850632062, "cost": 33.05997619849816 }, "jevbench_score": 63.48852106881859 }, { "members": [ "semif-qwen3.5-4b", "system-one-open", "simplejev-qwen3.6-35b-a3b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9794520547945206, "hard": 0.6045454545454545 }, "accuracy": 0.799625468164794, "ece_hard": 0.10224553000259155, "calibration": 76.16150417022877, "p50_s": 1.7357386276125908, "p95_s": 2.0568130582571027, "cost_usd_per_1000": 0.15287722846441948, "axes": { "intelligence": 82.60268783727689, "calibration": 76.16150417022877, "speed": 74.47320850632062, "cost": 34.46971597826156 }, "jevbench_score": 63.39308183993642 }, { "members": [ "open-alternative-jev", "system-one-open", "simplejev-qwen3.8-27b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.78125, "judge": 0.910958904109589, "hard": 0.6409090909090909 }, "accuracy": 0.7883895131086143, "ece_hard": 0.07462952000297095, "calibration": 80.61658756520598, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.1410488576779026, "axes": { "intelligence": 80.60912204234121, "calibration": 80.61658756520598, "speed": 69.75909261944061, "cost": 35.51891280235006 }, "jevbench_score": 63.34571933321863 }, { "members": [ "semif-qwen3.5-4b", "system-one-open", "openjev-sglang" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8020833333333334, "judge": 0.958904109589041, "hard": 0.6090909090909091 }, "accuracy": 0.7921348314606742, "ece_hard": 0.08679856617286502, "calibration": 78.71351078086829, "p50_s": 1.3879805654287338, "p95_s": 2.030777859687805, "cost_usd_per_1000": 0.16860101123595506, "axes": { "intelligence": 81.58037567455376, "calibration": 78.71351078086829, "speed": 75.49954194936167, "cost": 33.194194746698045 }, "jevbench_score": 63.33741867091613 }, { "members": [ "semif-qwen3.5-4b", "system-one-open", "simplejev-qwen3.6-35b-a3b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9794520547945206, "hard": 0.6090909090909091 }, "accuracy": 0.8014981273408239, "ece_hard": 0.1061896231365379, "calibration": 75.73935198588279, "p50_s": 1.7357386276125908, "p95_s": 2.0568130582571027, "cost_usd_per_1000": 0.15287722846441948, "axes": { "intelligence": 82.73905147364053, "calibration": 75.73935198588279, "speed": 74.47320850632062, "cost": 34.46971597826156 }, "jevbench_score": 63.331164423617324 }, { "members": [ "djev", "open-alternative-jev", "simplejev-qwen3.8-27b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9452054794520548, "hard": 0.6590909090909091 }, "accuracy": 0.8108614232209738, "ece_hard": 0.08826083612411861, "calibration": 80.3616168668002, "p50_s": 2.3997448533773422, "p95_s": 4.385688671469688, "cost_usd_per_1000": 0.15211917602996256, "axes": { "intelligence": 82.98848069738482, "calibration": 80.3616168668002, "speed": 69.7779713205367, "cost": 34.53448106971932 }, "jevbench_score": 63.31541216392697 }, { "members": [ "open-alternative-jev", "system-one-open", "openjev-sglang" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.7708333333333334, "judge": 0.9315068493150684, "hard": 0.6318181818181818 }, "accuracy": 0.7883895131086143, "ece_hard": 0.08042581131081421, "calibration": 79.45623805057694, "p50_s": 1.3879805654287338, "p95_s": 2.030777859687805, "cost_usd_per_1000": 0.16832780898876404, "axes": { "intelligence": 80.62007056870073, "calibration": 79.45623805057694, "speed": 75.49954194936167, "cost": 33.2153238838832 }, "jevbench_score": 63.30871452976128 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "djev", "open-alternative-jev", "system-one-open" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9726027397260274, "hard": 0.6363636363636364 }, "accuracy": 0.8108614232209738, "ece_hard": 0.14636363636363608, "calibration": 68.60263636363639, "p50_s": 1.3041210994124413, "p95_s": 2.030777859687805, "cost_usd_per_1000": 0.12536243820224718, "axes": { "intelligence": 83.36545246990454, "calibration": 68.60263636363639, "speed": 75.77019658655011, "cost": 37.05497709298896 }, "jevbench_score": 63.30207849902799 }, { "members": [ "jev-1.13.0", "djev", "simplejev-qwen3.6-35b-a3b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9657534246575342, "hard": 0.6681818181818182 }, "accuracy": 0.8202247191011236, "ece_hard": 0.111054722917709, "calibration": 79.1874325435558, "p50_s": 1.724735751748085, "p95_s": 1.9244587391614911, "cost_usd_per_1000": 0.18141717602996255, "axes": { "intelligence": 83.8365504358655, "calibration": 79.1874325435558, "speed": 74.78968829969378, "cost": 32.239647929208815 }, "jevbench_score": 63.25289670601287 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "djev", "system-one-open", "simplejev-qwen3.8-27b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.7 }, "accuracy": 0.8389513108614233, "ece_hard": 0.03109980423650019, "calibration": 87.34133443619709, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.20718755056179775, "axes": { "intelligence": 85.5662100456621, "calibration": 87.34133443619709, "speed": 69.75909261944061, "cost": 30.509090317912694 }, "jevbench_score": 63.15213685706244 }, { "members": [ "semif-qwen3.5-4b", "openjev-razorback16", "simplejev-qwen3.8-27b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9657534246575342, "hard": 0.6772727272727272 }, "accuracy": 0.8277153558052435, "ece_hard": 0.04304336798600484, "calibration": 85.08333045173836, "p50_s": 2.3997448533773422, "p95_s": 4.381843151897192, "cost_usd_per_1000": 0.19204567415730334, "axes": { "intelligence": 84.69261104192611, "calibration": 85.08333045173836, "speed": 69.78178103134502, "cost": 31.497864127097145 }, "jevbench_score": 63.085316987588136 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "djev", "system-one-open", "simplejev-qwen3.6-35b-a3b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6590909090909091 }, "accuracy": 0.8220973782771536, "ece_hard": 0.05149609170090978, "calibration": 84.60425789849006, "p50_s": 1.7357386276125908, "p95_s": 2.0568130582571027, "cost_usd_per_1000": 0.21874271910112358, "axes": { "intelligence": 84.33893731838938, "calibration": 84.60425789849006, "speed": 74.47320850632062, "cost": 29.80199180953467 }, "jevbench_score": 63.08359359443402 }, { "members": [ "system-one-open", "openjev-razorback16", "simplejev-qwen3.8-27b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.958904109589041, "hard": 0.6909090909090909 }, "accuracy": 0.8314606741573034, "ece_hard": 0.056769679544029214, "calibration": 83.491493361397, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.18448200374531834, "axes": { "intelligence": 84.90992112909922, "calibration": 83.491493361397, "speed": 69.75909261944061, "cost": 32.0213797884486 }, "jevbench_score": 63.082715903971305 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "djev", "open-alternative-jev", "openjev-sglang" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9726027397260274, "hard": 0.6863636363636364 }, "accuracy": 0.8314606741573034, "ece_hard": 0.05097223024250166, "calibration": 85.44129416499732, "p50_s": 1.3671194911003113, "p95_s": 1.6307271040976046, "cost_usd_per_1000": 0.2417569700374532, "axes": { "intelligence": 84.86545246990454, "calibration": 85.44129416499732, "speed": 76.51812235281348, "cost": 28.49862988071989 }, "jevbench_score": 63.05895343293836 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "open-alternative-jev", "system-one-open", "simplejev-qwen3.6-35b-a3b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9794520547945206, "hard": 0.6409090909090909 }, "accuracy": 0.8127340823970037, "ece_hard": 0.04657575858090858, "calibration": 84.30614994315054, "p50_s": 1.7357386276125908, "p95_s": 2.0568130582571027, "cost_usd_per_1000": 0.21496286891385769, "axes": { "intelligence": 83.40193026151931, "calibration": 84.30614994315054, "speed": 74.47320850632062, "cost": 30.029096512085246 }, "jevbench_score": 62.97155565896435 }, { "members": [ "semif-qwen3.5-4b", "openjev-razorback16", "simplejev-qwen3.8-27b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6772727272727272 }, "accuracy": 0.8295880149812734, "ece_hard": 0.04899359461242739, "calibration": 84.19278103362655, "p50_s": 2.3997448533773422, "p95_s": 4.381843151897192, "cost_usd_per_1000": 0.19204567415730334, "axes": { "intelligence": 84.88439186384392, "calibration": 84.19278103362655, "speed": 69.78178103134502, "cost": 31.497864127097145 }, "jevbench_score": 62.955178978405804 }, { "members": [ "system-one-open", "openjev-razorback16", "simplejev-qwen3.8-27b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.952054794520548, "hard": 0.7 }, "accuracy": 0.8333333333333334, "ece_hard": 0.06656473570732716, "calibration": 82.73602755478498, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.18448200374531834, "axes": { "intelligence": 84.99086757990868, "calibration": 82.73602755478498, "speed": 69.75909261944061, "cost": 32.0213797884486 }, "jevbench_score": 62.954524499586 }, { "members": [ "jev-1.13.0", "openjev-razorback16", "simplejev-qwen3.8-27b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9657534246575342, "hard": 0.7318181818181818 }, "accuracy": 0.850187265917603, "ece_hard": 0.0410136785618546, "calibration": 85.82259427677505, "p50_s": 2.3997448533773422, "p95_s": 4.381843151897192, "cost_usd_per_1000": 0.2095153033707865, "axes": { "intelligence": 86.32897467828975, "calibration": 85.82259427677505, "speed": 69.78178103134502, "cost": 30.363527496908077 }, "jevbench_score": 62.945273348912345 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "djev", "open-alternative-jev", "openjev-sglang" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9726027397260274, "hard": 0.6818181818181818 }, "accuracy": 0.8295880149812734, "ece_hard": 0.054333372044610605, "calibration": 84.83266552055315, "p50_s": 1.3671194911003113, "p95_s": 1.6307271040976046, "cost_usd_per_1000": 0.2417569700374532, "axes": { "intelligence": 84.72908883354089, "calibration": 84.83266552055315, "speed": 76.51812235281348, "cost": 28.49862988071989 }, "jevbench_score": 62.921053240848025 }, { "members": [ "semif-qwen3.5-4b", "open-alternative-jev", "simplejev-qwen3.8-27b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.910958904109589, "hard": 0.6272727272727273 }, "accuracy": 0.7846441947565543, "ece_hard": 0.08721048542737422, "calibration": 79.80777560439748, "p50_s": 2.3997448533773422, "p95_s": 4.385688671469688, "cost_usd_per_1000": 0.14861252808988762, "axes": { "intelligence": 80.49169779991698, "calibration": 79.80777560439748, "speed": 69.7779713205367, "cost": 34.838337335506964 }, "jevbench_score": 62.86269753578472 }, { "members": [ "djev", "open-alternative-jev", "simplejev-qwen3.6-35b-a3b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8020833333333334, "judge": 0.9315068493150684, "hard": 0.6409090909090909 }, "accuracy": 0.797752808988764, "ece_hard": 0.11428881143661364, "calibration": 76.05010957035618, "p50_s": 1.7256089895963669, "p95_s": 1.9284205481410026, "cost_usd_per_1000": 0.1636743445692884, "axes": { "intelligence": 81.76779784142798, "calibration": 76.05010957035618, "speed": 74.77855855005357, "cost": 33.580581685787266 }, "jevbench_score": 62.86180722032748 }, { "members": [ "jev-1.13.0", "openjev-razorback16", "simplejev-qwen3.8-27b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.958904109589041, "hard": 0.7227272727272728 }, "accuracy": 0.8445692883895131, "ece_hard": 0.03887243581938465, "calibration": 85.72736854179912, "p50_s": 2.3997448533773422, "p95_s": 4.381843151897192, "cost_usd_per_1000": 0.2095153033707865, "axes": { "intelligence": 85.86446658364467, "calibration": 85.72736854179912, "speed": 69.78178103134502, "cost": 30.363527496908077 }, "jevbench_score": 62.84298576893377 }, { "members": [ "djev", "open-alternative-jev", "simplejev-qwen3.6-35b-a3b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8020833333333334, "judge": 0.9315068493150684, "hard": 0.6409090909090909 }, "accuracy": 0.797752808988764, "ece_hard": 0.11548262905586434, "calibration": 75.9337440707474, "p50_s": 1.7256089895963669, "p95_s": 1.9284205481410026, "cost_usd_per_1000": 0.1636743445692884, "axes": { "intelligence": 81.76779784142798, "calibration": 75.9337440707474, "speed": 74.77855855005357, "cost": 33.580581685787266 }, "jevbench_score": 62.83774694393944 }, { "members": [ "open-alternative-jev", "system-one-open", "simplejev-qwen3.6-35b-a3b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.958904109589041, "hard": 0.6090909090909091 }, "accuracy": 0.7902621722846442, "ece_hard": 0.1156332787007377, "calibration": 74.6035430515663, "p50_s": 1.7357386276125908, "p95_s": 2.0568130582571027, "cost_usd_per_1000": 0.15260402621722846, "axes": { "intelligence": 81.28870900788709, "calibration": 74.6035430515663, "speed": 74.47320850632062, "cost": 34.49302024159303 }, "jevbench_score": 62.82467374659772 }, { "members": [ "djev", "system-one-open", "openjev-razorback16" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9246575342465754, "hard": 0.6409090909090909 }, "accuracy": 0.797752808988764, "ece_hard": 0.17121212121212095, "calibration": 63.95528787878791, "p50_s": 1.3041210994124413, "p95_s": 2.030777859687805, "cost_usd_per_1000": 0.1064367415730337, "axes": { "intelligence": 81.86768368617685, "calibration": 63.95528787878791, "speed": 75.77019658655011, "cost": 39.1872528788934 }, "jevbench_score": 62.792539670173205 }, { "members": [ "jev-1.13.0", "djev", "open-alternative-jev", "system-one-open", "openjev-sglang" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9657534246575342, "hard": 0.6818181818181818 }, "accuracy": 0.8258426966292135, "ece_hard": 0.06078117649204772, "calibration": 84.5364764726141, "p50_s": 1.3879805654287338, "p95_s": 2.030777859687805, "cost_usd_per_1000": 0.23419329962546817, "axes": { "intelligence": 84.24564134495643, "calibration": 84.5364764726141, "speed": 75.49954194936167, "cost": 28.912766033329746 }, "jevbench_score": 62.79229919596739 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "open-alternative-jev", "system-one-open", "openjev-sglang" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.958904109589041, "hard": 0.6409090909090909 }, "accuracy": 0.8033707865168539, "ece_hard": 0.03563755844186865, "calibration": 85.91402620942772, "p50_s": 1.3879805654287338, "p95_s": 2.030777859687805, "cost_usd_per_1000": 0.23068665168539326, "axes": { "intelligence": 82.24325446243255, "calibration": 85.91402620942772, "speed": 75.49954194936167, "cost": 29.10932603525056 }, "jevbench_score": 62.77478105631494 }, { "members": [ "semif-qwen3.5-4b", "open-alternative-jev", "openjev-sglang" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.78125, "judge": 0.9452054794520548, "hard": 0.6181818181818182 }, "accuracy": 0.7883895131086143, "ece_hard": 0.10264894035238806, "calibration": 76.83444671917795, "p50_s": 1.3671194911003113, "p95_s": 1.6307271040976046, "cost_usd_per_1000": 0.17589147940074906, "axes": { "intelligence": 80.88620797011208, "calibration": 76.83444671917795, "speed": 76.51812235281348, "cost": 32.64265594854422 }, "jevbench_score": 62.768978212975696 }, { "members": [ "open-alternative-jev", "system-one-open", "simplejev-qwen3.6-35b-a3b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.78125, "judge": 0.952054794520548, "hard": 0.6045454545454545 }, "accuracy": 0.7846441947565543, "ece_hard": 0.1151796470744981, "calibration": 74.63987339484127, "p50_s": 1.7357386276125908, "p95_s": 2.0568130582571027, "cost_usd_per_1000": 0.15260402621722846, "axes": { "intelligence": 80.66889788293898, "calibration": 74.63987339484127, "speed": 74.47320850632062, "cost": 34.49302024159303 }, "jevbench_score": 62.71220576887885 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "djev", "open-alternative-jev", "simplejev-qwen3.8-27b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6818181818181818 }, "accuracy": 0.8314606741573034, "ece_hard": 0.04198655308176018, "calibration": 86.66553989382217, "p50_s": 2.3997448533773422, "p95_s": 4.385688671469688, "cost_usd_per_1000": 0.21447801872659175, "axes": { "intelligence": 85.02075550020756, "calibration": 86.66553989382217, "speed": 69.7779713205367, "cost": 30.058516326044753 }, "jevbench_score": 62.69953521246622 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "open-alternative-jev", "system-one-open", "openjev-sglang" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8020833333333334, "judge": 0.9657534246575342, "hard": 0.6454545454545455 }, "accuracy": 0.8089887640449438, "ece_hard": 0.05298330875459179, "calibration": 84.58276675050713, "p50_s": 1.3879805654287338, "p95_s": 2.030777859687805, "cost_usd_per_1000": 0.23068665168539326, "axes": { "intelligence": 82.86306558738066, "calibration": 84.58276675050713, "speed": 75.49954194936167, "cost": 29.10932603525056 }, "jevbench_score": 62.64765764524476 }, { "members": [ "djev", "open-alternative-jev", "openjev-razorback16" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9246575342465754, "hard": 0.65 }, "accuracy": 0.8014981273408239, "ece_hard": 0.21515151515151487, "calibration": 60.12609848484851, "p50_s": 0.6516780346632004, "p95_s": 1.131779689341783, "cost_usd_per_1000": 0.11372720973782771, "axes": { "intelligence": 82.14041095890411, "calibration": 60.12609848484851, "speed": 81.3220502108525, "cost": 38.324068480148846 }, "jevbench_score": 62.63613557512727 }, { "members": [ "open-alternative-jev", "system-one-open", "openjev-sglang" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.7708333333333334, "judge": 0.9315068493150684, "hard": 0.6181818181818182 }, "accuracy": 0.7827715355805244, "ece_hard": 0.1086684207771313, "calibration": 76.4970950465669, "p50_s": 1.3879805654287338, "p95_s": 2.030777859687805, "cost_usd_per_1000": 0.16832780898876404, "axes": { "intelligence": 80.2109796596098, "calibration": 76.4970950465669, "speed": 75.49954194936167, "cost": 33.2153238838832 }, "jevbench_score": 62.63115101291932 }, { "members": [ "semif-qwen3.5-4b", "open-alternative-jev", "openjev-sglang" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.78125, "judge": 0.9452054794520548, "hard": 0.6090909090909091 }, "accuracy": 0.7846441947565543, "ece_hard": 0.10776309637847917, "calibration": 76.17524893024529, "p50_s": 1.3671194911003113, "p95_s": 1.6307271040976046, "cost_usd_per_1000": 0.17589147940074906, "axes": { "intelligence": 80.61348069738482, "calibration": 76.17524893024529, "speed": 76.51812235281348, "cost": 32.64265594854422 }, "jevbench_score": 62.581048750247845 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "open-alternative-jev", "system-one-open", "simplejev-qwen3.6-35b-a3b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9794520547945206, "hard": 0.6227272727272727 }, "accuracy": 0.8052434456928839, "ece_hard": 0.06120544289454078, "calibration": 82.4843691085041, "p50_s": 1.7357386276125908, "p95_s": 2.0568130582571027, "cost_usd_per_1000": 0.21496286891385769, "axes": { "intelligence": 82.85647571606476, "calibration": 82.4843691085041, "speed": 74.47320850632062, "cost": 30.029096512085246 }, "jevbench_score": 62.52592294523857 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "djev", "system-one-open", "openjev-sglang" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9657534246575342, "hard": 0.6636363636363637 }, "accuracy": 0.8220973782771536, "ece_hard": 0.07583628830978563, "calibration": 82.95413183924732, "p50_s": 1.3879805654287338, "p95_s": 2.030777859687805, "cost_usd_per_1000": 0.2344665018726592, "axes": { "intelligence": 84.2835201328352, "calibration": 82.95413183924732, "speed": 75.49954194936167, "cost": 28.897575879432267 }, "jevbench_score": 62.49519168729477 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "open-alternative-jev", "system-one-open", "simplejev-qwen3.8-27b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.958904109589041, "hard": 0.6545454545454545 }, "accuracy": 0.8146067415730337, "ece_hard": 0.05602283272627454, "calibration": 85.02959902138798, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.20340770037453182, "axes": { "intelligence": 83.52734537152347, "calibration": 85.02959902138798, "speed": 69.75909261944061, "cost": 30.748978302447497 }, "jevbench_score": 62.475189308895004 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "djev", "system-one-open", "simplejev-qwen3.6-35b-a3b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6454545454545455 }, "accuracy": 0.8164794007490637, "ece_hard": 0.07698924605465521, "calibration": 81.6788781315649, "p50_s": 1.7357386276125908, "p95_s": 2.0568130582571027, "cost_usd_per_1000": 0.21874271910112358, "axes": { "intelligence": 83.92984640929846, "calibration": 81.6788781315649, "speed": 74.47320850632062, "cost": 29.80199180953467 }, "jevbench_score": 62.4550969793025 }, { "members": [ "semif-qwen3.5-4b", "open-alternative-jev", "system-one-open" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9178082191780822, "hard": 0.5772727272727273 }, "accuracy": 0.7659176029962547, "ece_hard": 0.24090909090909077, "calibration": 54.19592424242426, "p50_s": 1.3041210994124413, "p95_s": 2.030777859687805, "cost_usd_per_1000": 0.05949694756554307, "axes": { "intelligence": 79.18347862183481, "calibration": 54.19592424242426, "speed": 75.77019658655011, "cost": 46.765159442990324 }, "jevbench_score": 62.4461271965557 }, { "members": [ "jev-1.13.0", "djev", "open-alternative-jev", "system-one-open", "simplejev-qwen3.8-27b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9726027397260274, "hard": 0.6818181818181818 }, "accuracy": 0.8295880149812734, "ece_hard": 0.061524824292706254, "calibration": 84.09532588768892, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.20691434831460673, "axes": { "intelligence": 84.72908883354089, "calibration": 84.09532588768892, "speed": 69.75909261944061, "cost": 30.526281774305716 }, "jevbench_score": 62.41224165818741 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "djev", "open-alternative-jev", "simplejev-qwen3.8-27b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6681818181818182 }, "accuracy": 0.8258426966292135, "ece_hard": 0.05388657897510735, "calibration": 85.33184429464879, "p50_s": 2.3997448533773422, "p95_s": 4.385688671469688, "cost_usd_per_1000": 0.21447801872659175, "axes": { "intelligence": 84.61166459111665, "calibration": 85.33184429464879, "speed": 69.7779713205367, "cost": 30.058516326044753 }, "jevbench_score": 62.38164352499589 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "openjev-razorback16" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.958904109589041, "hard": 0.6409090909090909 }, "accuracy": 0.8108614232209738, "ece_hard": 0.20303030303030284, "calibration": 60.830969696969724, "p50_s": 0.6811514683067799, "p95_s": 1.1347835324704645, "cost_usd_per_1000": 0.12796339325842696, "axes": { "intelligence": 83.40992112909922, "calibration": 60.830969696969724, "speed": 81.11843278006634, "cost": 36.78742756204445 }, "jevbench_score": 62.379279664976394 }, { "members": [ "jev-1.13.0", "openjev-razorback16", "openjev-sglang" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.958904109589041, "hard": 0.6863636363636364 }, "accuracy": 0.8277153558052435, "ece_hard": 0.08663095451524241, "calibration": 81.22247600534848, "p50_s": 1.3671194911003113, "p95_s": 1.6257096692919728, "cost_usd_per_1000": 0.23679425468164794, "axes": { "intelligence": 84.48189082606892, "calibration": 81.22247600534848, "speed": 76.53150536083959, "cost": 28.76886517211291 }, "jevbench_score": 62.344867915980295 }, { "members": [ "system-one-open", "openjev-razorback16", "openjev-sglang" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.958904109589041, "hard": 0.6545454545454545 }, "accuracy": 0.8127340823970037, "ece_hard": 0.08935793255092463, "calibration": 79.43968780186547, "p50_s": 1.3879805654287338, "p95_s": 2.030777859687805, "cost_usd_per_1000": 0.21176095505617978, "axes": { "intelligence": 83.2356787048568, "calibration": 79.43968780186547, "speed": 75.49954194936167, "cost": 30.2246233901472 }, "jevbench_score": 62.32511193814847 }, { "members": [ "semif-qwen3.5-4b", "djev", "open-alternative-jev", "system-one-open", "openjev-sglang" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.958904109589041, "hard": 0.6181818181818182 }, "accuracy": 0.797752808988764, "ece_hard": 0.0704911877124503, "calibration": 81.08103190284957, "p50_s": 1.3879805654287338, "p95_s": 2.030777859687805, "cost_usd_per_1000": 0.21672367041198504, "axes": { "intelligence": 82.1447696139477, "calibration": 81.08103190284957, "speed": 75.49954194936167, "cost": 29.922809582245947 }, "jevbench_score": 62.28184469127685 }, { "members": [ "jev-1.13.0", "djev", "open-alternative-jev", "system-one-open", "openjev-sglang" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9726027397260274, "hard": 0.6590909090909091 }, "accuracy": 0.8183520599250936, "ece_hard": 0.07916039933304747, "calibration": 82.21671347829691, "p50_s": 1.3879805654287338, "p95_s": 2.030777859687805, "cost_usd_per_1000": 0.23419329962546817, "axes": { "intelligence": 83.75560398505606, "calibration": 82.21671347829691, "speed": 75.49954194936167, "cost": 28.912766033329746 }, "jevbench_score": 62.26614611603291 }, { "members": [ "jev-1.13.0", "djev", "open-alternative-jev", "system-one-open", "simplejev-qwen3.6-35b-a3b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9726027397260274, "hard": 0.6363636363636364 }, "accuracy": 0.8108614232209738, "ece_hard": 0.07951928028622413, "calibration": 81.1750135817222, "p50_s": 1.7357386276125908, "p95_s": 2.0568130582571027, "cost_usd_per_1000": 0.21846951685393257, "axes": { "intelligence": 83.36545246990454, "calibration": 81.1750135817222, "speed": 74.47320850632062, "cost": 29.818274552891452 }, "jevbench_score": 62.261956531824495 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "djev", "system-one-open", "openjev-sglang" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9657534246575342, "hard": 0.65 }, "accuracy": 0.8164794007490637, "ece_hard": 0.08054787000958263, "calibration": 82.1057483608319, "p50_s": 1.3879805654287338, "p95_s": 2.030777859687805, "cost_usd_per_1000": 0.2344665018726592, "axes": { "intelligence": 83.8744292237443, "calibration": 82.1057483608319, "speed": 75.49954194936167, "cost": 28.897575879432267 }, "jevbench_score": 62.259011039694315 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "djev", "system-one-open", "simplejev-qwen3.8-27b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6818181818181818 }, "accuracy": 0.8314606741573034, "ece_hard": 0.07140325376564004, "calibration": 83.00888596962298, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.20718755056179775, "axes": { "intelligence": 85.02075550020756, "calibration": 83.00888596962298, "speed": 69.75909261944061, "cost": 30.509090317912694 }, "jevbench_score": 62.25437919810072 }, { "members": [ "system-one-open", "openjev-razorback16", "openjev-sglang" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.958904109589041, "hard": 0.6590909090909091 }, "accuracy": 0.8146067415730337, "ece_hard": 0.09387687057671629, "calibration": 78.90220694380466, "p50_s": 1.3879805654287338, "p95_s": 2.030777859687805, "cost_usd_per_1000": 0.21176095505617978, "axes": { "intelligence": 83.37204234122044, "calibration": 78.90220694380466, "speed": 75.49954194936167, "cost": 30.2246233901472 }, "jevbench_score": 62.24488967895895 }, { "members": [ "semif-qwen3.5-4b", "open-alternative-jev", "simplejev-qwen3.6-35b-a3b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.7708333333333334, "judge": 0.910958904109589, "hard": 0.6045454545454545 }, "accuracy": 0.7715355805243446, "ece_hard": 0.11151167743937015, "calibration": 74.72461424702773, "p50_s": 1.7256089895963669, "p95_s": 1.9284205481410026, "cost_usd_per_1000": 0.16016769662921349, "axes": { "intelligence": 79.22654628476546, "calibration": 74.72461424702773, "speed": 74.77855855005357, "cost": 33.86275209892408 }, "jevbench_score": 62.22407318912852 }, { "members": [ "jev-1.13.0", "djev", "open-alternative-jev", "system-one-open", "simplejev-qwen3.6-35b-a3b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9726027397260274, "hard": 0.6409090909090909 }, "accuracy": 0.8127340823970037, "ece_hard": 0.08795909677914016, "calibration": 80.77138362151732, "p50_s": 1.7357386276125908, "p95_s": 2.0568130582571027, "cost_usd_per_1000": 0.21846951685393257, "axes": { "intelligence": 83.50181610626817, "calibration": 80.77138362151732, "speed": 74.47320850632062, "cost": 29.818274552891452 }, "jevbench_score": 62.209828548138695 }, { "members": [ "semif-qwen3.5-4b", "djev", "open-alternative-jev", "system-one-open", "openjev-sglang" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9657534246575342, "hard": 0.6227272727272727 }, "accuracy": 0.8014981273408239, "ece_hard": 0.07876315302296835, "calibration": 80.36294253693114, "p50_s": 1.3879805654287338, "p95_s": 2.030777859687805, "cost_usd_per_1000": 0.21672367041198504, "axes": { "intelligence": 82.47291407222914, "calibration": 80.36294253693114, "speed": 75.49954194936167, "cost": 29.922809582245947 }, "jevbench_score": 62.20545399072777 }, { "members": [ "jev-1.13.0", "openjev-razorback16", "simplejev-qwen3.6-35b-a3b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9315068493150684, "hard": 0.6727272727272727 }, "accuracy": 0.8127340823970037, "ece_hard": 0.08595909155599049, "calibration": 81.2641991710193, "p50_s": 1.724735751748085, "p95_s": 1.9244587391614911, "cost_usd_per_1000": 0.22107047191011236, "axes": { "intelligence": 83.0140099626401, "calibration": 81.2641991710193, "speed": 74.78968829969378, "cost": 29.664077850575623 }, "jevbench_score": 62.19862829653094 }, { "members": [ "semif-qwen3.5-4b", "djev", "open-alternative-jev", "system-one-open", "simplejev-qwen3.8-27b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.958904109589041, "hard": 0.6409090909090909 }, "accuracy": 0.8089887640449438, "ece_hard": 0.07631851219975132, "calibration": 81.4378046267064, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.1894447191011236, "axes": { "intelligence": 83.11825446243255, "calibration": 81.4378046267064, "speed": 69.75909261944061, "cost": 31.675524894125246 }, "jevbench_score": 62.188743013610356 }, { "members": [ "open-alternative-jev", "system-one-open", "openjev-razorback16" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8020833333333334, "judge": 0.8698630136986302, "hard": 0.5863636363636363 }, "accuracy": 0.7584269662921348, "ece_hard": 0.17575757575757556, "calibration": 63.42490909090911, "p50_s": 1.3041210994124413, "p95_s": 2.030777859687805, "cost_usd_per_1000": 0.10265689138576778, "axes": { "intelligence": 78.40540680780407, "calibration": 63.42490909090911, "speed": 75.77019658655011, "cost": 39.658356730150174 }, "jevbench_score": 62.174134553547496 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "djev", "open-alternative-jev", "simplejev-qwen3.6-35b-a3b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9726027397260274, "hard": 0.6454545454545455 }, "accuracy": 0.8146067415730337, "ece_hard": 0.08214180347813439, "calibration": 81.20240868946735, "p50_s": 1.7256089895963669, "p95_s": 1.9284205481410026, "cost_usd_per_1000": 0.2260331872659176, "axes": { "intelligence": 83.6381797426318, "calibration": 81.20240868946735, "speed": 74.77855855005357, "cost": 29.374833729784655 }, "jevbench_score": 62.14862114857283 }, { "members": [ "jev-1.13.0", "djev", "openjev-razorback16" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9315068493150684, "hard": 0.6681818181818182 }, "accuracy": 0.8108614232209738, "ece_hard": 0.21818181818181814, "calibration": 60.72598484848485, "p50_s": 0.6792524389922618, "p95_s": 1.086352695524692, "cost_usd_per_1000": 0.13147004119850186, "axes": { "intelligence": 82.87764632627646, "calibration": 60.72598484848485, "speed": 81.31997947189848, "cost": 36.43519602918106 }, "jevbench_score": 62.14162225235766 }, { "members": [ "semif-qwen3.5-4b", "open-alternative-jev", "simplejev-qwen3.6-35b-a3b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.78125, "judge": 0.9041095890410958, "hard": 0.6 }, "accuracy": 0.7696629213483146, "ece_hard": 0.11683255552602001, "calibration": 74.14631399363722, "p50_s": 1.7256089895963669, "p95_s": 1.9284205481410026, "cost_usd_per_1000": 0.16016769662921349, "axes": { "intelligence": 79.19006849315069, "calibration": 74.14631399363722, "speed": 74.77855855005357, "cost": 33.86275209892408 }, "jevbench_score": 62.096183144754995 }, { "members": [ "jev-1.13.0", "djev", "open-alternative-jev", "system-one-open", "openjev-razorback16" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9315068493150684, "hard": 0.6454545454545455 }, "accuracy": 0.8052434456928839, "ece_hard": 0.1254545454545452, "calibration": 71.39354545454549, "p50_s": 1.3041210994124413, "p95_s": 2.030777859687805, "cost_usd_per_1000": 0.16852238202247188, "axes": { "intelligence": 82.77916147779162, "calibration": 71.39354545454549, "speed": 75.77019658655011, "cost": 33.20027232567968 }, "jevbench_score": 62.0947679131074 }, { "members": [ "semif-qwen3.5-4b", "openjev-razorback16", "openjev-sglang" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.952054794520548, "hard": 0.6363636363636364 }, "accuracy": 0.8052434456928839, "ece_hard": 0.09207641956377695, "calibration": 78.59306197853452, "p50_s": 1.3671194911003113, "p95_s": 1.6307271040976046, "cost_usd_per_1000": 0.21932462546816478, "axes": { "intelligence": 82.7901100041511, "calibration": 78.59306197853452, "speed": 76.51812235281348, "cost": 29.767378107879182 }, "jevbench_score": 62.046443002158334 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "djev", "open-alternative-jev", "simplejev-qwen3.6-35b-a3b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9726027397260274, "hard": 0.6409090909090909 }, "accuracy": 0.8127340823970037, "ece_hard": 0.08956735850100256, "calibration": 80.7402580163839, "p50_s": 1.7256089895963669, "p95_s": 1.9284205481410026, "cost_usd_per_1000": 0.2260331872659176, "axes": { "intelligence": 83.50181610626817, "calibration": 80.7402580163839, "speed": 74.77855855005357, "cost": 29.374833729784655 }, "jevbench_score": 62.03469340137808 }, { "members": [ "open-alternative-jev", "openjev-razorback16", "simplejev-qwen3.8-27b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9041095890410958, "hard": 0.6636363636363637 }, "accuracy": 0.8033707865168539, "ece_hard": 0.07300567378744893, "calibration": 81.77096506429308, "p50_s": 2.3997448533773422, "p95_s": 4.385688671469688, "cost_usd_per_1000": 0.19177247191011235, "axes": { "intelligence": 82.26582606890828, "calibration": 81.77096506429308, "speed": 69.7779713205367, "cost": 31.516412012284874 }, "jevbench_score": 62.018095242886844 }, { "members": [ "open-alternative-jev", "openjev-razorback16", "simplejev-qwen3.8-27b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9178082191780822, "hard": 0.6818181818181818 }, "accuracy": 0.8146067415730337, "ece_hard": 0.08533449016301778, "calibration": 80.8362479051948, "p50_s": 2.3997448533773422, "p95_s": 4.385688671469688, "cost_usd_per_1000": 0.19177247191011235, "axes": { "intelligence": 83.19484225819845, "calibration": 80.8362479051948, "speed": 69.7779713205367, "cost": 31.516412012284874 }, "jevbench_score": 62.01395281748643 }, { "members": [ "djev", "system-one-open", "openjev-sglang" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.958904109589041, "hard": 0.6909090909090909 }, "accuracy": 0.8277153558052435, "ece_hard": 0.11212121212121193, "calibration": 70.51404545454547, "p50_s": 1.3879805654287338, "p95_s": 2.030777859687805, "cost_usd_per_1000": 0.17210765917603, "axes": { "intelligence": 84.32658779576589, "calibration": 70.51404545454547, "speed": 75.49954194936167, "cost": 32.92599406497797 }, "jevbench_score": 62.00558977923514 }, { "members": [ "jev-1.13.0", "openjev-razorback16", "openjev-sglang" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9657534246575342, "hard": 0.6863636363636364 }, "accuracy": 0.8295880149812734, "ece_hard": 0.11056905972014008, "calibration": 79.2254651976323, "p50_s": 1.3671194911003113, "p95_s": 1.6257096692919728, "cost_usd_per_1000": 0.23679425468164794, "axes": { "intelligence": 84.67367164798672, "calibration": 79.2254651976323, "speed": 76.53150536083959, "cost": 28.76886517211291 }, "jevbench_score": 61.99319852276 }, { "members": [ "jev-1.13.0", "djev", "open-alternative-jev", "system-one-open", "simplejev-qwen3.8-27b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9657534246575342, "hard": 0.6636363636363637 }, "accuracy": 0.8202247191011236, "ece_hard": 0.0751238756447119, "calibration": 82.40638464195831, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.20691434831460673, "axes": { "intelligence": 83.99185346616854, "calibration": 82.40638464195831, "speed": 69.75909261944061, "cost": 30.526281774305716 }, "jevbench_score": 61.96096734653694 }, { "members": [ "system-one-open", "openjev-razorback16", "simplejev-qwen3.6-35b-a3b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9178082191780822, "hard": 0.6454545454545455 }, "accuracy": 0.799625468164794, "ece_hard": 0.10128140291200574, "calibration": 77.02534251949533, "p50_s": 1.7357386276125908, "p95_s": 2.0568130582571027, "cost_usd_per_1000": 0.19603717228464418, "axes": { "intelligence": 82.10393316728934, "calibration": 77.02534251949533, "speed": 74.47320850632062, "cost": 31.22984711633177 }, "jevbench_score": 61.928688229541976 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "open-alternative-jev", "system-one-open", "simplejev-qwen3.8-27b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8020833333333334, "judge": 0.958904109589041, "hard": 0.6409090909090909 }, "accuracy": 0.8052434456928839, "ece_hard": 0.07245057471553942, "calibration": 82.98410029093164, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.20340770037453182, "axes": { "intelligence": 82.53492112909922, "calibration": 82.98410029093164, "speed": 69.75909261944061, "cost": 30.748978302447497 }, "jevbench_score": 61.91074581661441 }, { "members": [ "semif-qwen3.5-4b", "djev", "open-alternative-jev", "system-one-open", "simplejev-qwen3.6-35b-a3b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9726027397260274, "hard": 0.6181818181818182 }, "accuracy": 0.8033707865168539, "ece_hard": 0.10572998946903187, "calibration": 76.90303246194799, "p50_s": 1.7357386276125908, "p95_s": 2.0568130582571027, "cost_usd_per_1000": 0.20099988764044946, "axes": { "intelligence": 82.81999792444998, "calibration": 76.90303246194799, "speed": 74.47320850632062, "cost": 30.904125560541516 }, "jevbench_score": 61.87622375560585 }, { "members": [ "jev-1.13.0", "open-alternative-jev", "system-one-open", "openjev-razorback16", "openjev-sglang" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9657534246575342, "hard": 0.6590909090909091 }, "accuracy": 0.8164794007490637, "ece_hard": 0.03896333173610071, "calibration": 86.42534848306781, "p50_s": 1.3879805654287338, "p95_s": 2.030777859687805, "cost_usd_per_1000": 0.27384659550561796, "axes": { "intelligence": 83.56382316313824, "calibration": 86.42534848306781, "speed": 75.49954194936167, "cost": 26.87477961711315 }, "jevbench_score": 61.87100453630144 }, { "members": [ "djev", "openjev-razorback16", "openjev-sglang" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9383561643835616, "hard": 0.6727272727272727 }, "accuracy": 0.8146067415730337, "ece_hard": 0.10755263143304067, "calibration": 77.8060145971117, "p50_s": 1.3671194911003113, "p95_s": 1.6257096692919728, "cost_usd_per_1000": 0.22283127340823972, "axes": { "intelligence": 83.2057907845579, "calibration": 77.8060145971117, "speed": 76.53150536083959, "cost": 29.5607157358955 }, "jevbench_score": 61.862929347644176 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "djev", "open-alternative-jev", "openjev-razorback16" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.65 }, "accuracy": 0.8108614232209738, "ece_hard": 0.1636363636363637, "calibration": 66.27036363636364, "p50_s": 0.6818747088313103, "p95_s": 1.1444025911390774, "cost_usd_per_1000": 0.1760860524344569, "axes": { "intelligence": 83.29908675799086, "calibration": 66.27036363636364, "speed": 81.07716585059153, "cost": 32.62825127850175 }, "jevbench_score": 61.817720661465465 }, { "members": [ "jev-1.13.0", "openjev-razorback16", "simplejev-qwen3.6-35b-a3b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9246575342465754, "hard": 0.6681818181818182 }, "accuracy": 0.8089887640449438, "ece_hard": 0.09997212474584798, "calibration": 79.45917721302935, "p50_s": 1.724735751748085, "p95_s": 1.9244587391614911, "cost_usd_per_1000": 0.22107047191011236, "axes": { "intelligence": 82.68586550435866, "calibration": 79.45917721302935, "speed": 74.78968829969378, "cost": 29.664077850575623 }, "jevbench_score": 61.78911557764233 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "djev", "system-one-open", "openjev-razorback16" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.958904109589041, "hard": 0.6681818181818182 }, "accuracy": 0.8220973782771536, "ece_hard": 0.14636363636363625, "calibration": 68.74013636363637, "p50_s": 1.3041210994124413, "p95_s": 2.030777859687805, "cost_usd_per_1000": 0.1687955842696629, "axes": { "intelligence": 84.22810294728103, "calibration": 68.74013636363637, "speed": 75.77019658655011, "cost": 33.179167564083926 }, "jevbench_score": 61.76718756086523 }, { "members": [ "djev", "openjev-razorback16", "openjev-sglang" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9383561643835616, "hard": 0.6727272727272727 }, "accuracy": 0.8146067415730337, "ece_hard": 0.11203344527991058, "calibration": 77.21483199064275, "p50_s": 1.3671194911003113, "p95_s": 1.6257096692919728, "cost_usd_per_1000": 0.22283127340823972, "axes": { "intelligence": 83.2057907845579, "calibration": 77.21483199064275, "speed": 76.53150536083959, "cost": 29.5607157358955 }, "jevbench_score": 61.74508190970858 }, { "members": [ "semif-qwen3.5-4b", "openjev-razorback16", "openjev-sglang" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.958904109589041, "hard": 0.6363636363636364 }, "accuracy": 0.8052434456928839, "ece_hard": 0.10856233191943636, "calibration": 77.08903389328982, "p50_s": 1.3671194911003113, "p95_s": 1.6307271040976046, "cost_usd_per_1000": 0.21932462546816478, "axes": { "intelligence": 82.69022415940225, "calibration": 77.08903389328982, "speed": 76.51812235281348, "cost": 29.767378107879182 }, "jevbench_score": 61.728811476772556 }, { "members": [ "jev-1.13.0", "djev", "system-one-open", "openjev-razorback16", "openjev-sglang" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9657534246575342, "hard": 0.6863636363636364 }, "accuracy": 0.8295880149812734, "ece_hard": 0.05370185194144459, "calibration": 85.06365472035503, "p50_s": 1.3879805654287338, "p95_s": 2.030777859687805, "cost_usd_per_1000": 0.2776264456928839, "axes": { "intelligence": 84.67367164798672, "calibration": 85.06365472035503, "speed": 75.49954194936167, "cost": 26.696175007669808 }, "jevbench_score": 61.7264707951279 }, { "members": [ "open-alternative-jev", "openjev-razorback16", "openjev-sglang" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.910958904109589, "hard": 0.65 }, "accuracy": 0.797752808988764, "ece_hard": 0.10024027835105517, "calibration": 77.6775879315613, "p50_s": 1.3671194911003113, "p95_s": 1.6307271040976046, "cost_usd_per_1000": 0.2190514232209738, "axes": { "intelligence": 81.75684931506851, "calibration": 77.6775879315613, "speed": 76.51812235281348, "cost": 29.783617623351972 }, "jevbench_score": 61.6794379361093 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "djev", "openjev-razorback16", "openjev-sglang" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9657534246575342, "hard": 0.6818181818181818 }, "accuracy": 0.8295880149812734, "ece_hard": 0.058027587217152544, "calibration": 84.60225338156177, "p50_s": 1.3671194911003113, "p95_s": 1.6307271040976046, "cost_usd_per_1000": 0.2851901161048689, "axes": { "intelligence": 84.82897467828975, "calibration": 84.60225338156177, "speed": 76.51812235281348, "cost": 26.34596589989596 }, "jevbench_score": 61.67386198407032 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "system-one-open", "openjev-razorback16", "simplejev-qwen3.8-27b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6863636363636364 }, "accuracy": 0.8333333333333334, "ece_hard": 0.04474478895669768, "calibration": 86.24643564854851, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.24684084644194754, "axes": { "intelligence": 85.1571191365712, "calibration": 86.24643564854851, "speed": 69.75909261944061, "cost": 28.227489190476646 }, "jevbench_score": 61.667848576590806 }, { "members": [ "semif-qwen3.5-4b", "djev", "openjev-razorback16" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9452054794520548, "hard": 0.6409090909090909 }, "accuracy": 0.8033707865168539, "ece_hard": 0.23181818181818167, "calibration": 56.28551515151517, "p50_s": 0.6500487640500069, "p95_s": 1.1367489621043205, "cost_usd_per_1000": 0.11400041198501872, "axes": { "intelligence": 82.44302615193027, "calibration": 56.28551515151517, "speed": 81.31389496755949, "cost": 38.2928073750382 }, "jevbench_score": 61.65355668903456 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "system-one-open", "openjev-razorback16", "openjev-sglang" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9657534246575342, "hard": 0.6636363636363637 }, "accuracy": 0.8220973782771536, "ece_hard": 0.057848370262717434, "calibration": 84.51190697147811, "p50_s": 1.3879805654287338, "p95_s": 2.030777859687805, "cost_usd_per_1000": 0.274119797752809, "axes": { "intelligence": 84.2835201328352, "calibration": 84.51190697147811, "speed": 75.49954194936167, "cost": 26.861787918386057 }, "jevbench_score": 61.65026532138758 }, { "members": [ "semif-qwen3.5-4b", "openjev-razorback16", "simplejev-qwen3.6-35b-a3b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9246575342465754, "hard": 0.6363636363636364 }, "accuracy": 0.795880149812734, "ece_hard": 0.10231275146575383, "calibration": 76.68292186385204, "p50_s": 1.7256089895963669, "p95_s": 1.9284205481410026, "cost_usd_per_1000": 0.2036008426966292, "axes": { "intelligence": 81.7313200498132, "calibration": 76.68292186385204, "speed": 74.77855855005357, "cost": 30.73661286406545 }, "jevbench_score": 61.607000646625956 }, { "members": [ "djev", "system-one-open", "simplejev-qwen3.8-27b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.952054794520548, "hard": 0.6954545454545454 }, "accuracy": 0.8314606741573034, "ece_hard": 0.11818181818181858, "calibration": 69.1409848484848, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.14482870786516855, "axes": { "intelligence": 84.85450394354504, "calibration": 69.1409848484848, "speed": 69.75909261944061, "cost": 35.1743603199228 }, "jevbench_score": 61.59697332055118 }, { "members": [ "semif-qwen3.5-4b", "djev", "open-alternative-jev", "system-one-open", "simplejev-qwen3.6-35b-a3b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9726027397260274, "hard": 0.6181818181818182 }, "accuracy": 0.8033707865168539, "ece_hard": 0.12194007512276023, "calibration": 75.37391619383173, "p50_s": 1.7357386276125908, "p95_s": 2.0568130582571027, "cost_usd_per_1000": 0.20099988764044946, "axes": { "intelligence": 82.81999792444998, "calibration": 75.37391619383173, "speed": 74.47320850632062, "cost": 30.904125560541516 }, "jevbench_score": 61.566321380497406 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "system-one-open", "openjev-razorback16", "simplejev-qwen3.6-35b-a3b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9657534246575342, "hard": 0.65 }, "accuracy": 0.8164794007490637, "ece_hard": 0.060690681623685, "calibration": 83.18932991212571, "p50_s": 1.7357386276125908, "p95_s": 2.0568130582571027, "cost_usd_per_1000": 0.2583960149812734, "axes": { "intelligence": 83.8744292237443, "calibration": 83.18932991212571, "speed": 74.47320850632062, "cost": 27.631425651228327 }, "jevbench_score": 61.556673180512455 }, { "members": [ "semif-qwen3.5-4b", "openjev-razorback16", "simplejev-qwen3.6-35b-a3b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9246575342465754, "hard": 0.6318181818181818 }, "accuracy": 0.7940074906367042, "ece_hard": 0.10446600508808795, "calibration": 76.46550840723776, "p50_s": 1.7256089895963669, "p95_s": 1.9284205481410026, "cost_usd_per_1000": 0.2036008426966292, "axes": { "intelligence": 81.59495641344957, "calibration": 76.46550840723776, "speed": 74.77855855005357, "cost": 30.73661286406545 }, "jevbench_score": 61.537591978052674 }, { "members": [ "semif-qwen3.5-4b", "system-one-open", "openjev-razorback16" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.958904109589041, "hard": 0.5909090909090909 }, "accuracy": 0.7883895131086143, "ece_hard": 0.215151515151515, "calibration": 58.36051515151517, "p50_s": 1.3041210994124413, "p95_s": 2.030777859687805, "cost_usd_per_1000": 0.1029300936329588, "axes": { "intelligence": 81.61825446243255, "calibration": 58.36051515151517, "speed": 75.77019658655011, "cost": 39.62372896471127 }, "jevbench_score": 61.49503377783112 }, { "members": [ "semif-qwen3.5-4b", "djev", "system-one-open", "openjev-razorback16", "simplejev-qwen3.8-27b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6545454545454545 }, "accuracy": 0.8202247191011236, "ece_hard": 0.054321088651836734, "calibration": 83.88647845003706, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.2328778651685393, "axes": { "intelligence": 84.20257368202574, "calibration": 83.88647845003706, "speed": 69.75909261944061, "cost": 28.98615366409723 }, "jevbench_score": 61.4755164061078 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "system-one-open", "openjev-razorback16", "simplejev-qwen3.6-35b-a3b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9657534246575342, "hard": 0.6454545454545455 }, "accuracy": 0.8146067415730337, "ece_hard": 0.06818709477423501, "calibration": 82.74925265306653, "p50_s": 1.7357386276125908, "p95_s": 2.0568130582571027, "cost_usd_per_1000": 0.2583960149812734, "axes": { "intelligence": 83.73806558738066, "calibration": 82.74925265306653, "speed": 74.47320850632062, "cost": 27.631425651228327 }, "jevbench_score": 61.45009951496833 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "open-alternative-jev", "openjev-razorback16", "openjev-sglang" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9726027397260274, "hard": 0.6681818181818182 }, "accuracy": 0.8239700374531835, "ece_hard": 0.06344710381260868, "calibration": 83.28040988688238, "p50_s": 1.3671194911003113, "p95_s": 1.6307271040976046, "cost_usd_per_1000": 0.281410265917603, "axes": { "intelligence": 84.31999792444998, "calibration": 83.28040988688238, "speed": 76.51812235281348, "cost": 26.519801903195614 }, "jevbench_score": 61.440112938528465 }, { "members": [ "semif-qwen3.5-4b", "djev", "system-one-open", "openjev-razorback16", "openjev-sglang" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9657534246575342, "hard": 0.6454545454545455 }, "accuracy": 0.8127340823970037, "ece_hard": 0.06617540784384371, "calibration": 81.95718165534397, "p50_s": 1.3879805654287338, "p95_s": 2.030777859687805, "cost_usd_per_1000": 0.26015681647940075, "axes": { "intelligence": 83.44639892071399, "calibration": 81.95718165534397, "speed": 75.49954194936167, "cost": 27.542943714532868 }, "jevbench_score": 61.409748907944724 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "djev", "openjev-razorback16", "openjev-sglang" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9657534246575342, "hard": 0.6772727272727272 }, "accuracy": 0.8258426966292135, "ece_hard": 0.065994479288283, "calibration": 83.57884479740858, "p50_s": 1.3671194911003113, "p95_s": 1.6307271040976046, "cost_usd_per_1000": 0.2851901161048689, "axes": { "intelligence": 84.40094437525944, "calibration": 83.57884479740858, "speed": 76.51812235281348, "cost": 26.34596589989596 }, "jevbench_score": 61.40878786494324 }, { "members": [ "open-alternative-jev", "openjev-razorback16", "openjev-sglang" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.910958904109589, "hard": 0.6454545454545455 }, "accuracy": 0.795880149812734, "ece_hard": 0.11193340831488245, "calibration": 76.39566149740118, "p50_s": 1.3671194911003113, "p95_s": 1.6307271040976046, "cost_usd_per_1000": 0.2190514232209738, "axes": { "intelligence": 81.62048567870485, "calibration": 76.39566149740118, "speed": 76.51812235281348, "cost": 29.783617623351972 }, "jevbench_score": 61.39774285703939 }, { "members": [ "semif-qwen3.5-4b", "djev", "system-one-open", "openjev-razorback16", "openjev-sglang" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9657534246575342, "hard": 0.65 }, "accuracy": 0.8146067415730337, "ece_hard": 0.06875838150833974, "calibration": 81.61932434634613, "p50_s": 1.3879805654287338, "p95_s": 2.030777859687805, "cost_usd_per_1000": 0.26015681647940075, "axes": { "intelligence": 83.58276255707764, "calibration": 81.61932434634613, "speed": 75.49954194936167, "cost": 27.542943714532868 }, "jevbench_score": 61.37140939726516 }, { "members": [ "jev-1.13.0", "djev", "system-one-open", "openjev-razorback16", "openjev-sglang" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9657534246575342, "hard": 0.6772727272727272 }, "accuracy": 0.8258426966292135, "ece_hard": 0.06879771364712453, "calibration": 83.30091904237568, "p50_s": 1.3879805654287338, "p95_s": 2.030777859687805, "cost_usd_per_1000": 0.2776264456928839, "axes": { "intelligence": 84.40094437525944, "calibration": 83.30091904237568, "speed": 75.49954194936167, "cost": 26.696175007669808 }, "jevbench_score": 61.35466852259277 }, { "members": [ "system-one-open", "openjev-razorback16", "simplejev-qwen3.6-35b-a3b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9178082191780822, "hard": 0.6363636363636364 }, "accuracy": 0.7940074906367042, "ece_hard": 0.12463457063127112, "calibration": 74.68162263510025, "p50_s": 1.7357386276125908, "p95_s": 2.0568130582571027, "cost_usd_per_1000": 0.19603717228464418, "axes": { "intelligence": 81.5395392278954, "calibration": 74.68162263510025, "speed": 74.47320850632062, "cost": 31.22984711633177 }, "jevbench_score": 61.34624438097087 }, { "members": [ "djev", "openjev-razorback16", "simplejev-qwen3.8-27b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.952054794520548, "hard": 0.6772727272727272 }, "accuracy": 0.8202247191011236, "ece_hard": 0.10587462503266487, "calibration": 77.49517478669117, "p50_s": 2.3997448533773422, "p95_s": 4.381843151897192, "cost_usd_per_1000": 0.19555232209737827, "axes": { "intelligence": 83.72571606475717, "calibration": 77.49517478669117, "speed": 69.78178103134502, "cost": 31.262110678787337 }, "jevbench_score": 61.33713260399733 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "system-one-open", "openjev-razorback16", "simplejev-qwen3.8-27b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6681818181818182 }, "accuracy": 0.8258426966292135, "ece_hard": 0.05580080700322828, "calibration": 84.95394366111509, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.24684084644194754, "axes": { "intelligence": 84.61166459111665, "calibration": 84.95394366111509, "speed": 69.75909261944061, "cost": 28.227489190476646 }, "jevbench_score": 61.336884643860785 }, { "members": [ "jev-1.13.0", "open-alternative-jev", "simplejev-qwen3.8-27b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8020833333333334, "judge": 0.9315068493150684, "hard": 0.7136363636363636 }, "accuracy": 0.8277153558052435, "ece_hard": 0.1272727272727277, "calibration": 72.35856060606056, "p50_s": 2.3997448533773422, "p95_s": 4.385688671469688, "cost_usd_per_1000": 0.16608215730337078, "axes": { "intelligence": 83.94961602324616, "calibration": 72.35856060606056, "speed": 69.7779713205367, "cost": 33.390310677437384 }, "jevbench_score": 61.33549266240975 }, { "members": [ "jev-1.13.0", "open-alternative-jev", "openjev-sglang" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.958904109589041, "hard": 0.7 }, "accuracy": 0.8314606741573034, "ece_hard": 0.1439393939393936, "calibration": 69.59114393939397, "p50_s": 1.3671194911003113, "p95_s": 1.6307271040976046, "cost_usd_per_1000": 0.19336110861423222, "axes": { "intelligence": 84.59931506849315, "calibration": 69.59114393939397, "speed": 76.51812235281348, "cost": 31.408926178436772 }, "jevbench_score": 61.33163662666127 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "djev", "openjev-razorback16", "simplejev-qwen3.8-27b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6954545454545454 }, "accuracy": 0.8370786516853933, "ece_hard": 0.04641461027468968, "calibration": 85.79686413946173, "p50_s": 2.3997448533773422, "p95_s": 4.381843151897192, "cost_usd_per_1000": 0.25791116479400744, "axes": { "intelligence": 85.42984640929848, "calibration": 85.79686413946173, "speed": 69.78178103134502, "cost": 27.65589571451909 }, "jevbench_score": 61.327139004587735 }, { "members": [ "djev", "open-alternative-jev", "system-one-open", "openjev-razorback16", "simplejev-qwen3.8-27b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9657534246575342, "hard": 0.6636363636363637 }, "accuracy": 0.8220973782771536, "ece_hard": 0.06345487560677193, "calibration": 82.82879585347075, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.2326046629213483, "axes": { "intelligence": 84.2835201328352, "calibration": 82.82879585347075, "speed": 69.75909261944061, "cost": 29.00144750240858 }, "jevbench_score": 61.3036210260935 }, { "members": [ "jev-1.13.0", "system-one-open", "simplejev-qwen3.8-27b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9383561643835616, "hard": 0.7136363636363636 }, "accuracy": 0.8333333333333334, "ece_hard": 0.11666666666666621, "calibration": 70.2974166666667, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.15879168913857677, "axes": { "intelligence": 84.72473017849731, "calibration": 70.2974166666667, "speed": 69.75909261944061, "cost": 33.975166944366165 }, "jevbench_score": 61.295415820581574 }, { "members": [ "semif-qwen3.5-4b", "djev", "open-alternative-jev", "system-one-open", "simplejev-qwen3.8-27b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9657534246575342, "hard": 0.6136363636363636 }, "accuracy": 0.799625468164794, "ece_hard": 0.11302125815431323, "calibration": 77.44100598048499, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.1894447191011236, "axes": { "intelligence": 82.49185346616854, "calibration": 77.44100598048499, "speed": 69.75909261944061, "cost": 31.675524894125246 }, "jevbench_score": 61.29522911195305 }, { "members": [ "jev-1.13.0", "djev", "open-alternative-jev", "openjev-razorback16", "openjev-sglang" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.952054794520548, "hard": 0.6681818181818182 }, "accuracy": 0.8202247191011236, "ece_hard": 0.07345597213026604, "calibration": 83.15044525005585, "p50_s": 1.3671194911003113, "p95_s": 1.6307271040976046, "cost_usd_per_1000": 0.2849169138576779, "axes": { "intelligence": 84.03632212536323, "calibration": 83.15044525005585, "speed": 76.51812235281348, "cost": 26.358453053841984 }, "jevbench_score": 61.27085769288307 }, { "members": [ "jev-1.13.0", "open-alternative-jev", "system-one-open", "openjev-razorback16", "simplejev-qwen3.8-27b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9726027397260274, "hard": 0.6818181818181818 }, "accuracy": 0.8295880149812734, "ece_hard": 0.06237968507933941, "calibration": 84.42026737298235, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.24656764419475652, "axes": { "intelligence": 84.72908883354089, "calibration": 84.42026737298235, "speed": 69.75909261944061, "cost": 28.241917427052528 }, "jevbench_score": 61.26939124241467 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "open-alternative-jev", "openjev-razorback16", "simplejev-qwen3.8-27b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6772727272727272 }, "accuracy": 0.8295880149812734, "ece_hard": 0.05061216047236898, "calibration": 85.41863187019518, "p50_s": 2.3997448533773422, "p95_s": 4.385688671469688, "cost_usd_per_1000": 0.25413131460674154, "axes": { "intelligence": 84.88439186384392, "calibration": 85.41863187019518, "speed": 69.7779713205367, "cost": 27.84825450858567 }, "jevbench_score": 61.26665826196086 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "open-alternative-jev", "openjev-razorback16", "openjev-sglang" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9657534246575342, "hard": 0.6681818181818182 }, "accuracy": 0.8220973782771536, "ece_hard": 0.0746278095799234, "calibration": 82.52442077411959, "p50_s": 1.3671194911003113, "p95_s": 1.6307271040976046, "cost_usd_per_1000": 0.281410265917603, "axes": { "intelligence": 84.12821710253218, "calibration": 82.52442077411959, "speed": 76.51812235281348, "cost": 26.519801903195614 }, "jevbench_score": 61.265317358676036 }, { "members": [ "semif-qwen3.5-4b", "openjev-sglang", "simplejev-qwen3.8-27b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.958904109589041, "hard": 0.6909090909090909 }, "accuracy": 0.8295880149812734, "ece_hard": 0.03566540787277966, "calibration": 86.18390628227083, "p50_s": 2.3997448533773422, "p95_s": 4.381843151897192, "cost_usd_per_1000": 0.2577165917602996, "axes": { "intelligence": 84.61825446243255, "calibration": 86.18390628227083, "speed": 69.78178103134502, "cost": 27.6657286218213 }, "jevbench_score": 61.25528992962414 }, { "members": [ "jev-1.13.0", "system-one-open", "openjev-sglang" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9452054794520548, "hard": 0.6909090909090909 }, "accuracy": 0.8239700374531835, "ece_hard": 0.1303030303030301, "calibration": 69.50461363636366, "p50_s": 1.3879805654287338, "p95_s": 2.030777859687805, "cost_usd_per_1000": 0.18607064044943822, "axes": { "intelligence": 83.94302615193027, "calibration": 69.50461363636366, "speed": 75.49954194936167, "cost": 31.90966442617821 }, "jevbench_score": 61.23027390699431 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "system-one-open", "openjev-razorback16", "openjev-sglang" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9657534246575342, "hard": 0.6454545454545455 }, "accuracy": 0.8146067415730337, "ece_hard": 0.07320330232929435, "calibration": 82.64485236579279, "p50_s": 1.3879805654287338, "p95_s": 2.030777859687805, "cost_usd_per_1000": 0.274119797752809, "axes": { "intelligence": 83.73806558738066, "calibration": 82.64485236579279, "speed": 75.49954194936167, "cost": 26.861787918386057 }, "jevbench_score": 61.20747886129957 }, { "members": [ "jev-1.13.0", "djev", "system-one-open", "openjev-razorback16", "simplejev-qwen3.6-35b-a3b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9657534246575342, "hard": 0.6636363636363637 }, "accuracy": 0.8202247191011236, "ece_hard": 0.08159295355610177, "calibration": 81.72268121682464, "p50_s": 1.7357386276125908, "p95_s": 2.0568130582571027, "cost_usd_per_1000": 0.26190266292134834, "axes": { "intelligence": 83.99185346616854, "calibration": 81.72268121682464, "speed": 74.47320850632062, "cost": 27.455802574600654 }, "jevbench_score": 61.20733845372926 }, { "members": [ "jev-1.13.0", "djev", "openjev-sglang" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9657534246575342, "hard": 0.7045454545454546 }, "accuracy": 0.8352059925093633, "ece_hard": 0.15454545454545449, "calibration": 69.21612121212121, "p50_s": 1.3671194911003113, "p95_s": 1.6257096692919728, "cost_usd_per_1000": 0.19714095880149812, "axes": { "intelligence": 84.92745952677461, "calibration": 69.21612121212121, "speed": 76.53150536083959, "cost": 31.156694067130275 }, "jevbench_score": 61.18736547045555 }, { "members": [ "jev-1.13.0", "open-alternative-jev", "system-one-open", "openjev-razorback16", "openjev-sglang" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8020833333333334, "judge": 0.9657534246575342, "hard": 0.65 }, "accuracy": 0.8108614232209738, "ece_hard": 0.06659481394070287, "calibration": 83.22206919651049, "p50_s": 1.3879805654287338, "p95_s": 2.030777859687805, "cost_usd_per_1000": 0.27384659550561796, "axes": { "intelligence": 82.9994292237443, "calibration": 83.22206919651049, "speed": 75.49954194936167, "cost": 26.87477961711315 }, "jevbench_score": 61.18580960650921 }, { "members": [ "jev-1.13.0", "djev", "open-alternative-jev", "openjev-razorback16", "simplejev-qwen3.8-27b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6909090909090909 }, "accuracy": 0.8352059925093633, "ece_hard": 0.054072234733221834, "calibration": 85.10069841863921, "p50_s": 2.3997448533773422, "p95_s": 4.385688671469688, "cost_usd_per_1000": 0.2576379625468165, "axes": { "intelligence": 85.29348277293484, "calibration": 85.10069841863921, "speed": 69.7779713205367, "cost": 27.669704319737818 }, "jevbench_score": 61.184717256041296 }, { "members": [ "djev", "open-alternative-jev", "system-one-open", "openjev-razorback16", "openjev-sglang" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.952054794520548, "hard": 0.65 }, "accuracy": 0.8108614232209738, "ece_hard": 0.0759089255938634, "calibration": 80.95532353139394, "p50_s": 1.3879805654287338, "p95_s": 2.030777859687805, "cost_usd_per_1000": 0.25988361423220974, "axes": { "intelligence": 83.19920091324202, "calibration": 80.95532353139394, "speed": 75.49954194936167, "cost": 27.55663306239589 }, "jevbench_score": 61.183421436103494 }, { "members": [ "semif-qwen3.5-4b", "djev", "system-one-open", "openjev-razorback16", "simplejev-qwen3.6-35b-a3b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.952054794520548, "hard": 0.6454545454545455 }, "accuracy": 0.8089887640449438, "ece_hard": 0.08039648693598983, "calibration": 79.8214001831809, "p50_s": 1.7357386276125908, "p95_s": 2.0568130582571027, "cost_usd_per_1000": 0.24443303370786518, "axes": { "intelligence": 83.06283727687838, "calibration": 79.8214001831809, "speed": 74.47320850632062, "cost": 28.355203062428018 }, "jevbench_score": 61.17017634431174 }, { "members": [ "open-alternative-jev", "openjev-razorback16", "simplejev-qwen3.6-35b-a3b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8020833333333334, "judge": 0.9315068493150684, "hard": 0.6454545454545455 }, "accuracy": 0.799625468164794, "ece_hard": 0.12350603390819581, "calibration": 74.24889723533006, "p50_s": 1.7256089895963669, "p95_s": 1.9284205481410026, "cost_usd_per_1000": 0.20332764044943819, "axes": { "intelligence": 81.90416147779162, "calibration": 74.24889723533006, "speed": 74.77855855005357, "cost": 30.75410737493995 }, "jevbench_score": 61.15318034503048 }, { "members": [ "jev-1.13.0", "open-alternative-jev", "system-one-open", "openjev-razorback16", "simplejev-qwen3.6-35b-a3b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9657534246575342, "hard": 0.6454545454545455 }, "accuracy": 0.8127340823970037, "ece_hard": 0.0802048360222696, "calibration": 81.37513712409618, "p50_s": 1.7357386276125908, "p95_s": 2.0568130582571027, "cost_usd_per_1000": 0.2581228127340824, "axes": { "intelligence": 83.44639892071399, "calibration": 81.37513712409618, "speed": 74.47320850632062, "cost": 27.645208332489517 }, "jevbench_score": 61.147656361370835 }, { "members": [ "semif-qwen3.5-4b", "open-alternative-jev", "system-one-open", "openjev-razorback16", "openjev-sglang" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9657534246575342, "hard": 0.6272727272727273 }, "accuracy": 0.8033707865168539, "ece_hard": 0.070340559236487, "calibration": 80.7969867807941, "p50_s": 1.3879805654287338, "p95_s": 2.030777859687805, "cost_usd_per_1000": 0.2563769662921348, "axes": { "intelligence": 82.60927770859278, "calibration": 80.7969867807941, "speed": 75.49954194936167, "cost": 27.733629873221304 }, "jevbench_score": 61.14257928806132 }, { "members": [ "semif-qwen3.5-4b", "djev", "system-one-open", "openjev-razorback16", "simplejev-qwen3.6-35b-a3b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.952054794520548, "hard": 0.6454545454545455 }, "accuracy": 0.8089887640449438, "ece_hard": 0.08257102296810934, "calibration": 79.62150934382481, "p50_s": 1.7357386276125908, "p95_s": 2.0568130582571027, "cost_usd_per_1000": 0.24443303370786518, "axes": { "intelligence": 83.06283727687838, "calibration": 79.62150934382481, "speed": 74.47320850632062, "cost": 28.355203062428018 }, "jevbench_score": 61.131844339380606 }, { "members": [ "system-one-open", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9657534246575342, "hard": 0.6909090909090909 }, "accuracy": 0.8295880149812734, "ece_hard": 0.05234601330069842, "calibration": 81.81739676359365, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.234429138576779, "axes": { "intelligence": 84.51836861768369, "calibration": 81.81739676359365, "speed": 69.75909261944061, "cost": 28.899652248482568 }, "jevbench_score": 61.10440937874417 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "open-alternative-jev", "openjev-razorback16", "simplejev-qwen3.8-27b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6681818181818182 }, "accuracy": 0.8258426966292135, "ece_hard": 0.05421058582640647, "calibration": 84.7538060262585, "p50_s": 2.3997448533773422, "p95_s": 4.385688671469688, "cost_usd_per_1000": 0.25413131460674154, "axes": { "intelligence": 84.61166459111665, "calibration": 84.7538060262585, "speed": 69.7779713205367, "cost": 27.84825450858567 }, "jevbench_score": 61.09792230453661 }, { "members": [ "jev-1.13.0", "djev", "open-alternative-jev", "openjev-razorback16", "openjev-sglang" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9657534246575342, "hard": 0.6727272727272727 }, "accuracy": 0.8239700374531835, "ece_hard": 0.08739626900104826, "calibration": 81.97376999343123, "p50_s": 1.3671194911003113, "p95_s": 1.6307271040976046, "cost_usd_per_1000": 0.2849169138576779, "axes": { "intelligence": 84.26458073889582, "calibration": 81.97376999343123, "speed": 76.51812235281348, "cost": 26.358453053841984 }, "jevbench_score": 61.094350061500435 }, { "members": [ "semif-qwen3.5-4b", "djev", "system-one-open", "openjev-razorback16", "simplejev-qwen3.8-27b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9657534246575342, "hard": 0.6545454545454545 }, "accuracy": 0.8183520599250936, "ece_hard": 0.07203322176106222, "calibration": 81.9784015606999, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.2328778651685393, "axes": { "intelligence": 84.01079286010793, "calibration": 81.9784015606999, "speed": 69.75909261944061, "cost": 28.98615366409723 }, "jevbench_score": 61.088081434369464 }, { "members": [ "open-alternative-jev", "openjev-razorback16", "simplejev-qwen3.6-35b-a3b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6454545454545455 }, "accuracy": 0.797752808988764, "ece_hard": 0.1252085226072714, "calibration": 74.08232510778154, "p50_s": 1.7256089895963669, "p95_s": 1.9284205481410026, "cost_usd_per_1000": 0.20332764044943819, "axes": { "intelligence": 81.61249481112495, "calibration": 74.08232510778154, "speed": 74.77855855005357, "cost": 30.75410737493995 }, "jevbench_score": 61.064368266299596 }, { "members": [ "semif-qwen3.5-4b", "open-alternative-jev", "system-one-open", "openjev-razorback16", "openjev-sglang" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.958904109589041, "hard": 0.6227272727272727 }, "accuracy": 0.799625468164794, "ece_hard": 0.07069645208670736, "calibration": 80.67696050600108, "p50_s": 1.3879805654287338, "p95_s": 2.030777859687805, "cost_usd_per_1000": 0.2563769662921348, "axes": { "intelligence": 82.28113325031133, "calibration": 80.67696050600108, "speed": 75.49954194936167, "cost": 27.733629873221304 }, "jevbench_score": 61.05907294024706 }, { "members": [ "jev-1.13.0", "djev", "system-one-open", "openjev-razorback16", "simplejev-qwen3.6-35b-a3b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.958904109589041, "hard": 0.6590909090909091 }, "accuracy": 0.8164794007490637, "ece_hard": 0.08262631452913081, "calibration": 81.22929891721239, "p50_s": 1.7357386276125908, "p95_s": 2.0568130582571027, "cost_usd_per_1000": 0.26190266292134834, "axes": { "intelligence": 83.66370900788709, "calibration": 81.22929891721239, "speed": 74.47320850632062, "cost": 27.455802574600654 }, "jevbench_score": 61.054967830091876 }, { "members": [ "jev-1.13.0", "djev", "open-alternative-jev", "system-one-open", "openjev-sglang" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9657534246575342, "hard": 0.6863636363636364 }, "accuracy": 0.8277153558052435, "ece_hard": 0.09363636363636339, "calibration": 75.40111363636366, "p50_s": 1.3879805654287338, "p95_s": 2.030777859687805, "cost_usd_per_1000": 0.23419329962546817, "axes": { "intelligence": 84.38200498132005, "calibration": 75.40111363636366, "speed": 75.49954194936167, "cost": 28.912766033329746 }, "jevbench_score": 61.04714946378541 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "simplejev-qwen3.8-27b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9726027397260274, "hard": 0.7136363636363636 }, "accuracy": 0.8426966292134831, "ece_hard": 0.14696969696969664, "calibration": 69.60586363636367, "p50_s": 2.3997448533773422, "p95_s": 4.381843151897192, "cost_usd_per_1000": 0.16635535955056177, "axes": { "intelligence": 85.68363428808635, "calibration": 69.60586363636367, "speed": 69.78178103134502, "cost": 33.36889608087208 }, "jevbench_score": 61.045955898593306 }, { "members": [ "jev-1.13.0", "djev", "system-one-open", "openjev-razorback16", "simplejev-qwen3.8-27b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9657534246575342, "hard": 0.6954545454545454 }, "accuracy": 0.8352059925093633, "ece_hard": 0.06675622554158489, "calibration": 83.27903133301771, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.2503474943820225, "axes": { "intelligence": 85.23806558738066, "calibration": 83.27903133301771, "speed": 69.75909261944061, "cost": 28.043702527174077 }, "jevbench_score": 61.045176845026816 }, { "members": [ "jev-1.13.0", "openjev-sglang", "simplejev-qwen3.8-27b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9657534246575342, "hard": 0.7272727272727273 }, "accuracy": 0.8445692883895131, "ece_hard": 0.043675795151469826, "calibration": 86.6869545431268, "p50_s": 2.3997448533773422, "p95_s": 4.381843151897192, "cost_usd_per_1000": 0.27518622097378276, "axes": { "intelligence": 85.60927770859278, "calibration": 86.6869545431268, "speed": 69.78178103134502, "cost": 26.811199471911323 }, "jevbench_score": 61.04262736422647 }, { "members": [ "jev-1.13.0", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9726027397260274, "hard": 0.7181818181818181 }, "accuracy": 0.8445692883895131, "ece_hard": 0.05600158752210355, "calibration": 84.05863272156529, "p50_s": 2.3997448533773422, "p95_s": 4.381843151897192, "cost_usd_per_1000": 0.2594624382022472, "axes": { "intelligence": 85.81999792444999, "calibration": 84.05863272156529, "speed": 69.78178103134502, "cost": 27.577765153220284 }, "jevbench_score": 61.04048630793842 }, { "members": [ "djev", "open-alternative-jev", "system-one-open", "openjev-razorback16", "openjev-sglang" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.952054794520548, "hard": 0.6636363636363637 }, "accuracy": 0.8164794007490637, "ece_hard": 0.08887572057867793, "calibration": 79.74132928622868, "p50_s": 1.3879805654287338, "p95_s": 2.030777859687805, "cost_usd_per_1000": 0.25988361423220974, "axes": { "intelligence": 83.60829182233293, "calibration": 79.74132928622868, "speed": 75.49954194936167, "cost": 27.55663306239589 }, "jevbench_score": 61.027534223758025 }, { "members": [ "semif-qwen3.5-4b", "open-alternative-jev", "system-one-open", "openjev-razorback16", "simplejev-qwen3.8-27b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.952054794520548, "hard": 0.6409090909090909 }, "accuracy": 0.8071161048689138, "ece_hard": 0.06887042959048502, "calibration": 82.08202771785153, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.22909801498127338, "axes": { "intelligence": 82.92647364051474, "calibration": 82.08202771785153, "speed": 69.75909261944061, "cost": 29.199360212106924 }, "jevbench_score": 61.02093505318747 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "open-alternative-jev", "system-one-open", "openjev-razorback16" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.958904109589041, "hard": 0.6181818181818182 }, "accuracy": 0.799625468164794, "ece_hard": 0.16181818181818164, "calibration": 66.29043181818184, "p50_s": 1.3041210994124413, "p95_s": 2.030777859687805, "cost_usd_per_1000": 0.165015734082397, "axes": { "intelligence": 82.43643628061437, "calibration": 66.29043181818184, "speed": 75.77019658655011, "cost": 33.47423932824064 }, "jevbench_score": 61.016149430055066 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "djev", "openjev-razorback16", "simplejev-qwen3.8-27b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9657534246575342, "hard": 0.6863636363636364 }, "accuracy": 0.8314606741573034, "ece_hard": 0.057539479997921424, "calibration": 84.47399145742511, "p50_s": 2.3997448533773422, "p95_s": 4.381843151897192, "cost_usd_per_1000": 0.25791116479400744, "axes": { "intelligence": 84.9653383146534, "calibration": 84.47399145742511, "speed": 69.78178103134502, "cost": 27.65589571451909 }, "jevbench_score": 61.00615393605959 }, { "members": [ "jev-1.13.0", "openjev-sglang", "simplejev-qwen3.8-27b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9657534246575342, "hard": 0.7272727272727273 }, "accuracy": 0.8445692883895131, "ece_hard": 0.04632165552779856, "calibration": 86.4297016097221, "p50_s": 2.3997448533773422, "p95_s": 4.381843151897192, "cost_usd_per_1000": 0.27518622097378276, "axes": { "intelligence": 85.60927770859278, "calibration": 86.4297016097221, "speed": 69.78178103134502, "cost": 26.811199471911323 }, "jevbench_score": 60.997289225052604 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "djev", "openjev-razorback16", "simplejev-qwen3.6-35b-a3b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.958904109589041, "hard": 0.6545454545454545 }, "accuracy": 0.8146067415730337, "ece_hard": 0.08560103783832783, "calibration": 81.62964904937687, "p50_s": 1.7256089895963669, "p95_s": 1.9284205481410026, "cost_usd_per_1000": 0.2694663333333333, "axes": { "intelligence": 83.52734537152347, "calibration": 81.62964904937687, "speed": 74.77855855005357, "cost": 27.084864613118356 }, "jevbench_score": 60.960018720384305 }, { "members": [ "djev", "openjev-sglang", "simplejev-qwen3.8-27b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.958904109589041, "hard": 0.7227272727272728 }, "accuracy": 0.8408239700374532, "ece_hard": 0.056265857315133797, "calibration": 84.36125315971097, "p50_s": 2.3997448533773422, "p95_s": 4.381843151897192, "cost_usd_per_1000": 0.26122323970037453, "axes": { "intelligence": 85.28113325031134, "calibration": 84.36125315971097, "speed": 69.78178103134502, "cost": 27.48964566134252 }, "jevbench_score": 60.95043285207745 }, { "members": [ "semif-qwen3.5-4b", "openjev-sglang", "simplejev-qwen3.8-27b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.958904109589041, "hard": 0.6863636363636364 }, "accuracy": 0.8277153558052435, "ece_hard": 0.050766803164228255, "calibration": 84.57462043760313, "p50_s": 2.3997448533773422, "p95_s": 4.381843151897192, "cost_usd_per_1000": 0.2577165917602996, "axes": { "intelligence": 84.48189082606892, "calibration": 84.57462043760313, "speed": 69.78178103134502, "cost": 27.6657286218213 }, "jevbench_score": 60.94273788949398 }, { "members": [ "system-one-open", "openjev-sglang", "simplejev-qwen3.8-27b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.952054794520548, "hard": 0.7045454545454546 }, "accuracy": 0.8314606741573034, "ece_hard": 0.06097104663388381, "calibration": 83.1174287086896, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.25015292134831457, "axes": { "intelligence": 84.54389788293899, "calibration": 83.1174287086896, "speed": 69.75909261944061, "cost": 28.05383262854474 }, "jevbench_score": 60.89643204016276 }, { "members": [ "semif-qwen3.5-4b", "open-alternative-jev", "system-one-open", "openjev-razorback16", "simplejev-qwen3.8-27b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.952054794520548, "hard": 0.6363636363636364 }, "accuracy": 0.8052434456928839, "ece_hard": 0.07234999097875354, "calibration": 81.46725368974367, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.22909801498127338, "axes": { "intelligence": 82.7901100041511, "calibration": 81.46725368974367, "speed": 69.75909261944061, "cost": 29.199360212106924 }, "jevbench_score": 60.88130087897734 }, { "members": [ "semif-qwen3.5-4b", "djev", "open-alternative-jev", "system-one-open", "openjev-razorback16" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9452054794520548, "hard": 0.6090909090909091 }, "accuracy": 0.7921348314606742, "ece_hard": 0.1763636363636363, "calibration": 63.920636363636376, "p50_s": 1.3041210994124413, "p95_s": 2.030777859687805, "cost_usd_per_1000": 0.15105275280898878, "axes": { "intelligence": 81.78014736405149, "calibration": 63.920636363636376, "speed": 75.77019658655011, "cost": 34.62614067004084 }, "jevbench_score": 60.85522471634985 }, { "members": [ "djev", "openjev-razorback16", "simplejev-qwen3.8-27b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.952054794520548, "hard": 0.6636363636363637 }, "accuracy": 0.8146067415730337, "ece_hard": 0.12504654758914582, "calibration": 75.29436603173704, "p50_s": 2.3997448533773422, "p95_s": 4.381843151897192, "cost_usd_per_1000": 0.19555232209737827, "axes": { "intelligence": 83.31662515566626, "calibration": 75.29436603173704, "speed": 69.78178103134502, "cost": 31.262110678787337 }, "jevbench_score": 60.822409971172 }, { "members": [ "jev-1.13.0", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9726027397260274, "hard": 0.7136363636363636 }, "accuracy": 0.8426966292134831, "ece_hard": 0.0660290076196075, "calibration": 82.86080568877652, "p50_s": 2.3997448533773422, "p95_s": 4.381843151897192, "cost_usd_per_1000": 0.2594624382022472, "axes": { "intelligence": 85.68363428808635, "calibration": 82.86080568877652, "speed": 69.78178103134502, "cost": 27.577765153220284 }, "jevbench_score": 60.7976842184843 }, { "members": [ "jev-1.13.0", "djev", "open-alternative-jev", "openjev-razorback16", "simplejev-qwen3.8-27b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6772727272727272 }, "accuracy": 0.8295880149812734, "ece_hard": 0.0699870326614458, "calibration": 83.36432091047996, "p50_s": 2.3997448533773422, "p95_s": 4.385688671469688, "cost_usd_per_1000": 0.2576379625468165, "axes": { "intelligence": 84.88439186384392, "calibration": 83.36432091047996, "speed": 69.7779713205367, "cost": 27.669704319737818 }, "jevbench_score": 60.79708107322625 }, { "members": [ "djev", "open-alternative-jev", "system-one-open", "openjev-razorback16", "simplejev-qwen3.8-27b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6590909090909091 }, "accuracy": 0.8220973782771536, "ece_hard": 0.08890257503346803, "calibration": 80.05844213162386, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.2326046629213483, "axes": { "intelligence": 84.33893731838938, "calibration": 80.05844213162386, "speed": 69.75909261944061, "cost": 29.00144750240858 }, "jevbench_score": 60.79445076291108 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "open-alternative-jev", "openjev-razorback16", "simplejev-qwen3.6-35b-a3b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9726027397260274, "hard": 0.6454545454545455 }, "accuracy": 0.8146067415730337, "ece_hard": 0.08935940726378327, "calibration": 80.08166118236034, "p50_s": 1.7256089895963669, "p95_s": 1.9284205481410026, "cost_usd_per_1000": 0.2656864831460674, "axes": { "intelligence": 83.6381797426318, "calibration": 80.08166118236034, "speed": 74.77855855005357, "cost": 27.26891619543717 }, "jevbench_score": 60.79189104200487 }, { "members": [ "djev", "open-alternative-jev", "system-one-open", "openjev-razorback16", "simplejev-qwen3.6-35b-a3b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9246575342465754, "hard": 0.6454545454545455 }, "accuracy": 0.8014981273408239, "ece_hard": 0.09268459321128142, "calibration": 78.53812413794896, "p50_s": 1.7357386276125908, "p95_s": 2.0568130582571027, "cost_usd_per_1000": 0.24415983146067416, "axes": { "intelligence": 82.29571398920714, "calibration": 78.53812413794896, "speed": 74.47320850632062, "cost": 28.3697735053333 }, "jevbench_score": 60.789478646315445 }, { "members": [ "semif-qwen3.5-4b", "djev", "open-alternative-jev", "openjev-razorback16", "openjev-sglang" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6363636363636364 }, "accuracy": 0.8052434456928839, "ece_hard": 0.09341679474580993, "calibration": 79.1703116433444, "p50_s": 1.3671194911003113, "p95_s": 1.6307271040976046, "cost_usd_per_1000": 0.26744728464419476, "axes": { "intelligence": 82.88999584889997, "calibration": 79.1703116433444, "speed": 76.51812235281348, "cost": 27.182854212376725 }, "jevbench_score": 60.782835819469014 }, { "members": [ "jev-1.13.0", "open-alternative-jev", "system-one-open", "openjev-razorback16", "simplejev-qwen3.8-27b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9726027397260274, "hard": 0.6681818181818182 }, "accuracy": 0.8239700374531835, "ece_hard": 0.08221912863467656, "calibration": 82.13240218944915, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.24656764419475652, "axes": { "intelligence": 84.31999792444998, "calibration": 82.13240218944915, "speed": 69.75909261944061, "cost": 28.241917427052528 }, "jevbench_score": 60.776407953301195 }, { "members": [ "semif-qwen3.5-4b", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.958904109589041, "hard": 0.6863636363636364 }, "accuracy": 0.8277153558052435, "ece_hard": 0.07109313096992215, "calibration": 81.23982948127366, "p50_s": 2.3997448533773422, "p95_s": 4.381843151897192, "cost_usd_per_1000": 0.24199280898876402, "axes": { "intelligence": 84.48189082606892, "calibration": 81.23982948127366, "speed": 69.78178103134502, "cost": 28.48592617714492 }, "jevbench_score": 60.775180266677296 }, { "members": [ "open-alternative-jev", "openjev-sglang", "simplejev-qwen3.8-27b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9383561643835616, "hard": 0.7 }, "accuracy": 0.8220973782771536, "ece_hard": 0.0502569946447274, "calibration": 84.65366026020868, "p50_s": 2.3997448533773422, "p95_s": 4.385688671469688, "cost_usd_per_1000": 0.25744338951310863, "axes": { "intelligence": 83.44063926940639, "calibration": 84.65366026020868, "speed": 69.7779713205367, "cost": 27.679547657907364 }, "jevbench_score": 60.77502872324429 }, { "members": [ "djev", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9726027397260274, "hard": 0.7 }, "accuracy": 0.8352059925093633, "ece_hard": 0.07298360068571917, "calibration": 81.27622902851539, "p50_s": 2.3997448533773422, "p95_s": 4.381843151897192, "cost_usd_per_1000": 0.24549945692883895, "axes": { "intelligence": 84.98287671232877, "calibration": 81.27622902851539, "speed": 69.78178103134502, "cost": 28.298483927379394 }, "jevbench_score": 60.771512884789296 }, { "members": [ "djev", "openjev-sglang", "simplejev-qwen3.8-27b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.958904109589041, "hard": 0.7318181818181818 }, "accuracy": 0.8445692883895131, "ece_hard": 0.07185843322185381, "calibration": 83.04190462129867, "p50_s": 2.3997448533773422, "p95_s": 4.381843151897192, "cost_usd_per_1000": 0.26122323970037453, "axes": { "intelligence": 85.55386052303862, "calibration": 83.04190462129867, "speed": 69.78178103134502, "cost": 27.48964566134252 }, "jevbench_score": 60.75919707711865 }, { "members": [ "djev", "open-alternative-jev", "system-one-open", "openjev-razorback16", "simplejev-qwen3.6-35b-a3b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9246575342465754, "hard": 0.6454545454545455 }, "accuracy": 0.8014981273408239, "ece_hard": 0.09542357880643523, "calibration": 78.31147911085779, "p50_s": 1.7357386276125908, "p95_s": 2.0568130582571027, "cost_usd_per_1000": 0.24415983146067416, "axes": { "intelligence": 82.29571398920714, "calibration": 78.31147911085779, "speed": 74.47320850632062, "cost": 28.3697735053333 }, "jevbench_score": 60.74557459294758 }, { "members": [ "jev-1.13.0", "open-alternative-jev", "system-one-open", "openjev-razorback16", "simplejev-qwen3.6-35b-a3b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9657534246575342, "hard": 0.6318181818181818 }, "accuracy": 0.8052434456928839, "ece_hard": 0.09148034754736563, "calibration": 79.88455544789007, "p50_s": 1.7357386276125908, "p95_s": 2.0568130582571027, "cost_usd_per_1000": 0.2581228127340824, "axes": { "intelligence": 82.74564134495643, "calibration": 79.88455544789007, "speed": 74.47320850632062, "cost": 27.645208332489517 }, "jevbench_score": 60.73750813931886 }, { "members": [ "semif-qwen3.5-4b", "djev", "open-alternative-jev", "openjev-razorback16", "simplejev-qwen3.8-27b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.65 }, "accuracy": 0.8183520599250936, "ece_hard": 0.08339115835779715, "calibration": 81.14690787936783, "p50_s": 2.3997448533773422, "p95_s": 4.385688671469688, "cost_usd_per_1000": 0.24016833333333332, "axes": { "intelligence": 84.0662100456621, "calibration": 81.14690787936783, "speed": 69.7779713205367, "cost": 28.584527672174644 }, "jevbench_score": 60.73453361526942 }, { "members": [ "semif-qwen3.5-4b", "open-alternative-jev", "openjev-razorback16" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.910958904109589, "hard": 0.6090909090909091 }, "accuracy": 0.7808988764044944, "ece_hard": 0.23939393939393905, "calibration": 53.61127272727276, "p50_s": 0.6506742924451828, "p95_s": 1.1367489621043205, "cost_usd_per_1000": 0.1102205617977528, "axes": { "intelligence": 80.52957658779577, "calibration": 53.61127272727276, "speed": 81.30971785161026, "cost": 38.73212139075841 }, "jevbench_score": 60.72343320146087 }, { "members": [ "jev-1.13.0", "djev", "open-alternative-jev", "openjev-razorback16", "simplejev-qwen3.6-35b-a3b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9383561643835616, "hard": 0.65 }, "accuracy": 0.8071161048689138, "ece_hard": 0.09115442497633269, "calibration": 80.95509470140448, "p50_s": 1.7256089895963669, "p95_s": 1.9284205481410026, "cost_usd_per_1000": 0.26919313108614235, "axes": { "intelligence": 82.8156392694064, "calibration": 80.95509470140448, "speed": 74.77855855005357, "cost": 27.098080781745452 }, "jevbench_score": 60.71109158466869 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "djev", "openjev-razorback16", "simplejev-qwen3.6-35b-a3b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.952054794520548, "hard": 0.65 }, "accuracy": 0.8108614232209738, "ece_hard": 0.09371758118356861, "calibration": 80.60208661083794, "p50_s": 1.7256089895963669, "p95_s": 1.9284205481410026, "cost_usd_per_1000": 0.2694663333333333, "axes": { "intelligence": 83.19920091324202, "calibration": 80.60208661083794, "speed": 74.77855855005357, "cost": 27.084864613118356 }, "jevbench_score": 60.70749316049181 }, { "members": [ "jev-1.13.0", "djev", "system-one-open", "openjev-razorback16", "simplejev-qwen3.8-27b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9657534246575342, "hard": 0.6772727272727272 }, "accuracy": 0.8277153558052435, "ece_hard": 0.07922760587350372, "calibration": 81.79704543583748, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.2503474943820225, "axes": { "intelligence": 84.69261104192611, "calibration": 81.79704543583748, "speed": 69.75909261944061, "cost": 28.043702527174077 }, "jevbench_score": 60.674307526961414 }, { "members": [ "jev-1.13.0", "djev", "simplejev-qwen3.8-27b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9657534246575342, "hard": 0.7227272727272728 }, "accuracy": 0.8445692883895131, "ece_hard": 0.14696969696969697, "calibration": 68.35394696969698, "p50_s": 2.3997448533773422, "p95_s": 4.381843151897192, "cost_usd_per_1000": 0.16986200749063668, "axes": { "intelligence": 85.76458073889582, "calibration": 68.35394696969698, "speed": 69.78178103134502, "cost": 33.097112427150094 }, "jevbench_score": 60.659793876957444 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "djev", "system-one-open", "openjev-sglang" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6909090909090909 }, "accuracy": 0.8352059925093633, "ece_hard": 0.11636363636363625, "calibration": 72.72038636363638, "p50_s": 1.3879805654287338, "p95_s": 2.030777859687805, "cost_usd_per_1000": 0.2344665018726592, "axes": { "intelligence": 85.29348277293484, "calibration": 72.72038636363638, "speed": 75.49954194936167, "cost": 28.897575879432267 }, "jevbench_score": 60.65190399473975 }, { "members": [ "system-one-open", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9657534246575342, "hard": 0.6681818181818182 }, "accuracy": 0.8202247191011236, "ece_hard": 0.06722981949540016, "calibration": 80.02208935262979, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.234429138576779, "axes": { "intelligence": 83.8365504358655, "calibration": 80.02208935262979, "speed": 69.75909261944061, "cost": 28.899652248482568 }, "jevbench_score": 60.64348929127021 }, { "members": [ "jev-1.13.0", "djev", "open-alternative-jev", "system-one-open", "simplejev-qwen3.8-27b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.958904109589041, "hard": 0.6863636363636364 }, "accuracy": 0.8277153558052435, "ece_hard": 0.08818181818181867, "calibration": 75.14656818181814, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.20691434831460673, "axes": { "intelligence": 84.48189082606892, "calibration": 75.14656818181814, "speed": 69.75909261944061, "cost": 30.526281774305716 }, "jevbench_score": 60.63688420955534 }, { "members": [ "djev", "openjev-razorback16", "simplejev-qwen3.6-35b-a3b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9246575342465754, "hard": 0.6454545454545455 }, "accuracy": 0.799625468164794, "ece_hard": 0.15253781469682146, "calibration": 72.2378864206397, "p50_s": 1.724735751748085, "p95_s": 1.9244587391614911, "cost_usd_per_1000": 0.2071074906367041, "axes": { "intelligence": 82.00404732254047, "calibration": 72.2378864206397, "speed": 74.78968829969378, "cost": 30.514125799489577 }, "jevbench_score": 60.63672590927409 }, { "members": [ "djev", "open-alternative-jev", "simplejev-qwen3.8-27b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8020833333333334, "judge": 0.910958904109589, "hard": 0.6727272727272727 }, "accuracy": 0.8052434456928839, "ece_hard": 0.14848484848484814, "calibration": 68.26059848484851, "p50_s": 2.3997448533773422, "p95_s": 4.385688671469688, "cost_usd_per_1000": 0.15211917602996256, "axes": { "intelligence": 82.14700083022001, "calibration": 68.26059848484851, "speed": 69.7779713205367, "cost": 34.53448106971932 }, "jevbench_score": 60.62939984530117 }, { "members": [ "djev", "system-one-open", "simplejev-qwen3.6-35b-a3b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9383561643835616, "hard": 0.6454545454545455 }, "accuracy": 0.8052434456928839, "ece_hard": 0.1681818181818179, "calibration": 63.986484848484885, "p50_s": 1.7357386276125908, "p95_s": 2.0568130582571027, "cost_usd_per_1000": 0.1563838764044944, "axes": { "intelligence": 82.67927563304276, "calibration": 63.986484848484885, "speed": 74.47320850632062, "cost": 34.174240776685366 }, "jevbench_score": 60.575354419995094 }, { "members": [ "semif-qwen3.5-4b", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.958904109589041, "hard": 0.6818181818181818 }, "accuracy": 0.8258426966292135, "ece_hard": 0.07847496667607876, "calibration": 80.25476521095104, "p50_s": 2.3997448533773422, "p95_s": 4.381843151897192, "cost_usd_per_1000": 0.24199280898876402, "axes": { "intelligence": 84.34552718970527, "calibration": 80.25476521095104, "speed": 69.78178103134502, "cost": 28.48592617714492 }, "jevbench_score": 60.56564127549108 }, { "members": [ "semif-qwen3.5-4b", "djev", "open-alternative-jev", "openjev-razorback16", "simplejev-qwen3.8-27b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9657534246575342, "hard": 0.6545454545454545 }, "accuracy": 0.8183520599250936, "ece_hard": 0.09083737581360533, "calibration": 80.25448606878507, "p50_s": 2.3997448533773422, "p95_s": 4.385688671469688, "cost_usd_per_1000": 0.24016833333333332, "axes": { "intelligence": 84.01079286010793, "calibration": 80.25448606878507, "speed": 69.7779713205367, "cost": 28.584527672174644 }, "jevbench_score": 60.55687285171945 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "openjev-sglang" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9726027397260274, "hard": 0.6863636363636364 }, "accuracy": 0.8314606741573034, "ece_hard": 0.17727272727272703, "calibration": 65.95889393939396, "p50_s": 1.3671194911003113, "p95_s": 1.6307271040976046, "cost_usd_per_1000": 0.19363431086142321, "axes": { "intelligence": 84.86545246990454, "calibration": 65.95889393939396, "speed": 76.51812235281348, "cost": 31.390530573402955 }, "jevbench_score": 60.55385744176667 }, { "members": [ "semif-qwen3.5-4b", "djev", "simplejev-qwen3.8-27b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9657534246575342, "hard": 0.6772727272727272 }, "accuracy": 0.8277153558052435, "ece_hard": 0.16212121212121183, "calibration": 65.91779545454548, "p50_s": 2.3997448533773422, "p95_s": 4.381843151897192, "cost_usd_per_1000": 0.15239237827715355, "axes": { "intelligence": 84.69261104192611, "calibration": 65.91779545454548, "speed": 69.78178103134502, "cost": 34.5111025951973 }, "jevbench_score": 60.553223902833444 }, { "members": [ "jev-1.13.0", "openjev-sglang", "simplejev-qwen3.6-35b-a3b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9657534246575342, "hard": 0.6818181818181818 }, "accuracy": 0.8258426966292135, "ece_hard": 0.08889392925112503, "calibration": 81.16290668021074, "p50_s": 1.7284078150987625, "p95_s": 1.928032819926739, "cost_usd_per_1000": 0.2867413895131086, "axes": { "intelligence": 84.24564134495643, "calibration": 81.16290668021074, "speed": 74.77239356263168, "cost": 26.275288436602636 }, "jevbench_score": 60.540841336325386 }, { "members": [ "semif-qwen3.5-4b", "djev", "open-alternative-jev", "openjev-razorback16", "openjev-sglang" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6318181818181818 }, "accuracy": 0.8033707865168539, "ece_hard": 0.10519601337210614, "calibration": 77.97561947022595, "p50_s": 1.3671194911003113, "p95_s": 1.6307271040976046, "cost_usd_per_1000": 0.26744728464419476, "axes": { "intelligence": 82.75363221253633, "calibration": 77.97561947022595, "speed": 76.51812235281348, "cost": 27.182854212376725 }, "jevbench_score": 60.527301865810585 }, { "members": [ "semif-qwen3.5-4b", "open-alternative-jev", "system-one-open", "openjev-razorback16", "simplejev-qwen3.6-35b-a3b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9383561643835616, "hard": 0.6136363636363636 }, "accuracy": 0.7921348314606742, "ece_hard": 0.09986523025058047, "calibration": 77.20665583557523, "p50_s": 1.7357386276125908, "p95_s": 2.0568130582571027, "cost_usd_per_1000": 0.24065318352059922, "axes": { "intelligence": 81.72473017849731, "calibration": 77.20665583557523, "speed": 74.47320850632062, "cost": 28.55825166435858 }, "jevbench_score": 60.5250252879171 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "open-alternative-jev", "openjev-razorback16", "simplejev-qwen3.6-35b-a3b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9383561643835616, "hard": 0.6454545454545455 }, "accuracy": 0.8033707865168539, "ece_hard": 0.08831680254212901, "calibration": 79.84958608377255, "p50_s": 1.7256089895963669, "p95_s": 1.9284205481410026, "cost_usd_per_1000": 0.2656864831460674, "axes": { "intelligence": 82.38760896637609, "calibration": 79.84958608377255, "speed": 74.77855855005357, "cost": 27.26891619543717 }, "jevbench_score": 60.519437625982924 }, { "members": [ "jev-1.13.0", "djev", "open-alternative-jev", "openjev-razorback16", "simplejev-qwen3.6-35b-a3b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9246575342465754, "hard": 0.65 }, "accuracy": 0.8014981273408239, "ece_hard": 0.09205733310861691, "calibration": 80.57660777969929, "p50_s": 1.7256089895963669, "p95_s": 1.9284205481410026, "cost_usd_per_1000": 0.26919313108614235, "axes": { "intelligence": 82.14041095890411, "calibration": 80.57660777969929, "speed": 74.77855855005357, "cost": 27.098080781745452 }, "jevbench_score": 60.516021490325855 }, { "members": [ "jev-1.13.0", "openjev-sglang", "simplejev-qwen3.6-35b-a3b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9657534246575342, "hard": 0.6818181818181818 }, "accuracy": 0.8258426966292135, "ece_hard": 0.08662786978350617, "calibration": 81.0031681390017, "p50_s": 1.7284078150987625, "p95_s": 1.928032819926739, "cost_usd_per_1000": 0.2867413895131086, "axes": { "intelligence": 84.24564134495643, "calibration": 81.0031681390017, "speed": 74.77239356263168, "cost": 26.275288436602636 }, "jevbench_score": 60.51103137847387 }, { "members": [ "system-one-open", "openjev-sglang", "simplejev-qwen3.8-27b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9452054794520548, "hard": 0.6954545454545454 }, "accuracy": 0.8258426966292135, "ece_hard": 0.07360209983948694, "calibration": 81.42367944101565, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.25015292134831457, "axes": { "intelligence": 84.0793897882939, "calibration": 81.42367944101565, "speed": 69.75909261944061, "cost": 28.05383262854474 }, "jevbench_score": 60.50041098371578 }, { "members": [ "jev-1.13.0", "djev", "simplejev-qwen3.6-35b-a3b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9452054794520548, "hard": 0.6772727272727272 }, "accuracy": 0.8183520599250936, "ece_hard": 0.18181818181818177, "calibration": 66.40059848484849, "p50_s": 1.724735751748085, "p95_s": 1.9244587391614911, "cost_usd_per_1000": 0.18141717602996255, "axes": { "intelligence": 83.53393524283935, "calibration": 66.40059848484849, "speed": 74.78968829969378, "cost": 32.239647929208815 }, "jevbench_score": 60.47372730132604 }, { "members": [ "djev", "openjev-razorback16", "simplejev-qwen3.6-35b-a3b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9246575342465754, "hard": 0.6409090909090909 }, "accuracy": 0.797752808988764, "ece_hard": 0.15930504278922325, "calibration": 71.55802628248783, "p50_s": 1.724735751748085, "p95_s": 1.9244587391614911, "cost_usd_per_1000": 0.2071074906367041, "axes": { "intelligence": 81.86768368617685, "calibration": 71.55802628248783, "speed": 74.78968829969378, "cost": 30.514125799489577 }, "jevbench_score": 60.468386169655155 }, { "members": [ "system-one-open", "openjev-sglang", "simplejev-qwen3.6-35b-a3b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.958904109589041, "hard": 0.65 }, "accuracy": 0.8071161048689138, "ece_hard": 0.07938296977297576, "calibration": 78.8790825231116, "p50_s": 1.7359912395477295, "p95_s": 2.0568130582571027, "cost_usd_per_1000": 0.26170808988764044, "axes": { "intelligence": 82.51598173515981, "calibration": 78.8790825231116, "speed": 74.47257649879765, "cost": 27.4654855688825 }, "jevbench_score": 60.4047227831445 }, { "members": [ "open-alternative-jev", "openjev-sglang", "simplejev-qwen3.8-27b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.78125, "judge": 0.9246575342465754, "hard": 0.6909090909090909 }, "accuracy": 0.8127340823970037, "ece_hard": 0.05954036769630828, "calibration": 83.4673811483927, "p50_s": 2.3997448533773422, "p95_s": 4.385688671469688, "cost_usd_per_1000": 0.25744338951310863, "axes": { "intelligence": 82.49268368617683, "calibration": 83.4673811483927, "speed": 69.7779713205367, "cost": 27.679547657907364 }, "jevbench_score": 60.38824200852824 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "djev", "system-one-open", "simplejev-qwen3.8-27b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6954545454545454 }, "accuracy": 0.8370786516853933, "ece_hard": 0.1072727272727272, "calibration": 73.11247727272729, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.20718755056179775, "axes": { "intelligence": 85.42984640929848, "calibration": 73.11247727272729, "speed": 69.75909261944061, "cost": 30.509090317912694 }, "jevbench_score": 60.38204279360719 }, { "members": [ "open-alternative-jev", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8020833333333334, "judge": 0.952054794520548, "hard": 0.6818181818181818 }, "accuracy": 0.8202247191011236, "ece_hard": 0.0823504844343548, "calibration": 79.8436586438812, "p50_s": 2.3997448533773422, "p95_s": 4.385688671469688, "cost_usd_per_1000": 0.241719606741573, "axes": { "intelligence": 83.57041303445413, "calibration": 79.8436586438812, "speed": 69.7779713205367, "cost": 28.500643629559633 }, "jevbench_score": 60.35545053304771 }, { "members": [ "djev", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.958904109589041, "hard": 0.6909090909090909 }, "accuracy": 0.8277153558052435, "ece_hard": 0.08865583580391989, "calibration": 79.49822667510637, "p50_s": 2.3997448533773422, "p95_s": 4.381843151897192, "cost_usd_per_1000": 0.24549945692883895, "axes": { "intelligence": 84.32658779576589, "calibration": 79.49822667510637, "speed": 69.78178103134502, "cost": 28.298483927379394 }, "jevbench_score": 60.319369984809306 }, { "members": [ "system-one-open", "openjev-sglang", "simplejev-qwen3.6-35b-a3b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.958904109589041, "hard": 0.6409090909090909 }, "accuracy": 0.8033707865168539, "ece_hard": 0.08196803845839204, "calibration": 78.52492107218362, "p50_s": 1.7359912395477295, "p95_s": 2.0568130582571027, "cost_usd_per_1000": 0.26170808988764044, "axes": { "intelligence": 82.24325446243255, "calibration": 78.52492107218362, "speed": 74.47257649879765, "cost": 27.4654855688825 }, "jevbench_score": 60.28688771578187 }, { "members": [ "system-one-open", "openjev-razorback16", "simplejev-qwen3.8-27b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6863636363636364 }, "accuracy": 0.8258426966292135, "ece_hard": 0.12575757575757554, "calibration": 69.92507575757578, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.18448200374531834, "axes": { "intelligence": 84.38999584889997, "calibration": 69.92507575757578, "speed": 69.75909261944061, "cost": 32.0213797884486 }, "jevbench_score": 60.25473682179833 }, { "members": [ "semif-qwen3.5-4b", "open-alternative-jev", "system-one-open", "openjev-razorback16", "simplejev-qwen3.6-35b-a3b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9383561643835616, "hard": 0.6136363636363636 }, "accuracy": 0.7921348314606742, "ece_hard": 0.11412585150119428, "calibration": 75.7737630064887, "p50_s": 1.7357386276125908, "p95_s": 2.0568130582571027, "cost_usd_per_1000": 0.24065318352059922, "axes": { "intelligence": 81.72473017849731, "calibration": 75.7737630064887, "speed": 74.47320850632062, "cost": 28.55825166435858 }, "jevbench_score": 60.242225578050274 }, { "members": [ "jev-1.13.0", "system-one-open", "simplejev-qwen3.6-35b-a3b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9452054794520548, "hard": 0.6590909090909091 }, "accuracy": 0.8127340823970037, "ece_hard": 0.17424242424242392, "calibration": 64.18917424242427, "p50_s": 1.7357386276125908, "p95_s": 2.0568130582571027, "cost_usd_per_1000": 0.1703468576779026, "axes": { "intelligence": 83.28014736405149, "calibration": 64.18917424242427, "speed": 74.47320850632062, "cost": 33.05997619849816 }, "jevbench_score": 60.23188692011913 }, { "members": [ "djev", "open-alternative-jev", "openjev-sglang" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8020833333333334, "judge": 0.9383561643835616, "hard": 0.6545454545454545 }, "accuracy": 0.8052434456928839, "ece_hard": 0.18181818181818143, "calibration": 64.46368181818187, "p50_s": 1.3671194911003113, "p95_s": 1.6307271040976046, "cost_usd_per_1000": 0.179398127340824, "axes": { "intelligence": 82.36866957243669, "calibration": 64.46368181818187, "speed": 76.51812235281348, "cost": 32.38546284039515 }, "jevbench_score": 60.2279238476012 }, { "members": [ "semif-qwen3.5-4b", "djev", "system-one-open", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9726027397260274, "hard": 0.6681818181818182 }, "accuracy": 0.8239700374531835, "ece_hard": 0.039782905288318036, "calibration": 84.5022023829655, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.282825, "axes": { "intelligence": 84.31999792444998, "calibration": 84.5022023829655, "speed": 69.75909261944061, "cost": 26.45446612608039 }, "jevbench_score": 60.21776187089054 }, { "members": [ "semif-qwen3.5-4b", "djev", "open-alternative-jev", "openjev-razorback16", "simplejev-qwen3.6-35b-a3b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9315068493150684, "hard": 0.6272727272727273 }, "accuracy": 0.7940074906367042, "ece_hard": 0.10977496812383251, "calibration": 76.93095719373305, "p50_s": 1.7256089895963669, "p95_s": 1.9284205481410026, "cost_usd_per_1000": 0.2517235018726592, "axes": { "intelligence": 81.65037359900374, "calibration": 76.93095719373305, "speed": 74.77855855005357, "cost": 27.9722870547499 }, "jevbench_score": 60.206183023789826 }, { "members": [ "semif-qwen3.5-4b", "djev", "system-one-open", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9726027397260274, "hard": 0.6727272727272727 }, "accuracy": 0.8258426966292135, "ece_hard": 0.04466048988064954, "calibration": 84.29906264050427, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.282825, "axes": { "intelligence": 84.45636156081362, "calibration": 84.29906264050427, "speed": 69.75909261944061, "cost": 26.45446612608039 }, "jevbench_score": 60.205855850440614 }, { "members": [ "jev-1.13.0", "djev", "open-alternative-jev", "openjev-sglang", "simplejev-qwen3.8-27b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9657534246575342, "hard": 0.7272727272727273 }, "accuracy": 0.846441947565543, "ece_hard": 0.02819270371931308, "calibration": 88.57005199091887, "p50_s": 2.3997448533773422, "p95_s": 4.385688671469688, "cost_usd_per_1000": 0.32330888014981274, "axes": { "intelligence": 85.90094437525944, "calibration": 88.57005199091887, "speed": 69.7779713205367, "cost": 24.711471000802888 }, "jevbench_score": 60.1832232742209 }, { "members": [ "jev-1.13.0", "open-alternative-jev", "system-one-open", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9657534246575342, "hard": 0.6954545454545454 }, "accuracy": 0.8314606741573034, "ece_hard": 0.035629485026692664, "calibration": 85.62788545930822, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.2965147790262172, "axes": { "intelligence": 84.65473225404733, "calibration": 85.62788545930822, "speed": 69.75909261944061, "cost": 25.838609663638195 }, "jevbench_score": 60.122095072362676 }, { "members": [ "semif-qwen3.5-4b", "djev", "open-alternative-jev", "openjev-razorback16", "simplejev-qwen3.6-35b-a3b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9315068493150684, "hard": 0.6272727272727273 }, "accuracy": 0.795880149812734, "ece_hard": 0.11697720037800781, "calibration": 76.22396640857106, "p50_s": 1.7256089895963669, "p95_s": 1.9284205481410026, "cost_usd_per_1000": 0.2517235018726592, "axes": { "intelligence": 81.94204026567041, "calibration": 76.22396640857106, "speed": 74.77855855005357, "cost": 27.9722870547499 }, "jevbench_score": 60.120951561506715 }, { "members": [ "jev-1.13.0", "open-alternative-jev", "system-one-open", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9657534246575342, "hard": 0.7090909090909091 }, "accuracy": 0.8389513108614233, "ece_hard": 0.046181459603494034, "calibration": 84.88961492191787, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.2965147790262172, "axes": { "intelligence": 85.35548982980491, "calibration": 84.88961492191787, "speed": 69.75909261944061, "cost": 25.838609663638195 }, "jevbench_score": 60.11585062732076 }, { "members": [ "open-alternative-jev", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8020833333333334, "judge": 0.952054794520548, "hard": 0.6681818181818182 }, "accuracy": 0.8146067415730337, "ece_hard": 0.08801354295480662, "calibration": 78.94018182458251, "p50_s": 2.3997448533773422, "p95_s": 4.385688671469688, "cost_usd_per_1000": 0.241719606741573, "axes": { "intelligence": 83.16132212536323, "calibration": 78.94018182458251, "speed": 69.7779713205367, "cost": 28.500643629559633 }, "jevbench_score": 60.11019385707561 }, { "members": [ "djev", "openjev-sglang", "simplejev-qwen3.6-35b-a3b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9452054794520548, "hard": 0.6863636363636364 }, "accuracy": 0.8220973782771536, "ece_hard": 0.11113530975321632, "calibration": 77.27269458766584, "p50_s": 1.7284078150987625, "p95_s": 1.928032819926739, "cost_usd_per_1000": 0.2727784082397004, "axes": { "intelligence": 83.80666251556663, "calibration": 77.27269458766584, "speed": 74.77239356263168, "cost": 26.925700276346674 }, "jevbench_score": 60.09014102472493 }, { "members": [ "open-alternative-jev", "system-one-open", "openjev-sglang" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9178082191780822, "hard": 0.6227272727272727 }, "accuracy": 0.7846441947565543, "ece_hard": 0.1651515151515151, "calibration": 64.54401515151515, "p50_s": 1.3879805654287338, "p95_s": 2.030777859687805, "cost_usd_per_1000": 0.16832780898876404, "axes": { "intelligence": 80.54711498547115, "calibration": 64.54401515151515, "speed": 75.49954194936167, "cost": 33.2153238838832 }, "jevbench_score": 60.089306824038765 }, { "members": [ "djev", "openjev-sglang", "simplejev-qwen3.6-35b-a3b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9452054794520548, "hard": 0.6818181818181818 }, "accuracy": 0.8202247191011236, "ece_hard": 0.11071345786581163, "calibration": 77.31027906531477, "p50_s": 1.7284078150987625, "p95_s": 1.928032819926739, "cost_usd_per_1000": 0.2727784082397004, "axes": { "intelligence": 83.67029887920299, "calibration": 77.31027906531477, "speed": 74.77239356263168, "cost": 26.925700276346674 }, "jevbench_score": 60.07298507170493 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "djev", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6909090909090909 }, "accuracy": 0.8352059925093633, "ece_hard": 0.042125117772199315, "calibration": 86.2768791893699, "p50_s": 2.3997448533773422, "p95_s": 4.381843151897192, "cost_usd_per_1000": 0.3078582996254682, "axes": { "intelligence": 85.29348277293484, "calibration": 86.2768791893699, "speed": 69.78178103134502, "cost": 25.349474009978593 }, "jevbench_score": 60.06622235148132 }, { "members": [ "semif-qwen3.5-4b", "system-one-open", "simplejev-qwen3.8-27b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9657534246575342, "hard": 0.6227272727272727 }, "accuracy": 0.8033707865168539, "ece_hard": 0.18181818181818157, "calibration": 63.45234848484851, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.1413220599250936, "axes": { "intelligence": 82.76458073889582, "calibration": 63.45234848484851, "speed": 69.75909261944061, "cost": 35.49370122608096 }, "jevbench_score": 60.049733324176884 }, { "members": [ "open-alternative-jev", "system-one-open", "simplejev-qwen3.8-27b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.7708333333333334, "judge": 0.8972602739726028, "hard": 0.6363636363636364 }, "accuracy": 0.7808988764044944, "ece_hard": 0.15454545454545437, "calibration": 65.74212121212122, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.1410488576779026, "axes": { "intelligence": 79.79753009547531, "calibration": 65.74212121212122, "speed": 69.75909261944061, "cost": 35.51891280235006 }, "jevbench_score": 60.044524816261614 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "simplejev-qwen3.6-35b-a3b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9794520547945206, "hard": 0.6636363636363637 }, "accuracy": 0.8220973782771536, "ece_hard": 0.20151515151515126, "calibration": 63.62015151515154, "p50_s": 1.7256089895963669, "p95_s": 1.9284205481410026, "cost_usd_per_1000": 0.17791052808988764, "axes": { "intelligence": 84.0837484433375, "calibration": 63.62015151515154, "speed": 74.77855855005357, "cost": 32.493950536225114 }, "jevbench_score": 60.044273145036996 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "system-one-open", "openjev-sglang", "simplejev-qwen3.6-35b-a3b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9726027397260274, "hard": 0.6590909090909091 }, "accuracy": 0.8202247191011236, "ece_hard": 0.05439340116629775, "calibration": 84.13302143732096, "p50_s": 1.7359912395477295, "p95_s": 2.0568130582571027, "cost_usd_per_1000": 0.32406693258426966, "axes": { "intelligence": 84.04727065172271, "calibration": 84.13302143732096, "speed": 74.47257649879765, "cost": 24.680958448439497 }, "jevbench_score": 60.04296431438425 }, { "members": [ "semif-qwen3.5-4b", "djev", "simplejev-qwen3.6-35b-a3b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.952054794520548, "hard": 0.65 }, "accuracy": 0.8089887640449438, "ece_hard": 0.19545454545454516, "calibration": 62.24946212121215, "p50_s": 1.7256089895963669, "p95_s": 1.9284205481410026, "cost_usd_per_1000": 0.16394754681647938, "axes": { "intelligence": 82.90753424657535, "calibration": 62.24946212121215, "speed": 74.77855855005357, "cost": 33.55885232172082 }, "jevbench_score": 59.989942803885825 }, { "members": [ "jev-1.13.0", "djev", "system-one-open", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9726027397260274, "hard": 0.7045454545454546 }, "accuracy": 0.8370786516853933, "ece_hard": 0.04437033778913478, "calibration": 84.79273797427618, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.30029462921348316, "axes": { "intelligence": 85.1192403486924, "calibration": 84.79273797427618, "speed": 69.75909261944061, "cost": 25.67357305339182 }, "jevbench_score": 59.96093327730258 }, { "members": [ "jev-1.13.0", "open-alternative-jev", "system-one-open", "openjev-sglang", "simplejev-qwen3.8-27b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.958904109589041, "hard": 0.7181818181818181 }, "accuracy": 0.8389513108614233, "ece_hard": 0.04288502527204208, "calibration": 86.46975920402494, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.3122385617977528, "axes": { "intelligence": 85.1447696139477, "calibration": 86.46975920402494, "speed": 69.75909261944061, "cost": 25.16540386378908 }, "jevbench_score": 59.959325394343935 }, { "members": [ "jev-1.13.0", "djev", "open-alternative-jev", "system-one-open", "simplejev-qwen3.6-35b-a3b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9452054794520548, "hard": 0.6545454545454545 }, "accuracy": 0.8108614232209738, "ece_hard": 0.13909090909090885, "calibration": 69.95665909090911, "p50_s": 1.7357386276125908, "p95_s": 2.0568130582571027, "cost_usd_per_1000": 0.21846951685393257, "axes": { "intelligence": 83.14378372768785, "calibration": 69.95665909090911, "speed": 74.47320850632062, "cost": 29.818274552891452 }, "jevbench_score": 59.94947095241792 }, { "members": [ "jev-1.13.0", "djev", "open-alternative-jev", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9726027397260274, "hard": 0.6909090909090909 }, "accuracy": 0.8333333333333334, "ece_hard": 0.045843721787928254, "calibration": 85.73843409810637, "p50_s": 2.3997448533773422, "p95_s": 4.385688671469688, "cost_usd_per_1000": 0.30758509737827716, "axes": { "intelligence": 85.00181610626817, "calibration": 85.73843409810637, "speed": 69.7779713205367, "cost": 25.36104130309799 }, "jevbench_score": 59.92696633580767 }, { "members": [ "jev-1.13.0", "open-alternative-jev", "system-one-open", "openjev-sglang", "simplejev-qwen3.6-35b-a3b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9657534246575342, "hard": 0.6681818181818182 }, "accuracy": 0.8202247191011236, "ece_hard": 0.05900872708921207, "calibration": 83.5420800966246, "p50_s": 1.7359912395477295, "p95_s": 2.0568130582571027, "cost_usd_per_1000": 0.32379373033707864, "axes": { "intelligence": 83.8365504358655, "calibration": 83.5420800966246, "speed": 74.47257649879765, "cost": 24.691946944166034 }, "jevbench_score": 59.90631417545384 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "djev", "openjev-sglang", "simplejev-qwen3.8-27b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6909090909090909 }, "accuracy": 0.8352059925093633, "ece_hard": 0.036927186323175196, "calibration": 87.53346949281995, "p50_s": 2.3997448533773422, "p95_s": 4.381843151897192, "cost_usd_per_1000": 0.32358208239700376, "axes": { "intelligence": 85.29348277293484, "calibration": 87.53346949281995, "speed": 69.78178103134502, "cost": 24.70046603313547 }, "jevbench_score": 59.89413497435287 }, { "members": [ "jev-1.13.0", "djev", "open-alternative-jev", "openjev-sglang", "simplejev-qwen3.6-35b-a3b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9726027397260274, "hard": 0.6772727272727272 }, "accuracy": 0.8277153558052435, "ece_hard": 0.06424132996939431, "calibration": 83.87479553154182, "p50_s": 1.7284078150987625, "p95_s": 1.9287681855261325, "cost_usd_per_1000": 0.3348640486891386, "axes": { "intelligence": 84.59272519717726, "calibration": 83.87479553154182, "speed": 74.77073744801535, "cost": 24.253944286318045 }, "jevbench_score": 59.89211262308485 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "djev", "system-one-open", "simplejev-qwen3.6-35b-a3b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6636363636363637 }, "accuracy": 0.8239700374531835, "ece_hard": 0.1527272727272725, "calibration": 68.62252272727275, "p50_s": 1.7357386276125908, "p95_s": 2.0568130582571027, "cost_usd_per_1000": 0.21874271910112358, "axes": { "intelligence": 84.47530095475302, "calibration": 68.62252272727275, "speed": 74.47320850632062, "cost": 29.80199180953467 }, "jevbench_score": 59.890845720001664 }, { "members": [ "system-one-open", "openjev-razorback16", "openjev-sglang" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.952054794520548, "hard": 0.6590909090909091 }, "accuracy": 0.8127340823970037, "ece_hard": 0.1393939393939393, "calibration": 67.7659393939394, "p50_s": 1.3879805654287338, "p95_s": 2.030777859687805, "cost_usd_per_1000": 0.21176095505617978, "axes": { "intelligence": 83.18026151930262, "calibration": 67.7659393939394, "speed": 75.49954194936167, "cost": 30.2246233901472 }, "jevbench_score": 59.887231928266836 }, { "members": [ "jev-1.13.0", "djev", "system-one-open", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9726027397260274, "hard": 0.7136363636363636 }, "accuracy": 0.8426966292134831, "ece_hard": 0.05866784702473189, "calibration": 83.76108508506468, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.30029462921348316, "axes": { "intelligence": 85.68363428808635, "calibration": 83.76108508506468, "speed": 69.75909261944061, "cost": 25.67357305339182 }, "jevbench_score": 59.87655811467913 }, { "members": [ "djev", "open-alternative-jev", "system-one-open", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9657534246575342, "hard": 0.6863636363636364 }, "accuracy": 0.8295880149812734, "ece_hard": 0.06293227429605838, "calibration": 82.17371219779861, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.282551797752809, "axes": { "intelligence": 84.67367164798672, "calibration": 82.17371219779861, "speed": 69.75909261944061, "cost": 26.467057753975837 }, "jevbench_score": 59.868302822734876 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "djev", "open-alternative-jev", "openjev-sglang" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9657534246575342, "hard": 0.6863636363636364 }, "accuracy": 0.8295880149812734, "ece_hard": 0.14727272727272744, "calibration": 69.49922727272725, "p50_s": 1.3671194911003113, "p95_s": 1.6307271040976046, "cost_usd_per_1000": 0.2417569700374532, "axes": { "intelligence": 84.67367164798672, "calibration": 69.49922727272725, "speed": 76.51812235281348, "cost": 28.49862988071989 }, "jevbench_score": 59.85206837811852 }, { "members": [ "semif-qwen3.5-4b", "openjev-sglang", "simplejev-qwen3.6-35b-a3b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9657534246575342, "hard": 0.6636363636363637 }, "accuracy": 0.8146067415730337, "ece_hard": 0.10957252284577218, "calibration": 76.07412166679708, "p50_s": 1.7284078150987625, "p95_s": 1.9287681855261325, "cost_usd_per_1000": 0.26927176029962546, "axes": { "intelligence": 83.11685346616854, "calibration": 76.07412166679708, "speed": 74.77073744801535, "cost": 27.09427571653272 }, "jevbench_score": 59.82515226867669 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "open-alternative-jev", "openjev-sglang", "simplejev-qwen3.6-35b-a3b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9657534246575342, "hard": 0.6727272727272727 }, "accuracy": 0.8220973782771536, "ece_hard": 0.05684682442337908, "calibration": 83.57179656741752, "p50_s": 1.7284078150987625, "p95_s": 1.9287681855261325, "cost_usd_per_1000": 0.33135740074906367, "axes": { "intelligence": 83.97291407222914, "calibration": 83.57179656741752, "speed": 74.77073744801535, "cost": 24.391099754577894 }, "jevbench_score": 59.81229991941737 }, { "members": [ "semif-qwen3.5-4b", "openjev-sglang", "simplejev-qwen3.6-35b-a3b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9657534246575342, "hard": 0.6681818181818182 }, "accuracy": 0.8164794007490637, "ece_hard": 0.1109601866448033, "calibration": 75.83761864021942, "p50_s": 1.7284078150987625, "p95_s": 1.9287681855261325, "cost_usd_per_1000": 0.26927176029962546, "axes": { "intelligence": 83.25321710253218, "calibration": 75.83761864021942, "speed": 74.77073744801535, "cost": 27.09427571653272 }, "jevbench_score": 59.803104613743486 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "open-alternative-jev", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.958904109589041, "hard": 0.6863636363636364 }, "accuracy": 0.8277153558052435, "ece_hard": 0.048408517089225875, "calibration": 84.99525106609205, "p50_s": 2.3997448533773422, "p95_s": 4.385688671469688, "cost_usd_per_1000": 0.3040784494382022, "axes": { "intelligence": 84.48189082606892, "calibration": 84.99525106609205, "speed": 69.7779713205367, "cost": 25.510430738824738 }, "jevbench_score": 59.79276049524032 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "system-one-open", "openjev-sglang", "simplejev-qwen3.8-27b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9657534246575342, "hard": 0.6727272727272727 }, "accuracy": 0.8258426966292135, "ece_hard": 0.04575770988288359, "calibration": 86.12811712432207, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.3125117640449438, "axes": { "intelligence": 84.55624740556247, "calibration": 86.12811712432207, "speed": 69.75909261944061, "cost": 25.15400888908762 }, "jevbench_score": 59.789465601937835 }, { "members": [ "jev-1.13.0", "djev", "open-alternative-jev", "openjev-sglang", "simplejev-qwen3.8-27b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9657534246575342, "hard": 0.7045454545454546 }, "accuracy": 0.8370786516853933, "ece_hard": 0.043539884708379814, "calibration": 86.89328761577192, "p50_s": 2.3997448533773422, "p95_s": 4.385688671469688, "cost_usd_per_1000": 0.32330888014981274, "axes": { "intelligence": 85.21912619344127, "calibration": 86.89328761577192, "speed": 69.7779713205367, "cost": 24.711471000802888 }, "jevbench_score": 59.77713011126648 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "djev", "openjev-sglang", "simplejev-qwen3.8-27b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6863636363636364 }, "accuracy": 0.8333333333333334, "ece_hard": 0.04139205285078347, "calibration": 86.96784226098748, "p50_s": 2.3997448533773422, "p95_s": 4.381843151897192, "cost_usd_per_1000": 0.32358208239700376, "axes": { "intelligence": 85.1571191365712, "calibration": 86.96784226098748, "speed": 69.78178103134502, "cost": 24.70046603313547 }, "jevbench_score": 59.77322842275391 }, { "members": [ "semif-qwen3.5-4b", "djev", "openjev-sglang" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9657534246575342, "hard": 0.6590909090909091 }, "accuracy": 0.8183520599250936, "ece_hard": 0.2030303030303028, "calibration": 61.40638636363639, "p50_s": 1.3671194911003113, "p95_s": 1.6307271040976046, "cost_usd_per_1000": 0.179671329588015, "axes": { "intelligence": 83.85548982980491, "calibration": 61.40638636363639, "speed": 76.51812235281348, "cost": 32.365636550689075 }, "jevbench_score": 59.758319318379385 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "djev", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6863636363636364 }, "accuracy": 0.8333333333333334, "ece_hard": 0.05658088895614088, "calibration": 84.64611733076188, "p50_s": 2.3997448533773422, "p95_s": 4.381843151897192, "cost_usd_per_1000": 0.3078582996254682, "axes": { "intelligence": 85.1571191365712, "calibration": 84.64611733076188, "speed": 69.78178103134502, "cost": 25.349474009978593 }, "jevbench_score": 59.75644479190168 }, { "members": [ "djev", "open-alternative-jev", "simplejev-qwen3.6-35b-a3b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.65 }, "accuracy": 0.799625468164794, "ece_hard": 0.19090909090909047, "calibration": 62.07759090909096, "p50_s": 1.7256089895963669, "p95_s": 1.9284205481410026, "cost_usd_per_1000": 0.1636743445692884, "axes": { "intelligence": 81.74885844748859, "calibration": 62.07759090909096, "speed": 74.77855855005357, "cost": 33.580581685787266 }, "jevbench_score": 59.74759987208256 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "system-one-open", "openjev-sglang", "simplejev-qwen3.6-35b-a3b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9726027397260274, "hard": 0.6409090909090909 }, "accuracy": 0.8127340823970037, "ece_hard": 0.06364479449141744, "calibration": 82.90703756483012, "p50_s": 1.7359912395477295, "p95_s": 2.0568130582571027, "cost_usd_per_1000": 0.32406693258426966, "axes": { "intelligence": 83.50181610626817, "calibration": 82.90703756483012, "speed": 74.47257649879765, "cost": 24.680958448439497 }, "jevbench_score": 59.72572490980267 }, { "members": [ "semif-qwen3.5-4b", "open-alternative-jev", "system-one-open", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.958904109589041, "hard": 0.65 }, "accuracy": 0.8108614232209738, "ece_hard": 0.054934790829852365, "calibration": 82.38188036614235, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.2790451498127341, "axes": { "intelligence": 83.09931506849315, "calibration": 82.38188036614235, "speed": 69.75909261944061, "cost": 26.629765651456538 }, "jevbench_score": 59.71718509112485 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "open-alternative-jev", "system-one-open", "openjev-sglang" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8020833333333334, "judge": 0.9726027397260274, "hard": 0.6454545454545455 }, "accuracy": 0.8108614232209738, "ece_hard": 0.13636363636363613, "calibration": 69.59013636363639, "p50_s": 1.3879805654287338, "p95_s": 2.030777859687805, "cost_usd_per_1000": 0.23068665168539326, "axes": { "intelligence": 83.05484640929846, "calibration": 69.59013636363639, "speed": 75.49954194936167, "cost": 29.10932603525056 }, "jevbench_score": 59.69971852936994 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "system-one-open", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9726027397260274, "hard": 0.6772727272727272 }, "accuracy": 0.8277153558052435, "ece_hard": 0.06203784422458355, "calibration": 83.34602084472863, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.2967879812734082, "axes": { "intelligence": 84.59272519717726, "calibration": 83.34602084472863, "speed": 69.75909261944061, "cost": 25.82661070680966 }, "jevbench_score": 59.6996151444039 }, { "members": [ "jev-1.13.0", "djev", "system-one-open", "openjev-sglang", "simplejev-qwen3.8-27b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9657534246575342, "hard": 0.7181818181818181 }, "accuracy": 0.8426966292134831, "ece_hard": 0.05908836303780027, "calibration": 84.92080155728698, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.31601841198501873, "axes": { "intelligence": 85.62821710253218, "calibration": 84.92080155728698, "speed": 69.75909261944061, "cost": 25.008628408420932 }, "jevbench_score": 59.68022072770548 }, { "members": [ "djev", "open-alternative-jev", "system-one-open", "openjev-sglang", "simplejev-qwen3.8-27b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.958904109589041, "hard": 0.6909090909090909 }, "accuracy": 0.8277153558052435, "ece_hard": 0.059673909541749115, "calibration": 83.67868724566566, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.29827558052434455, "axes": { "intelligence": 84.32658779576589, "calibration": 83.67868724566566, "speed": 69.75909261944061, "cost": 25.761469011035786 }, "jevbench_score": 59.67435152318763 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "open-alternative-jev", "openjev-sglang", "simplejev-qwen3.8-27b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.958904109589041, "hard": 0.7 }, "accuracy": 0.8333333333333334, "ece_hard": 0.0441348829927152, "calibration": 86.08886474479203, "p50_s": 2.3997448533773422, "p95_s": 4.385688671469688, "cost_usd_per_1000": 0.3198022322097378, "axes": { "intelligence": 84.89098173515983, "calibration": 86.08886474479203, "speed": 69.7779713205367, "cost": 24.853555276512637 }, "jevbench_score": 59.666264591386835 }, { "members": [ "semif-qwen3.5-4b", "open-alternative-jev", "system-one-open", "openjev-sglang", "simplejev-qwen3.8-27b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8020833333333334, "judge": 0.952054794520548, "hard": 0.6681818181818182 }, "accuracy": 0.8146067415730337, "ece_hard": 0.04645007235951735, "calibration": 84.26736866154889, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.29476893258426967, "axes": { "intelligence": 83.16132212536323, "calibration": 84.26736866154889, "speed": 69.75909261944061, "cost": 25.915548736826693 }, "jevbench_score": 59.66031094188132 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "open-alternative-jev", "openjev-sglang", "simplejev-qwen3.6-35b-a3b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9657534246575342, "hard": 0.6636363636363637 }, "accuracy": 0.8183520599250936, "ece_hard": 0.05902658669827327, "calibration": 82.99500016717127, "p50_s": 1.7284078150987625, "p95_s": 1.9287681855261325, "cost_usd_per_1000": 0.33135740074906367, "axes": { "intelligence": 83.70018679950188, "calibration": 82.99500016717127, "speed": 74.77073744801535, "cost": 24.391099754577894 }, "jevbench_score": 59.66028874618884 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "system-one-open", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9726027397260274, "hard": 0.6909090909090909 }, "accuracy": 0.8333333333333334, "ece_hard": 0.07255982401000771, "calibration": 82.72553007156968, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.2967879812734082, "axes": { "intelligence": 85.00181610626817, "calibration": 82.72553007156968, "speed": 69.75909261944061, "cost": 25.82661070680966 }, "jevbench_score": 59.66010325298487 }, { "members": [ "jev-1.13.0", "openjev-razorback16", "openjev-sglang" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.952054794520548, "hard": 0.7045454545454546 }, "accuracy": 0.8314606741573034, "ece_hard": 0.16363636363636358, "calibration": 68.02986363636364, "p50_s": 1.3671194911003113, "p95_s": 1.6257096692919728, "cost_usd_per_1000": 0.23679425468164794, "axes": { "intelligence": 84.54389788293899, "calibration": 68.02986363636364, "speed": 76.53150536083959, "cost": 28.76886517211291 }, "jevbench_score": 59.65353898354146 }, { "members": [ "jev-1.13.0", "djev", "system-one-open", "openjev-sglang", "simplejev-qwen3.6-35b-a3b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9657534246575342, "hard": 0.6590909090909091 }, "accuracy": 0.8164794007490637, "ece_hard": 0.06804920131881576, "calibration": 82.89536936376236, "p50_s": 1.7359912395477295, "p95_s": 2.0568130582571027, "cost_usd_per_1000": 0.3275735805243446, "axes": { "intelligence": 83.56382316313824, "calibration": 82.89536936376236, "speed": 74.47257649879765, "cost": 24.540733966979843 }, "jevbench_score": 59.64968090662658 }, { "members": [ "jev-1.13.0", "djev", "system-one-open", "openjev-sglang", "simplejev-qwen3.6-35b-a3b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9657534246575342, "hard": 0.6590909090909091 }, "accuracy": 0.8164794007490637, "ece_hard": 0.0729117275088453, "calibration": 82.85650930042351, "p50_s": 1.7359912395477295, "p95_s": 2.0568130582571027, "cost_usd_per_1000": 0.3275735805243446, "axes": { "intelligence": 83.56382316313824, "calibration": 82.85650930042351, "speed": 74.47257649879765, "cost": 24.540733966979843 }, "jevbench_score": 59.64268896607351 }, { "members": [ "open-alternative-jev", "openjev-sglang", "simplejev-qwen3.6-35b-a3b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.78125, "judge": 0.9246575342465754, "hard": 0.6590909090909091 }, "accuracy": 0.799625468164794, "ece_hard": 0.09947502501475769, "calibration": 76.54881488086994, "p50_s": 1.7284078150987625, "p95_s": 1.9287681855261325, "cost_usd_per_1000": 0.26899855805243444, "axes": { "intelligence": 81.53813823163138, "calibration": 76.54881488086994, "speed": 74.77073744801535, "cost": 27.107501439877495 }, "jevbench_score": 59.63896583119494 }, { "members": [ "jev-1.13.0", "open-alternative-jev", "simplejev-qwen3.6-35b-a3b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8020833333333334, "judge": 0.9315068493150684, "hard": 0.65 }, "accuracy": 0.8014981273408239, "ece_hard": 0.19090909090909053, "calibration": 63.41717424242428, "p50_s": 1.7256089895963669, "p95_s": 1.9284205481410026, "cost_usd_per_1000": 0.17763732584269662, "axes": { "intelligence": 82.04052511415524, "calibration": 63.41717424242428, "speed": 74.77855855005357, "cost": 32.5139731999152 }, "jevbench_score": 59.63766247401418 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "open-alternative-jev", "openjev-sglang", "simplejev-qwen3.8-27b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.958904109589041, "hard": 0.7045454545454546 }, "accuracy": 0.8352059925093633, "ece_hard": 0.050677493349059625, "calibration": 85.75119691676622, "p50_s": 2.3997448533773422, "p95_s": 4.385688671469688, "cost_usd_per_1000": 0.3198022322097378, "axes": { "intelligence": 85.02734537152345, "calibration": 85.75119691676622, "speed": 69.7779713205367, "cost": 24.853555276512637 }, "jevbench_score": 59.63159394974998 }, { "members": [ "open-alternative-jev", "openjev-sglang", "simplejev-qwen3.6-35b-a3b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.78125, "judge": 0.9178082191780822, "hard": 0.6590909090909091 }, "accuracy": 0.797752808988764, "ece_hard": 0.09699128217991279, "calibration": 76.68281137357324, "p50_s": 1.7284078150987625, "p95_s": 1.9287681855261325, "cost_usd_per_1000": 0.26899855805243444, "axes": { "intelligence": 81.34635740971359, "calibration": 76.68281137357324, "speed": 74.77073744801535, "cost": 27.107501439877495 }, "jevbench_score": 59.6299331831633 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "system-one-open", "openjev-sglang", "simplejev-qwen3.8-27b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9657534246575342, "hard": 0.6863636363636364 }, "accuracy": 0.8295880149812734, "ece_hard": 0.06013706757673379, "calibration": 85.01853715392346, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.3125117640449438, "axes": { "intelligence": 84.67367164798672, "calibration": 85.01853715392346, "speed": 69.75909261944061, "cost": 25.15400888908762 }, "jevbench_score": 59.61664244754412 }, { "members": [ "jev-1.13.0", "open-alternative-jev", "system-one-open", "openjev-razorback16", "openjev-sglang" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.952054794520548, "hard": 0.6636363636363637 }, "accuracy": 0.8146067415730337, "ece_hard": 0.1045454545454544, "calibration": 74.68520454545455, "p50_s": 1.3879805654287338, "p95_s": 2.030777859687805, "cost_usd_per_1000": 0.27384659550561796, "axes": { "intelligence": 83.31662515566626, "calibration": 74.68520454545455, "speed": 75.49954194936167, "cost": 26.87477961711315 }, "jevbench_score": 59.60927891750946 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "djev", "open-alternative-jev", "simplejev-qwen3.8-27b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9657534246575342, "hard": 0.6818181818181818 }, "accuracy": 0.8277153558052435, "ece_hard": 0.13090909090909114, "calibration": 71.14809090909088, "p50_s": 2.3997448533773422, "p95_s": 4.385688671469688, "cost_usd_per_1000": 0.21447801872659175, "axes": { "intelligence": 84.5373080116231, "calibration": 71.14809090909088, "speed": 69.7779713205367, "cost": 30.058516326044753 }, "jevbench_score": 59.59699827213199 }, { "members": [ "jev-1.13.0", "djev", "open-alternative-jev", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9726027397260274, "hard": 0.6727272727272727 }, "accuracy": 0.8258426966292135, "ece_hard": 0.05717655870803351, "calibration": 84.39488962392107, "p50_s": 2.3997448533773422, "p95_s": 4.385688671469688, "cost_usd_per_1000": 0.30758509737827716, "axes": { "intelligence": 84.45636156081362, "calibration": 84.39488962392107, "speed": 69.7779713205367, "cost": 25.36104130309799 }, "jevbench_score": 59.594816159957276 }, { "members": [ "djev", "open-alternative-jev", "system-one-open", "openjev-sglang", "simplejev-qwen3.6-35b-a3b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9657534246575342, "hard": 0.6454545454545455 }, "accuracy": 0.8108614232209738, "ece_hard": 0.07575506494380399, "calibration": 80.57767814757537, "p50_s": 1.7359912395477295, "p95_s": 2.0568130582571027, "cost_usd_per_1000": 0.3098307490636704, "axes": { "intelligence": 83.15473225404733, "calibration": 80.57767814757537, "speed": 74.47257649879765, "cost": 25.266264490204037 }, "jevbench_score": 59.58813375753566 }, { "members": [ "semif-qwen3.5-4b", "djev", "system-one-open", "openjev-sglang", "simplejev-qwen3.8-27b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9657534246575342, "hard": 0.6727272727272727 }, "accuracy": 0.8258426966292135, "ece_hard": 0.06598953545020647, "calibration": 83.00545422515239, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.29854878277153557, "axes": { "intelligence": 84.55624740556247, "calibration": 83.00545422515239, "speed": 69.75909261944061, "cost": 25.749540854883023 }, "jevbench_score": 59.58756805148907 }, { "members": [ "jev-1.13.0", "open-alternative-jev", "system-one-open", "openjev-razorback16", "simplejev-qwen3.8-27b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9452054794520548, "hard": 0.6909090909090909 }, "accuracy": 0.8258426966292135, "ece_hard": 0.08818181818181774, "calibration": 75.95906818181822, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.24656764419475652, "axes": { "intelligence": 84.23469281859694, "calibration": 75.95906818181822, "speed": 69.75909261944061, "cost": 28.241917427052528 }, "jevbench_score": 59.58561232552783 }, { "members": [ "semif-qwen3.5-4b", "djev", "system-one-open", "openjev-sglang", "simplejev-qwen3.8-27b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9657534246575342, "hard": 0.6772727272727272 }, "accuracy": 0.8277153558052435, "ece_hard": 0.06659515593229912, "calibration": 82.7182453460706, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.29854878277153557, "axes": { "intelligence": 84.69261104192611, "calibration": 82.7182453460706, "speed": 69.75909261944061, "cost": 25.749540854883023 }, "jevbench_score": 59.559944858741815 }, { "members": [ "jev-1.13.0", "openjev-razorback16", "simplejev-qwen3.8-27b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.952054794520548, "hard": 0.7318181818181818 }, "accuracy": 0.8445692883895131, "ece_hard": 0.1499999999999999, "calibration": 69.33625, "p50_s": 2.3997448533773422, "p95_s": 4.381843151897192, "cost_usd_per_1000": 0.2095153033707865, "axes": { "intelligence": 85.65374636778746, "calibration": 69.33625, "speed": 69.78178103134502, "cost": 30.363527496908077 }, "jevbench_score": 59.55939076583053 }, { "members": [ "jev-1.13.0", "djev", "open-alternative-jev", "openjev-sglang", "simplejev-qwen3.6-35b-a3b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9726027397260274, "hard": 0.6727272727272727 }, "accuracy": 0.8239700374531835, "ece_hard": 0.07602095966591964, "calibration": 82.39811712911444, "p50_s": 1.7284078150987625, "p95_s": 1.9287681855261325, "cost_usd_per_1000": 0.3348640486891386, "axes": { "intelligence": 84.16469489414695, "calibration": 82.39811712911444, "speed": 74.77073744801535, "cost": 24.253944286318045 }, "jevbench_score": 59.551172980347616 }, { "members": [ "semif-qwen3.5-4b", "djev", "open-alternative-jev", "openjev-sglang", "simplejev-qwen3.8-27b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.958904109589041, "hard": 0.6681818181818182 }, "accuracy": 0.8202247191011236, "ece_hard": 0.053338492493658324, "calibration": 84.38154941725747, "p50_s": 2.3997448533773422, "p95_s": 4.385688671469688, "cost_usd_per_1000": 0.3058392509363296, "axes": { "intelligence": 83.93643628061437, "calibration": 84.38154941725747, "speed": 69.7779713205367, "cost": 25.435203359945874 }, "jevbench_score": 59.543984614928654 }, { "members": [ "semif-qwen3.5-4b", "open-alternative-jev", "system-one-open", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.958904109589041, "hard": 0.6545454545454545 }, "accuracy": 0.8127340823970037, "ece_hard": 0.06789290805288649, "calibration": 81.27379658417708, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.2790451498127341, "axes": { "intelligence": 83.2356787048568, "calibration": 81.27379658417708, "speed": 69.75909261944061, "cost": 26.629765651456538 }, "jevbench_score": 59.5397573074775 }, { "members": [ "djev", "open-alternative-jev", "system-one-open", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9657534246575342, "hard": 0.6681818181818182 }, "accuracy": 0.8220973782771536, "ece_hard": 0.07368414648888992, "calibration": 80.86358118361957, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.282551797752809, "axes": { "intelligence": 84.12821710253218, "calibration": 80.86358118361957, "speed": 69.75909261944061, "cost": 26.467057753975837 }, "jevbench_score": 59.53197435774043 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "open-alternative-jev", "system-one-open", "simplejev-qwen3.8-27b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9657534246575342, "hard": 0.6454545454545455 }, "accuracy": 0.8108614232209738, "ece_hard": 0.13454545454545452, "calibration": 70.28445454545455, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.20340770037453182, "axes": { "intelligence": 83.15473225404733, "calibration": 70.28445454545455, "speed": 69.75909261944061, "cost": 30.748978302447497 }, "jevbench_score": 59.50376424251458 }, { "members": [ "jev-1.13.0", "open-alternative-jev", "system-one-open", "openjev-sglang", "simplejev-qwen3.6-35b-a3b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8020833333333334, "judge": 0.9657534246575342, "hard": 0.6636363636363637 }, "accuracy": 0.8164794007490637, "ece_hard": 0.07413865425703099, "calibration": 81.61236496944007, "p50_s": 1.7359912395477295, "p95_s": 2.0568130582571027, "cost_usd_per_1000": 0.32379373033707864, "axes": { "intelligence": 83.4085201328352, "calibration": 81.61236496944007, "speed": 74.47257649879765, "cost": 24.691946944166034 }, "jevbench_score": 59.481172843855006 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "djev", "openjev-sglang", "simplejev-qwen3.6-35b-a3b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9657534246575342, "hard": 0.6681818181818182 }, "accuracy": 0.8202247191011236, "ece_hard": 0.07687299778624036, "calibration": 82.3283185417962, "p50_s": 1.7284078150987625, "p95_s": 1.9287681855261325, "cost_usd_per_1000": 0.3351372509363296, "axes": { "intelligence": 83.8365504358655, "calibration": 82.3283185417962, "speed": 74.77073744801535, "cost": 24.24331891275166 }, "jevbench_score": 59.47392435359742 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "djev", "openjev-sglang", "simplejev-qwen3.6-35b-a3b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9657534246575342, "hard": 0.6590909090909091 }, "accuracy": 0.8164794007490637, "ece_hard": 0.07714253938170032, "calibration": 82.51525147802737, "p50_s": 1.7284078150987625, "p95_s": 1.9287681855261325, "cost_usd_per_1000": 0.3351372509363296, "axes": { "intelligence": 83.56382316313824, "calibration": 82.51525147802737, "speed": 74.77073744801535, "cost": 24.24331891275166 }, "jevbench_score": 59.45920071501112 }, { "members": [ "jev-1.13.0", "openjev-razorback16", "simplejev-qwen3.6-35b-a3b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.910958904109589, "hard": 0.6727272727272727 }, "accuracy": 0.8071161048689138, "ece_hard": 0.17575757575757558, "calibration": 68.29199242424244, "p50_s": 1.724735751748085, "p95_s": 1.9244587391614911, "cost_usd_per_1000": 0.22107047191011236, "axes": { "intelligence": 82.43866749688668, "calibration": 68.29199242424244, "speed": 74.78968829969378, "cost": 29.664077850575623 }, "jevbench_score": 59.44883257438648 }, { "members": [ "jev-1.13.0", "open-alternative-jev", "system-one-open", "openjev-sglang", "simplejev-qwen3.8-27b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.952054794520548, "hard": 0.6909090909090909 }, "accuracy": 0.8258426966292135, "ece_hard": 0.057933864063032116, "calibration": 84.54951287127423, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.3122385617977528, "axes": { "intelligence": 84.13480697384807, "calibration": 84.54951287127423, "speed": 69.75909261944061, "cost": 25.16540386378908 }, "jevbench_score": 59.446033922672136 }, { "members": [ "semif-qwen3.5-4b", "open-alternative-jev", "system-one-open", "openjev-sglang", "simplejev-qwen3.8-27b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9452054794520548, "hard": 0.6454545454545455 }, "accuracy": 0.8014981273408239, "ece_hard": 0.04494506038169408, "calibration": 84.1812793951261, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.29476893258426967, "axes": { "intelligence": 81.99605645496057, "calibration": 84.1812793951261, "speed": 69.75909261944061, "cost": 25.915548736826693 }, "jevbench_score": 59.43502229055472 }, { "members": [ "semif-qwen3.5-4b", "djev", "open-alternative-jev", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.958904109589041, "hard": 0.6545454545454545 }, "accuracy": 0.8146067415730337, "ece_hard": 0.06998137799296594, "calibration": 81.9467646955024, "p50_s": 2.3997448533773422, "p95_s": 4.385688671469688, "cost_usd_per_1000": 0.290115468164794, "axes": { "intelligence": 83.52734537152347, "calibration": 81.9467646955024, "speed": 69.7779713205367, "cost": 26.122873455514053 }, "jevbench_score": 59.43262933918366 }, { "members": [ "semif-qwen3.5-4b", "djev", "system-one-open", "openjev-sglang", "simplejev-qwen3.6-35b-a3b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9726027397260274, "hard": 0.6363636363636364 }, "accuracy": 0.8108614232209738, "ece_hard": 0.08605919018002611, "calibration": 79.32476791375755, "p50_s": 1.7359912395477295, "p95_s": 2.0568130582571027, "cost_usd_per_1000": 0.31010395131086144, "axes": { "intelligence": 83.36545246990454, "calibration": 79.32476791375755, "speed": 74.47257649879765, "cost": 25.254780999829194 }, "jevbench_score": 59.38595255236417 }, { "members": [ "semif-qwen3.5-4b", "djev", "open-alternative-jev", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9657534246575342, "hard": 0.65 }, "accuracy": 0.8146067415730337, "ece_hard": 0.07234645803005157, "calibration": 81.55790981847032, "p50_s": 2.3997448533773422, "p95_s": 4.385688671469688, "cost_usd_per_1000": 0.290115468164794, "axes": { "intelligence": 83.58276255707764, "calibration": 81.55790981847032, "speed": 69.7779713205367, "cost": 26.122873455514053 }, "jevbench_score": 59.37184205714714 }, { "members": [ "semif-qwen3.5-4b", "djev", "system-one-open", "openjev-sglang", "simplejev-qwen3.6-35b-a3b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9726027397260274, "hard": 0.6454545454545455 }, "accuracy": 0.8146067415730337, "ece_hard": 0.09202041593424758, "calibration": 78.80960844449305, "p50_s": 1.7359912395477295, "p95_s": 2.0568130582571027, "cost_usd_per_1000": 0.31010395131086144, "axes": { "intelligence": 83.6381797426318, "calibration": 78.80960844449305, "speed": 74.47257649879765, "cost": 25.254780999829194 }, "jevbench_score": 59.33773054260465 }, { "members": [ "openjev-razorback16", "openjev-sglang", "simplejev-qwen3.8-27b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.958904109589041, "hard": 0.7045454545454546 }, "accuracy": 0.8333333333333334, "ece_hard": 0.07914834898730214, "calibration": 81.7182171773643, "p50_s": 2.3997448533773422, "p95_s": 4.381843151897192, "cost_usd_per_1000": 0.30087653558052435, "axes": { "intelligence": 84.73567870485678, "calibration": 81.7182171773643, "speed": 69.78178103134502, "cost": 25.648350406207385 }, "jevbench_score": 59.33304961660633 }, { "members": [ "jev-1.13.0", "djev", "system-one-open", "openjev-sglang", "simplejev-qwen3.8-27b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9657534246575342, "hard": 0.7 }, "accuracy": 0.8370786516853933, "ece_hard": 0.0739497960147552, "calibration": 82.99134023947865, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.31601841198501873, "axes": { "intelligence": 85.3744292237443, "calibration": 82.99134023947865, "speed": 69.75909261944061, "cost": 25.008628408420932 }, "jevbench_score": 59.294283017657406 }, { "members": [ "semif-qwen3.5-4b", "open-alternative-jev", "system-one-open", "openjev-sglang", "simplejev-qwen3.6-35b-a3b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8020833333333334, "judge": 0.952054794520548, "hard": 0.6363636363636364 }, "accuracy": 0.8014981273408239, "ece_hard": 0.07802279207037248, "calibration": 79.40213631367638, "p50_s": 1.7359912395477295, "p95_s": 2.0568130582571027, "cost_usd_per_1000": 0.30632410112359554, "axes": { "intelligence": 82.20677667081777, "calibration": 79.40213631367638, "speed": 74.47257649879765, "cost": 25.4145649662033 }, "jevbench_score": 59.28635041442097 }, { "members": [ "semif-qwen3.5-4b", "djev", "open-alternative-jev", "openjev-sglang", "simplejev-qwen3.8-27b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.958904109589041, "hard": 0.6590909090909091 }, "accuracy": 0.8164794007490637, "ece_hard": 0.06412790549805247, "calibration": 83.14907006598463, "p50_s": 2.3997448533773422, "p95_s": 4.385688671469688, "cost_usd_per_1000": 0.3058392509363296, "axes": { "intelligence": 83.66370900788709, "calibration": 83.14907006598463, "speed": 69.7779713205367, "cost": 25.435203359945874 }, "jevbench_score": 59.27710918497017 }, { "members": [ "system-one-open", "openjev-razorback16", "simplejev-qwen3.6-35b-a3b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.910958904109589, "hard": 0.6318181818181818 }, "accuracy": 0.7921348314606742, "ece_hard": 0.17272727272727248, "calibration": 65.07360606060608, "p50_s": 1.7357386276125908, "p95_s": 2.0568130582571027, "cost_usd_per_1000": 0.19603717228464418, "axes": { "intelligence": 81.50306143628062, "calibration": 65.07360606060608, "speed": 74.47320850632062, "cost": 31.22984711633177 }, "jevbench_score": 59.26348405642471 }, { "members": [ "openjev-razorback16", "openjev-sglang", "simplejev-qwen3.8-27b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.958904109589041, "hard": 0.7 }, "accuracy": 0.8333333333333334, "ece_hard": 0.08241832811732093, "calibration": 81.1172309151026, "p50_s": 2.3997448533773422, "p95_s": 4.381843151897192, "cost_usd_per_1000": 0.30087653558052435, "axes": { "intelligence": 84.89098173515983, "calibration": 81.1172309151026, "speed": 69.78178103134502, "cost": 25.648350406207385 }, "jevbench_score": 59.2507756744737 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "system-one-open", "openjev-razorback16", "simplejev-qwen3.8-27b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.958904109589041, "hard": 0.6863636363636364 }, "accuracy": 0.8295880149812734, "ece_hard": 0.1072727272727273, "calibration": 73.48747727272728, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.24684084644194754, "axes": { "intelligence": 84.77355749273559, "calibration": 73.48747727272728, "speed": 69.75909261944061, "cost": 28.227489190476646 }, "jevbench_score": 59.18160037007864 }, { "members": [ "semif-qwen3.5-4b", "open-alternative-jev", "system-one-open", "openjev-sglang", "simplejev-qwen3.6-35b-a3b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8020833333333334, "judge": 0.952054794520548, "hard": 0.6454545454545455 }, "accuracy": 0.8052434456928839, "ece_hard": 0.08695537589279544, "calibration": 78.56136575502805, "p50_s": 1.7359912395477295, "p95_s": 2.0568130582571027, "cost_usd_per_1000": 0.30632410112359554, "axes": { "intelligence": 82.47950394354504, "calibration": 78.56136575502805, "speed": 74.47257649879765, "cost": 25.4145649662033 }, "jevbench_score": 59.177761215278814 }, { "members": [ "semif-qwen3.5-4b", "djev", "open-alternative-jev", "openjev-sglang", "simplejev-qwen3.6-35b-a3b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8020833333333334, "judge": 0.952054794520548, "hard": 0.6409090909090909 }, "accuracy": 0.8033707865168539, "ece_hard": 0.08307622694231523, "calibration": 79.8055138123608, "p50_s": 1.7284078150987625, "p95_s": 1.9287681855261325, "cost_usd_per_1000": 0.31739441947565544, "axes": { "intelligence": 82.3431403071814, "calibration": 79.8055138123608, "speed": 74.77073744801535, "cost": 24.95202140231555 }, "jevbench_score": 59.173114783542545 }, { "members": [ "semif-qwen3.5-4b", "djev", "open-alternative-jev", "system-one-open", "openjev-sglang" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9657534246575342, "hard": 0.6318181818181818 }, "accuracy": 0.8052434456928839, "ece_hard": 0.16181818181818153, "calibration": 65.57418181818184, "p50_s": 1.3879805654287338, "p95_s": 2.030777859687805, "cost_usd_per_1000": 0.21672367041198504, "axes": { "intelligence": 82.74564134495643, "calibration": 65.57418181818184, "speed": 75.49954194936167, "cost": 29.922809582245947 }, "jevbench_score": 59.17062725519932 }, { "members": [ "semif-qwen3.5-4b", "system-one-open", "openjev-razorback16", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9726027397260274, "hard": 0.6590909090909091 }, "accuracy": 0.8202247191011236, "ece_hard": 0.04103541583773958, "calibration": 84.45520827071792, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.3224782958801498, "axes": { "intelligence": 84.04727065172271, "calibration": 84.45520827071792, "speed": 69.75909261944061, "cost": 24.744985295669025 }, "jevbench_score": 59.16427114735916 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "djev", "open-alternative-jev", "simplejev-qwen3.6-35b-a3b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.952054794520548, "hard": 0.6545454545454545 }, "accuracy": 0.8127340823970037, "ece_hard": 0.1681818181818182, "calibration": 66.89131818181818, "p50_s": 1.7256089895963669, "p95_s": 1.9284205481410026, "cost_usd_per_1000": 0.2260331872659176, "axes": { "intelligence": 83.33556454960565, "calibration": 66.89131818181818, "speed": 74.77855855005357, "cost": 29.374833729784655 }, "jevbench_score": 59.154550777571735 }, { "members": [ "djev", "open-alternative-jev", "system-one-open", "openjev-sglang", "simplejev-qwen3.8-27b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.958904109589041, "hard": 0.6727272727272727 }, "accuracy": 0.8202247191011236, "ece_hard": 0.08001405317038326, "calibration": 81.29591471392007, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.29827558052434455, "axes": { "intelligence": 83.78113325031134, "calibration": 81.29591471392007, "speed": 69.75909261944061, "cost": 25.761469011035786 }, "jevbench_score": 59.14889013132087 }, { "members": [ "jev-1.13.0", "djev", "system-one-open", "openjev-razorback16", "openjev-sglang" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.952054794520548, "hard": 0.6818181818181818 }, "accuracy": 0.8239700374531835, "ece_hard": 0.12181818181818166, "calibration": 72.09718181818184, "p50_s": 1.3879805654287338, "p95_s": 2.030777859687805, "cost_usd_per_1000": 0.2776264456928839, "axes": { "intelligence": 84.15374636778748, "calibration": 72.09718181818184, "speed": 75.49954194936167, "cost": 26.696175007669808 }, "jevbench_score": 59.13522796135681 }, { "members": [ "jev-1.13.0", "system-one-open", "openjev-razorback16", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9657534246575342, "hard": 0.7045454545454546 }, "accuracy": 0.8370786516853933, "ece_hard": 0.04803090643351003, "calibration": 85.41835680211746, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.3399479250936329, "axes": { "intelligence": 85.21912619344127, "calibration": 85.41835680211746, "speed": 69.75909261944061, "cost": 24.05762815725552 }, "jevbench_score": 59.12014357544211 }, { "members": [ "djev", "system-one-open", "openjev-razorback16", "openjev-sglang", "simplejev-qwen3.8-27b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9657534246575342, "hard": 0.7045454545454546 }, "accuracy": 0.8370786516853933, "ece_hard": 0.04074823179535705, "calibration": 85.65673534877413, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.3417087265917603, "axes": { "intelligence": 85.21912619344127, "calibration": 85.65673534877413, "speed": 69.75909261944061, "cost": 23.990317900106177 }, "jevbench_score": 59.11992237712195 }, { "members": [ "semif-qwen3.5-4b", "system-one-open", "openjev-sglang" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9726027397260274, "hard": 0.6090909090909091 }, "accuracy": 0.797752808988764, "ece_hard": 0.20606060606060583, "calibration": 59.190939393939416, "p50_s": 1.3879805654287338, "p95_s": 2.030777859687805, "cost_usd_per_1000": 0.16860101123595506, "axes": { "intelligence": 82.25560398505604, "calibration": 59.190939393939416, "speed": 75.49954194936167, "cost": 33.194194746698045 }, "jevbench_score": 59.10262846561874 }, { "members": [ "semif-qwen3.5-4b", "system-one-open", "openjev-razorback16", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9726027397260274, "hard": 0.6590909090909091 }, "accuracy": 0.8202247191011236, "ece_hard": 0.046753314740477864, "calibration": 84.10230143103323, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.3224782958801498, "axes": { "intelligence": 84.04727065172271, "calibration": 84.10230143103323, "speed": 69.75909261944061, "cost": 24.744985295669025 }, "jevbench_score": 59.102367701164646 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "open-alternative-jev", "system-one-open", "simplejev-qwen3.6-35b-a3b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9794520547945206, "hard": 0.6318181818181818 }, "accuracy": 0.8089887640449438, "ece_hard": 0.17090909090909073, "calibration": 65.61859090909093, "p50_s": 1.7357386276125908, "p95_s": 2.0568130582571027, "cost_usd_per_1000": 0.21496286891385769, "axes": { "intelligence": 83.12920298879204, "calibration": 65.61859090909093, "speed": 74.47320850632062, "cost": 30.029096512085246 }, "jevbench_score": 59.09907885734304 }, { "members": [ "semif-qwen3.5-4b", "djev", "open-alternative-jev", "system-one-open", "simplejev-qwen3.8-27b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.958904109589041, "hard": 0.6318181818181818 }, "accuracy": 0.8052434456928839, "ece_hard": 0.15636363636363618, "calibration": 66.60263636363639, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.1894447191011236, "axes": { "intelligence": 82.84552718970528, "calibration": 66.60263636363639, "speed": 69.75909261944061, "cost": 31.675524894125246 }, "jevbench_score": 59.09099221206457 }, { "members": [ "openjev-razorback16", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9246575342465754, "hard": 0.6772727272727272 }, "accuracy": 0.8127340823970037, "ece_hard": 0.08775065245351754, "calibration": 79.92148395066893, "p50_s": 2.3997448533773422, "p95_s": 4.381843151897192, "cost_usd_per_1000": 0.2851527528089888, "axes": { "intelligence": 82.95859277708593, "calibration": 79.92148395066893, "speed": 69.78178103134502, "cost": 26.347672943935677 }, "jevbench_score": 59.088410478480625 }, { "members": [ "djev", "open-alternative-jev", "system-one-open", "openjev-sglang", "simplejev-qwen3.6-35b-a3b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8020833333333334, "judge": 0.9657534246575342, "hard": 0.6363636363636364 }, "accuracy": 0.8052434456928839, "ece_hard": 0.09627426829518855, "calibration": 78.4193919682005, "p50_s": 1.7359912395477295, "p95_s": 2.0568130582571027, "cost_usd_per_1000": 0.3098307490636704, "axes": { "intelligence": 82.5903383146534, "calibration": 78.4193919682005, "speed": 74.47257649879765, "cost": 25.266264490204037 }, "jevbench_score": 59.084359678372394 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "open-alternative-jev", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.958904109589041, "hard": 0.6772727272727272 }, "accuracy": 0.8239700374531835, "ece_hard": 0.08136844551974219, "calibration": 81.29196014043004, "p50_s": 2.3997448533773422, "p95_s": 4.385688671469688, "cost_usd_per_1000": 0.3040784494382022, "axes": { "intelligence": 84.20916355334164, "calibration": 81.29196014043004, "speed": 69.7779713205367, "cost": 25.510430738824738 }, "jevbench_score": 59.08275933400893 }, { "members": [ "jev-1.13.0", "open-alternative-jev", "openjev-razorback16", "openjev-sglang", "simplejev-qwen3.8-27b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.958904109589041, "hard": 0.7136363636363636 }, "accuracy": 0.8389513108614233, "ece_hard": 0.029809600293121106, "calibration": 88.07844436337939, "p50_s": 2.3997448533773422, "p95_s": 4.385688671469688, "cost_usd_per_1000": 0.36296217602996256, "axes": { "intelligence": 85.30007264425073, "calibration": 88.07844436337939, "speed": 69.7779713205367, "cost": 23.20415890159012 }, "jevbench_score": 59.05760103046775 }, { "members": [ "jev-1.13.0", "system-one-open", "openjev-razorback16", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9657534246575342, "hard": 0.7045454545454546 }, "accuracy": 0.8370786516853933, "ece_hard": 0.0483031147394982, "calibration": 85.05154715608063, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.3399479250936329, "axes": { "intelligence": 85.21912619344127, "calibration": 85.05154715608063, "speed": 69.75909261944061, "cost": 24.05762815725552 }, "jevbench_score": 59.056571618187505 }, { "members": [ "semif-qwen3.5-4b", "djev", "open-alternative-jev", "openjev-sglang", "simplejev-qwen3.6-35b-a3b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8020833333333334, "judge": 0.9452054794520548, "hard": 0.6363636363636364 }, "accuracy": 0.799625468164794, "ece_hard": 0.08745154801868092, "calibration": 79.33690309531289, "p50_s": 1.7284078150987625, "p95_s": 1.9287681855261325, "cost_usd_per_1000": 0.31739441947565544, "axes": { "intelligence": 82.01499584889996, "calibration": 79.33690309531289, "speed": 74.77073744801535, "cost": 24.95202140231555 }, "jevbench_score": 59.027104142246436 }, { "members": [ "semif-qwen3.5-4b", "system-one-open", "openjev-razorback16", "openjev-sglang", "simplejev-qwen3.8-27b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.958904109589041, "hard": 0.6636363636363637 }, "accuracy": 0.8202247191011236, "ece_hard": 0.03791335292444814, "calibration": 85.6459341668208, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.3382020786516854, "axes": { "intelligence": 84.0917393109174, "calibration": 85.6459341668208, "speed": 69.75909261944061, "cost": 24.12471182430835 }, "jevbench_score": 59.0039057764239 }, { "members": [ "semif-qwen3.5-4b", "system-one-open", "openjev-razorback16", "openjev-sglang", "simplejev-qwen3.8-27b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.958904109589041, "hard": 0.6590909090909091 }, "accuracy": 0.8183520599250936, "ece_hard": 0.035177860619374404, "calibration": 85.74129162014475, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.3382020786516854, "axes": { "intelligence": 83.95537567455376, "calibration": 85.74129162014475, "speed": 69.75909261944061, "cost": 24.12471182430835 }, "jevbench_score": 58.99638104979315 }, { "members": [ "open-alternative-jev", "system-one-open", "openjev-razorback16", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9657534246575342, "hard": 0.6727272727272727 }, "accuracy": 0.8239700374531835, "ece_hard": 0.05471738415643128, "calibration": 83.09149794902422, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.3222050936329588, "axes": { "intelligence": 84.26458073889582, "calibration": 83.09149794902422, "speed": 69.75909261944061, "cost": 24.756027947373553 }, "jevbench_score": 58.968605818752735 }, { "members": [ "openjev-razorback16", "openjev-sglang", "simplejev-qwen3.6-35b-a3b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9246575342465754, "hard": 0.6681818181818182 }, "accuracy": 0.8089887640449438, "ece_hard": 0.10105857408597478, "calibration": 77.6922490135228, "p50_s": 1.7284078150987625, "p95_s": 1.928032819926739, "cost_usd_per_1000": 0.3124317041198502, "axes": { "intelligence": 82.68586550435866, "calibration": 77.6922490135228, "speed": 74.77239356263168, "cost": 25.157347071081787 }, "jevbench_score": 58.95949752881324 }, { "members": [ "openjev-razorback16", "openjev-sglang", "simplejev-qwen3.6-35b-a3b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9246575342465754, "hard": 0.6681818181818182 }, "accuracy": 0.8089887640449438, "ece_hard": 0.10141412852683555, "calibration": 77.6111096919567, "p50_s": 1.7284078150987625, "p95_s": 1.928032819926739, "cost_usd_per_1000": 0.3124317041198502, "axes": { "intelligence": 82.68586550435866, "calibration": 77.6111096919567, "speed": 74.77239356263168, "cost": 25.157347071081787 }, "jevbench_score": 58.94409763885292 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "system-one-open", "openjev-razorback16", "openjev-sglang" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.958904109589041, "hard": 0.6681818181818182 }, "accuracy": 0.8220973782771536, "ece_hard": 0.13545454545454533, "calibration": 70.62379545454547, "p50_s": 1.3879805654287338, "p95_s": 2.030777859687805, "cost_usd_per_1000": 0.274119797752809, "axes": { "intelligence": 84.22810294728103, "calibration": 70.62379545454547, "speed": 75.49954194936167, "cost": 26.861787918386057 }, "jevbench_score": 58.93480167588876 }, { "members": [ "openjev-razorback16", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9315068493150684, "hard": 0.6863636363636364 }, "accuracy": 0.8183520599250936, "ece_hard": 0.10413640022344417, "calibration": 78.49327221533595, "p50_s": 2.3997448533773422, "p95_s": 4.381843151897192, "cost_usd_per_1000": 0.2851527528089888, "axes": { "intelligence": 83.42310087173101, "calibration": 78.49327221533595, "speed": 69.78178103134502, "cost": 26.347672943935677 }, "jevbench_score": 58.90481130783985 }, { "members": [ "semif-qwen3.5-4b", "djev", "open-alternative-jev", "system-one-open", "simplejev-qwen3.6-35b-a3b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.952054794520548, "hard": 0.6181818181818182 }, "accuracy": 0.797752808988764, "ece_hard": 0.17636363636363617, "calibration": 63.60263636363638, "p50_s": 1.7357386276125908, "p95_s": 2.0568130582571027, "cost_usd_per_1000": 0.20099988764044946, "axes": { "intelligence": 82.24465545869656, "calibration": 63.60263636363638, "speed": 74.47320850632062, "cost": 30.904125560541516 }, "jevbench_score": 58.90468419468217 }, { "members": [ "jev-1.13.0", "open-alternative-jev", "system-one-open", "openjev-razorback16", "simplejev-qwen3.6-35b-a3b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.910958904109589, "hard": 0.6409090909090909 }, "accuracy": 0.795880149812734, "ece_hard": 0.13181818181818158, "calibration": 71.49643181818185, "p50_s": 1.7357386276125908, "p95_s": 2.0568130582571027, "cost_usd_per_1000": 0.2581228127340824, "axes": { "intelligence": 81.77578870900788, "calibration": 71.49643181818185, "speed": 74.47320850632062, "cost": 27.645208332489517 }, "jevbench_score": 58.902300951719916 }, { "members": [ "jev-1.13.0", "system-one-open", "openjev-razorback16", "openjev-sglang", "simplejev-qwen3.6-35b-a3b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9657534246575342, "hard": 0.6681818181818182 }, "accuracy": 0.8202247191011236, "ece_hard": 0.06027091837036038, "calibration": 83.54837107813347, "p50_s": 1.7359912395477295, "p95_s": 2.0568130582571027, "cost_usd_per_1000": 0.36722687640449436, "axes": { "intelligence": 83.8365504358655, "calibration": 83.54837107813347, "speed": 74.47257649879765, "cost": 23.05196624185075 }, "jevbench_score": 58.88693296854332 }, { "members": [ "open-alternative-jev", "openjev-razorback16", "simplejev-qwen3.8-27b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8020833333333334, "judge": 0.8904109589041096, "hard": 0.6590909090909091 }, "accuracy": 0.7940074906367042, "ece_hard": 0.16363636363636327, "calibration": 67.31203030303034, "p50_s": 2.3997448533773422, "p95_s": 4.385688671469688, "cost_usd_per_1000": 0.19177247191011235, "axes": { "intelligence": 81.1625674553757, "calibration": 67.31203030303034, "speed": 69.7779713205367, "cost": 31.516412012284874 }, "jevbench_score": 58.8743186123869 }, { "members": [ "open-alternative-jev", "system-one-open", "simplejev-qwen3.6-35b-a3b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.7708333333333334, "judge": 0.8904109589041096, "hard": 0.6045454545454545 }, "accuracy": 0.7659176029962547, "ece_hard": 0.20151515151515162, "calibration": 59.4306515151515, "p50_s": 1.7357386276125908, "p95_s": 2.0568130582571027, "cost_usd_per_1000": 0.15260402621722846, "axes": { "intelligence": 78.65120381901205, "calibration": 59.4306515151515, "speed": 74.47320850632062, "cost": 34.49302024159303 }, "jevbench_score": 58.86562892317323 }, { "members": [ "jev-1.13.0", "system-one-open", "openjev-razorback16", "openjev-sglang", "simplejev-qwen3.6-35b-a3b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9657534246575342, "hard": 0.6727272727272727 }, "accuracy": 0.8220973782771536, "ece_hard": 0.06809674848473898, "calibration": 83.1342207543425, "p50_s": 1.7359912395477295, "p95_s": 2.0568130582571027, "cost_usd_per_1000": 0.36722687640449436, "axes": { "intelligence": 83.97291407222914, "calibration": 83.1342207543425, "speed": 74.47257649879765, "cost": 23.05196624185075 }, "jevbench_score": 58.83772231317531 }, { "members": [ "jev-1.13.0", "open-alternative-jev", "openjev-razorback16", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9726027397260274, "hard": 0.6863636363636364 }, "accuracy": 0.8314606741573034, "ece_hard": 0.05227671753982764, "calibration": 84.90152014854753, "p50_s": 2.3997448533773422, "p95_s": 4.385688671469688, "cost_usd_per_1000": 0.3472383932584269, "axes": { "intelligence": 84.86545246990454, "calibration": 84.90152014854753, "speed": 69.7779713205367, "cost": 23.781167860415607 }, "jevbench_score": 58.80299935966425 }, { "members": [ "jev-1.13.0", "open-alternative-jev", "openjev-razorback16", "openjev-sglang", "simplejev-qwen3.8-27b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.958904109589041, "hard": 0.7090909090909091 }, "accuracy": 0.8370786516853933, "ece_hard": 0.041364438789567165, "calibration": 86.67338964485324, "p50_s": 2.3997448533773422, "p95_s": 4.385688671469688, "cost_usd_per_1000": 0.36296217602996256, "axes": { "intelligence": 85.1637090078871, "calibration": 86.67338964485324, "speed": 69.7779713205367, "cost": 23.20415890159012 }, "jevbench_score": 58.79713023993588 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "openjev-razorback16", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6818181818181818 }, "accuracy": 0.8314606741573034, "ece_hard": 0.05628101907579171, "calibration": 84.71379337777378, "p50_s": 2.3997448533773422, "p95_s": 4.381843151897192, "cost_usd_per_1000": 0.34751159550561794, "axes": { "intelligence": 85.02075550020756, "calibration": 84.71379337777378, "speed": 69.78178103134502, "cost": 23.77092098861135 }, "jevbench_score": 58.7918040428234 }, { "members": [ "jev-1.13.0", "system-one-open", "openjev-razorback16", "openjev-sglang", "simplejev-qwen3.8-27b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9657534246575342, "hard": 0.7 }, "accuracy": 0.8352059925093633, "ece_hard": 0.05348602992188112, "calibration": 85.67922984563108, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.3556717078651685, "axes": { "intelligence": 85.08276255707763, "calibration": 85.67922984563108, "speed": 69.75909261944061, "cost": 23.468520390228022 }, "jevbench_score": 58.776120514906495 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "system-one-open", "openjev-razorback16", "simplejev-qwen3.6-35b-a3b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.958904109589041, "hard": 0.6545454545454545 }, "accuracy": 0.8164794007490637, "ece_hard": 0.15090909090909077, "calibration": 69.17934090909093, "p50_s": 1.7357386276125908, "p95_s": 2.0568130582571027, "cost_usd_per_1000": 0.2583960149812734, "axes": { "intelligence": 83.81901203819014, "calibration": 69.17934090909093, "speed": 74.47320850632062, "cost": 27.631425651228327 }, "jevbench_score": 58.77336803461405 }, { "members": [ "djev", "system-one-open", "openjev-razorback16", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.958904109589041, "hard": 0.6727272727272727 }, "accuracy": 0.8220973782771536, "ece_hard": 0.06163325465482573, "calibration": 82.60669006696236, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.3259849438202247, "axes": { "intelligence": 84.072799916978, "calibration": 82.60669006696236, "speed": 69.75909261944061, "cost": 24.604073743372496 }, "jevbench_score": 58.758357029438635 }, { "members": [ "jev-1.13.0", "djev", "system-one-open", "openjev-razorback16", "simplejev-qwen3.6-35b-a3b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9315068493150684, "hard": 0.6636363636363637 }, "accuracy": 0.8108614232209738, "ece_hard": 0.1445454545454549, "calibration": 70.14070454545451, "p50_s": 1.7357386276125908, "p95_s": 2.0568130582571027, "cost_usd_per_1000": 0.26190266292134834, "axes": { "intelligence": 83.0329493565795, "calibration": 70.14070454545451, "speed": 74.47320850632062, "cost": 27.455802574600654 }, "jevbench_score": 58.74402488018098 }, { "members": [ "djev", "open-alternative-jev", "system-one-open", "openjev-razorback16", "simplejev-qwen3.8-27b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9383561643835616, "hard": 0.6590909090909091 }, "accuracy": 0.8127340823970037, "ece_hard": 0.12454545454545432, "calibration": 70.48445454545455, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.2326046629213483, "axes": { "intelligence": 83.38003320880034, "calibration": 70.48445454545455, "speed": 69.75909261944061, "cost": 29.00144750240858 }, "jevbench_score": 58.72107210589887 }, { "members": [ "jev-1.13.0", "system-one-open", "openjev-razorback16", "openjev-sglang", "simplejev-qwen3.8-27b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9657534246575342, "hard": 0.7 }, "accuracy": 0.8370786516853933, "ece_hard": 0.0577494264786445, "calibration": 85.00658995379015, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.3556717078651685, "axes": { "intelligence": 85.3744292237443, "calibration": 85.00658995379015, "speed": 69.75909261944061, "cost": 23.468520390228022 }, "jevbench_score": 58.71062925738846 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "openjev-razorback16", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6727272727272727 }, "accuracy": 0.8277153558052435, "ece_hard": 0.05841082819248307, "calibration": 84.19579127358368, "p50_s": 2.3997448533773422, "p95_s": 4.381843151897192, "cost_usd_per_1000": 0.34751159550561794, "axes": { "intelligence": 84.74802822748029, "calibration": 84.19579127358368, "speed": 69.78178103134502, "cost": 23.77092098861135 }, "jevbench_score": 58.65459101400207 }, { "members": [ "djev", "system-one-open", "openjev-razorback16", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.958904109589041, "hard": 0.6681818181818182 }, "accuracy": 0.8202247191011236, "ece_hard": 0.06558306093344685, "calibration": 81.99598082033694, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.3259849438202247, "axes": { "intelligence": 83.93643628061437, "calibration": 81.99598082033694, "speed": 69.75909261944061, "cost": 24.604073743372496 }, "jevbench_score": 58.62565846434005 }, { "members": [ "semif-qwen3.5-4b", "system-one-open", "simplejev-qwen3.6-35b-a3b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9794520547945206, "hard": 0.5863636363636363 }, "accuracy": 0.7902621722846442, "ece_hard": 0.23636363636363628, "calibration": 56.246136363636374, "p50_s": 1.7357386276125908, "p95_s": 2.0568130582571027, "cost_usd_per_1000": 0.15287722846441948, "axes": { "intelligence": 81.76556662515567, "calibration": 56.246136363636374, "speed": 74.47320850632062, "cost": 34.46971597826156 }, "jevbench_score": 58.617221535996244 }, { "members": [ "open-alternative-jev", "system-one-open", "openjev-razorback16", "openjev-sglang", "simplejev-qwen3.8-27b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.958904109589041, "hard": 0.6863636363636364 }, "accuracy": 0.8258426966292135, "ece_hard": 0.06106052998424248, "calibration": 83.25864343850691, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.33792887640449437, "axes": { "intelligence": 84.19022415940225, "calibration": 83.25864343850691, "speed": 69.75909261944061, "cost": 24.135240870550234 }, "jevbench_score": 58.61190866617968 }, { "members": [ "jev-1.13.0", "open-alternative-jev", "openjev-razorback16", "openjev-sglang", "simplejev-qwen3.6-35b-a3b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.952054794520548, "hard": 0.6727272727272727 }, "accuracy": 0.8202247191011236, "ece_hard": 0.07286391849216306, "calibration": 82.5001980866374, "p50_s": 1.7284078150987625, "p95_s": 1.9287681855261325, "cost_usd_per_1000": 0.3745173445692884, "axes": { "intelligence": 83.8810190950602, "calibration": 82.5001980866374, "speed": 74.77073744801535, "cost": 22.79584193629418 }, "jevbench_score": 58.60389716757958 }, { "members": [ "djev", "open-alternative-jev", "system-one-open", "openjev-razorback16", "openjev-sglang" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9452054794520548, "hard": 0.6409090909090909 }, "accuracy": 0.8033707865168539, "ece_hard": 0.14636363636363614, "calibration": 68.69463636363639, "p50_s": 1.3879805654287338, "p95_s": 2.030777859687805, "cost_usd_per_1000": 0.25988361423220974, "axes": { "intelligence": 82.44302615193027, "calibration": 68.69463636363639, "speed": 75.49954194936167, "cost": 27.55663306239589 }, "jevbench_score": 58.58842265823762 }, { "members": [ "open-alternative-jev", "system-one-open", "openjev-razorback16", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9657534246575342, "hard": 0.6681818181818182 }, "accuracy": 0.8202247191011236, "ece_hard": 0.07055646192321605, "calibration": 81.27764536021317, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.3222050936329588, "axes": { "intelligence": 83.8365504358655, "calibration": 81.27764536021317, "speed": 69.75909261944061, "cost": 24.756027947373553 }, "jevbench_score": 58.56950792276229 }, { "members": [ "jev-1.13.0", "djev", "openjev-razorback16", "openjev-sglang", "simplejev-qwen3.6-35b-a3b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.952054794520548, "hard": 0.6863636363636364 }, "accuracy": 0.8239700374531835, "ece_hard": 0.07744800365483456, "calibration": 82.66068621400152, "p50_s": 1.7284078150987625, "p95_s": 1.928032819926739, "cost_usd_per_1000": 0.3782971947565543, "axes": { "intelligence": 83.99844333748445, "calibration": 82.66068621400152, "speed": 74.77239356263168, "cost": 22.66500637551316 }, "jevbench_score": 58.56886975322332 }, { "members": [ "djev", "open-alternative-jev", "openjev-razorback16", "openjev-sglang", "simplejev-qwen3.8-27b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.958904109589041, "hard": 0.7 }, "accuracy": 0.8314606741573034, "ece_hard": 0.06079504497288242, "calibration": 83.9596799541093, "p50_s": 2.3997448533773422, "p95_s": 4.385688671469688, "cost_usd_per_1000": 0.3489991947565543, "axes": { "intelligence": 84.59931506849315, "calibration": 83.9596799541093, "speed": 69.7779713205367, "cost": 23.715267252530197 }, "jevbench_score": 58.552574570201465 }, { "members": [ "jev-1.13.0", "djev", "openjev-razorback16", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9657534246575342, "hard": 0.6954545454545454 }, "accuracy": 0.8333333333333334, "ece_hard": 0.06383548292378885, "calibration": 83.85961925727452, "p50_s": 2.3997448533773422, "p95_s": 4.381843151897192, "cost_usd_per_1000": 0.3510182434456929, "axes": { "intelligence": 84.94639892071399, "calibration": 83.85961925727452, "speed": 69.78178103134502, "cost": 23.640109341760308 }, "jevbench_score": 58.54938632239577 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "openjev-razorback16", "openjev-sglang", "simplejev-qwen3.8-27b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9657534246575342, "hard": 0.6863636363636364 }, "accuracy": 0.8314606741573034, "ece_hard": 0.05501293721552285, "calibration": 85.41052144238817, "p50_s": 2.3997448533773422, "p95_s": 4.381843151897192, "cost_usd_per_1000": 0.3632353782771535, "axes": { "intelligence": 84.9653383146534, "calibration": 85.41052144238817, "speed": 69.78178103134502, "cost": 23.194355764692247 }, "jevbench_score": 58.542246235565536 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "openjev-razorback16", "openjev-sglang", "simplejev-qwen3.6-35b-a3b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9657534246575342, "hard": 0.6681818181818182 }, "accuracy": 0.8220973782771536, "ece_hard": 0.0765997285077929, "calibration": 81.88031568730136, "p50_s": 1.7284078150987625, "p95_s": 1.9287681855261325, "cost_usd_per_1000": 0.3747905468164794, "axes": { "intelligence": 84.12821710253218, "calibration": 81.88031568730136, "speed": 74.77073744801535, "cost": 22.786341150167118 }, "jevbench_score": 58.530450011968924 }, { "members": [ "semif-qwen3.5-4b", "open-alternative-jev", "openjev-razorback16", "openjev-sglang", "simplejev-qwen3.8-27b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.958904109589041, "hard": 0.6727272727272727 }, "accuracy": 0.8239700374531835, "ece_hard": 0.05668421927695331, "calibration": 83.59842561276567, "p50_s": 2.3997448533773422, "p95_s": 4.385688671469688, "cost_usd_per_1000": 0.3454925468164794, "axes": { "intelligence": 84.36446658364467, "calibration": 83.59842561276567, "speed": 69.7779713205367, "cost": 23.846839511911085 }, "jevbench_score": 58.52975521907663 }, { "members": [ "jev-1.13.0", "djev", "openjev-razorback16", "openjev-sglang", "simplejev-qwen3.8-27b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9657534246575342, "hard": 0.7181818181818181 }, "accuracy": 0.8426966292134831, "ece_hard": 0.054517867314106475, "calibration": 85.00939726893014, "p50_s": 2.3997448533773422, "p95_s": 4.381843151897192, "cost_usd_per_1000": 0.36674202621722846, "axes": { "intelligence": 85.62821710253218, "calibration": 85.00939726893014, "speed": 69.78178103134502, "cost": 23.06917959713782 }, "jevbench_score": 58.50789988281842 }, { "members": [ "semif-qwen3.5-4b", "open-alternative-jev", "openjev-razorback16", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.958904109589041, "hard": 0.6590909090909091 }, "accuracy": 0.8164794007490637, "ece_hard": 0.06413018654097975, "calibration": 82.020539948123, "p50_s": 2.3997448533773422, "p95_s": 4.385688671469688, "cost_usd_per_1000": 0.3297687640449438, "axes": { "intelligence": 83.66370900788709, "calibration": 82.020539948123, "speed": 69.7779713205367, "cost": 24.45371450368343 }, "jevbench_score": 58.49661404643271 }, { "members": [ "jev-1.13.0", "open-alternative-jev", "openjev-razorback16", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9726027397260274, "hard": 0.6727272727272727 }, "accuracy": 0.8258426966292135, "ece_hard": 0.06403926562420141, "calibration": 83.42941250586745, "p50_s": 2.3997448533773422, "p95_s": 4.385688671469688, "cost_usd_per_1000": 0.3472383932584269, "axes": { "intelligence": 84.45636156081362, "calibration": 83.42941250586745, "speed": 69.7779713205367, "cost": 23.781167860415607 }, "jevbench_score": 58.4757455449608 }, { "members": [ "djev", "open-alternative-jev", "system-one-open", "openjev-razorback16", "simplejev-qwen3.6-35b-a3b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9246575342465754, "hard": 0.6272727272727273 }, "accuracy": 0.7940074906367042, "ece_hard": 0.15181818181818163, "calibration": 67.68768181818183, "p50_s": 1.7357386276125908, "p95_s": 2.0568130582571027, "cost_usd_per_1000": 0.24415983146067416, "axes": { "intelligence": 81.75025944375261, "calibration": 67.68768181818183, "speed": 74.47320850632062, "cost": 28.3697735053333 }, "jevbench_score": 58.47411780818955 }, { "members": [ "semif-qwen3.5-4b", "openjev-razorback16", "simplejev-qwen3.8-27b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9657534246575342, "hard": 0.6681818181818182 }, "accuracy": 0.8239700374531835, "ece_hard": 0.18181818181818152, "calibration": 62.95218181818184, "p50_s": 2.3997448533773422, "p95_s": 4.381843151897192, "cost_usd_per_1000": 0.19204567415730334, "axes": { "intelligence": 84.41988376919885, "calibration": 62.95218181818184, "speed": 69.78178103134502, "cost": 31.497864127097145 }, "jevbench_score": 58.46146212093844 }, { "members": [ "open-alternative-jev", "system-one-open", "openjev-razorback16", "openjev-sglang", "simplejev-qwen3.6-35b-a3b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8020833333333334, "judge": 0.9178082191780822, "hard": 0.65 }, "accuracy": 0.797752808988764, "ece_hard": 0.06718188973270682, "calibration": 80.96444005409782, "p50_s": 1.7359912395477295, "p95_s": 2.0568130582571027, "cost_usd_per_1000": 0.34948404494382024, "axes": { "intelligence": 81.65696347031965, "calibration": 80.96444005409782, "speed": 74.47257649879765, "cost": 23.697179391628367 }, "jevbench_score": 58.44470145654107 }, { "members": [ "open-alternative-jev", "openjev-razorback16", "simplejev-qwen3.6-35b-a3b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6363636363636364 }, "accuracy": 0.7940074906367042, "ece_hard": 0.19393939393939358, "calibration": 62.34114393939397, "p50_s": 1.7256089895963669, "p95_s": 1.9284205481410026, "cost_usd_per_1000": 0.20332764044943819, "axes": { "intelligence": 81.33976753839768, "calibration": 62.34114393939397, "speed": 74.77855855005357, "cost": 30.75410737493995 }, "jevbench_score": 58.437210596967645 }, { "members": [ "jev-1.13.0", "djev", "open-alternative-jev", "openjev-razorback16", "openjev-sglang" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9452054794520548, "hard": 0.6636363636363637 }, "accuracy": 0.8127340823970037, "ece_hard": 0.15545454545454568, "calibration": 69.55504545454544, "p50_s": 1.3671194911003113, "p95_s": 1.6307271040976046, "cost_usd_per_1000": 0.2849169138576779, "axes": { "intelligence": 83.12484433374846, "calibration": 69.55504545454544, "speed": 76.51812235281348, "cost": 26.358453053841984 }, "jevbench_score": 58.43673282214442 }, { "members": [ "semif-qwen3.5-4b", "djev", "openjev-razorback16", "openjev-sglang", "simplejev-qwen3.6-35b-a3b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9452054794520548, "hard": 0.6727272727272727 }, "accuracy": 0.8183520599250936, "ece_hard": 0.08391719906121317, "calibration": 80.03272862220028, "p50_s": 1.7284078150987625, "p95_s": 1.9287681855261325, "cost_usd_per_1000": 0.36082756554307116, "axes": { "intelligence": 83.68923827314238, "calibration": 80.03272862220028, "speed": 74.77073744801535, "cost": 23.28100875369647 }, "jevbench_score": 58.43427690939231 }, { "members": [ "semif-qwen3.5-4b", "djev", "system-one-open", "openjev-razorback16", "openjev-sglang" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9657534246575342, "hard": 0.6454545454545455 }, "accuracy": 0.8127340823970037, "ece_hard": 0.15272727272727266, "calibration": 67.05827272727274, "p50_s": 1.3879805654287338, "p95_s": 2.030777859687805, "cost_usd_per_1000": 0.26015681647940075, "axes": { "intelligence": 83.44639892071399, "calibration": 67.05827272727274, "speed": 75.49954194936167, "cost": 27.542943714532868 }, "jevbench_score": 58.40548820082228 }, { "members": [ "semif-qwen3.5-4b", "djev", "system-one-open", "openjev-razorback16", "simplejev-qwen3.8-27b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9657534246575342, "hard": 0.6545454545454545 }, "accuracy": 0.8183520599250936, "ece_hard": 0.13909090909090913, "calibration": 68.46740909090909, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.2328778651685393, "axes": { "intelligence": 84.01079286010793, "calibration": 68.46740909090909, "speed": 69.75909261944061, "cost": 28.98615366409723 }, "jevbench_score": 58.39862175414908 }, { "members": [ "jev-1.13.0", "djev", "open-alternative-jev", "openjev-razorback16", "simplejev-qwen3.6-35b-a3b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9246575342465754, "hard": 0.6636363636363637 }, "accuracy": 0.8071161048689138, "ece_hard": 0.15454545454545449, "calibration": 69.51770454545456, "p50_s": 1.7256089895963669, "p95_s": 1.9284205481410026, "cost_usd_per_1000": 0.26919313108614235, "axes": { "intelligence": 82.54950186799502, "calibration": 69.51770454545456, "speed": 74.77855855005357, "cost": 27.098080781745452 }, "jevbench_score": 58.39576816082595 }, { "members": [ "semif-qwen3.5-4b", "djev", "openjev-razorback16", "openjev-sglang", "simplejev-qwen3.6-35b-a3b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9383561643835616, "hard": 0.6727272727272727 }, "accuracy": 0.8164794007490637, "ece_hard": 0.0850749401942941, "calibration": 79.91925146495734, "p50_s": 1.7284078150987625, "p95_s": 1.9287681855261325, "cost_usd_per_1000": 0.36082756554307116, "axes": { "intelligence": 83.49745745122458, "calibration": 79.91925146495734, "speed": 74.77073744801535, "cost": 23.28100875369647 }, "jevbench_score": 58.38005895454937 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "djev", "openjev-razorback16", "simplejev-qwen3.6-35b-a3b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9452054794520548, "hard": 0.6681818181818182 }, "accuracy": 0.8164794007490637, "ece_hard": 0.15818181818181826, "calibration": 68.61381818181817, "p50_s": 1.7256089895963669, "p95_s": 1.9284205481410026, "cost_usd_per_1000": 0.2694663333333333, "axes": { "intelligence": 83.55287463677875, "calibration": 68.61381818181817, "speed": 74.77855855005357, "cost": 27.084864613118356 }, "jevbench_score": 58.37396394547448 }, { "members": [ "semif-qwen3.5-4b", "djev", "openjev-razorback16", "openjev-sglang", "simplejev-qwen3.8-27b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9657534246575342, "hard": 0.6863636363636364 }, "accuracy": 0.8295880149812734, "ece_hard": 0.06866229920178343, "calibration": 82.87882114462238, "p50_s": 2.3997448533773422, "p95_s": 4.381843151897192, "cost_usd_per_1000": 0.3492723970037453, "axes": { "intelligence": 84.67367164798672, "calibration": 82.87882114462238, "speed": 69.78178103134502, "cost": 23.70507205892477 }, "jevbench_score": 58.3705548672392 }, { "members": [ "jev-1.13.0", "open-alternative-jev", "openjev-razorback16", "openjev-sglang", "simplejev-qwen3.6-35b-a3b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9246575342465754, "hard": 0.6681818181818182 }, "accuracy": 0.8108614232209738, "ece_hard": 0.0740449940920034, "calibration": 82.0401254323759, "p50_s": 1.7284078150987625, "p95_s": 1.9287681855261325, "cost_usd_per_1000": 0.3745173445692884, "axes": { "intelligence": 82.97753217102533, "calibration": 82.0401254323759, "speed": 74.77073744801535, "cost": 22.79584193629418 }, "jevbench_score": 58.363796264666554 }, { "members": [ "semif-qwen3.5-4b", "djev", "openjev-razorback16", "openjev-sglang", "simplejev-qwen3.8-27b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9657534246575342, "hard": 0.6909090909090909 }, "accuracy": 0.8314606741573034, "ece_hard": 0.0695641582414862, "calibration": 82.6542598620504, "p50_s": 2.3997448533773422, "p95_s": 4.381843151897192, "cost_usd_per_1000": 0.3492723970037453, "axes": { "intelligence": 84.81003528435036, "calibration": 82.6542598620504, "speed": 69.78178103134502, "cost": 23.70507205892477 }, "jevbench_score": 58.35444644708745 }, { "members": [ "open-alternative-jev", "system-one-open", "openjev-razorback16", "openjev-sglang", "simplejev-qwen3.8-27b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.958904109589041, "hard": 0.6681818181818182 }, "accuracy": 0.8183520599250936, "ece_hard": 0.06852524237016946, "calibration": 82.26428072260592, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.33792887640449437, "axes": { "intelligence": 83.6447696139477, "calibration": 82.26428072260592, "speed": 69.75909261944061, "cost": 24.135240870550234 }, "jevbench_score": 58.34123773427821 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "openjev-razorback16", "openjev-sglang", "simplejev-qwen3.6-35b-a3b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9657534246575342, "hard": 0.6590909090909091 }, "accuracy": 0.8164794007490637, "ece_hard": 0.08510091069541113, "calibration": 81.3097453049096, "p50_s": 1.7284078150987625, "p95_s": 1.9287681855261325, "cost_usd_per_1000": 0.3747905468164794, "axes": { "intelligence": 83.56382316313824, "calibration": 81.3097453049096, "speed": 74.77073744801535, "cost": 22.786341150167118 }, "jevbench_score": 58.32997500326949 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "openjev-razorback16", "openjev-sglang", "simplejev-qwen3.8-27b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9657534246575342, "hard": 0.6863636363636364 }, "accuracy": 0.8314606741573034, "ece_hard": 0.06528282146234977, "calibration": 84.15044253707117, "p50_s": 2.3997448533773422, "p95_s": 4.381843151897192, "cost_usd_per_1000": 0.3632353782771535, "axes": { "intelligence": 84.9653383146534, "calibration": 84.15044253707117, "speed": 69.78178103134502, "cost": 23.194355764692247 }, "jevbench_score": 58.32511983924771 }, { "members": [ "djev", "system-one-open", "openjev-razorback16", "openjev-sglang", "simplejev-qwen3.8-27b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9657534246575342, "hard": 0.6954545454545454 }, "accuracy": 0.8333333333333334, "ece_hard": 0.08259156163836842, "calibration": 81.25157177475458, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.3417087265917603, "axes": { "intelligence": 84.94639892071399, "calibration": 81.25157177475458, "speed": 69.75909261944061, "cost": 23.990317900106177 }, "jevbench_score": 58.297964631326096 }, { "members": [ "semif-qwen3.5-4b", "djev", "openjev-razorback16", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.958904109589041, "hard": 0.6636363636363637 }, "accuracy": 0.8183520599250936, "ece_hard": 0.08180310351685512, "calibration": 81.23446112971442, "p50_s": 2.3997448533773422, "p95_s": 4.381843151897192, "cost_usd_per_1000": 0.3335486142322097, "axes": { "intelligence": 83.80007264425073, "calibration": 81.23446112971442, "speed": 69.78178103134502, "cost": 24.305225780087525 }, "jevbench_score": 58.2916840264891 }, { "members": [ "jev-1.13.0", "djev", "openjev-razorback16", "openjev-sglang", "simplejev-qwen3.6-35b-a3b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9452054794520548, "hard": 0.6772727272727272 }, "accuracy": 0.8202247191011236, "ece_hard": 0.08877554112850548, "calibration": 81.26848801796933, "p50_s": 1.7284078150987625, "p95_s": 1.928032819926739, "cost_usd_per_1000": 0.3782971947565543, "axes": { "intelligence": 83.82560190950602, "calibration": 81.26848801796933, "speed": 74.77239356263168, "cost": 22.66500637551316 }, "jevbench_score": 58.29066330510384 }, { "members": [ "semif-qwen3.5-4b", "djev", "system-one-open", "openjev-razorback16", "simplejev-qwen3.6-35b-a3b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9452054794520548, "hard": 0.6409090909090909 }, "accuracy": 0.8052434456928839, "ece_hard": 0.16454545454545452, "calibration": 65.97745454545455, "p50_s": 1.7357386276125908, "p95_s": 2.0568130582571027, "cost_usd_per_1000": 0.24443303370786518, "axes": { "intelligence": 82.73469281859693, "calibration": 65.97745454545455, "speed": 74.47320850632062, "cost": 28.355203062428018 }, "jevbench_score": 58.26785225939537 }, { "members": [ "djev", "open-alternative-jev", "openjev-razorback16", "openjev-sglang", "simplejev-qwen3.6-35b-a3b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9246575342465754, "hard": 0.6681818181818182 }, "accuracy": 0.8089887640449438, "ece_hard": 0.08382304550604387, "calibration": 80.04191597867717, "p50_s": 1.7284078150987625, "p95_s": 1.9287681855261325, "cost_usd_per_1000": 0.36055436329588014, "axes": { "intelligence": 82.68586550435866, "calibration": 80.04191597867717, "speed": 74.77073744801535, "cost": 23.29087733198999 }, "jevbench_score": 58.26618268322629 }, { "members": [ "djev", "system-one-open", "openjev-razorback16", "openjev-sglang", "simplejev-qwen3.6-35b-a3b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9452054794520548, "hard": 0.6636363636363637 }, "accuracy": 0.8146067415730337, "ece_hard": 0.09531971827463012, "calibration": 78.73751901495196, "p50_s": 1.7359912395477295, "p95_s": 2.0568130582571027, "cost_usd_per_1000": 0.35326389513108614, "axes": { "intelligence": 83.41651100041511, "calibration": 78.73751901495196, "speed": 74.47257649879765, "cost": 23.55702240369645 }, "jevbench_score": 58.262300702269954 }, { "members": [ "djev", "system-one-open", "openjev-razorback16", "openjev-sglang", "simplejev-qwen3.6-35b-a3b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9452054794520548, "hard": 0.6590909090909091 }, "accuracy": 0.8127340823970037, "ece_hard": 0.09566327000663073, "calibration": 78.79791422293283, "p50_s": 1.7359912395477295, "p95_s": 2.0568130582571027, "cost_usd_per_1000": 0.35326389513108614, "axes": { "intelligence": 83.28014736405149, "calibration": 78.79791422293283, "speed": 74.47257649879765, "cost": 23.55702240369645 }, "jevbench_score": 58.24963995012959 }, { "members": [ "semif-qwen3.5-4b", "open-alternative-jev", "openjev-razorback16", "openjev-sglang", "simplejev-qwen3.8-27b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.958904109589041, "hard": 0.6772727272727272 }, "accuracy": 0.8239700374531835, "ece_hard": 0.06990778487418844, "calibration": 82.09130592997619, "p50_s": 2.3997448533773422, "p95_s": 4.385688671469688, "cost_usd_per_1000": 0.3454925468164794, "axes": { "intelligence": 84.20916355334164, "calibration": 82.09130592997619, "speed": 69.7779713205367, "cost": 23.846839511911085 }, "jevbench_score": 58.237325410975075 }, { "members": [ "djev", "openjev-sglang", "simplejev-qwen3.8-27b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.958904109589041, "hard": 0.7318181818181818 }, "accuracy": 0.8445692883895131, "ece_hard": 0.13484848484848472, "calibration": 70.08306818181819, "p50_s": 2.3997448533773422, "p95_s": 4.381843151897192, "cost_usd_per_1000": 0.26122323970037453, "axes": { "intelligence": 85.55386052303862, "calibration": 70.08306818181819, "speed": 69.78178103134502, "cost": 27.48964566134252 }, "jevbench_score": 58.23592491237001 }, { "members": [ "jev-1.13.0", "djev", "openjev-razorback16", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.958904109589041, "hard": 0.6863636363636364 }, "accuracy": 0.8277153558052435, "ece_hard": 0.07577831882662801, "calibration": 82.48991844709784, "p50_s": 2.3997448533773422, "p95_s": 4.381843151897192, "cost_usd_per_1000": 0.3510182434456929, "axes": { "intelligence": 84.48189082606892, "calibration": 82.48991844709784, "speed": 69.78178103134502, "cost": 23.640109341760308 }, "jevbench_score": 58.2289565313716 }, { "members": [ "djev", "open-alternative-jev", "openjev-razorback16", "openjev-sglang", "simplejev-qwen3.6-35b-a3b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9246575342465754, "hard": 0.6636363636363637 }, "accuracy": 0.8071161048689138, "ece_hard": 0.08441796732408605, "calibration": 79.94756597510826, "p50_s": 1.7284078150987625, "p95_s": 1.9287681855261325, "cost_usd_per_1000": 0.36055436329588014, "axes": { "intelligence": 82.54950186799502, "calibration": 79.94756597510826, "speed": 74.77073744801535, "cost": 23.29087733198999 }, "jevbench_score": 58.22497406622083 }, { "members": [ "semif-qwen3.5-4b", "open-alternative-jev", "openjev-razorback16", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9246575342465754, "hard": 0.6681818181818182 }, "accuracy": 0.8089887640449438, "ece_hard": 0.0697052707885655, "calibration": 81.2798816717748, "p50_s": 2.3997448533773422, "p95_s": 4.385688671469688, "cost_usd_per_1000": 0.3297687640449438, "axes": { "intelligence": 82.68586550435866, "calibration": 81.2798816717748, "speed": 69.7779713205367, "cost": 24.45371450368343 }, "jevbench_score": 58.19281671925562 }, { "members": [ "semif-qwen3.5-4b", "djev", "openjev-razorback16", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.958904109589041, "hard": 0.6590909090909091 }, "accuracy": 0.8164794007490637, "ece_hard": 0.08538382393703325, "calibration": 80.74029777297747, "p50_s": 2.3997448533773422, "p95_s": 4.381843151897192, "cost_usd_per_1000": 0.3335486142322097, "axes": { "intelligence": 83.66370900788709, "calibration": 80.74029777297747, "speed": 69.78178103134502, "cost": 24.305225780087525 }, "jevbench_score": 58.17913936987425 }, { "members": [ "djev", "open-alternative-jev", "openjev-razorback16", "openjev-sglang", "simplejev-qwen3.8-27b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9452054794520548, "hard": 0.6818181818181818 }, "accuracy": 0.8220973782771536, "ece_hard": 0.07435752141342679, "calibration": 82.44342313862771, "p50_s": 2.3997448533773422, "p95_s": 4.385688671469688, "cost_usd_per_1000": 0.3489991947565543, "axes": { "intelligence": 83.96196554586966, "calibration": 82.44342313862771, "speed": 69.7779713205367, "cost": 23.715267252530197 }, "jevbench_score": 58.176319683111196 }, { "members": [ "open-alternative-jev", "openjev-razorback16", "openjev-sglang" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8020833333333334, "judge": 0.9178082191780822, "hard": 0.6318181818181818 }, "accuracy": 0.7902621722846442, "ece_hard": 0.19696969696969668, "calibration": 61.952613636363665, "p50_s": 1.3671194911003113, "p95_s": 1.6307271040976046, "cost_usd_per_1000": 0.2190514232209738, "axes": { "intelligence": 81.1115089248651, "calibration": 61.952613636363665, "speed": 76.51812235281348, "cost": 29.783617623351972 }, "jevbench_score": 58.17293374316375 }, { "members": [ "open-alternative-jev", "system-one-open", "openjev-razorback16", "openjev-sglang", "simplejev-qwen3.6-35b-a3b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8020833333333334, "judge": 0.9246575342465754, "hard": 0.6409090909090909 }, "accuracy": 0.795880149812734, "ece_hard": 0.08133760553718619, "calibration": 79.49542954615595, "p50_s": 1.7359912395477295, "p95_s": 2.0568130582571027, "cost_usd_per_1000": 0.34948404494382024, "axes": { "intelligence": 81.57601701951018, "calibration": 79.49542954615595, "speed": 74.47257649879765, "cost": 23.697179391628367 }, "jevbench_score": 58.16335144727902 }, { "members": [ "semif-qwen3.5-4b", "system-one-open", "openjev-razorback16", "openjev-sglang", "simplejev-qwen3.6-35b-a3b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9657534246575342, "hard": 0.6409090909090909 }, "accuracy": 0.8089887640449438, "ece_hard": 0.09706557154885422, "calibration": 78.08869393917719, "p50_s": 1.7359912395477295, "p95_s": 2.0568130582571027, "cost_usd_per_1000": 0.34975724719101126, "axes": { "intelligence": 83.01836861768369, "calibration": 78.08869393917719, "speed": 74.47257649879765, "cost": 23.686998336610245 }, "jevbench_score": 58.15233941737236 }, { "members": [ "jev-1.13.0", "djev", "openjev-razorback16", "openjev-sglang", "simplejev-qwen3.8-27b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9657534246575342, "hard": 0.7090909090909091 }, "accuracy": 0.8389513108614233, "ece_hard": 0.07002961320077393, "calibration": 83.21691460659756, "p50_s": 2.3997448533773422, "p95_s": 4.381843151897192, "cost_usd_per_1000": 0.36674202621722846, "axes": { "intelligence": 85.35548982980491, "calibration": 83.21691460659756, "speed": 69.78178103134502, "cost": 23.06917959713782 }, "jevbench_score": 58.15061565863241 }, { "members": [ "system-one-open", "openjev-sglang", "simplejev-qwen3.8-27b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.952054794520548, "hard": 0.6954545454545454 }, "accuracy": 0.8277153558052435, "ece_hard": 0.11818181818181782, "calibration": 69.24623484848487, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.25015292134831457, "axes": { "intelligence": 84.27117061021171, "calibration": 69.24623484848487, "speed": 69.75909261944061, "cost": 28.05383262854474 }, "jevbench_score": 58.132239468907066 }, { "members": [ "djev", "open-alternative-jev", "openjev-razorback16", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9315068493150684, "hard": 0.6727272727272727 }, "accuracy": 0.8127340823970037, "ece_hard": 0.08456492558262153, "calibration": 80.77319023541511, "p50_s": 2.3997448533773422, "p95_s": 4.385688671469688, "cost_usd_per_1000": 0.3332754119850187, "axes": { "intelligence": 83.0140099626401, "calibration": 80.77319023541511, "speed": 69.7779713205367, "cost": 24.31590178128478 }, "jevbench_score": 58.07735605694422 }, { "members": [ "semif-qwen3.5-4b", "open-alternative-jev", "simplejev-qwen3.8-27b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.78125, "judge": 0.9178082191780822, "hard": 0.6227272727272727 }, "accuracy": 0.7827715355805244, "ece_hard": 0.22727272727272693, "calibration": 58.23456060606064, "p50_s": 2.3997448533773422, "p95_s": 4.385688671469688, "cost_usd_per_1000": 0.14861252808988762, "axes": { "intelligence": 80.2554483188045, "calibration": 58.23456060606064, "speed": 69.7779713205367, "cost": 34.838337335506964 }, "jevbench_score": 58.05742526006052 }, { "members": [ "openjev-sglang", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.958904109589041, "hard": 0.6954545454545454 }, "accuracy": 0.8295880149812734, "ece_hard": 0.07259271592963874, "calibration": 81.51265694334423, "p50_s": 2.3997448533773422, "p95_s": 4.381843151897192, "cost_usd_per_1000": 0.35082367041198503, "axes": { "intelligence": 84.46295143212951, "calibration": 81.51265694334423, "speed": 69.78178103134502, "cost": 23.64733336147438 }, "jevbench_score": 58.05690520308325 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "open-alternative-jev", "openjev-razorback16", "openjev-sglang" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9657534246575342, "hard": 0.6636363636363637 }, "accuracy": 0.8202247191011236, "ece_hard": 0.16454545454545472, "calibration": 66.44270454545452, "p50_s": 1.3671194911003113, "p95_s": 1.6307271040976046, "cost_usd_per_1000": 0.281410265917603, "axes": { "intelligence": 83.99185346616854, "calibration": 66.44270454545452, "speed": 76.51812235281348, "cost": 26.519801903195614 }, "jevbench_score": 58.01025330177151 }, { "members": [ "jev-1.13.0", "djev", "open-alternative-jev", "openjev-razorback16", "simplejev-qwen3.8-27b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9452054794520548, "hard": 0.6727272727272727 }, "accuracy": 0.8183520599250936, "ece_hard": 0.14636363636363645, "calibration": 69.96938636363635, "p50_s": 2.3997448533773422, "p95_s": 4.385688671469688, "cost_usd_per_1000": 0.2576379625468165, "axes": { "intelligence": 83.68923827314238, "calibration": 69.96938636363635, "speed": 69.7779713205367, "cost": 27.669704319737818 }, "jevbench_score": 57.98626478188713 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "djev", "openjev-razorback16", "openjev-sglang" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9657534246575342, "hard": 0.6727272727272727 }, "accuracy": 0.8239700374531835, "ece_hard": 0.17454545454545473, "calibration": 66.54320454545453, "p50_s": 1.3671194911003113, "p95_s": 1.6307271040976046, "cost_usd_per_1000": 0.2851901161048689, "axes": { "intelligence": 84.26458073889582, "calibration": 66.54320454545453, "speed": 76.51812235281348, "cost": 26.34596589989596 }, "jevbench_score": 57.98381705510917 }, { "members": [ "jev-1.13.0", "djev", "open-alternative-jev", "openjev-sglang", "simplejev-qwen3.8-27b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.958904109589041, "hard": 0.7227272727272728 }, "accuracy": 0.8426966292134831, "ece_hard": 0.0936363636363638, "calibration": 76.56786363636363, "p50_s": 2.3997448533773422, "p95_s": 4.385688671469688, "cost_usd_per_1000": 0.32330888014981274, "axes": { "intelligence": 85.572799916978, "calibration": 76.56786363636363, "speed": 69.7779713205367, "cost": 24.711471000802888 }, "jevbench_score": 57.97620745349077 }, { "members": [ "semif-qwen3.5-4b", "open-alternative-jev", "openjev-razorback16", "openjev-sglang", "simplejev-qwen3.6-35b-a3b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8020833333333334, "judge": 0.9246575342465754, "hard": 0.6409090909090909 }, "accuracy": 0.795880149812734, "ece_hard": 0.08600347194370975, "calibration": 79.02767896252651, "p50_s": 1.7284078150987625, "p95_s": 1.9287681855261325, "cost_usd_per_1000": 0.35704771535580526, "axes": { "intelligence": 81.57601701951018, "calibration": 79.02767896252651, "speed": 74.77073744801535, "cost": 23.418212245121367 }, "jevbench_score": 57.96379094228175 }, { "members": [ "semif-qwen3.5-4b", "open-alternative-jev", "system-one-open", "openjev-razorback16", "openjev-sglang" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9657534246575342, "hard": 0.6272727272727273 }, "accuracy": 0.8033707865168539, "ece_hard": 0.1618181818181815, "calibration": 65.16793181818184, "p50_s": 1.3879805654287338, "p95_s": 2.030777859687805, "cost_usd_per_1000": 0.2563769662921348, "axes": { "intelligence": 82.60927770859278, "calibration": 65.16793181818184, "speed": 75.49954194936167, "cost": 27.733629873221304 }, "jevbench_score": 57.943329783298395 }, { "members": [ "openjev-sglang", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8020833333333334, "judge": 0.958904109589041, "hard": 0.6954545454545454 }, "accuracy": 0.8277153558052435, "ece_hard": 0.07387558614851436, "calibration": 81.1180665776696, "p50_s": 2.3997448533773422, "p95_s": 4.381843151897192, "cost_usd_per_1000": 0.35082367041198503, "axes": { "intelligence": 84.17128476546284, "calibration": 81.1180665776696, "speed": 69.78178103134502, "cost": 23.64733336147438 }, "jevbench_score": 57.936391557003006 }, { "members": [ "semif-qwen3.5-4b", "openjev-razorback16", "openjev-sglang" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9657534246575342, "hard": 0.6454545454545455 }, "accuracy": 0.8127340823970037, "ece_hard": 0.21515151515151482, "calibration": 59.19834848484852, "p50_s": 1.3671194911003113, "p95_s": 1.6307271040976046, "cost_usd_per_1000": 0.21932462546816478, "axes": { "intelligence": 83.44639892071399, "calibration": 59.19834848484852, "speed": 76.51812235281348, "cost": 29.767378107879182 }, "jevbench_score": 57.916928213494 }, { "members": [ "semif-qwen3.5-4b", "openjev-razorback16", "simplejev-qwen3.6-35b-a3b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9315068493150684, "hard": 0.6363636363636364 }, "accuracy": 0.797752808988764, "ece_hard": 0.2106060606060603, "calibration": 59.73839393939397, "p50_s": 1.7256089895963669, "p95_s": 1.9284205481410026, "cost_usd_per_1000": 0.2036008426966292, "axes": { "intelligence": 81.92310087173101, "calibration": 59.73839393939397, "speed": 74.77855855005357, "cost": 30.73661286406545 }, "jevbench_score": 57.912625630097644 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "djev", "openjev-razorback16", "simplejev-qwen3.8-27b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9657534246575342, "hard": 0.6818181818181818 }, "accuracy": 0.8295880149812734, "ece_hard": 0.15000000000000013, "calibration": 68.54324999999997, "p50_s": 2.3997448533773422, "p95_s": 4.381843151897192, "cost_usd_per_1000": 0.25791116479400744, "axes": { "intelligence": 84.82897467828975, "calibration": 68.54324999999997, "speed": 69.78178103134502, "cost": 27.65589571451909 }, "jevbench_score": 57.87748782461144 }, { "members": [ "jev-1.13.0", "djev", "system-one-open", "openjev-sglang", "simplejev-qwen3.8-27b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9657534246575342, "hard": 0.7318181818181818 }, "accuracy": 0.848314606741573, "ece_hard": 0.08818181818181789, "calibration": 74.73381818181821, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.31601841198501873, "axes": { "intelligence": 86.03730801162308, "calibration": 74.73381818181821, "speed": 69.75909261944061, "cost": 25.008628408420932 }, "jevbench_score": 57.87269011463164 }, { "members": [ "semif-qwen3.5-4b", "system-one-open", "openjev-razorback16", "openjev-sglang", "simplejev-qwen3.6-35b-a3b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9657534246575342, "hard": 0.6409090909090909 }, "accuracy": 0.8089887640449438, "ece_hard": 0.11249516919728607, "calibration": 76.5908416971034, "p50_s": 1.7359912395477295, "p95_s": 2.0568130582571027, "cost_usd_per_1000": 0.34975724719101126, "axes": { "intelligence": 83.01836861768369, "calibration": 76.5908416971034, "speed": 74.47257649879765, "cost": 23.686998336610245 }, "jevbench_score": 57.87144966560866 }, { "members": [ "semif-qwen3.5-4b", "open-alternative-jev", "openjev-razorback16", "openjev-sglang", "simplejev-qwen3.6-35b-a3b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8020833333333334, "judge": 0.9246575342465754, "hard": 0.6363636363636364 }, "accuracy": 0.7940074906367042, "ece_hard": 0.08928094741352204, "calibration": 78.64681468577577, "p50_s": 1.7284078150987625, "p95_s": 1.9287681855261325, "cost_usd_per_1000": 0.35704771535580526, "axes": { "intelligence": 81.43965338314653, "calibration": 78.64681468577577, "speed": 74.77073744801535, "cost": 23.418212245121367 }, "jevbench_score": 57.86961779505729 }, { "members": [ "jev-1.13.0", "djev", "system-one-open", "openjev-razorback16", "simplejev-qwen3.8-27b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.952054794520548, "hard": 0.6818181818181818 }, "accuracy": 0.8258426966292135, "ece_hard": 0.1563636363636367, "calibration": 67.75063636363633, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.2503474943820225, "axes": { "intelligence": 84.44541303445415, "calibration": 67.75063636363633, "speed": 69.75909261944061, "cost": 28.043702527174077 }, "jevbench_score": 57.84041554467908 }, { "members": [ "semif-qwen3.5-4b", "djev", "system-one-open", "openjev-sglang", "simplejev-qwen3.8-27b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9657534246575342, "hard": 0.7 }, "accuracy": 0.8370786516853933, "ece_hard": 0.10818181818181805, "calibration": 72.53856818181819, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.29854878277153557, "axes": { "intelligence": 85.3744292237443, "calibration": 72.53856818181819, "speed": 69.75909261944061, "cost": 25.749540854883023 }, "jevbench_score": 57.751966399148586 }, { "members": [ "djev", "system-one-open", "openjev-sglang", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9657534246575342, "hard": 0.6909090909090909 }, "accuracy": 0.8295880149812734, "ece_hard": 0.044044741288041506, "calibration": 84.77903592385883, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.391655861423221, "axes": { "intelligence": 84.51836861768369, "calibration": 84.77903592385883, "speed": 69.75909261944061, "cost": 22.212861085919087 }, "jevbench_score": 57.72463901627288 }, { "members": [ "semif-qwen3.5-4b", "open-alternative-jev", "system-one-open", "openjev-razorback16", "simplejev-qwen3.8-27b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.958904109589041, "hard": 0.6227272727272727 }, "accuracy": 0.8014981273408239, "ece_hard": 0.16454545454545444, "calibration": 65.94070454545455, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.22909801498127338, "axes": { "intelligence": 82.572799916978, "calibration": 65.94070454545455, "speed": 69.75909261944061, "cost": 29.199360212106924 }, "jevbench_score": 57.708686328177066 }, { "members": [ "djev", "open-alternative-jev", "system-one-open", "openjev-sglang", "simplejev-qwen3.8-27b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.952054794520548, "hard": 0.6818181818181818 }, "accuracy": 0.8220973782771536, "ece_hard": 0.10454545454545427, "calibration": 73.33970454545457, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.29827558052434455, "axes": { "intelligence": 83.8620797011208, "calibration": 73.33970454545457, "speed": 69.75909261944061, "cost": 25.761469011035786 }, "jevbench_score": 57.65925914765117 }, { "members": [ "djev", "openjev-razorback16", "simplejev-qwen3.6-35b-a3b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9246575342465754, "hard": 0.65 }, "accuracy": 0.8014981273408239, "ece_hard": 0.22878787878787868, "calibration": 58.89221212121214, "p50_s": 1.724735751748085, "p95_s": 1.9244587391614911, "cost_usd_per_1000": 0.2071074906367041, "axes": { "intelligence": 82.14041095890411, "calibration": 58.89221212121214, "speed": 74.78968829969378, "cost": 30.514125799489577 }, "jevbench_score": 57.64204082200466 }, { "members": [ "djev", "system-one-open", "openjev-sglang", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.958904109589041, "hard": 0.6954545454545454 }, "accuracy": 0.8295880149812734, "ece_hard": 0.044988430416424786, "calibration": 84.33226827064439, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.391655861423221, "axes": { "intelligence": 84.46295143212951, "calibration": 84.33226827064439, "speed": 69.75909261944061, "cost": 22.212861085919087 }, "jevbench_score": 57.638987008211295 }, { "members": [ "djev", "openjev-razorback16", "openjev-sglang" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9452054794520548, "hard": 0.6545454545454545 }, "accuracy": 0.8089887640449438, "ece_hard": 0.23030303030303015, "calibration": 58.724696969696986, "p50_s": 1.3671194911003113, "p95_s": 1.6257096692919728, "cost_usd_per_1000": 0.22283127340823972, "axes": { "intelligence": 82.85211706102118, "calibration": 58.724696969696986, "speed": 76.53150536083959, "cost": 29.5607157358955 }, "jevbench_score": 57.599657218512 }, { "members": [ "semif-qwen3.5-4b", "system-one-open", "openjev-sglang", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9657534246575342, "hard": 0.6818181818181818 }, "accuracy": 0.8258426966292135, "ece_hard": 0.04797880110707982, "calibration": 83.82541036916811, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.38814921348314607, "axes": { "intelligence": 84.24564134495643, "calibration": 83.82541036916811, "speed": 69.75909261944061, "cost": 22.33003868565656 }, "jevbench_score": 57.59083254968916 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "open-alternative-jev", "openjev-razorback16", "simplejev-qwen3.8-27b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9657534246575342, "hard": 0.6636363636363637 }, "accuracy": 0.8202247191011236, "ece_hard": 0.16090909090909108, "calibration": 67.3813409090909, "p50_s": 2.3997448533773422, "p95_s": 4.385688671469688, "cost_usd_per_1000": 0.25413131460674154, "axes": { "intelligence": 83.99185346616854, "calibration": 67.3813409090909, "speed": 69.7779713205367, "cost": 27.84825450858567 }, "jevbench_score": 57.586844679982335 }, { "members": [ "jev-1.13.0", "open-alternative-jev", "system-one-open", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8020833333333334, "judge": 0.9657534246575342, "hard": 0.6863636363636364 }, "accuracy": 0.8258426966292135, "ece_hard": 0.10909090909090965, "calibration": 72.54765909090904, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.2965147790262172, "axes": { "intelligence": 84.0903383146534, "calibration": 72.54765909090904, "speed": 69.75909261944061, "cost": 25.838609663638195 }, "jevbench_score": 57.585065725824 }, { "members": [ "jev-1.13.0", "open-alternative-jev", "openjev-sglang", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.958904109589041, "hard": 0.7 }, "accuracy": 0.8314606741573034, "ece_hard": 0.03737827106953008, "calibration": 86.53180840229675, "p50_s": 2.3997448533773422, "p95_s": 4.385688671469688, "cost_usd_per_1000": 0.41290931086142324, "axes": { "intelligence": 84.59931506849315, "calibration": 86.53180840229675, "speed": 69.7779713205367, "cost": 21.52435971791394 }, "jevbench_score": 57.58346598832961 }, { "members": [ "jev-1.13.0", "system-one-open", "openjev-sglang", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9657534246575342, "hard": 0.6909090909090909 }, "accuracy": 0.8314606741573034, "ece_hard": 0.045302055195838534, "calibration": 85.40548946270616, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.40561884269662923, "axes": { "intelligence": 84.81003528435036, "calibration": 85.40548946270616, "speed": 69.75909261944061, "cost": 21.756456352172563 }, "jevbench_score": 57.58117239015174 }, { "members": [ "djev", "open-alternative-jev", "system-one-open", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8020833333333334, "judge": 0.9452054794520548, "hard": 0.6818181818181818 }, "accuracy": 0.8183520599250936, "ece_hard": 0.1154545454545452, "calibration": 71.34979545454547, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.282551797752809, "axes": { "intelligence": 83.37863221253632, "calibration": 71.34979545454547, "speed": 69.75909261944061, "cost": 26.467057753975837 }, "jevbench_score": 57.56897495869577 }, { "members": [ "open-alternative-jev", "system-one-open", "openjev-sglang", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8020833333333334, "judge": 0.952054794520548, "hard": 0.6772727272727272 }, "accuracy": 0.8183520599250936, "ece_hard": 0.03952629716971819, "calibration": 84.47323084142617, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.38787601123595505, "axes": { "intelligence": 83.4340493980905, "calibration": 84.47323084142617, "speed": 69.75909261944061, "cost": 22.339212374566742 }, "jevbench_score": 57.56821673830777 }, { "members": [ "semif-qwen3.5-4b", "djev", "system-one-open", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6818181818181818 }, "accuracy": 0.8314606741573034, "ece_hard": 0.12909090909090887, "calibration": 69.98615909090911, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.282825, "axes": { "intelligence": 85.02075550020756, "calibration": 69.98615909090911, "speed": 69.75909261944061, "cost": 26.45446612608039 }, "jevbench_score": 57.56509636101748 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "system-one-open", "openjev-sglang", "simplejev-qwen3.8-27b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9726027397260274, "hard": 0.6909090909090909 }, "accuracy": 0.8333333333333334, "ece_hard": 0.10454545454545441, "calibration": 73.49320454545456, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.3125117640449438, "axes": { "intelligence": 85.00181610626817, "calibration": 73.49320454545456, "speed": 69.75909261944061, "cost": 25.15400888908762 }, "jevbench_score": 57.540132225021324 }, { "members": [ "djev", "open-alternative-jev", "openjev-razorback16", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9315068493150684, "hard": 0.6681818181818182 }, "accuracy": 0.8108614232209738, "ece_hard": 0.11291208370369404, "calibration": 77.81472454175275, "p50_s": 2.3997448533773422, "p95_s": 4.385688671469688, "cost_usd_per_1000": 0.3332754119850187, "axes": { "intelligence": 82.87764632627646, "calibration": 77.81472454175275, "speed": 69.7779713205367, "cost": 24.31590178128478 }, "jevbench_score": 57.51445094038164 }, { "members": [ "jev-1.13.0", "djev", "system-one-open", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9657534246575342, "hard": 0.6954545454545454 }, "accuracy": 0.8333333333333334, "ece_hard": 0.11636363636363617, "calibration": 71.79438636363638, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.30029462921348316, "axes": { "intelligence": 84.94639892071399, "calibration": 71.79438636363638, "speed": 69.75909261944061, "cost": 25.67357305339182 }, "jevbench_score": 57.488456645391025 }, { "members": [ "jev-1.13.0", "open-alternative-jev", "system-one-open", "openjev-sglang", "simplejev-qwen3.8-27b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.952054794520548, "hard": 0.6954545454545454 }, "accuracy": 0.8277153558052435, "ece_hard": 0.10545454545454602, "calibration": 73.81029545454538, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.3122385617977528, "axes": { "intelligence": 84.27117061021171, "calibration": 73.81029545454538, "speed": 69.75909261944061, "cost": 25.16540386378908 }, "jevbench_score": 57.48442273568411 }, { "members": [ "semif-qwen3.5-4b", "open-alternative-jev", "system-one-open", "openjev-razorback16", "simplejev-qwen3.6-35b-a3b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9315068493150684, "hard": 0.6136363636363636 }, "accuracy": 0.7902621722846442, "ece_hard": 0.18636363636363612, "calibration": 62.75888636363639, "p50_s": 1.7357386276125908, "p95_s": 2.0568130582571027, "cost_usd_per_1000": 0.24065318352059922, "axes": { "intelligence": 81.5329493565795, "calibration": 62.75888636363639, "speed": 74.47320850632062, "cost": 28.55825166435858 }, "jevbench_score": 57.43610960879795 }, { "members": [ "jev-1.13.0", "openjev-sglang", "simplejev-qwen3.8-27b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.958904109589041, "hard": 0.7181818181818181 }, "accuracy": 0.8408239700374532, "ece_hard": 0.1636363636363635, "calibration": 68.01303030303032, "p50_s": 2.3997448533773422, "p95_s": 4.381843151897192, "cost_usd_per_1000": 0.27518622097378276, "axes": { "intelligence": 85.43643628061437, "calibration": 68.01303030303032, "speed": 69.78178103134502, "cost": 26.811199471911323 }, "jevbench_score": 57.4213482979534 }, { "members": [ "open-alternative-jev", "openjev-sglang", "simplejev-qwen3.8-27b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6818181818181818 }, "accuracy": 0.8127340823970037, "ece_hard": 0.15757575757575723, "calibration": 68.01825757575762, "p50_s": 2.3997448533773422, "p95_s": 4.385688671469688, "cost_usd_per_1000": 0.25744338951310863, "axes": { "intelligence": 82.70340390203404, "calibration": 68.01825757575762, "speed": 69.7779713205367, "cost": 27.679547657907364 }, "jevbench_score": 57.41251258366296 }, { "members": [ "semif-qwen3.5-4b", "djev", "open-alternative-jev", "openjev-razorback16", "simplejev-qwen3.8-27b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6454545454545455 }, "accuracy": 0.8089887640449438, "ece_hard": 0.17090909090909107, "calibration": 65.4660909090909, "p50_s": 2.3997448533773422, "p95_s": 4.385688671469688, "cost_usd_per_1000": 0.24016833333333332, "axes": { "intelligence": 83.16272312162724, "calibration": 65.4660909090909, "speed": 69.7779713205367, "cost": 28.584527672174644 }, "jevbench_score": 57.4048541078344 }, { "members": [ "jev-1.13.0", "open-alternative-jev", "system-one-open", "openjev-sglang", "simplejev-qwen3.6-35b-a3b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9657534246575342, "hard": 0.6818181818181818 }, "accuracy": 0.8258426966292135, "ece_hard": 0.12818181818181787, "calibration": 70.09306818181821, "p50_s": 1.7359912395477295, "p95_s": 2.0568130582571027, "cost_usd_per_1000": 0.32379373033707864, "axes": { "intelligence": 84.24564134495643, "calibration": 70.09306818181821, "speed": 74.47257649879765, "cost": 24.691946944166034 }, "jevbench_score": 57.404185143682255 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "system-one-open", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9794520547945206, "hard": 0.6863636363636364 }, "accuracy": 0.8333333333333334, "ece_hard": 0.12636363636363634, "calibration": 70.84988636363636, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.2967879812734082, "axes": { "intelligence": 85.05723329182234, "calibration": 70.84988636363636, "speed": 69.75909261944061, "cost": 25.82661070680966 }, "jevbench_score": 57.40234867155044 }, { "members": [ "semif-qwen3.5-4b", "system-one-open", "openjev-sglang", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9657534246575342, "hard": 0.6681818181818182 }, "accuracy": 0.8202247191011236, "ece_hard": 0.05263553909983084, "calibration": 83.13371827265513, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.38814921348314607, "axes": { "intelligence": 83.8365504358655, "calibration": 83.13371827265513, "speed": 69.75909261944061, "cost": 22.33003868565656 }, "jevbench_score": 57.40176231337348 }, { "members": [ "semif-qwen3.5-4b", "djev", "open-alternative-jev", "openjev-razorback16", "openjev-sglang" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9452054794520548, "hard": 0.6318181818181818 }, "accuracy": 0.8014981273408239, "ece_hard": 0.18818181818181837, "calibration": 63.193318181818164, "p50_s": 1.3671194911003113, "p95_s": 1.6307271040976046, "cost_usd_per_1000": 0.26744728464419476, "axes": { "intelligence": 82.46196554586966, "calibration": 63.193318181818164, "speed": 76.51812235281348, "cost": 27.182854212376725 }, "jevbench_score": 57.37808515678451 }, { "members": [ "jev-1.13.0", "djev", "open-alternative-jev", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8020833333333334, "judge": 0.952054794520548, "hard": 0.6818181818181818 }, "accuracy": 0.8202247191011236, "ece_hard": 0.120909090909091, "calibration": 73.2748409090909, "p50_s": 2.3997448533773422, "p95_s": 4.385688671469688, "cost_usd_per_1000": 0.30758509737827716, "axes": { "intelligence": 83.57041303445413, "calibration": 73.2748409090909, "speed": 69.7779713205367, "cost": 25.36104130309799 }, "jevbench_score": 57.37506799953209 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "open-alternative-jev", "openjev-razorback16", "simplejev-qwen3.6-35b-a3b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9315068493150684, "hard": 0.6454545454545455 }, "accuracy": 0.8033707865168539, "ece_hard": 0.1854545454545455, "calibration": 64.39729545454546, "p50_s": 1.7256089895963669, "p95_s": 1.9284205481410026, "cost_usd_per_1000": 0.2656864831460674, "axes": { "intelligence": 82.48749481112495, "calibration": 64.39729545454546, "speed": 74.77855855005357, "cost": 27.26891619543717 }, "jevbench_score": 57.368722264012206 }, { "members": [ "djev", "open-alternative-jev", "system-one-open", "openjev-sglang", "simplejev-qwen3.6-35b-a3b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8020833333333334, "judge": 0.9452054794520548, "hard": 0.6636363636363637 }, "accuracy": 0.8108614232209738, "ece_hard": 0.13363636363636344, "calibration": 69.45661363636366, "p50_s": 1.7359912395477295, "p95_s": 2.0568130582571027, "cost_usd_per_1000": 0.3098307490636704, "axes": { "intelligence": 82.83317766708178, "calibration": 69.45661363636366, "speed": 74.47257649879765, "cost": 25.266264490204037 }, "jevbench_score": 57.360619912289884 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "openjev-sglang", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9657534246575342, "hard": 0.7090909090909091 }, "accuracy": 0.8389513108614233, "ece_hard": 0.05939720626584547, "calibration": 84.41580849762701, "p50_s": 2.3997448533773422, "p95_s": 4.381843151897192, "cost_usd_per_1000": 0.4131825131086142, "axes": { "intelligence": 85.35548982980491, "calibration": 84.41580849762701, "speed": 69.78178103134502, "cost": 21.515742014945715 }, "jevbench_score": 57.35065803336644 }, { "members": [ "system-one-open", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8020833333333334, "judge": 0.9657534246575342, "hard": 0.6863636363636364 }, "accuracy": 0.8258426966292135, "ece_hard": 0.163636363636364, "calibration": 63.77928030303026, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.234429138576779, "axes": { "intelligence": 84.0903383146534, "calibration": 63.77928030303026, "speed": 69.75909261944061, "cost": 28.899652248482568 }, "jevbench_score": 57.34291792316614 }, { "members": [ "jev-1.13.0", "open-alternative-jev", "openjev-sglang", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.958904109589041, "hard": 0.6863636363636364 }, "accuracy": 0.8258426966292135, "ece_hard": 0.04468877576991892, "calibration": 85.4665591996505, "p50_s": 2.3997448533773422, "p95_s": 4.385688671469688, "cost_usd_per_1000": 0.41290931086142324, "axes": { "intelligence": 84.19022415940225, "calibration": 85.4665591996505, "speed": 69.7779713205367, "cost": 21.52435971791394 }, "jevbench_score": 57.33589769564344 }, { "members": [ "semif-qwen3.5-4b", "open-alternative-jev", "openjev-sglang" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9452054794520548, "hard": 0.6090909090909091 }, "accuracy": 0.7865168539325843, "ece_hard": 0.2530303030303028, "calibration": 53.45880303030306, "p50_s": 1.3671194911003113, "p95_s": 1.6307271040976046, "cost_usd_per_1000": 0.17589147940074906, "axes": { "intelligence": 80.90514736405147, "calibration": 53.45880303030306, "speed": 76.51812235281348, "cost": 32.64265594854422 }, "jevbench_score": 57.33057452265406 }, { "members": [ "open-alternative-jev", "system-one-open", "openjev-sglang", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.6727272727272727 }, "accuracy": 0.8146067415730337, "ece_hard": 0.048236543251886796, "calibration": 83.36176592426182, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.38787601123595505, "axes": { "intelligence": 83.0060190950602, "calibration": 83.36176592426182, "speed": 69.75909261944061, "cost": 22.339212374566742 }, "jevbench_score": 57.304178764448636 }, { "members": [ "semif-qwen3.5-4b", "djev", "open-alternative-jev", "openjev-razorback16", "simplejev-qwen3.6-35b-a3b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9315068493150684, "hard": 0.6318181818181818 }, "accuracy": 0.797752808988764, "ece_hard": 0.1927272727272728, "calibration": 62.78427272727272, "p50_s": 1.7256089895963669, "p95_s": 1.9284205481410026, "cost_usd_per_1000": 0.2517235018726592, "axes": { "intelligence": 82.07840390203404, "calibration": 62.78427272727272, "speed": 74.77855855005357, "cost": 27.9722870547499 }, "jevbench_score": 57.29889002434988 }, { "members": [ "jev-1.13.0", "djev", "system-one-open", "openjev-sglang", "simplejev-qwen3.6-35b-a3b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9657534246575342, "hard": 0.6727272727272727 }, "accuracy": 0.8220973782771536, "ece_hard": 0.13636363636363605, "calibration": 70.2193863636364, "p50_s": 1.7359912395477295, "p95_s": 2.0568130582571027, "cost_usd_per_1000": 0.3275735805243446, "axes": { "intelligence": 83.97291407222914, "calibration": 70.2193863636364, "speed": 74.47257649879765, "cost": 24.540733966979843 }, "jevbench_score": 57.29543820122508 }, { "members": [ "djev", "open-alternative-jev", "openjev-sglang", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9657534246575342, "hard": 0.6954545454545454 }, "accuracy": 0.8314606741573034, "ece_hard": 0.0650591272294599, "calibration": 82.90184309938567, "p50_s": 2.3997448533773422, "p95_s": 4.385688671469688, "cost_usd_per_1000": 0.398946329588015, "axes": { "intelligence": 84.65473225404733, "calibration": 82.90184309938567, "speed": 69.7779713205367, "cost": 21.9725657859062 }, "jevbench_score": 57.27348825231051 }, { "members": [ "jev-1.13.0", "system-one-open", "openjev-sglang", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9657534246575342, "hard": 0.6863636363636364 }, "accuracy": 0.8295880149812734, "ece_hard": 0.05833599539288054, "calibration": 83.69125695575411, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.40561884269662923, "axes": { "intelligence": 84.67367164798672, "calibration": 83.69125695575411, "speed": 69.75909261944061, "cost": 21.756456352172563 }, "jevbench_score": 57.26699125680851 }, { "members": [ "semif-qwen3.5-4b", "open-alternative-jev", "openjev-sglang", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8020833333333334, "judge": 0.958904109589041, "hard": 0.6863636363636364 }, "accuracy": 0.8239700374531835, "ece_hard": 0.05230667938776917, "calibration": 83.15239587111974, "p50_s": 2.3997448533773422, "p95_s": 4.385688671469688, "cost_usd_per_1000": 0.3954396816479401, "axes": { "intelligence": 83.89855749273558, "calibration": 83.15239587111974, "speed": 69.7779713205367, "cost": 22.087592565387524 }, "jevbench_score": 57.26298664290935 }, { "members": [ "semif-qwen3.5-4b", "open-alternative-jev", "simplejev-qwen3.6-35b-a3b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.7708333333333334, "judge": 0.910958904109589, "hard": 0.5954545454545455 }, "accuracy": 0.7677902621722846, "ece_hard": 0.25151515151515114, "calibration": 53.69581818181822, "p50_s": 1.7256089895963669, "p95_s": 1.9284205481410026, "cost_usd_per_1000": 0.16016769662921349, "axes": { "intelligence": 78.9538190120382, "calibration": 53.69581818181822, "speed": 74.77855855005357, "cost": 33.86275209892408 }, "jevbench_score": 57.240476006176735 }, { "members": [ "jev-1.13.0", "djev", "openjev-sglang", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9657534246575342, "hard": 0.6954545454545454 }, "accuracy": 0.8314606741573034, "ece_hard": 0.06010732269951966, "calibration": 84.8710999727096, "p50_s": 2.3997448533773422, "p95_s": 4.381843151897192, "cost_usd_per_1000": 0.41668916104868914, "axes": { "intelligence": 84.65473225404733, "calibration": 84.8710999727096, "speed": 69.78178103134502, "cost": 21.405633888943044 }, "jevbench_score": 57.23613555440268 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "openjev-sglang", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9657534246575342, "hard": 0.7045454545454546 }, "accuracy": 0.8370786516853933, "ece_hard": 0.06228586261778751, "calibration": 83.83378657227547, "p50_s": 2.3997448533773422, "p95_s": 4.381843151897192, "cost_usd_per_1000": 0.4131825131086142, "axes": { "intelligence": 85.21912619344127, "calibration": 83.83378657227547, "speed": 69.78178103134502, "cost": 21.515742014945715 }, "jevbench_score": 57.22866754364795 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "djev", "openjev-sglang", "simplejev-qwen3.8-27b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9657534246575342, "hard": 0.7090909090909091 }, "accuracy": 0.8389513108614233, "ece_hard": 0.12181818181818202, "calibration": 72.8416818181818, "p50_s": 2.3997448533773422, "p95_s": 4.381843151897192, "cost_usd_per_1000": 0.32358208239700376, "axes": { "intelligence": 85.35548982980491, "calibration": 72.8416818181818, "speed": 69.78178103134502, "cost": 24.70046603313547 }, "jevbench_score": 57.21562615618846 }, { "members": [ "djev", "open-alternative-jev", "openjev-sglang", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8020833333333334, "judge": 0.9657534246575342, "hard": 0.6863636363636364 }, "accuracy": 0.8258426966292135, "ece_hard": 0.0626153059529273, "calibration": 83.01183445720116, "p50_s": 2.3997448533773422, "p95_s": 4.385688671469688, "cost_usd_per_1000": 0.398946329588015, "axes": { "intelligence": 84.0903383146534, "calibration": 83.01183445720116, "speed": 69.7779713205367, "cost": 21.9725657859062 }, "jevbench_score": 57.19674388666945 }, { "members": [ "semif-qwen3.5-4b", "djev", "openjev-sglang", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9657534246575342, "hard": 0.6818181818181818 }, "accuracy": 0.8258426966292135, "ece_hard": 0.06909257061074472, "calibration": 82.53371334034618, "p50_s": 2.3997448533773422, "p95_s": 4.381843151897192, "cost_usd_per_1000": 0.399219531835206, "axes": { "intelligence": 84.24564134495643, "calibration": 82.53371334034618, "speed": 69.78178103134502, "cost": 21.963646569580348 }, "jevbench_score": 57.135539243102265 }, { "members": [ "jev-1.13.0", "djev", "openjev-sglang", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9657534246575342, "hard": 0.6909090909090909 }, "accuracy": 0.8295880149812734, "ece_hard": 0.06612185868870593, "calibration": 84.11453520389139, "p50_s": 2.3997448533773422, "p95_s": 4.381843151897192, "cost_usd_per_1000": 0.41668916104868914, "axes": { "intelligence": 84.51836861768369, "calibration": 84.11453520389139, "speed": 69.78178103134502, "cost": 21.405633888943044 }, "jevbench_score": 57.08514052808157 }, { "members": [ "system-one-open", "openjev-sglang", "simplejev-qwen3.6-35b-a3b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9657534246575342, "hard": 0.6545454545454545 }, "accuracy": 0.8108614232209738, "ece_hard": 0.16969696969696937, "calibration": 62.660719696969736, "p50_s": 1.7359912395477295, "p95_s": 2.0568130582571027, "cost_usd_per_1000": 0.26170808988764044, "axes": { "intelligence": 82.84412619344126, "calibration": 62.660719696969736, "speed": 74.47257649879765, "cost": 27.4654855688825 }, "jevbench_score": 57.08344699935262 }, { "members": [ "semif-qwen3.5-4b", "openjev-sglang", "simplejev-qwen3.8-27b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.958904109589041, "hard": 0.6818181818181818 }, "accuracy": 0.8258426966292135, "ece_hard": 0.1727272727272724, "calibration": 65.19927272727276, "p50_s": 2.3997448533773422, "p95_s": 4.381843151897192, "cost_usd_per_1000": 0.2577165917602996, "axes": { "intelligence": 84.34552718970527, "calibration": 65.19927272727276, "speed": 69.78178103134502, "cost": 27.6657286218213 }, "jevbench_score": 57.0817452049903 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "djev", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9726027397260274, "hard": 0.6909090909090909 }, "accuracy": 0.8333333333333334, "ece_hard": 0.14363636363636373, "calibration": 70.53161363636363, "p50_s": 2.3997448533773422, "p95_s": 4.381843151897192, "cost_usd_per_1000": 0.3078582996254682, "axes": { "intelligence": 85.00181610626817, "calibration": 70.53161363636363, "speed": 69.78178103134502, "cost": 25.349474009978593 }, "jevbench_score": 57.066436851175744 }, { "members": [ "jev-1.13.0", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8020833333333334, "judge": 0.9726027397260274, "hard": 0.6954545454545454 }, "accuracy": 0.8314606741573034, "ece_hard": 0.19242424242424228, "calibration": 65.04599242424243, "p50_s": 2.3997448533773422, "p95_s": 4.381843151897192, "cost_usd_per_1000": 0.2594624382022472, "axes": { "intelligence": 84.55484640929846, "calibration": 65.04599242424243, "speed": 69.78178103134502, "cost": 27.577765153220284 }, "jevbench_score": 57.03809890794552 }, { "members": [ "djev", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.6954545454545454 }, "accuracy": 0.8239700374531835, "ece_hard": 0.1803030303030302, "calibration": 63.99111363636365, "p50_s": 2.3997448533773422, "p95_s": 4.381843151897192, "cost_usd_per_1000": 0.24549945692883895, "axes": { "intelligence": 83.68783727687837, "calibration": 63.99111363636365, "speed": 69.78178103134502, "cost": 28.298483927379394 }, "jevbench_score": 57.02585532355081 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "open-alternative-jev", "openjev-sglang", "simplejev-qwen3.8-27b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.958904109589041, "hard": 0.6954545454545454 }, "accuracy": 0.8314606741573034, "ece_hard": 0.12818181818181837, "calibration": 71.90831818181816, "p50_s": 2.3997448533773422, "p95_s": 4.385688671469688, "cost_usd_per_1000": 0.3198022322097378, "axes": { "intelligence": 84.75461809879619, "calibration": 71.90831818181816, "speed": 69.7779713205367, "cost": 24.853555276512637 }, "jevbench_score": 57.018047459311475 }, { "members": [ "open-alternative-jev", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8020833333333334, "judge": 0.8904109589041096, "hard": 0.6590909090909091 }, "accuracy": 0.7940074906367042, "ece_hard": 0.1681818181818178, "calibration": 65.48065151515155, "p50_s": 2.3997448533773422, "p95_s": 4.385688671469688, "cost_usd_per_1000": 0.241719606741573, "axes": { "intelligence": 81.1625674553757, "calibration": 65.48065151515155, "speed": 69.7779713205367, "cost": 28.500643629559633 }, "jevbench_score": 57.01779836690686 }, { "members": [ "semif-qwen3.5-4b", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9657534246575342, "hard": 0.6681818181818182 }, "accuracy": 0.8164794007490637, "ece_hard": 0.18787878787878765, "calibration": 63.76962121212124, "p50_s": 2.3997448533773422, "p95_s": 4.381843151897192, "cost_usd_per_1000": 0.24199280898876402, "axes": { "intelligence": 83.25321710253218, "calibration": 63.76962121212124, "speed": 69.78178103134502, "cost": 28.48592617714492 }, "jevbench_score": 56.99631972810035 }, { "members": [ "semif-qwen3.5-4b", "djev", "openjev-sglang", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9657534246575342, "hard": 0.6772727272727272 }, "accuracy": 0.8239700374531835, "ece_hard": 0.07673676124205396, "calibration": 81.64369973949832, "p50_s": 2.3997448533773422, "p95_s": 4.381843151897192, "cost_usd_per_1000": 0.399219531835206, "axes": { "intelligence": 84.10927770859277, "calibration": 81.64369973949832, "speed": 69.78178103134502, "cost": 21.963646569580348 }, "jevbench_score": 56.957808244115284 }, { "members": [ "jev-1.13.0", "system-one-open", "openjev-razorback16", "openjev-sglang", "simplejev-qwen3.8-27b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.952054794520548, "hard": 0.7227272727272728 }, "accuracy": 0.8408239700374532, "ece_hard": 0.0899999999999998, "calibration": 75.20825000000002, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.3556717078651685, "axes": { "intelligence": 85.3810190950602, "calibration": 75.20825000000002, "speed": 69.75909261944061, "cost": 23.468520390228022 }, "jevbench_score": 56.94142579339267 }, { "members": [ "semif-qwen3.5-4b", "djev", "system-one-open", "openjev-sglang", "simplejev-qwen3.6-35b-a3b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9657534246575342, "hard": 0.6545454545454545 }, "accuracy": 0.8164794007490637, "ece_hard": 0.1572727272727271, "calibration": 66.68497727272728, "p50_s": 1.7359912395477295, "p95_s": 2.0568130582571027, "cost_usd_per_1000": 0.31010395131086144, "axes": { "intelligence": 83.71912619344127, "calibration": 66.68497727272728, "speed": 74.47257649879765, "cost": 25.254780999829194 }, "jevbench_score": 56.92436177458378 }, { "members": [ "jev-1.13.0", "open-alternative-jev", "openjev-razorback16", "openjev-sglang", "simplejev-qwen3.8-27b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.958904109589041, "hard": 0.7090909090909091 }, "accuracy": 0.8370786516853933, "ece_hard": 0.10272727272727292, "calibration": 76.03377272727272, "p50_s": 2.3997448533773422, "p95_s": 4.385688671469688, "cost_usd_per_1000": 0.36296217602996256, "axes": { "intelligence": 85.1637090078871, "calibration": 76.03377272727272, "speed": 69.7779713205367, "cost": 23.20415890159012 }, "jevbench_score": 56.903151436272516 }, { "members": [ "jev-1.13.0", "djev", "open-alternative-jev", "openjev-sglang", "simplejev-qwen3.6-35b-a3b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9452054794520548, "hard": 0.6727272727272727 }, "accuracy": 0.8164794007490637, "ece_hard": 0.1527272727272727, "calibration": 69.17977272727273, "p50_s": 1.7284078150987625, "p95_s": 1.9287681855261325, "cost_usd_per_1000": 0.3348640486891386, "axes": { "intelligence": 83.39757160647572, "calibration": 69.17977272727273, "speed": 74.77073744801535, "cost": 24.253944286318045 }, "jevbench_score": 56.87372109018665 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "system-one-open", "openjev-sglang", "simplejev-qwen3.6-35b-a3b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9726027397260274, "hard": 0.6636363636363637 }, "accuracy": 0.8220973782771536, "ece_hard": 0.15545454545454526, "calibration": 67.42829545454546, "p50_s": 1.7359912395477295, "p95_s": 2.0568130582571027, "cost_usd_per_1000": 0.32406693258426966, "axes": { "intelligence": 84.18363428808635, "calibration": 67.42829545454546, "speed": 74.47257649879765, "cost": 24.680958448439497 }, "jevbench_score": 56.83384820216175 }, { "members": [ "djev", "openjev-sglang", "simplejev-qwen3.6-35b-a3b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8020833333333334, "judge": 0.9452054794520548, "hard": 0.6681818181818182 }, "accuracy": 0.8127340823970037, "ece_hard": 0.20303030303030287, "calibration": 62.338053030303044, "p50_s": 1.7284078150987625, "p95_s": 1.928032819926739, "cost_usd_per_1000": 0.2727784082397004, "axes": { "intelligence": 82.96954130344541, "calibration": 62.338053030303044, "speed": 74.77239356263168, "cost": 26.925700276346674 }, "jevbench_score": 56.80611038449694 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "djev", "openjev-sglang", "simplejev-qwen3.6-35b-a3b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9657534246575342, "hard": 0.6727272727272727 }, "accuracy": 0.8220973782771536, "ece_hard": 0.16545454545454552, "calibration": 68.33729545454545, "p50_s": 1.7284078150987625, "p95_s": 1.9287681855261325, "cost_usd_per_1000": 0.3351372509363296, "axes": { "intelligence": 83.97291407222914, "calibration": 68.33729545454545, "speed": 74.77073744801535, "cost": 24.24331891275166 }, "jevbench_score": 56.79108781347363 }, { "members": [ "djev", "system-one-open", "openjev-razorback16", "openjev-sglang", "simplejev-qwen3.8-27b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9657534246575342, "hard": 0.7 }, "accuracy": 0.8352059925093633, "ece_hard": 0.10272727272727251, "calibration": 73.00627272727274, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.3417087265917603, "axes": { "intelligence": 85.08276255707763, "calibration": 73.00627272727274, "speed": 69.75909261944061, "cost": 23.990317900106177 }, "jevbench_score": 56.78186413827475 }, { "members": [ "semif-qwen3.5-4b", "open-alternative-jev", "openjev-sglang", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8020833333333334, "judge": 0.958904109589041, "hard": 0.6727272727272727 }, "accuracy": 0.8183520599250936, "ece_hard": 0.07387792705546839, "calibration": 80.76852778368523, "p50_s": 2.3997448533773422, "p95_s": 4.385688671469688, "cost_usd_per_1000": 0.3954396816479401, "axes": { "intelligence": 83.48946658364467, "calibration": 80.76852778368523, "speed": 69.7779713205367, "cost": 22.087592565387524 }, "jevbench_score": 56.77866063747892 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "open-alternative-jev", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8020833333333334, "judge": 0.9657534246575342, "hard": 0.6818181818181818 }, "accuracy": 0.8239700374531835, "ece_hard": 0.1472727272727275, "calibration": 69.48222727272726, "p50_s": 2.3997448533773422, "p95_s": 4.385688671469688, "cost_usd_per_1000": 0.3040784494382022, "axes": { "intelligence": 83.95397467828975, "calibration": 69.48222727272726, "speed": 69.7779713205367, "cost": 25.510430738824738 }, "jevbench_score": 56.76593309412532 }, { "members": [ "openjev-razorback16", "openjev-sglang", "simplejev-qwen3.8-27b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.958904109589041, "hard": 0.7 }, "accuracy": 0.8314606741573034, "ece_hard": 0.14999999999999974, "calibration": 68.54708333333336, "p50_s": 2.3997448533773422, "p95_s": 4.381843151897192, "cost_usd_per_1000": 0.30087653558052435, "axes": { "intelligence": 84.59931506849315, "calibration": 68.54708333333336, "speed": 69.78178103134502, "cost": 25.648350406207385 }, "jevbench_score": 56.75960034337848 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "open-alternative-jev", "openjev-sglang", "simplejev-qwen3.6-35b-a3b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9726027397260274, "hard": 0.6636363636363637 }, "accuracy": 0.8220973782771536, "ece_hard": 0.16272727272727278, "calibration": 66.95377272727272, "p50_s": 1.7284078150987625, "p95_s": 1.9287681855261325, "cost_usd_per_1000": 0.33135740074906367, "axes": { "intelligence": 84.18363428808635, "calibration": 66.95377272727272, "speed": 74.77073744801535, "cost": 24.391099754577894 }, "jevbench_score": 56.622813205018765 }, { "members": [ "system-one-open", "openjev-razorback16", "openjev-sglang", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.958904109589041, "hard": 0.7045454545454546 }, "accuracy": 0.8333333333333334, "ece_hard": 0.06064484244047272, "calibration": 82.72871763454019, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.4313091573033708, "axes": { "intelligence": 84.73567870485678, "calibration": 82.72871763454019, "speed": 69.75909261944061, "cost": 20.956339632103877 }, "jevbench_score": 56.57957931223532 }, { "members": [ "djev", "system-one-open", "openjev-razorback16", "openjev-sglang", "simplejev-qwen3.6-35b-a3b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9452054794520548, "hard": 0.6772727272727272 }, "accuracy": 0.8202247191011236, "ece_hard": 0.13999999999999985, "calibration": 69.61225000000002, "p50_s": 1.7359912395477295, "p95_s": 2.0568130582571027, "cost_usd_per_1000": 0.35326389513108614, "axes": { "intelligence": 83.82560190950602, "calibration": 69.61225000000002, "speed": 74.47257649879765, "cost": 23.55702240369645 }, "jevbench_score": 56.564607965556455 }, { "members": [ "djev", "openjev-razorback16", "simplejev-qwen3.8-27b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9452054794520548, "hard": 0.6545454545454545 }, "accuracy": 0.8089887640449438, "ece_hard": 0.23484848484848475, "calibration": 56.55848484848486, "p50_s": 2.3997448533773422, "p95_s": 4.381843151897192, "cost_usd_per_1000": 0.19555232209737827, "axes": { "intelligence": 82.85211706102118, "calibration": 56.55848484848486, "speed": 69.78178103134502, "cost": 31.262110678787337 }, "jevbench_score": 56.54450814279836 }, { "members": [ "open-alternative-jev", "system-one-open", "openjev-razorback16", "openjev-sglang", "simplejev-qwen3.8-27b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.952054794520548, "hard": 0.6636363636363637 }, "accuracy": 0.8146067415730337, "ece_hard": 0.1145454545454544, "calibration": 72.71470454545457, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.33792887640449437, "axes": { "intelligence": 83.31662515566626, "calibration": 72.71470454545457, "speed": 69.75909261944061, "cost": 24.135240870550234 }, "jevbench_score": 56.513420447077 }, { "members": [ "jev-1.13.0", "open-alternative-jev", "openjev-razorback16", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8020833333333334, "judge": 0.9315068493150684, "hard": 0.6909090909090909 }, "accuracy": 0.8183520599250936, "ece_hard": 0.12363636363636378, "calibration": 73.81461363636362, "p50_s": 2.3997448533773422, "p95_s": 4.385688671469688, "cost_usd_per_1000": 0.3472383932584269, "axes": { "intelligence": 83.26779784142798, "calibration": 73.81461363636362, "speed": 69.7779713205367, "cost": 23.781167860415607 }, "jevbench_score": 56.51226796144915 }, { "members": [ "jev-1.13.0", "system-one-open", "openjev-razorback16", "openjev-sglang", "simplejev-qwen3.6-35b-a3b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.952054794520548, "hard": 0.6727272727272727 }, "accuracy": 0.8183520599250936, "ece_hard": 0.13454545454545433, "calibration": 71.05745454545456, "p50_s": 1.7359912395477295, "p95_s": 2.0568130582571027, "cost_usd_per_1000": 0.36722687640449436, "axes": { "intelligence": 83.58935242839352, "calibration": 71.05745454545456, "speed": 74.47257649879765, "cost": 23.05196624185075 }, "jevbench_score": 56.5088202097725 }, { "members": [ "open-alternative-jev", "system-one-open", "openjev-razorback16", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8020833333333334, "judge": 0.9246575342465754, "hard": 0.6681818181818182 }, "accuracy": 0.8071161048689138, "ece_hard": 0.12272727272727255, "calibration": 71.43502272727275, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.3222050936329588, "axes": { "intelligence": 82.39419883769199, "calibration": 71.43502272727275, "speed": 69.75909261944061, "cost": 24.756027947373553 }, "jevbench_score": 56.46409899348799 }, { "members": [ "semif-qwen3.5-4b", "system-one-open", "openjev-razorback16", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6681818181818182 }, "accuracy": 0.8258426966292135, "ece_hard": 0.13818181818181807, "calibration": 69.45206818181819, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.3224782958801498, "axes": { "intelligence": 84.61166459111665, "calibration": 69.45206818181819, "speed": 69.75909261944061, "cost": 24.744985295669025 }, "jevbench_score": 56.43530302159974 }, { "members": [ "jev-1.13.0", "openjev-razorback16", "openjev-sglang", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9657534246575342, "hard": 0.7 }, "accuracy": 0.8333333333333334, "ece_hard": 0.060625065786167304, "calibration": 84.72949976972265, "p50_s": 2.3997448533773422, "p95_s": 4.381843151897192, "cost_usd_per_1000": 0.45634245692883896, "axes": { "intelligence": 84.79109589041096, "calibration": 84.72949976972265, "speed": 69.78178103134502, "cost": 20.22127371117014 }, "jevbench_score": 56.42659734041625 }, { "members": [ "jev-1.13.0", "system-one-open", "openjev-razorback16", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.952054794520548, "hard": 0.6818181818181818 }, "accuracy": 0.8239700374531835, "ece_hard": 0.12818181818181862, "calibration": 71.70631818181815, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.3399479250936329, "axes": { "intelligence": 84.15374636778748, "calibration": 71.70631818181815, "speed": 69.75909261944061, "cost": 24.05762815725552 }, "jevbench_score": 56.411951958908446 }, { "members": [ "semif-qwen3.5-4b", "djev", "open-alternative-jev", "openjev-sglang", "simplejev-qwen3.8-27b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.958904109589041, "hard": 0.6727272727272727 }, "accuracy": 0.8220973782771536, "ece_hard": 0.1545454545454547, "calibration": 67.80145454545453, "p50_s": 2.3997448533773422, "p95_s": 4.385688671469688, "cost_usd_per_1000": 0.3058392509363296, "axes": { "intelligence": 84.072799916978, "calibration": 67.80145454545453, "speed": 69.7779713205367, "cost": 25.435203359945874 }, "jevbench_score": 56.3977812635475 }, { "members": [ "djev", "openjev-razorback16", "openjev-sglang", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9452054794520548, "hard": 0.7 }, "accuracy": 0.8295880149812734, "ece_hard": 0.06391630435212176, "calibration": 83.12091461485913, "p50_s": 2.3997448533773422, "p95_s": 4.381843151897192, "cost_usd_per_1000": 0.44237947565543073, "axes": { "intelligence": 84.5074200913242, "calibration": 83.12091461485913, "speed": 69.78178103134502, "cost": 20.62615091423565 }, "jevbench_score": 56.388604826619705 }, { "members": [ "open-alternative-jev", "openjev-razorback16", "openjev-sglang", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9452054794520548, "hard": 0.7 }, "accuracy": 0.8277153558052435, "ece_hard": 0.0602507815449363, "calibration": 82.93633937551843, "p50_s": 2.3997448533773422, "p95_s": 4.385688671469688, "cost_usd_per_1000": 0.4385996254681648, "axes": { "intelligence": 84.21575342465754, "calibration": 82.93633937551843, "speed": 69.7779713205367, "cost": 20.73795230541691 }, "jevbench_score": 56.38396370177937 }, { "members": [ "jev-1.13.0", "openjev-razorback16", "openjev-sglang", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9657534246575342, "hard": 0.7 }, "accuracy": 0.8333333333333334, "ece_hard": 0.061101243423026655, "calibration": 84.43567021707187, "p50_s": 2.3997448533773422, "p95_s": 4.381843151897192, "cost_usd_per_1000": 0.45634245692883896, "axes": { "intelligence": 84.79109589041096, "calibration": 84.43567021707187, "speed": 69.78178103134502, "cost": 20.22127371117014 }, "jevbench_score": 56.37761379057314 }, { "members": [ "djev", "openjev-razorback16", "openjev-sglang", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.952054794520548, "hard": 0.7045454545454546 }, "accuracy": 0.8314606741573034, "ece_hard": 0.06692165376562664, "calibration": 82.96814754617974, "p50_s": 2.3997448533773422, "p95_s": 4.381843151897192, "cost_usd_per_1000": 0.44237947565543073, "axes": { "intelligence": 84.54389788293899, "calibration": 82.96814754617974, "speed": 69.78178103134502, "cost": 20.62615091423565 }, "jevbench_score": 56.368759222939325 }, { "members": [ "system-one-open", "openjev-razorback16", "openjev-sglang", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.958904109589041, "hard": 0.7090909090909091 }, "accuracy": 0.8352059925093633, "ece_hard": 0.07692441560759831, "calibration": 81.3098955162518, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.4313091573033708, "axes": { "intelligence": 84.87204234122044, "calibration": 81.3098955162518, "speed": 69.75909261944061, "cost": 20.956339632103877 }, "jevbench_score": 56.35806541705665 }, { "members": [ "semif-qwen3.5-4b", "openjev-razorback16", "openjev-sglang", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "calibration_weighted", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9657534246575342, "hard": 0.6863636363636364 }, "accuracy": 0.8277153558052435, "ece_hard": 0.06924665028804328, "calibration": 82.07074179391333, "p50_s": 2.3997448533773422, "p95_s": 4.381843151897192, "cost_usd_per_1000": 0.4388728277153558, "axes": { "intelligence": 84.38200498132005, "calibration": 82.07074179391333, "speed": 69.78178103134502, "cost": 20.729839214335655 }, "jevbench_score": 56.25926391082368 }, { "members": [ "open-alternative-jev", "system-one-open", "openjev-razorback16", "openjev-sglang", "simplejev-qwen3.6-35b-a3b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8020833333333334, "judge": 0.9246575342465754, "hard": 0.6454545454545455 }, "accuracy": 0.797752808988764, "ece_hard": 0.13727272727272707, "calibration": 69.4679772727273, "p50_s": 1.7359912395477295, "p95_s": 2.0568130582571027, "cost_usd_per_1000": 0.34948404494382024, "axes": { "intelligence": 81.7123806558738, "calibration": 69.4679772727273, "speed": 74.47257649879765, "cost": 23.697179391628367 }, "jevbench_score": 56.25891966164086 }, { "members": [ "semif-qwen3.5-4b", "system-one-open", "openjev-razorback16", "openjev-sglang", "simplejev-qwen3.8-27b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9657534246575342, "hard": 0.6727272727272727 }, "accuracy": 0.8258426966292135, "ece_hard": 0.12818181818181815, "calibration": 70.35106818181818, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.3382020786516854, "axes": { "intelligence": 84.55624740556247, "calibration": 70.35106818181818, "speed": 69.75909261944061, "cost": 24.12471182430835 }, "jevbench_score": 56.24965181361576 }, { "members": [ "semif-qwen3.5-4b", "djev", "open-alternative-jev", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8020833333333334, "judge": 0.9452054794520548, "hard": 0.6545454545454545 }, "accuracy": 0.8071161048689138, "ece_hard": 0.16363636363636375, "calibration": 66.37536363636363, "p50_s": 2.3997448533773422, "p95_s": 4.385688671469688, "cost_usd_per_1000": 0.290115468164794, "axes": { "intelligence": 82.56045039435452, "calibration": 66.37536363636363, "speed": 69.7779713205367, "cost": 26.122873455514053 }, "jevbench_score": 56.21854004450148 }, { "members": [ "open-alternative-jev", "openjev-sglang", "simplejev-qwen3.6-35b-a3b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.7708333333333334, "judge": 0.9246575342465754, "hard": 0.6590909090909091 }, "accuracy": 0.797752808988764, "ece_hard": 0.1999999999999996, "calibration": 60.61475000000004, "p50_s": 1.7284078150987625, "p95_s": 1.9287681855261325, "cost_usd_per_1000": 0.26899855805243444, "axes": { "intelligence": 81.24647156496472, "calibration": 60.61475000000004, "speed": 74.77073744801535, "cost": 27.107501439877495 }, "jevbench_score": 56.20837051606258 }, { "members": [ "semif-qwen3.5-4b", "djev", "open-alternative-jev", "openjev-sglang", "simplejev-qwen3.6-35b-a3b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.952054794520548, "hard": 0.65 }, "accuracy": 0.8089887640449438, "ece_hard": 0.1772727272727273, "calibration": 64.49922727272727, "p50_s": 1.7284078150987625, "p95_s": 1.9287681855261325, "cost_usd_per_1000": 0.31739441947565544, "axes": { "intelligence": 82.90753424657535, "calibration": 64.49922727272727, "speed": 74.77073744801535, "cost": 24.95202140231555 }, "jevbench_score": 56.20131498751867 }, { "members": [ "semif-qwen3.5-4b", "open-alternative-jev", "system-one-open", "openjev-sglang", "simplejev-qwen3.6-35b-a3b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8020833333333334, "judge": 0.9726027397260274, "hard": 0.65 }, "accuracy": 0.8127340823970037, "ece_hard": 0.17909090909090847, "calibration": 63.31740909090915, "p50_s": 1.7359912395477295, "p95_s": 2.0568130582571027, "cost_usd_per_1000": 0.30632410112359554, "axes": { "intelligence": 83.1912100456621, "calibration": 63.31740909090915, "speed": 74.47257649879765, "cost": 25.4145649662033 }, "jevbench_score": 56.19140729910973 }, { "members": [ "semif-qwen3.5-4b", "openjev-sglang", "simplejev-qwen3.6-35b-a3b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9726027397260274, "hard": 0.6681818181818182 }, "accuracy": 0.8183520599250936, "ece_hard": 0.21515151515151495, "calibration": 58.84234848484851, "p50_s": 1.7284078150987625, "p95_s": 1.9287681855261325, "cost_usd_per_1000": 0.26927176029962546, "axes": { "intelligence": 83.44499792444998, "calibration": 58.84234848484851, "speed": 74.77073744801535, "cost": 27.09427571653272 }, "jevbench_score": 56.15971290669399 }, { "members": [ "semif-qwen3.5-4b", "open-alternative-jev", "system-one-open", "openjev-sglang", "simplejev-qwen3.8-27b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8020833333333334, "judge": 0.952054794520548, "hard": 0.6363636363636364 }, "accuracy": 0.8014981273408239, "ece_hard": 0.15363636363636335, "calibration": 66.89236363636367, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.29476893258426967, "axes": { "intelligence": 82.20677667081777, "calibration": 66.89236363636367, "speed": 69.75909261944061, "cost": 25.915548736826693 }, "jevbench_score": 56.15149821565454 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "openjev-razorback16", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9726027397260274, "hard": 0.6818181818181818 }, "accuracy": 0.8295880149812734, "ece_hard": 0.14727272727272742, "calibration": 70.54297727272726, "p50_s": 2.3997448533773422, "p95_s": 4.381843151897192, "cost_usd_per_1000": 0.34751159550561794, "axes": { "intelligence": 84.72908883354089, "calibration": 70.54297727272726, "speed": 69.78178103134502, "cost": 23.77092098861135 }, "jevbench_score": 56.11365980350295 }, { "members": [ "jev-1.13.0", "open-alternative-jev", "openjev-razorback16", "openjev-sglang", "simplejev-qwen3.6-35b-a3b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9246575342465754, "hard": 0.6681818181818182 }, "accuracy": 0.8089887640449438, "ece_hard": 0.14727272727272733, "calibration": 70.10022727272727, "p50_s": 1.7284078150987625, "p95_s": 1.9287681855261325, "cost_usd_per_1000": 0.3745173445692884, "axes": { "intelligence": 82.68586550435866, "calibration": 70.10022727272727, "speed": 74.77073744801535, "cost": 22.79584193629418 }, "jevbench_score": 56.06405431764852 }, { "members": [ "openjev-razorback16", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6727272727272727 }, "accuracy": 0.8089887640449438, "ece_hard": 0.18484848484848462, "calibration": 65.07823484848487, "p50_s": 2.3997448533773422, "p95_s": 4.381843151897192, "cost_usd_per_1000": 0.2851527528089888, "axes": { "intelligence": 82.43067662930676, "calibration": 65.07823484848487, "speed": 69.78178103134502, "cost": 26.347672943935677 }, "jevbench_score": 56.04053023694929 }, { "members": [ "jev-1.13.0", "openjev-sglang", "simplejev-qwen3.6-35b-a3b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8020833333333334, "judge": 0.9657534246575342, "hard": 0.6681818181818182 }, "accuracy": 0.8183520599250936, "ece_hard": 0.22727272727272724, "calibration": 60.08539393939394, "p50_s": 1.7284078150987625, "p95_s": 1.928032819926739, "cost_usd_per_1000": 0.2867413895131086, "axes": { "intelligence": 83.54488376919885, "calibration": 60.08539393939394, "speed": 74.77239356263168, "cost": 26.275288436602636 }, "jevbench_score": 56.03951743592406 }, { "members": [ "open-alternative-jev", "openjev-razorback16", "openjev-sglang", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9178082191780822, "hard": 0.6909090909090909 }, "accuracy": 0.8164794007490637, "ece_hard": 0.07134734780115437, "calibration": 81.65737801181443, "p50_s": 2.3997448533773422, "p95_s": 4.385688671469688, "cost_usd_per_1000": 0.4385996254681648, "axes": { "intelligence": 83.17590286425904, "calibration": 81.65737801181443, "speed": 69.7779713205367, "cost": 20.73795230541691 }, "jevbench_score": 55.99113757822506 }, { "members": [ "djev", "system-one-open", "openjev-razorback16", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9452054794520548, "hard": 0.6681818181818182 }, "accuracy": 0.8164794007490637, "ece_hard": 0.1500000000000004, "calibration": 68.13699999999994, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.3259849438202247, "axes": { "intelligence": 83.55287463677875, "calibration": 68.13699999999994, "speed": 69.75909261944061, "cost": 24.604073743372496 }, "jevbench_score": 55.90981585560211 }, { "members": [ "jev-1.13.0", "djev", "openjev-razorback16", "openjev-sglang", "simplejev-qwen3.8-27b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9657534246575342, "hard": 0.7045454545454546 }, "accuracy": 0.8352059925093633, "ece_hard": 0.13454545454545466, "calibration": 71.32445454545453, "p50_s": 2.3997448533773422, "p95_s": 4.381843151897192, "cost_usd_per_1000": 0.36674202621722846, "axes": { "intelligence": 84.92745952677461, "calibration": 71.32445454545453, "speed": 69.78178103134502, "cost": 23.06917959713782 }, "jevbench_score": 55.88113236242433 }, { "members": [ "jev-1.13.0", "djev", "openjev-razorback16", "openjev-sglang", "simplejev-qwen3.6-35b-a3b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9452054794520548, "hard": 0.6727272727272727 }, "accuracy": 0.8164794007490637, "ece_hard": 0.16454545454545472, "calibration": 68.92720454545453, "p50_s": 1.7284078150987625, "p95_s": 1.928032819926739, "cost_usd_per_1000": 0.3782971947565543, "axes": { "intelligence": 83.39757160647572, "calibration": 68.92720454545453, "speed": 74.77239356263168, "cost": 22.66500637551316 }, "jevbench_score": 55.86763662679778 }, { "members": [ "semif-qwen3.5-4b", "openjev-razorback16", "openjev-sglang", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "probability_average", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.958904109589041, "hard": 0.6681818181818182 }, "accuracy": 0.8202247191011236, "ece_hard": 0.0889231664104011, "calibration": 79.98358407045305, "p50_s": 2.3997448533773422, "p95_s": 4.381843151897192, "cost_usd_per_1000": 0.4388728277153558, "axes": { "intelligence": 83.93643628061437, "calibration": 79.98358407045305, "speed": 69.78178103134502, "cost": 20.729839214335655 }, "jevbench_score": 55.82417873977902 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "openjev-razorback16", "openjev-sglang", "simplejev-qwen3.8-27b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9657534246575342, "hard": 0.6954545454545454 }, "accuracy": 0.8333333333333334, "ece_hard": 0.14454545454545475, "calibration": 70.38145454545452, "p50_s": 2.3997448533773422, "p95_s": 4.381843151897192, "cost_usd_per_1000": 0.3632353782771535, "axes": { "intelligence": 84.94639892071399, "calibration": 70.38145454545452, "speed": 69.78178103134502, "cost": 23.194355764692247 }, "jevbench_score": 55.77401301576039 }, { "members": [ "semif-qwen3.5-4b", "open-alternative-jev", "system-one-open", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.958904109589041, "hard": 0.6227272727272727 }, "accuracy": 0.795880149812734, "ece_hard": 0.17999999999999983, "calibration": 63.73900000000002, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.2790451498127341, "axes": { "intelligence": 81.697799916978, "calibration": 63.73900000000002, "speed": 69.75909261944061, "cost": 26.629765651456538 }, "jevbench_score": 55.769396337418094 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "openjev-razorback16", "openjev-sglang", "simplejev-qwen3.6-35b-a3b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9657534246575342, "hard": 0.6636363636363637 }, "accuracy": 0.8183520599250936, "ece_hard": 0.17363636363636364, "calibration": 67.33161363636364, "p50_s": 1.7284078150987625, "p95_s": 1.9287681855261325, "cost_usd_per_1000": 0.3747905468164794, "axes": { "intelligence": 83.70018679950188, "calibration": 67.33161363636364, "speed": 74.77073744801535, "cost": 22.786341150167118 }, "jevbench_score": 55.66573237295112 }, { "members": [ "semif-qwen3.5-4b", "system-one-open", "openjev-razorback16", "openjev-sglang", "simplejev-qwen3.6-35b-a3b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9657534246575342, "hard": 0.6454545454545455 }, "accuracy": 0.8127340823970037, "ece_hard": 0.17272727272727256, "calibration": 64.95202272727275, "p50_s": 1.7359912395477295, "p95_s": 2.0568130582571027, "cost_usd_per_1000": 0.34975724719101126, "axes": { "intelligence": 83.44639892071399, "calibration": 64.95202272727275, "speed": 74.47257649879765, "cost": 23.686998336610245 }, "jevbench_score": 55.606642348146146 }, { "members": [ "jev-1.13.0", "djev", "openjev-razorback16", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9452054794520548, "hard": 0.6954545454545454 }, "accuracy": 0.8258426966292135, "ece_hard": 0.16000000000000014, "calibration": 68.80374999999998, "p50_s": 2.3997448533773422, "p95_s": 4.381843151897192, "cost_usd_per_1000": 0.3510182434456929, "axes": { "intelligence": 84.0793897882939, "calibration": 68.80374999999998, "speed": 69.78178103134502, "cost": 23.640109341760308 }, "jevbench_score": 55.580608573866 }, { "members": [ "djev", "open-alternative-jev", "openjev-razorback16", "openjev-sglang", "simplejev-qwen3.8-27b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9383561643835616, "hard": 0.6772727272727272 }, "accuracy": 0.8164794007490637, "ece_hard": 0.1536363636363638, "calibration": 69.13411363636362, "p50_s": 2.3997448533773422, "p95_s": 4.385688671469688, "cost_usd_per_1000": 0.3489991947565543, "axes": { "intelligence": 83.34215442092155, "calibration": 69.13411363636362, "speed": 69.7779713205367, "cost": 23.715267252530197 }, "jevbench_score": 55.56814139393792 }, { "members": [ "openjev-razorback16", "openjev-sglang", "simplejev-qwen3.6-35b-a3b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8020833333333334, "judge": 0.9246575342465754, "hard": 0.65 }, "accuracy": 0.799625468164794, "ece_hard": 0.20757575757575736, "calibration": 61.86267424242426, "p50_s": 1.7284078150987625, "p95_s": 1.928032819926739, "cost_usd_per_1000": 0.3124317041198502, "axes": { "intelligence": 81.84874429223746, "calibration": 61.86267424242426, "speed": 74.77239356263168, "cost": 25.157347071081787 }, "jevbench_score": 55.55353622351479 }, { "members": [ "djev", "system-one-open", "openjev-sglang", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9657534246575342, "hard": 0.7090909090909091 }, "accuracy": 0.8370786516853933, "ece_hard": 0.11090909090909058, "calibration": 71.79434090909095, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.391655861423221, "axes": { "intelligence": 85.06382316313824, "calibration": 71.79434090909095, "speed": 69.75909261944061, "cost": 22.212861085919087 }, "jevbench_score": 55.463862256865184 }, { "members": [ "djev", "open-alternative-jev", "openjev-razorback16", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6545454545454545 }, "accuracy": 0.8014981273408239, "ece_hard": 0.16000000000000003, "calibration": 67.91125, "p50_s": 2.3997448533773422, "p95_s": 4.385688671469688, "cost_usd_per_1000": 0.3332754119850187, "axes": { "intelligence": 81.88522208385223, "calibration": 67.91125, "speed": 69.7779713205367, "cost": 24.31590178128478 }, "jevbench_score": 55.42286774633081 }, { "members": [ "djev", "open-alternative-jev", "openjev-razorback16", "openjev-sglang", "simplejev-qwen3.6-35b-a3b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9246575342465754, "hard": 0.6545454545454545 }, "accuracy": 0.8033707865168539, "ece_hard": 0.17818181818181827, "calibration": 65.67106818181817, "p50_s": 1.7284078150987625, "p95_s": 1.9287681855261325, "cost_usd_per_1000": 0.36055436329588014, "axes": { "intelligence": 82.27677459526774, "calibration": 65.67106818181817, "speed": 74.77073744801535, "cost": 23.29087733198999 }, "jevbench_score": 55.385006711303696 }, { "members": [ "semif-qwen3.5-4b", "djev", "openjev-razorback16", "openjev-sglang", "simplejev-qwen3.6-35b-a3b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9452054794520548, "hard": 0.6590909090909091 }, "accuracy": 0.8127340823970037, "ece_hard": 0.18818181818181828, "calibration": 64.06831818181817, "p50_s": 1.7284078150987625, "p95_s": 1.9287681855261325, "cost_usd_per_1000": 0.36082756554307116, "axes": { "intelligence": 83.28014736405149, "calibration": 64.06831818181817, "speed": 74.77073744801535, "cost": 23.28100875369647 }, "jevbench_score": 55.205146500023574 }, { "members": [ "semif-qwen3.5-4b", "djev", "openjev-razorback16", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.952054794520548, "hard": 0.6545454545454545 }, "accuracy": 0.8127340823970037, "ece_hard": 0.17636363636363644, "calibration": 65.33738636363636, "p50_s": 2.3997448533773422, "p95_s": 4.381843151897192, "cost_usd_per_1000": 0.3335486142322097, "axes": { "intelligence": 83.33556454960565, "calibration": 65.33738636363636, "speed": 69.78178103134502, "cost": 24.305225780087525 }, "jevbench_score": 55.12625151761061 }, { "members": [ "open-alternative-jev", "system-one-open", "openjev-sglang", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.958904109589041, "hard": 0.6818181818181818 }, "accuracy": 0.8202247191011236, "ece_hard": 0.11545454545454523, "calibration": 70.86579545454546, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.38787601123595505, "axes": { "intelligence": 83.47052718970527, "calibration": 70.86579545454546, "speed": 69.75909261944061, "cost": 22.339212374566742 }, "jevbench_score": 55.10101839080564 }, { "members": [ "semif-qwen3.5-4b", "open-alternative-jev", "openjev-razorback16", "openjev-sglang", "simplejev-qwen3.6-35b-a3b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9315068493150684, "hard": 0.6454545454545455 }, "accuracy": 0.8014981273408239, "ece_hard": 0.18000000000000005, "calibration": 63.82025, "p50_s": 1.7284078150987625, "p95_s": 1.9287681855261325, "cost_usd_per_1000": 0.35704771535580526, "axes": { "intelligence": 82.19582814445829, "calibration": 63.82025, "speed": 74.77073744801535, "cost": 23.418212245121367 }, "jevbench_score": 55.052040359762444 }, { "members": [ "semif-qwen3.5-4b", "djev", "openjev-razorback16", "openjev-sglang", "simplejev-qwen3.8-27b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9657534246575342, "hard": 0.6636363636363637 }, "accuracy": 0.8202247191011236, "ece_hard": 0.16909090909090926, "calibration": 66.08865909090906, "p50_s": 2.3997448533773422, "p95_s": 4.381843151897192, "cost_usd_per_1000": 0.3492723970037453, "axes": { "intelligence": 83.99185346616854, "calibration": 66.08865909090906, "speed": 69.78178103134502, "cost": 23.70507205892477 }, "jevbench_score": 55.047406025055786 }, { "members": [ "djev", "open-alternative-jev", "openjev-sglang", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9452054794520548, "hard": 0.6954545454545454 }, "accuracy": 0.8220973782771536, "ece_hard": 0.12818181818181823, "calibration": 71.50206818181817, "p50_s": 2.3997448533773422, "p95_s": 4.385688671469688, "cost_usd_per_1000": 0.398946329588015, "axes": { "intelligence": 83.49605645496057, "calibration": 71.50206818181817, "speed": 69.7779713205367, "cost": 21.9725657859062 }, "jevbench_score": 55.004208553687484 }, { "members": [ "semif-qwen3.5-4b", "open-alternative-jev", "openjev-razorback16", "openjev-sglang", "simplejev-qwen3.8-27b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.958904109589041, "hard": 0.6636363636363637 }, "accuracy": 0.8183520599250936, "ece_hard": 0.1727272727272728, "calibration": 65.57702272727272, "p50_s": 2.3997448533773422, "p95_s": 4.385688671469688, "cost_usd_per_1000": 0.3454925468164794, "axes": { "intelligence": 83.80007264425073, "calibration": 65.57702272727272, "speed": 69.7779713205367, "cost": 23.846839511911085 }, "jevbench_score": 54.99032856258428 }, { "members": [ "semif-qwen3.5-4b", "open-alternative-jev", "openjev-razorback16", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8020833333333334, "judge": 0.9246575342465754, "hard": 0.6545454545454545 }, "accuracy": 0.8014981273408239, "ece_hard": 0.17818181818181827, "calibration": 65.07706818181816, "p50_s": 2.3997448533773422, "p95_s": 4.385688671469688, "cost_usd_per_1000": 0.3297687640449438, "axes": { "intelligence": 81.98510792860108, "calibration": 65.07706818181816, "speed": 69.7779713205367, "cost": 24.45371450368343 }, "jevbench_score": 54.92961208935613 }, { "members": [ "semif-qwen3.5-4b", "system-one-open", "openjev-sglang", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9726027397260274, "hard": 0.6772727272727272 }, "accuracy": 0.8277153558052435, "ece_hard": 0.13636363636363585, "calibration": 68.36688636363641, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.38814921348314607, "axes": { "intelligence": 84.59272519717726, "calibration": 68.36688636363641, "speed": 69.75909261944061, "cost": 22.33003868565656 }, "jevbench_score": 54.7857080660046 }, { "members": [ "semif-qwen3.5-4b", "djev", "openjev-sglang", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9726027397260274, "hard": 0.6954545454545454 }, "accuracy": 0.8333333333333334, "ece_hard": 0.1463636363636363, "calibration": 69.27588636363637, "p50_s": 2.3997448533773422, "p95_s": 4.381843151897192, "cost_usd_per_1000": 0.399219531835206, "axes": { "intelligence": 84.84651307596513, "calibration": 69.27588636363637, "speed": 69.78178103134502, "cost": 21.963646569580348 }, "jevbench_score": 54.785502143315995 }, { "members": [ "jev-1.13.0", "djev", "openjev-sglang", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9657534246575342, "hard": 0.7 }, "accuracy": 0.8333333333333334, "ece_hard": 0.14636363636363647, "calibration": 70.38613636363635, "p50_s": 2.3997448533773422, "p95_s": 4.381843151897192, "cost_usd_per_1000": 0.41668916104868914, "axes": { "intelligence": 84.79109589041096, "calibration": 70.38613636363635, "speed": 69.78178103134502, "cost": 21.405633888943044 }, "jevbench_score": 54.64203680178989 }, { "members": [ "system-one-open", "openjev-razorback16", "openjev-sglang", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9657534246575342, "hard": 0.6909090909090909 }, "accuracy": 0.8295880149812734, "ece_hard": 0.1154545454545452, "calibration": 71.99604545454548, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.4313091573033708, "axes": { "intelligence": 84.51836861768369, "calibration": 71.99604545454548, "speed": 69.75909261944061, "cost": 20.956339632103877 }, "jevbench_score": 54.61274422294187 }, { "members": [ "jev-1.13.0", "open-alternative-jev", "openjev-sglang", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8020833333333334, "judge": 0.9657534246575342, "hard": 0.6909090909090909 }, "accuracy": 0.8277153558052435, "ece_hard": 0.14181818181818184, "calibration": 70.22943181818181, "p50_s": 2.3997448533773422, "p95_s": 4.385688671469688, "cost_usd_per_1000": 0.41290931086142324, "axes": { "intelligence": 84.22670195101702, "calibration": 70.22943181818181, "speed": 69.7779713205367, "cost": 21.52435971791394 }, "jevbench_score": 54.59519011356115 }, { "members": [ "jev-1.13.0", "system-one-open", "openjev-sglang", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9657534246575342, "hard": 0.6909090909090909 }, "accuracy": 0.8314606741573034, "ece_hard": 0.14999999999999963, "calibration": 68.35575000000003, "p50_s": 2.405150428414345, "p95_s": 4.394894900918007, "cost_usd_per_1000": 0.40561884269662923, "axes": { "intelligence": 84.81003528435036, "calibration": 68.35575000000003, "speed": 69.75909261944061, "cost": 21.756456352172563 }, "jevbench_score": 54.46315798855425 }, { "members": [ "jev-1.13.0", "semif-qwen3.5-4b", "openjev-sglang", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9726027397260274, "hard": 0.6863636363636364 }, "accuracy": 0.8314606741573034, "ece_hard": 0.16454545454545452, "calibration": 68.01920454545454, "p50_s": 2.3997448533773422, "p95_s": 4.381843151897192, "cost_usd_per_1000": 0.4131825131086142, "axes": { "intelligence": 84.86545246990454, "calibration": 68.01920454545454, "speed": 69.78178103134502, "cost": 21.515742014945715 }, "jevbench_score": 54.25817905573954 }, { "members": [ "openjev-sglang", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.78125, "judge": 0.9657534246575342, "hard": 0.6772727272727272 }, "accuracy": 0.8183520599250936, "ece_hard": 0.19848484848484838, "calibration": 62.4985984848485, "p50_s": 2.3997448533773422, "p95_s": 4.381843151897192, "cost_usd_per_1000": 0.35082367041198503, "axes": { "intelligence": 83.23427770859277, "calibration": 62.4985984848485, "speed": 69.78178103134502, "cost": 23.64733336147438 }, "jevbench_score": 54.12827405116949 }, { "members": [ "semif-qwen3.5-4b", "open-alternative-jev", "openjev-sglang", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8020833333333334, "judge": 0.958904109589041, "hard": 0.6772727272727272 }, "accuracy": 0.8202247191011236, "ece_hard": 0.16454545454545422, "calibration": 66.27195454545458, "p50_s": 2.3997448533773422, "p95_s": 4.385688671469688, "cost_usd_per_1000": 0.3954396816479401, "axes": { "intelligence": 83.6258302200083, "calibration": 66.27195454545458, "speed": 69.7779713205367, "cost": 22.087592565387524 }, "jevbench_score": 54.06102003464228 }, { "members": [ "open-alternative-jev", "openjev-razorback16", "openjev-sglang", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9246575342465754, "hard": 0.6818181818181818 }, "accuracy": 0.8108614232209738, "ece_hard": 0.1363636363636365, "calibration": 71.05888636363636, "p50_s": 2.3997448533773422, "p95_s": 4.385688671469688, "cost_usd_per_1000": 0.4385996254681648, "axes": { "intelligence": 82.51162308011624, "calibration": 71.05888636363636, "speed": 69.7779713205367, "cost": 20.73795230541691 }, "jevbench_score": 53.970250563168065 }, { "members": [ "jev-1.13.0", "openjev-razorback16", "openjev-sglang", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9657534246575342, "hard": 0.6954545454545454 }, "accuracy": 0.8314606741573034, "ece_hard": 0.150909090909091, "calibration": 70.5878409090909, "p50_s": 2.3997448533773422, "p95_s": 4.381843151897192, "cost_usd_per_1000": 0.45634245692883896, "axes": { "intelligence": 84.65473225404733, "calibration": 70.5878409090909, "speed": 69.78178103134502, "cost": 20.22127371117014 }, "jevbench_score": 53.88686572466083 }, { "members": [ "djev", "openjev-razorback16", "openjev-sglang", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9452054794520548, "hard": 0.7090909090909091 }, "accuracy": 0.8314606741573034, "ece_hard": 0.1500000000000002, "calibration": 69.32074999999998, "p50_s": 2.3997448533773422, "p95_s": 4.381843151897192, "cost_usd_per_1000": 0.44237947565543073, "axes": { "intelligence": 84.48848069738482, "calibration": 69.32074999999998, "speed": 69.78178103134502, "cost": 20.62615091423565 }, "jevbench_score": 53.88343188128679 }, { "members": [ "semif-qwen3.5-4b", "openjev-razorback16", "openjev-sglang", "simplejev-qwen3.8-27b", "simplejev-qwen3.6-35b-a3b" ], "method": "majority", "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9726027397260274, "hard": 0.6818181818181818 }, "accuracy": 0.8277153558052435, "ece_hard": 0.16636363636363638, "calibration": 67.08838636363636, "p50_s": 2.3997448533773422, "p95_s": 4.381843151897192, "cost_usd_per_1000": 0.4388728277153558, "axes": { "intelligence": 84.43742216687423, "calibration": 67.08838636363636, "speed": 69.78178103134502, "cost": 20.729839214335655 }, "jevbench_score": 53.50323894774292 } ], "cascades": [ { "cheap": "classifier-dev-fast", "backend": "jev-1.13.0", "chosen": { "threshold": 0.42, "public": { "n": 231, "escalation": 0.047619047619047616, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.7027027027027027 }, "accuracy": 0.8008658008658008, "ece_hard": 0.12020202020202014, "calibration": 74.125797979798, "p50_s": 0.3823304697871208, "p95_s": 0.6449073478579521, "cost_usd_per_1000": 0.005234011235955055, "axes": { "intelligence": 80.59100767434103, "calibration": 74.125797979798, "speed": 86.08063764376911, "cost": 78.43496048159845 }, "jevbench_score": 79.69256058741296 }, "heldout": { "n": 303, "escalation": 0.0297029702970297, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6513761467889908 }, "accuracy": 0.8481848184818482, "ece_hard": 0.13549902696691687, "calibration": 74.25959730330831, "p50_s": 0.3878225088119507, "p95_s": 0.4766422070562839, "cost_usd_per_1000": 0.004518904698335002, "axes": { "intelligence": 84.10749444933184, "calibration": 74.25959730330831, "speed": 87.3317449168534, "cost": 80.34900452941774 }, "jevbench_score": 81.36450391759082 } }, "full": { "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9726027397260274, "hard": 0.6772727272727272 }, "accuracy": 0.8277153558052435, "ece_hard": 0.12750826446280986, "calibration": 74.22692355371902, "p50_s": 0.38611557707190514, "p95_s": 0.49209726192057135, "cost_usd_per_1000": 0.004828248537642553, "axes": { "intelligence": 84.59272519717726, "calibration": 74.22692355371902, "speed": 87.21231728685717, "cost": 79.48631147114776 }, "jevbench_score": 81.2252696657347 }, "full_escalation": 0.03745318352059925, "backend_heldout_accuracy": 0.8514851485148515, "useful": true, "curve": [ { "threshold": 0.0, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6846846846846847 }, "accuracy": 0.7922077922077922, "ece_hard": 0.11600782600782594, "calibration": 74.5452173992174, "p50_s": 0.37952230870723724, "p95_s": 0.45826196670532227, "cost_usd_per_1000": 0.003333333333333333, "axes": { "intelligence": 79.84025692359026, "calibration": 74.5452173992174, "speed": 87.5964887612797, "cost": 84.31363764158988 }, "jevbench_score": 81.42477349817814 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6513761467889908 }, "accuracy": 0.8481848184818482, "ece_hard": 0.14760911871003615, "calibration": 73.04858812899639, "p50_s": 0.38682974874973297, "p95_s": 0.4573646493256092, "cost_usd_per_1000": 0.003333333333333333, "axes": { "intelligence": 84.10749444933184, "calibration": 73.04858812899639, "speed": 87.52217540292565, "cost": 84.31363764158988 }, "jevbench_score": 82.05699584888464 } }, { "threshold": 0.01, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6846846846846847 }, "accuracy": 0.7922077922077922, "ece_hard": 0.11600782600782594, "calibration": 74.5452173992174, "p50_s": 0.37952230870723724, "p95_s": 0.45826196670532227, "cost_usd_per_1000": 0.003333333333333333, "axes": { "intelligence": 79.84025692359026, "calibration": 74.5452173992174, "speed": 87.5964887612797, "cost": 84.31363764158988 }, "jevbench_score": 81.42477349817814 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6513761467889908 }, "accuracy": 0.8481848184818482, "ece_hard": 0.14760911871003615, "calibration": 73.04858812899639, "p50_s": 0.38682974874973297, "p95_s": 0.4573646493256092, "cost_usd_per_1000": 0.003333333333333333, "axes": { "intelligence": 84.10749444933184, "calibration": 73.04858812899639, "speed": 87.52217540292565, "cost": 84.31363764158988 }, "jevbench_score": 82.05699584888464 } }, { "threshold": 0.02, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6846846846846847 }, "accuracy": 0.7922077922077922, "ece_hard": 0.11600782600782594, "calibration": 74.5452173992174, "p50_s": 0.37952230870723724, "p95_s": 0.45826196670532227, "cost_usd_per_1000": 0.003333333333333333, "axes": { "intelligence": 79.84025692359026, "calibration": 74.5452173992174, "speed": 87.5964887612797, "cost": 84.31363764158988 }, "jevbench_score": 81.42477349817814 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6513761467889908 }, "accuracy": 0.8481848184818482, "ece_hard": 0.14760911871003615, "calibration": 73.04858812899639, "p50_s": 0.38682974874973297, "p95_s": 0.4573646493256092, "cost_usd_per_1000": 0.003333333333333333, "axes": { "intelligence": 84.10749444933184, "calibration": 73.04858812899639, "speed": 87.52217540292565, "cost": 84.31363764158988 }, "jevbench_score": 82.05699584888464 } }, { "threshold": 0.03, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6846846846846847 }, "accuracy": 0.7922077922077922, "ece_hard": 0.11600782600782594, "calibration": 74.5452173992174, "p50_s": 0.37952230870723724, "p95_s": 0.45826196670532227, "cost_usd_per_1000": 0.003333333333333333, "axes": { "intelligence": 79.84025692359026, "calibration": 74.5452173992174, "speed": 87.5964887612797, "cost": 84.31363764158988 }, "jevbench_score": 81.42477349817814 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6513761467889908 }, "accuracy": 0.8481848184818482, "ece_hard": 0.14760911871003615, "calibration": 73.04858812899639, "p50_s": 0.38682974874973297, "p95_s": 0.4573646493256092, "cost_usd_per_1000": 0.003333333333333333, "axes": { "intelligence": 84.10749444933184, "calibration": 73.04858812899639, "speed": 87.52217540292565, "cost": 84.31363764158988 }, "jevbench_score": 82.05699584888464 } }, { "threshold": 0.04, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6846846846846847 }, "accuracy": 0.7922077922077922, "ece_hard": 0.11600782600782594, "calibration": 74.5452173992174, "p50_s": 0.37952230870723724, "p95_s": 0.45826196670532227, "cost_usd_per_1000": 0.003333333333333333, "axes": { "intelligence": 79.84025692359026, "calibration": 74.5452173992174, "speed": 87.5964887612797, "cost": 84.31363764158988 }, "jevbench_score": 81.42477349817814 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6513761467889908 }, "accuracy": 0.8481848184818482, "ece_hard": 0.14760911871003615, "calibration": 73.04858812899639, "p50_s": 0.38682974874973297, "p95_s": 0.4573646493256092, "cost_usd_per_1000": 0.003333333333333333, "axes": { "intelligence": 84.10749444933184, "calibration": 73.04858812899639, "speed": 87.52217540292565, "cost": 84.31363764158988 }, "jevbench_score": 82.05699584888464 } }, { "threshold": 0.05, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6846846846846847 }, "accuracy": 0.7922077922077922, "ece_hard": 0.11600782600782594, "calibration": 74.5452173992174, "p50_s": 0.37952230870723724, "p95_s": 0.45826196670532227, "cost_usd_per_1000": 0.003333333333333333, "axes": { "intelligence": 79.84025692359026, "calibration": 74.5452173992174, "speed": 87.5964887612797, "cost": 84.31363764158988 }, "jevbench_score": 81.42477349817814 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6513761467889908 }, "accuracy": 0.8481848184818482, "ece_hard": 0.14760911871003615, "calibration": 73.04858812899639, "p50_s": 0.38682974874973297, "p95_s": 0.4573646493256092, "cost_usd_per_1000": 0.003333333333333333, "axes": { "intelligence": 84.10749444933184, "calibration": 73.04858812899639, "speed": 87.52217540292565, "cost": 84.31363764158988 }, "jevbench_score": 82.05699584888464 } }, { "threshold": 0.06, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6846846846846847 }, "accuracy": 0.7922077922077922, "ece_hard": 0.11600782600782594, "calibration": 74.5452173992174, "p50_s": 0.37952230870723724, "p95_s": 0.45826196670532227, "cost_usd_per_1000": 0.003333333333333333, "axes": { "intelligence": 79.84025692359026, "calibration": 74.5452173992174, "speed": 87.5964887612797, "cost": 84.31363764158988 }, "jevbench_score": 81.42477349817814 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6513761467889908 }, "accuracy": 0.8481848184818482, "ece_hard": 0.14760911871003615, "calibration": 73.04858812899639, "p50_s": 0.38682974874973297, "p95_s": 0.4573646493256092, "cost_usd_per_1000": 0.003333333333333333, "axes": { "intelligence": 84.10749444933184, "calibration": 73.04858812899639, "speed": 87.52217540292565, "cost": 84.31363764158988 }, "jevbench_score": 82.05699584888464 } }, { "threshold": 0.07, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6846846846846847 }, "accuracy": 0.7922077922077922, "ece_hard": 0.11600782600782594, "calibration": 74.5452173992174, "p50_s": 0.37952230870723724, "p95_s": 0.45826196670532227, "cost_usd_per_1000": 0.003333333333333333, "axes": { "intelligence": 79.84025692359026, "calibration": 74.5452173992174, "speed": 87.5964887612797, "cost": 84.31363764158988 }, "jevbench_score": 81.42477349817814 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6513761467889908 }, "accuracy": 0.8481848184818482, "ece_hard": 0.14760911871003615, "calibration": 73.04858812899639, "p50_s": 0.38682974874973297, "p95_s": 0.4573646493256092, "cost_usd_per_1000": 0.003333333333333333, "axes": { "intelligence": 84.10749444933184, "calibration": 73.04858812899639, "speed": 87.52217540292565, "cost": 84.31363764158988 }, "jevbench_score": 82.05699584888464 } }, { "threshold": 0.08, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6846846846846847 }, "accuracy": 0.7922077922077922, "ece_hard": 0.11600782600782594, "calibration": 74.5452173992174, "p50_s": 0.37952230870723724, "p95_s": 0.45826196670532227, "cost_usd_per_1000": 0.003333333333333333, "axes": { "intelligence": 79.84025692359026, "calibration": 74.5452173992174, "speed": 87.5964887612797, "cost": 84.31363764158988 }, "jevbench_score": 81.42477349817814 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6513761467889908 }, "accuracy": 0.8481848184818482, "ece_hard": 0.14760911871003615, "calibration": 73.04858812899639, "p50_s": 0.38682974874973297, "p95_s": 0.4573646493256092, "cost_usd_per_1000": 0.003333333333333333, "axes": { "intelligence": 84.10749444933184, "calibration": 73.04858812899639, "speed": 87.52217540292565, "cost": 84.31363764158988 }, "jevbench_score": 82.05699584888464 } }, { "threshold": 0.09, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6846846846846847 }, "accuracy": 0.7922077922077922, "ece_hard": 0.11600782600782594, "calibration": 74.5452173992174, "p50_s": 0.37952230870723724, "p95_s": 0.45826196670532227, "cost_usd_per_1000": 0.003333333333333333, "axes": { "intelligence": 79.84025692359026, "calibration": 74.5452173992174, "speed": 87.5964887612797, "cost": 84.31363764158988 }, "jevbench_score": 81.42477349817814 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6513761467889908 }, "accuracy": 0.8481848184818482, "ece_hard": 0.14760911871003615, "calibration": 73.04858812899639, "p50_s": 0.38682974874973297, "p95_s": 0.4573646493256092, "cost_usd_per_1000": 0.003333333333333333, "axes": { "intelligence": 84.10749444933184, "calibration": 73.04858812899639, "speed": 87.52217540292565, "cost": 84.31363764158988 }, "jevbench_score": 82.05699584888464 } }, { "threshold": 0.1, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6846846846846847 }, "accuracy": 0.7922077922077922, "ece_hard": 0.11600782600782594, "calibration": 74.5452173992174, "p50_s": 0.37952230870723724, "p95_s": 0.45826196670532227, "cost_usd_per_1000": 0.003333333333333333, "axes": { "intelligence": 79.84025692359026, "calibration": 74.5452173992174, "speed": 87.5964887612797, "cost": 84.31363764158988 }, "jevbench_score": 81.42477349817814 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6513761467889908 }, "accuracy": 0.8481848184818482, "ece_hard": 0.14760911871003615, "calibration": 73.04858812899639, "p50_s": 0.38682974874973297, "p95_s": 0.4573646493256092, "cost_usd_per_1000": 0.003333333333333333, "axes": { "intelligence": 84.10749444933184, "calibration": 73.04858812899639, "speed": 87.52217540292565, "cost": 84.31363764158988 }, "jevbench_score": 82.05699584888464 } }, { "threshold": 0.11, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6846846846846847 }, "accuracy": 0.7922077922077922, "ece_hard": 0.11600782600782594, "calibration": 74.5452173992174, "p50_s": 0.37952230870723724, "p95_s": 0.45826196670532227, "cost_usd_per_1000": 0.003333333333333333, "axes": { "intelligence": 79.84025692359026, "calibration": 74.5452173992174, "speed": 87.5964887612797, "cost": 84.31363764158988 }, "jevbench_score": 81.42477349817814 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6513761467889908 }, "accuracy": 0.8481848184818482, "ece_hard": 0.14760911871003615, "calibration": 73.04858812899639, "p50_s": 0.38682974874973297, "p95_s": 0.4573646493256092, "cost_usd_per_1000": 0.003333333333333333, "axes": { "intelligence": 84.10749444933184, "calibration": 73.04858812899639, "speed": 87.52217540292565, "cost": 84.31363764158988 }, "jevbench_score": 82.05699584888464 } }, { "threshold": 0.12, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6846846846846847 }, "accuracy": 0.7922077922077922, "ece_hard": 0.11600782600782594, "calibration": 74.5452173992174, "p50_s": 0.37952230870723724, "p95_s": 0.45826196670532227, "cost_usd_per_1000": 0.003333333333333333, "axes": { "intelligence": 79.84025692359026, "calibration": 74.5452173992174, "speed": 87.5964887612797, "cost": 84.31363764158988 }, "jevbench_score": 81.42477349817814 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6513761467889908 }, "accuracy": 0.8481848184818482, "ece_hard": 0.14760911871003615, "calibration": 73.04858812899639, "p50_s": 0.38682974874973297, "p95_s": 0.4573646493256092, "cost_usd_per_1000": 0.003333333333333333, "axes": { "intelligence": 84.10749444933184, "calibration": 73.04858812899639, "speed": 87.52217540292565, "cost": 84.31363764158988 }, "jevbench_score": 82.05699584888464 } }, { "threshold": 0.13, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6846846846846847 }, "accuracy": 0.7922077922077922, "ece_hard": 0.11600782600782594, "calibration": 74.5452173992174, "p50_s": 0.37952230870723724, "p95_s": 0.45826196670532227, "cost_usd_per_1000": 0.003333333333333333, "axes": { "intelligence": 79.84025692359026, "calibration": 74.5452173992174, "speed": 87.5964887612797, "cost": 84.31363764158988 }, "jevbench_score": 81.42477349817814 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6513761467889908 }, "accuracy": 0.8481848184818482, "ece_hard": 0.14760911871003615, "calibration": 73.04858812899639, "p50_s": 0.38682974874973297, "p95_s": 0.4573646493256092, "cost_usd_per_1000": 0.003333333333333333, "axes": { "intelligence": 84.10749444933184, "calibration": 73.04858812899639, "speed": 87.52217540292565, "cost": 84.31363764158988 }, "jevbench_score": 82.05699584888464 } }, { "threshold": 0.14, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6846846846846847 }, "accuracy": 0.7922077922077922, "ece_hard": 0.11600782600782594, "calibration": 74.5452173992174, "p50_s": 0.37952230870723724, "p95_s": 0.45826196670532227, "cost_usd_per_1000": 0.003333333333333333, "axes": { "intelligence": 79.84025692359026, "calibration": 74.5452173992174, "speed": 87.5964887612797, "cost": 84.31363764158988 }, "jevbench_score": 81.42477349817814 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6513761467889908 }, "accuracy": 0.8481848184818482, "ece_hard": 0.14760911871003615, "calibration": 73.04858812899639, "p50_s": 0.38682974874973297, "p95_s": 0.4573646493256092, "cost_usd_per_1000": 0.003333333333333333, "axes": { "intelligence": 84.10749444933184, "calibration": 73.04858812899639, "speed": 87.52217540292565, "cost": 84.31363764158988 }, "jevbench_score": 82.05699584888464 } }, { "threshold": 0.15, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6846846846846847 }, "accuracy": 0.7922077922077922, "ece_hard": 0.11600782600782594, "calibration": 74.5452173992174, "p50_s": 0.37952230870723724, "p95_s": 0.45826196670532227, "cost_usd_per_1000": 0.003333333333333333, "axes": { "intelligence": 79.84025692359026, "calibration": 74.5452173992174, "speed": 87.5964887612797, "cost": 84.31363764158988 }, "jevbench_score": 81.42477349817814 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6513761467889908 }, "accuracy": 0.8481848184818482, "ece_hard": 0.14760911871003615, "calibration": 73.04858812899639, "p50_s": 0.38682974874973297, "p95_s": 0.4573646493256092, "cost_usd_per_1000": 0.003333333333333333, "axes": { "intelligence": 84.10749444933184, "calibration": 73.04858812899639, "speed": 87.52217540292565, "cost": 84.31363764158988 }, "jevbench_score": 82.05699584888464 } }, { "threshold": 0.16, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6846846846846847 }, "accuracy": 0.7922077922077922, "ece_hard": 0.11600782600782594, "calibration": 74.5452173992174, "p50_s": 0.37952230870723724, "p95_s": 0.45826196670532227, "cost_usd_per_1000": 0.003333333333333333, "axes": { "intelligence": 79.84025692359026, "calibration": 74.5452173992174, "speed": 87.5964887612797, "cost": 84.31363764158988 }, "jevbench_score": 81.42477349817814 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6513761467889908 }, "accuracy": 0.8481848184818482, "ece_hard": 0.14760911871003615, "calibration": 73.04858812899639, "p50_s": 0.38682974874973297, "p95_s": 0.4573646493256092, "cost_usd_per_1000": 0.003333333333333333, "axes": { "intelligence": 84.10749444933184, "calibration": 73.04858812899639, "speed": 87.52217540292565, "cost": 84.31363764158988 }, "jevbench_score": 82.05699584888464 } }, { "threshold": 0.17, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6846846846846847 }, "accuracy": 0.7922077922077922, "ece_hard": 0.11600782600782594, "calibration": 74.5452173992174, "p50_s": 0.37952230870723724, "p95_s": 0.45826196670532227, "cost_usd_per_1000": 0.003333333333333333, "axes": { "intelligence": 79.84025692359026, "calibration": 74.5452173992174, "speed": 87.5964887612797, "cost": 84.31363764158988 }, "jevbench_score": 81.42477349817814 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6513761467889908 }, "accuracy": 0.8481848184818482, "ece_hard": 0.14760911871003615, "calibration": 73.04858812899639, "p50_s": 0.38682974874973297, "p95_s": 0.4573646493256092, "cost_usd_per_1000": 0.003333333333333333, "axes": { "intelligence": 84.10749444933184, "calibration": 73.04858812899639, "speed": 87.52217540292565, "cost": 84.31363764158988 }, "jevbench_score": 82.05699584888464 } }, { "threshold": 0.18, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6846846846846847 }, "accuracy": 0.7922077922077922, "ece_hard": 0.11600782600782594, "calibration": 74.5452173992174, "p50_s": 0.37952230870723724, "p95_s": 0.45826196670532227, "cost_usd_per_1000": 0.003333333333333333, "axes": { "intelligence": 79.84025692359026, "calibration": 74.5452173992174, "speed": 87.5964887612797, "cost": 84.31363764158988 }, "jevbench_score": 81.42477349817814 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6513761467889908 }, "accuracy": 0.8481848184818482, "ece_hard": 0.14760911871003615, "calibration": 73.04858812899639, "p50_s": 0.38682974874973297, "p95_s": 0.4573646493256092, "cost_usd_per_1000": 0.003333333333333333, "axes": { "intelligence": 84.10749444933184, "calibration": 73.04858812899639, "speed": 87.52217540292565, "cost": 84.31363764158988 }, "jevbench_score": 82.05699584888464 } }, { "threshold": 0.19, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6846846846846847 }, "accuracy": 0.7922077922077922, "ece_hard": 0.11600782600782594, "calibration": 74.5452173992174, "p50_s": 0.37952230870723724, "p95_s": 0.45826196670532227, "cost_usd_per_1000": 0.003333333333333333, "axes": { "intelligence": 79.84025692359026, "calibration": 74.5452173992174, "speed": 87.5964887612797, "cost": 84.31363764158988 }, "jevbench_score": 81.42477349817814 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6513761467889908 }, "accuracy": 0.8481848184818482, "ece_hard": 0.14760911871003615, "calibration": 73.04858812899639, "p50_s": 0.38682974874973297, "p95_s": 0.4573646493256092, "cost_usd_per_1000": 0.003333333333333333, "axes": { "intelligence": 84.10749444933184, "calibration": 73.04858812899639, "speed": 87.52217540292565, "cost": 84.31363764158988 }, "jevbench_score": 82.05699584888464 } }, { "threshold": 0.2, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6846846846846847 }, "accuracy": 0.7922077922077922, "ece_hard": 0.11600782600782594, "calibration": 74.5452173992174, "p50_s": 0.37952230870723724, "p95_s": 0.45826196670532227, "cost_usd_per_1000": 0.003333333333333333, "axes": { "intelligence": 79.84025692359026, "calibration": 74.5452173992174, "speed": 87.5964887612797, "cost": 84.31363764158988 }, "jevbench_score": 81.42477349817814 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6513761467889908 }, "accuracy": 0.8481848184818482, "ece_hard": 0.14760911871003615, "calibration": 73.04858812899639, "p50_s": 0.38682974874973297, "p95_s": 0.4573646493256092, "cost_usd_per_1000": 0.003333333333333333, "axes": { "intelligence": 84.10749444933184, "calibration": 73.04858812899639, "speed": 87.52217540292565, "cost": 84.31363764158988 }, "jevbench_score": 82.05699584888464 } }, { "threshold": 0.21, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6846846846846847 }, "accuracy": 0.7922077922077922, "ece_hard": 0.11600782600782594, "calibration": 74.5452173992174, "p50_s": 0.37952230870723724, "p95_s": 0.45826196670532227, "cost_usd_per_1000": 0.003333333333333333, "axes": { "intelligence": 79.84025692359026, "calibration": 74.5452173992174, "speed": 87.5964887612797, "cost": 84.31363764158988 }, "jevbench_score": 81.42477349817814 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6513761467889908 }, "accuracy": 0.8481848184818482, "ece_hard": 0.14760911871003615, "calibration": 73.04858812899639, "p50_s": 0.38682974874973297, "p95_s": 0.4573646493256092, "cost_usd_per_1000": 0.003333333333333333, "axes": { "intelligence": 84.10749444933184, "calibration": 73.04858812899639, "speed": 87.52217540292565, "cost": 84.31363764158988 }, "jevbench_score": 82.05699584888464 } }, { "threshold": 0.22, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6846846846846847 }, "accuracy": 0.7922077922077922, "ece_hard": 0.11600782600782594, "calibration": 74.5452173992174, "p50_s": 0.37952230870723724, "p95_s": 0.45826196670532227, "cost_usd_per_1000": 0.003333333333333333, "axes": { "intelligence": 79.84025692359026, "calibration": 74.5452173992174, "speed": 87.5964887612797, "cost": 84.31363764158988 }, "jevbench_score": 81.42477349817814 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6513761467889908 }, "accuracy": 0.8481848184818482, "ece_hard": 0.14760911871003615, "calibration": 73.04858812899639, "p50_s": 0.38682974874973297, "p95_s": 0.4573646493256092, "cost_usd_per_1000": 0.003333333333333333, "axes": { "intelligence": 84.10749444933184, "calibration": 73.04858812899639, "speed": 87.52217540292565, "cost": 84.31363764158988 }, "jevbench_score": 82.05699584888464 } }, { "threshold": 0.23, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6846846846846847 }, "accuracy": 0.7922077922077922, "ece_hard": 0.11600782600782594, "calibration": 74.5452173992174, "p50_s": 0.37952230870723724, "p95_s": 0.45826196670532227, "cost_usd_per_1000": 0.003333333333333333, "axes": { "intelligence": 79.84025692359026, "calibration": 74.5452173992174, "speed": 87.5964887612797, "cost": 84.31363764158988 }, "jevbench_score": 81.42477349817814 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6513761467889908 }, "accuracy": 0.8481848184818482, "ece_hard": 0.14760911871003615, "calibration": 73.04858812899639, "p50_s": 0.38682974874973297, "p95_s": 0.4573646493256092, "cost_usd_per_1000": 0.003333333333333333, "axes": { "intelligence": 84.10749444933184, "calibration": 73.04858812899639, "speed": 87.52217540292565, "cost": 84.31363764158988 }, "jevbench_score": 82.05699584888464 } }, { "threshold": 0.24, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6846846846846847 }, "accuracy": 0.7922077922077922, "ece_hard": 0.11600782600782594, "calibration": 74.5452173992174, "p50_s": 0.37952230870723724, "p95_s": 0.45826196670532227, "cost_usd_per_1000": 0.003333333333333333, "axes": { "intelligence": 79.84025692359026, "calibration": 74.5452173992174, "speed": 87.5964887612797, "cost": 84.31363764158988 }, "jevbench_score": 81.42477349817814 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6513761467889908 }, "accuracy": 0.8481848184818482, "ece_hard": 0.14760911871003615, "calibration": 73.04858812899639, "p50_s": 0.38682974874973297, "p95_s": 0.4573646493256092, "cost_usd_per_1000": 0.003333333333333333, "axes": { "intelligence": 84.10749444933184, "calibration": 73.04858812899639, "speed": 87.52217540292565, "cost": 84.31363764158988 }, "jevbench_score": 82.05699584888464 } }, { "threshold": 0.25, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6846846846846847 }, "accuracy": 0.7922077922077922, "ece_hard": 0.11600782600782594, "calibration": 74.5452173992174, "p50_s": 0.37952230870723724, "p95_s": 0.45826196670532227, "cost_usd_per_1000": 0.003333333333333333, "axes": { "intelligence": 79.84025692359026, "calibration": 74.5452173992174, "speed": 87.5964887612797, "cost": 84.31363764158988 }, "jevbench_score": 81.42477349817814 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6513761467889908 }, "accuracy": 0.8481848184818482, "ece_hard": 0.14760911871003615, "calibration": 73.04858812899639, "p50_s": 0.38682974874973297, "p95_s": 0.4573646493256092, "cost_usd_per_1000": 0.003333333333333333, "axes": { "intelligence": 84.10749444933184, "calibration": 73.04858812899639, "speed": 87.52217540292565, "cost": 84.31363764158988 }, "jevbench_score": 82.05699584888464 } }, { "threshold": 0.26, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6846846846846847 }, "accuracy": 0.7922077922077922, "ece_hard": 0.11600782600782594, "calibration": 74.5452173992174, "p50_s": 0.37952230870723724, "p95_s": 0.45826196670532227, "cost_usd_per_1000": 0.003333333333333333, "axes": { "intelligence": 79.84025692359026, "calibration": 74.5452173992174, "speed": 87.5964887612797, "cost": 84.31363764158988 }, "jevbench_score": 81.42477349817814 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6513761467889908 }, "accuracy": 0.8481848184818482, "ece_hard": 0.14760911871003615, "calibration": 73.04858812899639, "p50_s": 0.38682974874973297, "p95_s": 0.4573646493256092, "cost_usd_per_1000": 0.003333333333333333, "axes": { "intelligence": 84.10749444933184, "calibration": 73.04858812899639, "speed": 87.52217540292565, "cost": 84.31363764158988 }, "jevbench_score": 82.05699584888464 } }, { "threshold": 0.27, "public": { "n": 231, "escalation": 0.004329004329004329, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6936936936936937 }, "accuracy": 0.7965367965367965, "ece_hard": 0.11970151970151963, "calibration": 74.17584802984803, "p50_s": 0.37952230870723724, "p95_s": 0.45826196670532227, "cost_usd_per_1000": 0.0035061222335716715, "axes": { "intelligence": 80.21563229896564, "calibration": 74.17584802984803, "speed": 87.5964887612797, "cost": 83.6551884164597 }, "jevbench_score": 81.25971181881896 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6513761467889908 }, "accuracy": 0.8481848184818482, "ece_hard": 0.14760911871003615, "calibration": 73.04858812899639, "p50_s": 0.38682974874973297, "p95_s": 0.4573646493256092, "cost_usd_per_1000": 0.003333333333333333, "axes": { "intelligence": 84.10749444933184, "calibration": 73.04858812899639, "speed": 87.52217540292565, "cost": 84.31363764158988 }, "jevbench_score": 82.05699584888464 } }, { "threshold": 0.28, "public": { "n": 231, "escalation": 0.004329004329004329, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6936936936936937 }, "accuracy": 0.7965367965367965, "ece_hard": 0.11970151970151963, "calibration": 74.17584802984803, "p50_s": 0.37952230870723724, "p95_s": 0.45826196670532227, "cost_usd_per_1000": 0.0035061222335716715, "axes": { "intelligence": 80.21563229896564, "calibration": 74.17584802984803, "speed": 87.5964887612797, "cost": 83.6551884164597 }, "jevbench_score": 81.25971181881896 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6513761467889908 }, "accuracy": 0.8481848184818482, "ece_hard": 0.14760911871003615, "calibration": 73.04858812899639, "p50_s": 0.38682974874973297, "p95_s": 0.4573646493256092, "cost_usd_per_1000": 0.003333333333333333, "axes": { "intelligence": 84.10749444933184, "calibration": 73.04858812899639, "speed": 87.52217540292565, "cost": 84.31363764158988 }, "jevbench_score": 82.05699584888464 } }, { "threshold": 0.29, "public": { "n": 231, "escalation": 0.004329004329004329, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6936936936936937 }, "accuracy": 0.7965367965367965, "ece_hard": 0.11970151970151963, "calibration": 74.17584802984803, "p50_s": 0.37952230870723724, "p95_s": 0.45826196670532227, "cost_usd_per_1000": 0.0035061222335716715, "axes": { "intelligence": 80.21563229896564, "calibration": 74.17584802984803, "speed": 87.5964887612797, "cost": 83.6551884164597 }, "jevbench_score": 81.25971181881896 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6513761467889908 }, "accuracy": 0.8481848184818482, "ece_hard": 0.14760911871003615, "calibration": 73.04858812899639, "p50_s": 0.38682974874973297, "p95_s": 0.4573646493256092, "cost_usd_per_1000": 0.003333333333333333, "axes": { "intelligence": 84.10749444933184, "calibration": 73.04858812899639, "speed": 87.52217540292565, "cost": 84.31363764158988 }, "jevbench_score": 82.05699584888464 } }, { "threshold": 0.3, "public": { "n": 231, "escalation": 0.004329004329004329, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6936936936936937 }, "accuracy": 0.7965367965367965, "ece_hard": 0.11970151970151963, "calibration": 74.17584802984803, "p50_s": 0.37952230870723724, "p95_s": 0.45826196670532227, "cost_usd_per_1000": 0.0035061222335716715, "axes": { "intelligence": 80.21563229896564, "calibration": 74.17584802984803, "speed": 87.5964887612797, "cost": 83.6551884164597 }, "jevbench_score": 81.25971181881896 }, "heldout": { "n": 303, "escalation": 0.0033003300330033004, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6513761467889908 }, "accuracy": 0.8481848184818482, "ece_hard": 0.1477008618293022, "calibration": 73.03941381706977, "p50_s": 0.38682974874973297, "p95_s": 0.4617097586393355, "cost_usd_per_1000": 0.003465063485000185, "axes": { "intelligence": 84.10749444933184, "calibration": 73.03941381706977, "speed": 87.48111080486143, "cost": 83.80866412219129 }, "jevbench_score": 81.92166971672712 } }, { "threshold": 0.31, "public": { "n": 231, "escalation": 0.004329004329004329, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6936936936936937 }, "accuracy": 0.7965367965367965, "ece_hard": 0.11970151970151963, "calibration": 74.17584802984803, "p50_s": 0.37952230870723724, "p95_s": 0.45826196670532227, "cost_usd_per_1000": 0.0035061222335716715, "axes": { "intelligence": 80.21563229896564, "calibration": 74.17584802984803, "speed": 87.5964887612797, "cost": 83.6551884164597 }, "jevbench_score": 81.25971181881896 }, "heldout": { "n": 303, "escalation": 0.009900990099009901, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6422018348623854 }, "accuracy": 0.8448844884488449, "ece_hard": 0.15155407283847652, "calibration": 72.65409271615235, "p50_s": 0.38682974874973297, "p95_s": 0.46461645364761345, "cost_usd_per_1000": 0.003728523788333889, "axes": { "intelligence": 83.83226509153367, "calibration": 72.65409271615235, "speed": 87.45385548747693, "cost": 82.85389245174045 }, "jevbench_score": 81.50622711444174 } }, { "threshold": 0.32, "public": { "n": 231, "escalation": 0.008658008658008658, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6936936936936937 }, "accuracy": 0.7965367965367965, "ece_hard": 0.12663845663845658, "calibration": 73.48215433615434, "p50_s": 0.3799804672598839, "p95_s": 0.46032172814011574, "cost_usd_per_1000": 0.00367891113381001, "axes": { "intelligence": 80.21563229896564, "calibration": 73.48215433615434, "speed": 87.57177252561257, "cost": 83.02842108016867 }, "jevbench_score": 80.91107153958086 }, "heldout": { "n": 303, "escalation": 0.019801980198019802, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6513761467889908 }, "accuracy": 0.8481848184818482, "ece_hard": 0.14357242146232974, "calibration": 73.45225785376701, "p50_s": 0.3868839740753174, "p95_s": 0.4686349973082542, "cost_usd_per_1000": 0.004123714243334445, "axes": { "intelligence": 84.10749444933184, "calibration": 73.45225785376701, "speed": 87.41584541463484, "cost": 81.54134311574016 }, "jevbench_score": 81.46141828859678 } }, { "threshold": 0.33, "public": { "n": 231, "escalation": 0.008658008658008658, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6936936936936937 }, "accuracy": 0.7965367965367965, "ece_hard": 0.12663845663845658, "calibration": 73.48215433615434, "p50_s": 0.3799804672598839, "p95_s": 0.46032172814011574, "cost_usd_per_1000": 0.00367891113381001, "axes": { "intelligence": 80.21563229896564, "calibration": 73.48215433615434, "speed": 87.57177252561257, "cost": 83.02842108016867 }, "jevbench_score": 80.91107153958086 }, "heldout": { "n": 303, "escalation": 0.0231023102310231, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6513761467889908 }, "accuracy": 0.8481848184818482, "ece_hard": 0.13898526549902696, "calibration": 73.91097345009729, "p50_s": 0.3868839740753174, "p95_s": 0.4715754613280296, "cost_usd_per_1000": 0.004255444395001298, "axes": { "intelligence": 84.10749444933184, "calibration": 73.91097345009729, "speed": 87.3886806144598, "cost": 81.13165239838685 }, "jevbench_score": 81.47929833649593 } }, { "threshold": 0.34, "public": { "n": 231, "escalation": 0.012987012987012988, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6936936936936937 }, "accuracy": 0.7965367965367965, "ece_hard": 0.12618800618800613, "calibration": 73.52719938119938, "p50_s": 0.3799804672598839, "p95_s": 0.4620703458786011, "cost_usd_per_1000": 0.0038517000340483483, "axes": { "intelligence": 80.21563229896564, "calibration": 73.52719938119938, "speed": 87.5553062968599, "cost": 82.43042627750147 }, "jevbench_score": 80.77356722996103 }, "heldout": { "n": 303, "escalation": 0.0231023102310231, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6513761467889908 }, "accuracy": 0.8481848184818482, "ece_hard": 0.13898526549902696, "calibration": 73.91097345009729, "p50_s": 0.3868839740753174, "p95_s": 0.4715754613280296, "cost_usd_per_1000": 0.004255444395001298, "axes": { "intelligence": 84.10749444933184, "calibration": 73.91097345009729, "speed": 87.3886806144598, "cost": 81.13165239838685 }, "jevbench_score": 81.47929833649593 } }, { "threshold": 0.35, "public": { "n": 231, "escalation": 0.012987012987012988, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6936936936936937 }, "accuracy": 0.7965367965367965, "ece_hard": 0.12618800618800613, "calibration": 73.52719938119938, "p50_s": 0.3799804672598839, "p95_s": 0.4620703458786011, "cost_usd_per_1000": 0.0038517000340483483, "axes": { "intelligence": 80.21563229896564, "calibration": 73.52719938119938, "speed": 87.5553062968599, "cost": 82.43042627750147 }, "jevbench_score": 80.77356722996103 }, "heldout": { "n": 303, "escalation": 0.0231023102310231, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6513761467889908 }, "accuracy": 0.8481848184818482, "ece_hard": 0.13898526549902696, "calibration": 73.91097345009729, "p50_s": 0.3868839740753174, "p95_s": 0.4715754613280296, "cost_usd_per_1000": 0.004255444395001298, "axes": { "intelligence": 84.10749444933184, "calibration": 73.91097345009729, "speed": 87.3886806144598, "cost": 81.13165239838685 }, "jevbench_score": 81.47929833649593 } }, { "threshold": 0.36, "public": { "n": 231, "escalation": 0.017316017316017316, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6936936936936937 }, "accuracy": 0.7965367965367965, "ece_hard": 0.13177359177359171, "calibration": 72.96864082264082, "p50_s": 0.38095298409461975, "p95_s": 0.4637797400355339, "cost_usd_per_1000": 0.004024488934286687, "axes": { "intelligence": 80.21563229896564, "calibration": 72.96864082264082, "speed": 87.5281684734722, "cost": 81.85867787213589 }, "jevbench_score": 80.4733273743367 }, "heldout": { "n": 303, "escalation": 0.026402640264026403, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6513761467889908 }, "accuracy": 0.8481848184818482, "ece_hard": 0.1350403113705866, "calibration": 74.30546886294132, "p50_s": 0.3868839740753174, "p95_s": 0.4743887178599834, "cost_usd_per_1000": 0.00438717454666815, "axes": { "intelligence": 84.10749444933184, "calibration": 74.30546886294132, "speed": 87.36284907840614, "cost": 80.73445259666298 }, "jevbench_score": 81.48173953294231 } }, { "threshold": 0.37, "public": { "n": 231, "escalation": 0.025974025974025976, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6936936936936937 }, "accuracy": 0.7965367965367965, "ece_hard": 0.12663845663845658, "calibration": 73.48215433615434, "p50_s": 0.3810563385486603, "p95_s": 0.4835945852100849, "cost_usd_per_1000": 0.004370066734763364, "axes": { "intelligence": 80.21563229896564, "calibration": 73.48215433615434, "speed": 87.34529380419866, "cost": 80.78535792760462 }, "jevbench_score": 80.30697436018168 }, "heldout": { "n": 303, "escalation": 0.026402640264026403, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6513761467889908 }, "accuracy": 0.8481848184818482, "ece_hard": 0.1350403113705866, "calibration": 74.30546886294132, "p50_s": 0.3868839740753174, "p95_s": 0.4743887178599834, "cost_usd_per_1000": 0.00438717454666815, "axes": { "intelligence": 84.10749444933184, "calibration": 74.30546886294132, "speed": 87.36284907840614, "cost": 80.73445259666298 }, "jevbench_score": 81.48173953294231 } }, { "threshold": 0.38, "public": { "n": 231, "escalation": 0.025974025974025976, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6936936936936937 }, "accuracy": 0.7965367965367965, "ece_hard": 0.12663845663845658, "calibration": 73.48215433615434, "p50_s": 0.3810563385486603, "p95_s": 0.4835945852100849, "cost_usd_per_1000": 0.004370066734763364, "axes": { "intelligence": 80.21563229896564, "calibration": 73.48215433615434, "speed": 87.34529380419866, "cost": 80.78535792760462 }, "jevbench_score": 80.30697436018168 }, "heldout": { "n": 303, "escalation": 0.026402640264026403, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6513761467889908 }, "accuracy": 0.8481848184818482, "ece_hard": 0.1350403113705866, "calibration": 74.30546886294132, "p50_s": 0.3868839740753174, "p95_s": 0.4743887178599834, "cost_usd_per_1000": 0.00438717454666815, "axes": { "intelligence": 84.10749444933184, "calibration": 74.30546886294132, "speed": 87.36284907840614, "cost": 80.73445259666298 }, "jevbench_score": 81.48173953294231 } }, { "threshold": 0.39, "public": { "n": 231, "escalation": 0.025974025974025976, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6936936936936937 }, "accuracy": 0.7965367965367965, "ece_hard": 0.12663845663845658, "calibration": 73.48215433615434, "p50_s": 0.3810563385486603, "p95_s": 0.4835945852100849, "cost_usd_per_1000": 0.004370066734763364, "axes": { "intelligence": 80.21563229896564, "calibration": 73.48215433615434, "speed": 87.34529380419866, "cost": 80.78535792760462 }, "jevbench_score": 80.30697436018168 }, "heldout": { "n": 303, "escalation": 0.026402640264026403, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6513761467889908 }, "accuracy": 0.8481848184818482, "ece_hard": 0.1350403113705866, "calibration": 74.30546886294132, "p50_s": 0.3868839740753174, "p95_s": 0.4743887178599834, "cost_usd_per_1000": 0.00438717454666815, "axes": { "intelligence": 84.10749444933184, "calibration": 74.30546886294132, "speed": 87.36284907840614, "cost": 80.73445259666298 }, "jevbench_score": 81.48173953294231 } }, { "threshold": 0.4, "public": { "n": 231, "escalation": 0.03463203463203463, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6936936936936937 }, "accuracy": 0.7965367965367965, "ece_hard": 0.11654836654836648, "calibration": 74.49116334516336, "p50_s": 0.3822959288954735, "p95_s": 0.49219903722405434, "cost_usd_per_1000": 0.00471564453524004, "axes": { "intelligence": 80.21563229896564, "calibration": 74.49116334516336, "speed": 87.25459566941197, "cost": 79.79376818238512 }, "jevbench_score": 80.31196708974745 }, "heldout": { "n": 303, "escalation": 0.026402640264026403, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6513761467889908 }, "accuracy": 0.8481848184818482, "ece_hard": 0.1350403113705866, "calibration": 74.30546886294132, "p50_s": 0.3868839740753174, "p95_s": 0.4743887178599834, "cost_usd_per_1000": 0.00438717454666815, "axes": { "intelligence": 84.10749444933184, "calibration": 74.30546886294132, "speed": 87.36284907840614, "cost": 80.73445259666298 }, "jevbench_score": 81.48173953294231 } }, { "threshold": 0.41, "public": { "n": 231, "escalation": 0.03896103896103896, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6936936936936937 }, "accuracy": 0.7965367965367965, "ece_hard": 0.11794976794976789, "calibration": 74.35102320502321, "p50_s": 0.3822959288954735, "p95_s": 0.5056085176765919, "cost_usd_per_1000": 0.004888433435478378, "axes": { "intelligence": 80.21563229896564, "calibration": 74.35102320502321, "speed": 87.13785945169523, "cost": 79.32490882340409 }, "jevbench_score": 80.12916320591279 }, "heldout": { "n": 303, "escalation": 0.026402640264026403, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6513761467889908 }, "accuracy": 0.8481848184818482, "ece_hard": 0.1350403113705866, "calibration": 74.30546886294132, "p50_s": 0.3868839740753174, "p95_s": 0.4743887178599834, "cost_usd_per_1000": 0.00438717454666815, "axes": { "intelligence": 84.10749444933184, "calibration": 74.30546886294132, "speed": 87.36284907840614, "cost": 80.73445259666298 }, "jevbench_score": 81.48173953294231 } }, { "threshold": 0.42, "public": { "n": 231, "escalation": 0.047619047619047616, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.7027027027027027 }, "accuracy": 0.8008658008658008, "ece_hard": 0.12020202020202014, "calibration": 74.125797979798, "p50_s": 0.3823304697871208, "p95_s": 0.6449073478579521, "cost_usd_per_1000": 0.005234011235955055, "axes": { "intelligence": 80.59100767434103, "calibration": 74.125797979798, "speed": 86.08063764376911, "cost": 78.43496048159845 }, "jevbench_score": 79.69256058741296 }, "heldout": { "n": 303, "escalation": 0.0297029702970297, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6513761467889908 }, "accuracy": 0.8481848184818482, "ece_hard": 0.13549902696691687, "calibration": 74.25959730330831, "p50_s": 0.3878225088119507, "p95_s": 0.4766422070562839, "cost_usd_per_1000": 0.004518904698335002, "axes": { "intelligence": 84.10749444933184, "calibration": 74.25959730330831, "speed": 87.3317449168534, "cost": 80.34900452941774 }, "jevbench_score": 81.36450391759082 } }, { "threshold": 0.43, "public": { "n": 231, "escalation": 0.05194805194805195, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.7027027027027027 }, "accuracy": 0.8008658008658008, "ece_hard": 0.12002184002183997, "calibration": 74.143815997816, "p50_s": 0.3823304697871208, "p95_s": 0.871529832482338, "cost_usd_per_1000": 0.005406800136193394, "axes": { "intelligence": 80.59100767434103, "calibration": 74.143815997816, "speed": 84.7727883193944, "cost": 78.01179051816464 }, "jevbench_score": 79.2856440854687 }, "heldout": { "n": 303, "escalation": 0.033003300330033, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6513761467889908 }, "accuracy": 0.8481848184818482, "ece_hard": 0.13531554072838478, "calibration": 74.27794592716151, "p50_s": 0.3878225088119507, "p95_s": 0.47853453680872915, "cost_usd_per_1000": 0.0046506348500018535, "axes": { "intelligence": 84.10749444933184, "calibration": 74.27794592716151, "speed": 87.31453701371083, "cost": 79.97463274857031 }, "jevbench_score": 81.2705778547526 } }, { "threshold": 0.44, "public": { "n": 231, "escalation": 0.05194805194805195, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.7027027027027027 }, "accuracy": 0.8008658008658008, "ece_hard": 0.12002184002183997, "calibration": 74.143815997816, "p50_s": 0.3823304697871208, "p95_s": 0.871529832482338, "cost_usd_per_1000": 0.005406800136193394, "axes": { "intelligence": 80.59100767434103, "calibration": 74.143815997816, "speed": 84.7727883193944, "cost": 78.01179051816464 }, "jevbench_score": 79.2856440854687 }, "heldout": { "n": 303, "escalation": 0.033003300330033, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6513761467889908 }, "accuracy": 0.8481848184818482, "ece_hard": 0.13531554072838478, "calibration": 74.27794592716151, "p50_s": 0.3878225088119507, "p95_s": 0.47853453680872915, "cost_usd_per_1000": 0.0046506348500018535, "axes": { "intelligence": 84.10749444933184, "calibration": 74.27794592716151, "speed": 87.31453701371083, "cost": 79.97463274857031 }, "jevbench_score": 81.2705778547526 } }, { "threshold": 0.45, "public": { "n": 231, "escalation": 0.05194805194805195, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.7027027027027027 }, "accuracy": 0.8008658008658008, "ece_hard": 0.12002184002183997, "calibration": 74.143815997816, "p50_s": 0.3823304697871208, "p95_s": 0.871529832482338, "cost_usd_per_1000": 0.005406800136193394, "axes": { "intelligence": 80.59100767434103, "calibration": 74.143815997816, "speed": 84.7727883193944, "cost": 78.01179051816464 }, "jevbench_score": 79.2856440854687 }, "heldout": { "n": 303, "escalation": 0.036303630363036306, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6513761467889908 }, "accuracy": 0.8481848184818482, "ece_hard": 0.13513205448985266, "calibration": 74.29629455101473, "p50_s": 0.3878225088119507, "p95_s": 0.47853453680872915, "cost_usd_per_1000": 0.004782365001668705, "axes": { "intelligence": 84.10749444933184, "calibration": 74.29629455101473, "speed": 87.31453701371083, "cost": 79.61071841648685 }, "jevbench_score": 81.18297964670548 } }, { "threshold": 0.46, "public": { "n": 231, "escalation": 0.05194805194805195, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.7027027027027027 }, "accuracy": 0.8008658008658008, "ece_hard": 0.12002184002183997, "calibration": 74.143815997816, "p50_s": 0.3823304697871208, "p95_s": 0.871529832482338, "cost_usd_per_1000": 0.005406800136193394, "axes": { "intelligence": 80.59100767434103, "calibration": 74.143815997816, "speed": 84.7727883193944, "cost": 78.01179051816464 }, "jevbench_score": 79.2856440854687 }, "heldout": { "n": 303, "escalation": 0.036303630363036306, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6513761467889908 }, "accuracy": 0.8481848184818482, "ece_hard": 0.13513205448985266, "calibration": 74.29629455101473, "p50_s": 0.3878225088119507, "p95_s": 0.47853453680872915, "cost_usd_per_1000": 0.004782365001668705, "axes": { "intelligence": 84.10749444933184, "calibration": 74.29629455101473, "speed": 87.31453701371083, "cost": 79.61071841648685 }, "jevbench_score": 81.18297964670548 } }, { "threshold": 0.47, "public": { "n": 231, "escalation": 0.05194805194805195, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.7027027027027027 }, "accuracy": 0.8008658008658008, "ece_hard": 0.12002184002183997, "calibration": 74.143815997816, "p50_s": 0.3823304697871208, "p95_s": 0.871529832482338, "cost_usd_per_1000": 0.005406800136193394, "axes": { "intelligence": 80.59100767434103, "calibration": 74.143815997816, "speed": 84.7727883193944, "cost": 78.01179051816464 }, "jevbench_score": 79.2856440854687 }, "heldout": { "n": 303, "escalation": 0.039603960396039604, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6513761467889908 }, "accuracy": 0.8481848184818482, "ece_hard": 0.13439810953572423, "calibration": 74.36968904642757, "p50_s": 0.3878225088119507, "p95_s": 0.48337363377213466, "cost_usd_per_1000": 0.004914095153335558, "axes": { "intelligence": 84.10749444933184, "calibration": 74.36968904642757, "speed": 87.27084030954856, "cost": 79.25669315137732 }, "jevbench_score": 81.1024440044659 } }, { "threshold": 0.48, "public": { "n": 231, "escalation": 0.05194805194805195, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.7027027027027027 }, "accuracy": 0.8008658008658008, "ece_hard": 0.12002184002183997, "calibration": 74.143815997816, "p50_s": 0.3823304697871208, "p95_s": 0.871529832482338, "cost_usd_per_1000": 0.005406800136193394, "axes": { "intelligence": 80.59100767434103, "calibration": 74.143815997816, "speed": 84.7727883193944, "cost": 78.01179051816464 }, "jevbench_score": 79.2856440854687 }, "heldout": { "n": 303, "escalation": 0.039603960396039604, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6513761467889908 }, "accuracy": 0.8481848184818482, "ece_hard": 0.13439810953572423, "calibration": 74.36968904642757, "p50_s": 0.3878225088119507, "p95_s": 0.48337363377213466, "cost_usd_per_1000": 0.004914095153335558, "axes": { "intelligence": 84.10749444933184, "calibration": 74.36968904642757, "speed": 87.27084030954856, "cost": 79.25669315137732 }, "jevbench_score": 81.1024440044659 } }, { "threshold": 0.49, "public": { "n": 231, "escalation": 0.05194805194805195, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.7027027027027027 }, "accuracy": 0.8008658008658008, "ece_hard": 0.12002184002183997, "calibration": 74.143815997816, "p50_s": 0.3823304697871208, "p95_s": 0.871529832482338, "cost_usd_per_1000": 0.005406800136193394, "axes": { "intelligence": 80.59100767434103, "calibration": 74.143815997816, "speed": 84.7727883193944, "cost": 78.01179051816464 }, "jevbench_score": 79.2856440854687 }, "heldout": { "n": 303, "escalation": 0.039603960396039604, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6513761467889908 }, "accuracy": 0.8481848184818482, "ece_hard": 0.13439810953572423, "calibration": 74.36968904642757, "p50_s": 0.3878225088119507, "p95_s": 0.48337363377213466, "cost_usd_per_1000": 0.004914095153335558, "axes": { "intelligence": 84.10749444933184, "calibration": 74.36968904642757, "speed": 87.27084030954856, "cost": 79.25669315137732 }, "jevbench_score": 81.1024440044659 } }, { "threshold": 0.5, "public": { "n": 231, "escalation": 0.05627705627705628, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.7027027027027027 }, "accuracy": 0.8008658008658008, "ece_hard": 0.11984165984165979, "calibration": 74.16183401583402, "p50_s": 0.3823304697871208, "p95_s": 0.9914946109056473, "cost_usd_per_1000": 0.005579589036431732, "axes": { "intelligence": 80.59100767434103, "calibration": 74.16183401583402, "speed": 84.21270731531501, "cost": 77.60193363285504 }, "jevbench_score": 79.05499306935997 }, "heldout": { "n": 303, "escalation": 0.039603960396039604, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6513761467889908 }, "accuracy": 0.8481848184818482, "ece_hard": 0.13439810953572423, "calibration": 74.36968904642757, "p50_s": 0.3878225088119507, "p95_s": 0.48337363377213466, "cost_usd_per_1000": 0.004914095153335558, "axes": { "intelligence": 84.10749444933184, "calibration": 74.36968904642757, "speed": 87.27084030954856, "cost": 79.25669315137732 }, "jevbench_score": 81.1024440044659 } }, { "threshold": 0.51, "public": { "n": 231, "escalation": 0.06493506493506493, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.7027027027027027 }, "accuracy": 0.8008658008658008, "ece_hard": 0.1112831012831012, "calibration": 75.01768987168988, "p50_s": 0.38274315744638443, "p95_s": 1.007637556642294, "cost_usd_per_1000": 0.005925166836908409, "axes": { "intelligence": 80.59100767434103, "calibration": 75.01768987168988, "speed": 84.13788218590159, "cost": 76.81898249711246 }, "jevbench_score": 79.06378470159942 }, "heldout": { "n": 303, "escalation": 0.039603960396039604, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6513761467889908 }, "accuracy": 0.8481848184818482, "ece_hard": 0.13439810953572423, "calibration": 74.36968904642757, "p50_s": 0.3878225088119507, "p95_s": 0.48337363377213466, "cost_usd_per_1000": 0.004914095153335558, "axes": { "intelligence": 84.10749444933184, "calibration": 74.36968904642757, "speed": 87.27084030954856, "cost": 79.25669315137732 }, "jevbench_score": 81.1024440044659 } }, { "threshold": 0.52, "public": { "n": 231, "escalation": 0.06926406926406926, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.7027027027027027 }, "accuracy": 0.8008658008658008, "ece_hard": 0.11200382200382195, "calibration": 74.9456177996178, "p50_s": 0.38274315744638443, "p95_s": 1.0190754160284996, "cost_usd_per_1000": 0.006097955737146748, "axes": { "intelligence": 80.59100767434103, "calibration": 74.9456177996178, "speed": 84.08886239848334, "cost": 76.44447197036884 }, "jevbench_score": 78.93676935983687 }, "heldout": { "n": 303, "escalation": 0.052805280528052806, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6605504587155964 }, "accuracy": 0.8514851485148515, "ece_hard": 0.1278843480678343, "calibration": 75.90106519321657, "p50_s": 0.3884039744734764, "p95_s": 1.0087750829756224, "cost_usd_per_1000": 0.0054410157600029655, "axes": { "intelligence": 84.38272380713, "calibration": 75.90106519321657, "speed": 84.06921988174533, "cost": 77.92960048450779 }, "jevbench_score": 80.48412362816174 } }, { "threshold": 0.53, "public": { "n": 231, "escalation": 0.0735930735930736, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.7027027027027027 }, "accuracy": 0.8008658008658008, "ece_hard": 0.10407589407589402, "calibration": 75.7384105924106, "p50_s": 0.38274315744638443, "p95_s": 1.0220291279256344, "cost_usd_per_1000": 0.006270744637385086, "axes": { "intelligence": 80.59100767434103, "calibration": 75.7384105924106, "speed": 84.07629291322792, "cost": 76.08042653725288 }, "jevbench_score": 79.04735026488143 }, "heldout": { "n": 303, "escalation": 0.0627062706270627, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6605504587155964 }, "accuracy": 0.8514851485148515, "ece_hard": 0.1283430636641646, "calibration": 75.67869363358355, "p50_s": 0.3886619731783867, "p95_s": 1.0376297369599339, "cost_usd_per_1000": 0.005836206215003522, "axes": { "intelligence": 84.38272380713, "calibration": 75.67869363358355, "speed": 83.94385539309273, "cost": 77.01608113775158 }, "jevbench_score": 80.15846160698234 } }, { "threshold": 0.54, "public": { "n": 231, "escalation": 0.0735930735930736, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.7027027027027027 }, "accuracy": 0.8008658008658008, "ece_hard": 0.10407589407589402, "calibration": 75.7384105924106, "p50_s": 0.38274315744638443, "p95_s": 1.0220291279256344, "cost_usd_per_1000": 0.006270744637385086, "axes": { "intelligence": 80.59100767434103, "calibration": 75.7384105924106, "speed": 84.07629291322792, "cost": 76.08042653725288 }, "jevbench_score": 79.04735026488143 }, "heldout": { "n": 303, "escalation": 0.066006600660066, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6605504587155964 }, "accuracy": 0.8514851485148515, "ece_hard": 0.1301779260494857, "calibration": 75.49520739505144, "p50_s": 0.38873713463544846, "p95_s": 1.039823915064335, "cost_usd_per_1000": 0.005967936366670374, "axes": { "intelligence": 84.38272380713, "calibration": 75.49520739505144, "speed": 83.93384169209432, "cost": 76.72527448572276 }, "jevbench_score": 80.03171411251284 } }, { "threshold": 0.55, "public": { "n": 231, "escalation": 0.07792207792207792, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.7027027027027027 }, "accuracy": 0.8008658008658008, "ece_hard": 0.10542724542724535, "calibration": 75.60327545727546, "p50_s": 0.3835790231823921, "p95_s": 1.0302141197025776, "cost_usd_per_1000": 0.0064435335376234245, "axes": { "intelligence": 80.59100767434103, "calibration": 75.60327545727546, "speed": 84.03217653298277, "cost": 75.72627720107005 }, "jevbench_score": 78.90960223109724 }, "heldout": { "n": 303, "escalation": 0.066006600660066, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6605504587155964 }, "accuracy": 0.8514851485148515, "ece_hard": 0.1301779260494857, "calibration": 75.49520739505144, "p50_s": 0.38873713463544846, "p95_s": 1.039823915064335, "cost_usd_per_1000": 0.005967936366670374, "axes": { "intelligence": 84.38272380713, "calibration": 75.49520739505144, "speed": 83.93384169209432, "cost": 76.72527448572276 }, "jevbench_score": 80.03171411251284 } }, { "threshold": 0.56, "public": { "n": 231, "escalation": 0.09090909090909091, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.7027027027027027 }, "accuracy": 0.8008658008658008, "ece_hard": 0.10975156975156969, "calibration": 74.97084302484302, "p50_s": 0.3852977529168129, "p95_s": 1.0302141197025776, "cost_usd_per_1000": 0.006961900238338439, "axes": { "intelligence": 80.59100767434103, "calibration": 74.97084302484302, "speed": 84.01276025963342, "cost": 74.71816612897885 }, "jevbench_score": 78.47613571565142 }, "heldout": { "n": 303, "escalation": 0.06930693069306931, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6605504587155964 }, "accuracy": 0.8514851485148515, "ece_hard": 0.12981095357242148, "calibration": 75.53190464275785, "p50_s": 0.38873713463544846, "p95_s": 1.0501402437686918, "cost_usd_per_1000": 0.0060996665183372266, "axes": { "intelligence": 84.38272380713, "calibration": 75.53190464275785, "speed": 83.89096668871568, "cost": 76.44081724412754 }, "jevbench_score": 79.95693246634008 } }, { "threshold": 0.57, "public": { "n": 231, "escalation": 0.09956709956709957, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.7027027027027027 }, "accuracy": 0.8008658008658008, "ece_hard": 0.11173355173355165, "calibration": 74.77264482664484, "p50_s": 0.3855065032839775, "p95_s": 1.0476276613771915, "cost_usd_per_1000": 0.007307478038815116, "axes": { "intelligence": 80.59100767434103, "calibration": 74.77264482664484, "speed": 83.9376133430074, "cost": 74.08697443465081 }, "jevbench_score": 78.24056016393858 }, "heldout": { "n": 303, "escalation": 0.06930693069306931, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6605504587155964 }, "accuracy": 0.8514851485148515, "ece_hard": 0.12981095357242148, "calibration": 75.53190464275785, "p50_s": 0.38873713463544846, "p95_s": 1.0501402437686918, "cost_usd_per_1000": 0.0060996665183372266, "axes": { "intelligence": 84.38272380713, "calibration": 75.53190464275785, "speed": 83.89096668871568, "cost": 76.44081724412754 }, "jevbench_score": 79.95693246634008 } }, { "threshold": 0.58, "public": { "n": 231, "escalation": 0.1038961038961039, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.7027027027027027 }, "accuracy": 0.8008658008658008, "ece_hard": 0.11047229047229039, "calibration": 74.89877095277096, "p50_s": 0.3855883702635765, "p95_s": 1.0476276613771915, "cost_usd_per_1000": 0.007480266939053455, "axes": { "intelligence": 80.59100767434103, "calibration": 74.89877095277096, "speed": 83.93669116393856, "cost": 73.78248711183608 }, "jevbench_score": 78.19277087095824 }, "heldout": { "n": 303, "escalation": 0.08250825082508251, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9657534246575342, "hard": 0.6697247706422018 }, "accuracy": 0.8514851485148515, "ece_hard": 0.12329719210453156, "calibration": 76.18328078954684, "p50_s": 0.38873713463544846, "p95_s": 1.0609071180224416, "cost_usd_per_1000": 0.006626587125004635, "axes": { "intelligence": 84.46617234301034, "calibration": 76.18328078954684, "speed": 83.84666607560868, "cost": 75.3613026292933 }, "jevbench_score": 79.85353937206726 } }, { "threshold": 0.59, "public": { "n": 231, "escalation": 0.10822510822510822, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.7027027027027027 }, "accuracy": 0.8008658008658008, "ece_hard": 0.11011193011193003, "calibration": 74.934806988807, "p50_s": 0.38687043637037277, "p95_s": 1.0476276613771915, "cost_usd_per_1000": 0.007653055839291793, "axes": { "intelligence": 80.59100767434103, "calibration": 74.934806988807, "speed": 83.92227499722267, "cost": 73.48495353697324 }, "jevbench_score": 78.11986117377339 }, "heldout": { "n": 303, "escalation": 0.0891089108910891, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9657534246575342, "hard": 0.6788990825688074 }, "accuracy": 0.8547854785478548, "ece_hard": 0.11678343063664166, "calibration": 77.48865693633584, "p50_s": 0.38921745866537094, "p95_s": 1.0731464132666586, "cost_usd_per_1000": 0.006890047428338339, "axes": { "intelligence": 84.74140170080852, "calibration": 77.48865693633584, "speed": 83.79148708951487, "cost": 74.8533336571628 }, "jevbench_score": 80.10990292434155 } }, { "threshold": 0.6, "public": { "n": 231, "escalation": 0.10822510822510822, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.7027027027027027 }, "accuracy": 0.8008658008658008, "ece_hard": 0.11011193011193003, "calibration": 74.934806988807, "p50_s": 0.38687043637037277, "p95_s": 1.0476276613771915, "cost_usd_per_1000": 0.007653055839291793, "axes": { "intelligence": 80.59100767434103, "calibration": 74.934806988807, "speed": 83.92227499722267, "cost": 73.48495353697324 }, "jevbench_score": 78.11986117377339 }, "heldout": { "n": 303, "escalation": 0.09900990099009901, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9657534246575342, "hard": 0.6788990825688074 }, "accuracy": 0.8547854785478548, "ece_hard": 0.11724214623297192, "calibration": 77.4427853767028, "p50_s": 0.38939036428928375, "p95_s": 1.083354840427637, "cost_usd_per_1000": 0.007285237883338895, "axes": { "intelligence": 84.74140170080852, "calibration": 77.4427853767028, "speed": 83.74844071481766, "cost": 74.12668788208522 }, "jevbench_score": 79.89267881264858 } }, { "threshold": 0.61, "public": { "n": 231, "escalation": 0.11688311688311688, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.7027027027027027 }, "accuracy": 0.8008658008658008, "ece_hard": 0.11164346164346158, "calibration": 74.78165383565384, "p50_s": 0.3878612667322159, "p95_s": 1.0530537962913513, "cost_usd_per_1000": 0.007998633639768469, "axes": { "intelligence": 80.59100767434103, "calibration": 74.78165383565384, "speed": 83.888730303818, "cost": 72.9095258404532 }, "jevbench_score": 77.91882353165111 }, "heldout": { "n": 303, "escalation": 0.11221122112211221, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9657534246575342, "hard": 0.6880733944954128 }, "accuracy": 0.858085808580858, "ece_hard": 0.10650820127884351, "calibration": 78.51617987211566, "p50_s": 0.39043400436639786, "p95_s": 1.0957592256367206, "cost_usd_per_1000": 0.0078121584900063025, "axes": { "intelligence": 85.01663105860669, "calibration": 78.51617987211566, "speed": 83.68737229176769, "cost": 73.21686863496268 }, "jevbench_score": 79.97118543905125 } }, { "threshold": 0.62, "public": { "n": 231, "escalation": 0.11688311688311688, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.7027027027027027 }, "accuracy": 0.8008658008658008, "ece_hard": 0.11164346164346158, "calibration": 74.78165383565384, "p50_s": 0.3878612667322159, "p95_s": 1.0530537962913513, "cost_usd_per_1000": 0.007998633639768469, "axes": { "intelligence": 80.59100767434103, "calibration": 74.78165383565384, "speed": 83.888730303818, "cost": 72.9095258404532 }, "jevbench_score": 77.91882353165111 }, "heldout": { "n": 303, "escalation": 0.1188118811881188, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9657534246575342, "hard": 0.6880733944954128 }, "accuracy": 0.858085808580858, "ece_hard": 0.10540728384765084, "calibration": 78.62627161523491, "p50_s": 0.39046943187713623, "p95_s": 1.0957592256367206, "cost_usd_per_1000": 0.008075618793340008, "axes": { "intelligence": 85.01663105860669, "calibration": 78.62627161523491, "speed": 83.68697823606969, "cost": 72.78472569874772 }, "jevbench_score": 79.88080401324913 } }, { "threshold": 0.63, "public": { "n": 231, "escalation": 0.12554112554112554, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.7027027027027027 }, "accuracy": 0.8008658008658008, "ece_hard": 0.11182364182364177, "calibration": 74.76363581763582, "p50_s": 0.3878612667322159, "p95_s": 1.0698232911527157, "cost_usd_per_1000": 0.008344211440245147, "axes": { "intelligence": 80.59100767434103, "calibration": 74.76363581763582, "speed": 83.82011539796571, "cost": 72.35844098583406 }, "jevbench_score": 77.75057575279476 }, "heldout": { "n": 303, "escalation": 0.12211221122112212, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9657534246575342, "hard": 0.6880733944954128 }, "accuracy": 0.858085808580858, "ece_hard": 0.10540728384765084, "calibration": 78.62627161523491, "p50_s": 0.39095765352249146, "p95_s": 1.0957592256367206, "cost_usd_per_1000": 0.008207348945006859, "axes": { "intelligence": 85.01663105860669, "calibration": 78.62627161523491, "speed": 83.68155144710693, "cost": 72.57391304944335 }, "jevbench_score": 79.82160558929792 } }, { "threshold": 0.64, "public": { "n": 231, "escalation": 0.12987012987012986, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.7027027027027027 }, "accuracy": 0.8008658008658008, "ece_hard": 0.11966147966147961, "calibration": 73.97985203385204, "p50_s": 0.38910215348005295, "p95_s": 1.0698232911527157, "cost_usd_per_1000": 0.008517000340483485, "axes": { "intelligence": 80.59100767434103, "calibration": 73.97985203385204, "speed": 83.80624316894082, "cost": 72.09140006178166 }, "jevbench_score": 77.47114401774449 }, "heldout": { "n": 303, "escalation": 0.12211221122112212, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9657534246575342, "hard": 0.6880733944954128 }, "accuracy": 0.858085808580858, "ece_hard": 0.10540728384765084, "calibration": 78.62627161523491, "p50_s": 0.39095765352249146, "p95_s": 1.0957592256367206, "cost_usd_per_1000": 0.008207348945006859, "axes": { "intelligence": 85.01663105860669, "calibration": 78.62627161523491, "speed": 83.68155144710693, "cost": 72.57391304944335 }, "jevbench_score": 79.82160558929792 } }, { "threshold": 0.65, "public": { "n": 231, "escalation": 0.1341991341991342, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.7027027027027027 }, "accuracy": 0.8008658008658008, "ece_hard": 0.12191373191373188, "calibration": 73.75462680862681, "p50_s": 0.38922879844903946, "p95_s": 1.0698232911527157, "cost_usd_per_1000": 0.008689789240721824, "axes": { "intelligence": 80.59100767434103, "calibration": 73.75462680862681, "speed": 83.8048298572502, "cost": 71.82972269975578 }, "jevbench_score": 77.34144352248612 }, "heldout": { "n": 303, "escalation": 0.1254125412541254, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.958904109589041, "hard": 0.6880733944954128 }, "accuracy": 0.8547854785478548, "ece_hard": 0.10540728384765084, "calibration": 78.62627161523491, "p50_s": 0.39095765352249146, "p95_s": 1.1019696988165377, "cost_usd_per_1000": 0.00833907909667371, "axes": { "intelligence": 84.82485023668887, "calibration": 78.62627161523491, "speed": 83.65700627794217, "cost": 72.36645720502423 }, "jevbench_score": 79.71363314814587 } }, { "threshold": 0.66, "public": { "n": 231, "escalation": 0.1341991341991342, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.7027027027027027 }, "accuracy": 0.8008658008658008, "ece_hard": 0.12191373191373188, "calibration": 73.75462680862681, "p50_s": 0.38922879844903946, "p95_s": 1.0698232911527157, "cost_usd_per_1000": 0.008689789240721824, "axes": { "intelligence": 80.59100767434103, "calibration": 73.75462680862681, "speed": 83.8048298572502, "cost": 71.82972269975578 }, "jevbench_score": 77.34144352248612 }, "heldout": { "n": 303, "escalation": 0.132013201320132, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.958904109589041, "hard": 0.6880733944954128 }, "accuracy": 0.8547854785478548, "ece_hard": 0.10504031137058663, "calibration": 78.66296886294134, "p50_s": 0.39180759340524673, "p95_s": 1.1019696988165377, "cost_usd_per_1000": 0.008602539400007415, "axes": { "intelligence": 84.82485023668887, "calibration": 78.66296886294134, "speed": 83.64757498682278, "cost": 71.96119988844158 }, "jevbench_score": 79.608840137007 } }, { "threshold": 0.67, "public": { "n": 231, "escalation": 0.1471861471861472, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.7117117117117117 }, "accuracy": 0.8051948051948052, "ece_hard": 0.11137319137319134, "calibration": 74.80868086268086, "p50_s": 0.389875203371048, "p95_s": 1.0812386125326157, "cost_usd_per_1000": 0.009208155941436839, "axes": { "intelligence": 80.9663830497164, "calibration": 74.80868086268086, "speed": 83.75152839130439, "cost": 71.07482003430606 }, "jevbench_score": 77.4892239293938 }, "heldout": { "n": 303, "escalation": 0.132013201320132, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.958904109589041, "hard": 0.6880733944954128 }, "accuracy": 0.8547854785478548, "ece_hard": 0.10504031137058663, "calibration": 78.66296886294134, "p50_s": 0.39180759340524673, "p95_s": 1.1019696988165377, "cost_usd_per_1000": 0.008602539400007415, "axes": { "intelligence": 84.82485023668887, "calibration": 78.66296886294134, "speed": 83.64757498682278, "cost": 71.96119988844158 }, "jevbench_score": 79.608840137007 } }, { "threshold": 0.68, "public": { "n": 231, "escalation": 0.15151515151515152, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.7117117117117117 }, "accuracy": 0.8051948051948052, "ece_hard": 0.11137319137319134, "calibration": 74.80868086268086, "p50_s": 0.39158909022808075, "p95_s": 1.0812386125326157, "cost_usd_per_1000": 0.009380944841675178, "axes": { "intelligence": 80.9663830497164, "calibration": 74.80868086268086, "speed": 83.73247869692602, "cost": 70.8326025281785 }, "jevbench_score": 77.41871712317396 }, "heldout": { "n": 303, "escalation": 0.1353135313531353, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.952054794520548, "hard": 0.6880733944954128 }, "accuracy": 0.8514851485148515, "ece_hard": 0.10504031137058663, "calibration": 78.66296886294134, "p50_s": 0.39212508499622345, "p95_s": 1.1019696988165377, "cost_usd_per_1000": 0.008734269551674268, "axes": { "intelligence": 84.63306941477107, "calibration": 78.66296886294134, "speed": 83.64405721399994, "cost": 71.7632022783312 }, "jevbench_score": 79.50818350181088 } }, { "threshold": 0.69, "public": { "n": 231, "escalation": 0.16017316017316016, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.7117117117117117 }, "accuracy": 0.8051948051948052, "ece_hard": 0.11119301119301114, "calibration": 74.87669888069888, "p50_s": 0.39193499088287354, "p95_s": 1.0856791250407696, "cost_usd_per_1000": 0.009726522642151854, "axes": { "intelligence": 80.9663830497164, "calibration": 74.87669888069888, "speed": 83.7108447468334, "cost": 70.3612719366065 }, "jevbench_score": 77.30217411318344 }, "heldout": { "n": 303, "escalation": 0.14521452145214522, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6880733944954128 }, "accuracy": 0.8481848184818482, "ece_hard": 0.10935223797609119, "calibration": 78.23177620239088, "p50_s": 0.3926578313112259, "p95_s": 1.1019696988165377, "cost_usd_per_1000": 0.009129460006674823, "axes": { "intelligence": 84.44128859285325, "calibration": 78.23177620239088, "speed": 83.63816083640066, "cost": 71.1866472864252 }, "jevbench_score": 79.19272079217745 } }, { "threshold": 0.7, "public": { "n": 231, "escalation": 0.1645021645021645, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.7117117117117117 }, "accuracy": 0.8051948051948052, "ece_hard": 0.11146328146328142, "calibration": 74.84967185367185, "p50_s": 0.3930845111608505, "p95_s": 1.0856791250407696, "cost_usd_per_1000": 0.009899311542390193, "axes": { "intelligence": 80.9663830497164, "calibration": 74.84967185367185, "speed": 83.69812581006597, "cost": 70.13185023400568 }, "jevbench_score": 77.22917888602628 }, "heldout": { "n": 303, "escalation": 0.1551155115511551, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6880733944954128 }, "accuracy": 0.8481848184818482, "ece_hard": 0.11559077008618295, "calibration": 77.60792299138171, "p50_s": 0.3926578313112259, "p95_s": 1.1050984039902687, "cost_usd_per_1000": 0.00952465046167538, "axes": { "intelligence": 84.44128859285325, "calibration": 77.60792299138171, "speed": 83.62584784704703, "cost": 70.63452859666009 }, "jevbench_score": 78.87777009684197 } }, { "threshold": 0.71, "public": { "n": 231, "escalation": 0.17316017316017315, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.7117117117117117 }, "accuracy": 0.8051948051948052, "ece_hard": 0.11191373191373188, "calibration": 74.90462680862682, "p50_s": 0.3938719406723976, "p95_s": 1.0908589139580727, "cost_usd_per_1000": 0.01024488934286687, "axes": { "intelligence": 80.9663830497164, "calibration": 74.90462680862682, "speed": 83.66876372285617, "cost": 69.68478184421402 }, "jevbench_score": 77.11319013298365 }, "heldout": { "n": 303, "escalation": 0.16501650165016502, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6880733944954128 }, "accuracy": 0.8481848184818482, "ece_hard": 0.1148568251320545, "calibration": 77.68131748679454, "p50_s": 0.3928951770067215, "p95_s": 1.1062885060906409, "cost_usd_per_1000": 0.009919840916675936, "axes": { "intelligence": 84.44128859285325, "calibration": 77.68131748679454, "speed": 83.61854902083515, "cost": 70.1048587755877 }, "jevbench_score": 78.74637038182465 } }, { "threshold": 0.72, "public": { "n": 231, "escalation": 0.17316017316017315, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.7117117117117117 }, "accuracy": 0.8051948051948052, "ece_hard": 0.11191373191373188, "calibration": 74.90462680862682, "p50_s": 0.3938719406723976, "p95_s": 1.0908589139580727, "cost_usd_per_1000": 0.01024488934286687, "axes": { "intelligence": 80.9663830497164, "calibration": 74.90462680862682, "speed": 83.66876372285617, "cost": 69.68478184421402 }, "jevbench_score": 77.11319013298365 }, "heldout": { "n": 303, "escalation": 0.16501650165016502, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6880733944954128 }, "accuracy": 0.8481848184818482, "ece_hard": 0.1148568251320545, "calibration": 77.68131748679454, "p50_s": 0.3928951770067215, "p95_s": 1.1062885060906409, "cost_usd_per_1000": 0.009919840916675936, "axes": { "intelligence": 84.44128859285325, "calibration": 77.68131748679454, "speed": 83.61854902083515, "cost": 70.1048587755877 }, "jevbench_score": 78.74637038182465 } }, { "threshold": 0.73, "public": { "n": 231, "escalation": 0.17316017316017315, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.7117117117117117 }, "accuracy": 0.8051948051948052, "ece_hard": 0.11191373191373188, "calibration": 74.90462680862682, "p50_s": 0.3938719406723976, "p95_s": 1.0908589139580727, "cost_usd_per_1000": 0.01024488934286687, "axes": { "intelligence": 80.9663830497164, "calibration": 74.90462680862682, "speed": 83.66876372285617, "cost": 69.68478184421402 }, "jevbench_score": 77.11319013298365 }, "heldout": { "n": 303, "escalation": 0.16501650165016502, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6880733944954128 }, "accuracy": 0.8481848184818482, "ece_hard": 0.1148568251320545, "calibration": 77.68131748679454, "p50_s": 0.3928951770067215, "p95_s": 1.1062885060906409, "cost_usd_per_1000": 0.009919840916675936, "axes": { "intelligence": 84.44128859285325, "calibration": 77.68131748679454, "speed": 83.61854902083515, "cost": 70.1048587755877 }, "jevbench_score": 78.74637038182465 } }, { "threshold": 0.74, "public": { "n": 231, "escalation": 0.17316017316017315, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.7117117117117117 }, "accuracy": 0.8051948051948052, "ece_hard": 0.11191373191373188, "calibration": 74.90462680862682, "p50_s": 0.3938719406723976, "p95_s": 1.0908589139580727, "cost_usd_per_1000": 0.01024488934286687, "axes": { "intelligence": 80.9663830497164, "calibration": 74.90462680862682, "speed": 83.66876372285617, "cost": 69.68478184421402 }, "jevbench_score": 77.11319013298365 }, "heldout": { "n": 303, "escalation": 0.16501650165016502, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6880733944954128 }, "accuracy": 0.8481848184818482, "ece_hard": 0.1148568251320545, "calibration": 77.68131748679454, "p50_s": 0.3928951770067215, "p95_s": 1.1062885060906409, "cost_usd_per_1000": 0.009919840916675936, "axes": { "intelligence": 84.44128859285325, "calibration": 77.68131748679454, "speed": 83.61854902083515, "cost": 70.1048587755877 }, "jevbench_score": 78.74637038182465 } }, { "threshold": 0.75, "public": { "n": 231, "escalation": 0.17316017316017315, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.7117117117117117 }, "accuracy": 0.8051948051948052, "ece_hard": 0.11191373191373188, "calibration": 74.90462680862682, "p50_s": 0.3938719406723976, "p95_s": 1.0908589139580727, "cost_usd_per_1000": 0.01024488934286687, "axes": { "intelligence": 80.9663830497164, "calibration": 74.90462680862682, "speed": 83.66876372285617, "cost": 69.68478184421402 }, "jevbench_score": 77.11319013298365 }, "heldout": { "n": 303, "escalation": 0.16501650165016502, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6880733944954128 }, "accuracy": 0.8481848184818482, "ece_hard": 0.1148568251320545, "calibration": 77.68131748679454, "p50_s": 0.3928951770067215, "p95_s": 1.1062885060906409, "cost_usd_per_1000": 0.009919840916675936, "axes": { "intelligence": 84.44128859285325, "calibration": 77.68131748679454, "speed": 83.61854902083515, "cost": 70.1048587755877 }, "jevbench_score": 78.74637038182465 } }, { "threshold": 0.76, "public": { "n": 231, "escalation": 0.18614718614718614, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.7117117117117117 }, "accuracy": 0.8051948051948052, "ece_hard": 0.11263445263445257, "calibration": 74.73255473655473, "p50_s": 0.3938719406723976, "p95_s": 1.1284871771931648, "cost_usd_per_1000": 0.010763256043581883, "axes": { "intelligence": 80.9663830497164, "calibration": 74.73255473655473, "speed": 83.52148328479194, "cost": 69.04168984935862 }, "jevbench_score": 76.85657815278927 }, "heldout": { "n": 303, "escalation": 0.1716171617161716, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6972477064220184 }, "accuracy": 0.8514851485148515, "ece_hard": 0.1053155407283848, "calibration": 78.63544592716153, "p50_s": 0.39295361191034317, "p95_s": 1.1062885060906409, "cost_usd_per_1000": 0.01018330122000964, "axes": { "intelligence": 84.71651795065142, "calibration": 78.63544592716153, "speed": 83.61790314705897, "cost": 69.76334229095981 }, "jevbench_score": 78.95474780325324 } }, { "threshold": 0.77, "public": { "n": 231, "escalation": 0.19047619047619047, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.7117117117117117 }, "accuracy": 0.8051948051948052, "ece_hard": 0.11263445263445257, "calibration": 74.73255473655473, "p50_s": 0.3942127153277397, "p95_s": 1.1284871771931648, "cost_usd_per_1000": 0.010936044943820222, "axes": { "intelligence": 80.9663830497164, "calibration": 74.73255473655473, "speed": 83.51772743038, "cost": 68.83419140854807 }, "jevbench_score": 76.79790321456078 }, "heldout": { "n": 303, "escalation": 0.1716171617161716, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6972477064220184 }, "accuracy": 0.8514851485148515, "ece_hard": 0.1053155407283848, "calibration": 78.63544592716153, "p50_s": 0.39295361191034317, "p95_s": 1.1062885060906409, "cost_usd_per_1000": 0.01018330122000964, "axes": { "intelligence": 84.71651795065142, "calibration": 78.63544592716153, "speed": 83.61790314705897, "cost": 69.76334229095981 }, "jevbench_score": 78.95474780325324 } }, { "threshold": 0.78, "public": { "n": 231, "escalation": 0.19047619047619047, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.7117117117117117 }, "accuracy": 0.8051948051948052, "ece_hard": 0.11263445263445257, "calibration": 74.73255473655473, "p50_s": 0.3942127153277397, "p95_s": 1.1284871771931648, "cost_usd_per_1000": 0.010936044943820222, "axes": { "intelligence": 80.9663830497164, "calibration": 74.73255473655473, "speed": 83.51772743038, "cost": 68.83419140854807 }, "jevbench_score": 76.79790321456078 }, "heldout": { "n": 303, "escalation": 0.1782178217821782, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6972477064220184 }, "accuracy": 0.8514851485148515, "ece_hard": 0.10669168751737561, "calibration": 78.49783124826243, "p50_s": 0.39392561465501785, "p95_s": 1.1062885060906409, "cost_usd_per_1000": 0.010446761523343343, "axes": { "intelligence": 84.71651795065142, "calibration": 78.49783124826243, "speed": 83.60717378405474, "cost": 69.43054957515645 }, "jevbench_score": 78.82336574863777 } }, { "threshold": 0.79, "public": { "n": 231, "escalation": 0.19047619047619047, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.7117117117117117 }, "accuracy": 0.8051948051948052, "ece_hard": 0.11263445263445257, "calibration": 74.73255473655473, "p50_s": 0.3942127153277397, "p95_s": 1.1284871771931648, "cost_usd_per_1000": 0.010936044943820222, "axes": { "intelligence": 80.9663830497164, "calibration": 74.73255473655473, "speed": 83.51772743038, "cost": 68.83419140854807 }, "jevbench_score": 76.79790321456078 }, "heldout": { "n": 303, "escalation": 0.1782178217821782, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6972477064220184 }, "accuracy": 0.8514851485148515, "ece_hard": 0.10669168751737561, "calibration": 78.49783124826243, "p50_s": 0.39392561465501785, "p95_s": 1.1062885060906409, "cost_usd_per_1000": 0.010446761523343343, "axes": { "intelligence": 84.71651795065142, "calibration": 78.49783124826243, "speed": 83.60717378405474, "cost": 69.43054957515645 }, "jevbench_score": 78.82336574863777 } }, { "threshold": 0.8, "public": { "n": 231, "escalation": 0.19913419913419914, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.7117117117117117 }, "accuracy": 0.8051948051948052, "ece_hard": 0.11218400218400211, "calibration": 74.77759978159979, "p50_s": 0.3942127153277397, "p95_s": 1.1284871771931648, "cost_usd_per_1000": 0.0112816227442969, "axes": { "intelligence": 80.9663830497164, "calibration": 74.77759978159979, "speed": 83.51772743038, "cost": 68.42885281322432 }, "jevbench_score": 76.69614701945575 }, "heldout": { "n": 303, "escalation": 0.1848184818481848, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6972477064220184 }, "accuracy": 0.8514851485148515, "ece_hard": 0.10797609118710035, "calibration": 77.86939088128997, "p50_s": 0.39403771609067917, "p95_s": 1.126306684315204, "cost_usd_per_1000": 0.010710221826677049, "axes": { "intelligence": 84.71651795065142, "calibration": 77.86939088128997, "speed": 83.52805545301862, "cost": 69.10604602321209 }, "jevbench_score": 78.55445549824395 } }, { "threshold": 0.81, "public": { "n": 231, "escalation": 0.2077922077922078, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.7117117117117117 }, "accuracy": 0.8051948051948052, "ece_hard": 0.11310219310219305, "calibration": 74.6857806897807, "p50_s": 0.3962126150727272, "p95_s": 1.1284871771931648, "cost_usd_per_1000": 0.011627200544773578, "axes": { "intelligence": 80.9663830497164, "calibration": 74.6857806897807, "speed": 83.49575072382476, "cost": 68.03574510407958 }, "jevbench_score": 76.5572008082423 }, "heldout": { "n": 303, "escalation": 0.18811881188118812, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6972477064220184 }, "accuracy": 0.8514851485148515, "ece_hard": 0.10797609118710035, "calibration": 77.86939088128997, "p50_s": 0.39494913071393967, "p95_s": 1.126306684315204, "cost_usd_per_1000": 0.010841951978343901, "axes": { "intelligence": 84.71651795065142, "calibration": 77.86939088128997, "speed": 83.51802176185443, "cost": 68.94677561942821 }, "jevbench_score": 78.50679692399305 } }, { "threshold": 0.82, "public": { "n": 231, "escalation": 0.22077922077922077, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.7207207207207207 }, "accuracy": 0.8095238095238095, "ece_hard": 0.11274183274183269, "calibration": 74.72181672581672, "p50_s": 0.39779794961214066, "p95_s": 1.1284871771931648, "cost_usd_per_1000": 0.01214556724548859, "axes": { "intelligence": 81.34175842509177, "calibration": 74.72181672581672, "speed": 83.47840831081513, "cost": 67.46746591403387 }, "jevbench_score": 76.49047965246521 }, "heldout": { "n": 303, "escalation": 0.19141914191419143, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6972477064220184 }, "accuracy": 0.8514851485148515, "ece_hard": 0.10843480678343063, "calibration": 77.82351932165693, "p50_s": 0.39519932121038437, "p95_s": 1.1310784369707108, "cost_usd_per_1000": 0.010973682130010752, "axes": { "intelligence": 84.71651795065142, "calibration": 77.82351932165693, "speed": 83.49691087005014, "cost": 68.78942871944608 }, "jevbench_score": 78.44545179431726 } }, { "threshold": 0.83, "public": { "n": 231, "escalation": 0.22077922077922077, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.7207207207207207 }, "accuracy": 0.8095238095238095, "ece_hard": 0.11274183274183269, "calibration": 74.72181672581672, "p50_s": 0.39779794961214066, "p95_s": 1.1284871771931648, "cost_usd_per_1000": 0.01214556724548859, "axes": { "intelligence": 81.34175842509177, "calibration": 74.72181672581672, "speed": 83.47840831081513, "cost": 67.46746591403387 }, "jevbench_score": 76.49047965246521 }, "heldout": { "n": 303, "escalation": 0.19801980198019803, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6972477064220184 }, "accuracy": 0.8514851485148515, "ece_hard": 0.1078843480678343, "calibration": 77.87856519321657, "p50_s": 0.3956253454089165, "p95_s": 1.1310784369707108, "cost_usd_per_1000": 0.011237142433344456, "axes": { "intelligence": 84.71651795065142, "calibration": 77.87856519321657, "speed": 83.49223170450847, "cost": 68.48032342958626 }, "jevbench_score": 78.36993341287075 } }, { "threshold": 0.84, "public": { "n": 231, "escalation": 0.22943722943722944, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.7207207207207207 }, "accuracy": 0.8095238095238095, "ece_hard": 0.10940849940849937, "calibration": 75.05515005915007, "p50_s": 0.3989381045103073, "p95_s": 1.1284871771931648, "cost_usd_per_1000": 0.012491145045965268, "axes": { "intelligence": 81.34175842509177, "calibration": 75.05515005915007, "speed": 83.4659785151197, "cost": 67.1019324588222 }, "jevbench_score": 76.46886475021883 }, "heldout": { "n": 303, "escalation": 0.20462046204620463, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6972477064220184 }, "accuracy": 0.8514851485148515, "ece_hard": 0.10898526549902691, "calibration": 77.7684734500973, "p50_s": 0.39571626484394073, "p95_s": 1.1310784369707108, "cost_usd_per_1000": 0.011500602736678162, "axes": { "intelligence": 84.71651795065142, "calibration": 77.7684734500973, "speed": 83.4912337585243, "cost": 68.17838194151216 }, "jevbench_score": 78.25548906341446 } }, { "threshold": 0.85, "public": { "n": 231, "escalation": 0.22943722943722944, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.7207207207207207 }, "accuracy": 0.8095238095238095, "ece_hard": 0.10940849940849937, "calibration": 75.05515005915007, "p50_s": 0.3989381045103073, "p95_s": 1.1284871771931648, "cost_usd_per_1000": 0.012491145045965268, "axes": { "intelligence": 81.34175842509177, "calibration": 75.05515005915007, "speed": 83.4659785151197, "cost": 67.1019324588222 }, "jevbench_score": 76.46886475021883 }, "heldout": { "n": 303, "escalation": 0.22772277227722773, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6972477064220184 }, "accuracy": 0.8514851485148515, "ece_hard": 0.09733388935223791, "calibration": 78.93361106477622, "p50_s": 0.3963628485798836, "p95_s": 1.1310784369707108, "cost_usd_per_1000": 0.012422713798346124, "axes": { "intelligence": 84.71651795065142, "calibration": 78.93361106477622, "speed": 83.4841433604511, "cost": 67.17350560569275 }, "jevbench_score": 78.25426546309595 } }, { "threshold": 0.86, "public": { "n": 231, "escalation": 0.24242424242424243, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.7207207207207207 }, "accuracy": 0.8095238095238095, "ece_hard": 0.10985894985894981, "calibration": 75.01010501410502, "p50_s": 0.4004004970192909, "p95_s": 1.1531940624117851, "cost_usd_per_1000": 0.013009511746680285, "axes": { "intelligence": 81.34175842509177, "calibration": 75.01010501410502, "speed": 83.35602994482329, "cost": 66.5721700724873 }, "jevbench_score": 76.28089264126436 }, "heldout": { "n": 303, "escalation": 0.23432343234323433, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6972477064220184 }, "accuracy": 0.8514851485148515, "ece_hard": 0.09733388935223791, "calibration": 78.93361106477622, "p50_s": 0.3967704549431801, "p95_s": 1.1310784369707108, "cost_usd_per_1000": 0.01268617410167983, "axes": { "intelligence": 84.71651795065142, "calibration": 78.93361106477622, "speed": 83.47967951536563, "cost": 66.90007998267029 }, "jevbench_score": 78.17346632161782 } }, { "threshold": 0.87, "public": { "n": 231, "escalation": 0.24675324675324675, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.7207207207207207 }, "accuracy": 0.8095238095238095, "ece_hard": 0.10940849940849937, "calibration": 75.05515005915007, "p50_s": 0.40055521577596664, "p95_s": 1.1531940624117851, "cost_usd_per_1000": 0.013182300646918624, "axes": { "intelligence": 81.34175842509177, "calibration": 75.05515005915007, "speed": 83.35435211165391, "cost": 66.4002636306505 }, "jevbench_score": 76.24265906978503 }, "heldout": { "n": 303, "escalation": 0.24092409240924093, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6972477064220184 }, "accuracy": 0.8514851485148515, "ece_hard": 0.0969669168751737, "calibration": 79.22030831248264, "p50_s": 0.3967770114541054, "p95_s": 1.1310784369707108, "cost_usd_per_1000": 0.012949634405013535, "axes": { "intelligence": 84.71651795065142, "calibration": 79.22030831248264, "speed": 83.47960775011816, "cost": 66.63227477325755 }, "jevbench_score": 78.16591503472638 } }, { "threshold": 0.88, "public": { "n": 231, "escalation": 0.2597402597402597, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.7207207207207207 }, "accuracy": 0.8095238095238095, "ece_hard": 0.11066976066976068, "calibration": 74.92902393302393, "p50_s": 0.40208370983600616, "p95_s": 1.1844173558056355, "cost_usd_per_1000": 0.013700667347633637, "axes": { "intelligence": 81.34175842509177, "calibration": 74.92902393302393, "speed": 83.22178759025218, "cost": 65.89774834661557 }, "jevbench_score": 76.03574611243515 }, "heldout": { "n": 303, "escalation": 0.25742574257425743, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6972477064220184 }, "accuracy": 0.8514851485148515, "ece_hard": 0.09568251320544893, "calibration": 80.04874867945512, "p50_s": 0.39877817034721375, "p95_s": 1.1310784369707108, "cost_usd_per_1000": 0.013608285163347795, "axes": { "intelligence": 84.71651795065142, "calibration": 80.04874867945512, "speed": 83.45775900555344, "cost": 65.9858979581195 }, "jevbench_score": 78.17360203942133 } }, { "threshold": 0.89, "public": { "n": 231, "escalation": 0.2683982683982684, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.7207207207207207 }, "accuracy": 0.8095238095238095, "ece_hard": 0.1142733642733643, "calibration": 75.56866357266357, "p50_s": 0.4030695781111717, "p95_s": 1.1844173558056355, "cost_usd_per_1000": 0.014046245148110315, "axes": { "intelligence": 81.34175842509177, "calibration": 75.56866357266357, "speed": 83.21115216536671, "cost": 65.5731926842932 }, "jevbench_score": 76.10107486017729 }, "heldout": { "n": 303, "escalation": 0.264026402640264, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6972477064220184 }, "accuracy": 0.8514851485148515, "ece_hard": 0.09549902696691683, "calibration": 80.06709730330832, "p50_s": 0.39877817034721375, "p95_s": 1.137706685066223, "cost_usd_per_1000": 0.013871745466681497, "axes": { "intelligence": 84.71651795065142, "calibration": 80.06709730330832, "speed": 83.43238313602367, "cost": 65.73606666054688 }, "jevbench_score": 78.09804025200785 } }, { "threshold": 0.9, "public": { "n": 231, "escalation": 0.27705627705627706, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.7207207207207207 }, "accuracy": 0.8095238095238095, "ece_hard": 0.09931840931840934, "calibration": 77.06415906815906, "p50_s": 0.4035724848508835, "p95_s": 1.1844173558056355, "cost_usd_per_1000": 0.014391822948586992, "axes": { "intelligence": 81.34175842509177, "calibration": 77.06415906815906, "speed": 83.20573688490228, "cost": 65.2565257725141 }, "jevbench_score": 76.38108178251198 }, "heldout": { "n": 303, "escalation": 0.2739273927392739, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6972477064220184 }, "accuracy": 0.8514851485148515, "ece_hard": 0.09403113705865995, "calibration": 80.213886294134, "p50_s": 0.3995610922574997, "p95_s": 1.137706685066223, "cost_usd_per_1000": 0.014266935921682052, "axes": { "intelligence": 84.71651795065142, "calibration": 80.213886294134, "speed": 83.42386498364587, "cost": 65.37007868012566 }, "jevbench_score": 78.02283782829697 } }, { "threshold": 0.91, "public": { "n": 231, "escalation": 0.2857142857142857, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.7207207207207207 }, "accuracy": 0.8095238095238095, "ece_hard": 0.09832741832741831, "calibration": 77.16325816725816, "p50_s": 0.40393005311489105, "p95_s": 1.1844173558056355, "cost_usd_per_1000": 0.014737400749063666, "axes": { "intelligence": 81.34175842509177, "calibration": 77.16325816725816, "speed": 83.20189070667223, "cost": 64.94737320098527 }, "jevbench_score": 76.31408913925064 }, "heldout": { "n": 303, "escalation": 0.28052805280528054, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6972477064220184 }, "accuracy": 0.8514851485148515, "ece_hard": 0.09348067834306362, "calibration": 80.26893216569364, "p50_s": 0.39969054609537125, "p95_s": 1.137706685066223, "cost_usd_per_1000": 0.014530396225015757, "axes": { "intelligence": 84.71651795065142, "calibration": 80.26893216569364, "speed": 83.42245814041296, "cost": 65.1316762868369 }, "jevbench_score": 77.9646449053614 } }, { "threshold": 0.92, "public": { "n": 231, "escalation": 0.2943722943722944, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.7207207207207207 }, "accuracy": 0.8095238095238095, "ece_hard": 0.09868777868777864, "calibration": 77.12722213122214, "p50_s": 0.4045165404677391, "p95_s": 1.1844173558056355, "cost_usd_per_1000": 0.015082978549540344, "axes": { "intelligence": 81.34175842509177, "calibration": 77.12722213122214, "speed": 83.19558952938353, "cost": 64.64538659769605 }, "jevbench_score": 76.21488027487807 }, "heldout": { "n": 303, "escalation": 0.30033003300330036, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6972477064220184 }, "accuracy": 0.8514851485148515, "ece_hard": 0.09348067834306367, "calibration": 80.26893216569363, "p50_s": 0.3999708816409111, "p95_s": 1.1412913136184213, "cost_usd_per_1000": 0.01532077713501687, "axes": { "intelligence": 84.71651795065142, "calibration": 80.26893216569363, "speed": 83.40575112821251, "cost": 64.44157614641017 }, "jevbench_score": 77.75340799437703 } }, { "threshold": 0.93, "public": { "n": 231, "escalation": 0.3116883116883117, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.7207207207207207 }, "accuracy": 0.8095238095238095, "ece_hard": 0.10039949039949024, "calibration": 76.95605096005099, "p50_s": 0.4066837579011917, "p95_s": 1.1844173558056355, "cost_usd_per_1000": 0.0157741341504937, "axes": { "intelligence": 81.34175842509177, "calibration": 76.95605096005099, "speed": 83.172384093842, "cost": 64.06163410162651 }, "jevbench_score": 75.99471197134426 }, "heldout": { "n": 303, "escalation": 0.31683168316831684, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6972477064220184 }, "accuracy": 0.8514851485148515, "ece_hard": 0.09329719210453148, "calibration": 80.13728078954685, "p50_s": 0.40096187591552734, "p95_s": 1.1449913650751111, "cost_usd_per_1000": 0.01597942789335113, "axes": { "intelligence": 84.71651795065142, "calibration": 80.13728078954685, "speed": 83.38094707350939, "cost": 63.89316320975049 }, "jevbench_score": 77.5498528433556 } }, { "threshold": 0.94, "public": { "n": 231, "escalation": 0.3333333333333333, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.7207207207207207 }, "accuracy": 0.8095238095238095, "ece_hard": 0.10455455455455454, "calibration": 76.54054454454456, "p50_s": 0.4094219505786896, "p95_s": 1.210436686873436, "cost_usd_per_1000": 0.01663807865168539, "axes": { "intelligence": 81.34175842509177, "calibration": 76.54054454454456, "speed": 83.0488679880944, "cost": 63.36690481097774 }, "jevbench_score": 75.65721123534553 }, "heldout": { "n": 303, "escalation": 0.36303630363036304, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6972477064220184 }, "accuracy": 0.8514851485148515, "ece_hard": 0.08917431192660548, "calibration": 81.08956880733945, "p50_s": 0.40668586641550064, "p95_s": 1.1480483673512936, "cost_usd_per_1000": 0.017823650016687058, "axes": { "intelligence": 84.71651795065142, "calibration": 81.08956880733945, "speed": 83.30780735100197, "cost": 62.47010062549927 }, "jevbench_score": 77.3255022224787 } }, { "threshold": 0.95, "public": { "n": 231, "escalation": 0.3463203463203463, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.7207207207207207 }, "accuracy": 0.8095238095238095, "ece_hard": 0.09536536536536534, "calibration": 77.30946346346347, "p50_s": 0.409928597509861, "p95_s": 1.210436686873436, "cost_usd_per_1000": 0.017156445352400407, "axes": { "intelligence": 81.34175842509177, "calibration": 77.30946346346347, "speed": 83.04349705144125, "cost": 62.967180662019665 }, "jevbench_score": 75.72539082170127 }, "heldout": { "n": 303, "escalation": 0.36633663366336633, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6972477064220184 }, "accuracy": 0.8514851485148515, "ece_hard": 0.088348623853211, "calibration": 81.17213761467889, "p50_s": 0.4092683419585228, "p95_s": 1.1480483673512936, "cost_usd_per_1000": 0.01795538016835391, "axes": { "intelligence": 84.71651795065142, "calibration": 81.17213761467889, "speed": 83.28031662636582, "cost": 62.37416185436547 }, "jevbench_score": 77.30908668298979 } }, { "threshold": 0.96, "public": { "n": 231, "escalation": 0.3593073593073593, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.7207207207207207 }, "accuracy": 0.8095238095238095, "ece_hard": 0.0915815815815815, "calibration": 78.36284184184186, "p50_s": 0.4113641306757927, "p95_s": 1.210436686873436, "cost_usd_per_1000": 0.01767481205311542, "axes": { "intelligence": 81.34175842509177, "calibration": 78.36284184184186, "speed": 83.02831501594865, "cost": 62.57935586861168 }, "jevbench_score": 75.86129668543015 }, "heldout": { "n": 303, "escalation": 0.38283828382838286, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6972477064220184 }, "accuracy": 0.8514851485148515, "ece_hard": 0.086788990825688, "calibration": 81.32810091743119, "p50_s": 0.4161411300301552, "p95_s": 1.149820913374424, "cost_usd_per_1000": 0.01861403092668817, "axes": { "intelligence": 84.71651795065142, "calibration": 81.32810091743119, "speed": 83.20129154344635, "cost": 61.904787065635304 }, "jevbench_score": 77.18195190964732 } }, { "threshold": 0.97, "public": { "n": 231, "escalation": 0.37662337662337664, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.7207207207207207 }, "accuracy": 0.8095238095238095, "ece_hard": 0.09041041041041031, "calibration": 78.47995895895897, "p50_s": 0.41320376843214035, "p95_s": 1.2149329222738743, "cost_usd_per_1000": 0.018365967654068775, "axes": { "intelligence": 81.34175842509177, "calibration": 78.47995895895897, "speed": 82.99283425613694, "cost": 62.079585546784 }, "jevbench_score": 75.72955977905909 }, "heldout": { "n": 303, "escalation": 0.40264026402640263, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6972477064220184 }, "accuracy": 0.8514851485148515, "ece_hard": 0.08587155963302745, "calibration": 81.41984403669726, "p50_s": 0.41730349510908127, "p95_s": 1.149820913374424, "cost_usd_per_1000": 0.019404411836689284, "axes": { "intelligence": 84.71651795065142, "calibration": 81.41984403669726, "speed": 83.18917774363766, "cost": 61.36298549617274 }, "jevbench_score": 77.03142270531016 } }, { "threshold": 0.98, "public": { "n": 231, "escalation": 0.3939393939393939, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.7207207207207207 }, "accuracy": 0.8095238095238095, "ece_hard": 0.09023023023023015, "calibration": 78.39797697697699, "p50_s": 0.41639333218336105, "p95_s": 1.2149329222738743, "cost_usd_per_1000": 0.019057123255022127, "axes": { "intelligence": 81.34175842509177, "calibration": 78.39797697697699, "speed": 82.95943932837795, "cost": 61.59827971443925 }, "jevbench_score": 75.55499847278793 }, "heldout": { "n": 303, "escalation": 0.4389438943894389, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6972477064220184 }, "accuracy": 0.8514851485148515, "ece_hard": 0.08550458715596324, "calibration": 81.45654128440367, "p50_s": 0.430687740445137, "p95_s": 1.1517731554806232, "cost_usd_per_1000": 0.020853443505024657, "axes": { "intelligence": 84.71651795065142, "calibration": 81.45654128440367, "speed": 83.04470538125808, "cost": 60.4246666066792 }, "jevbench_score": 76.71054504728303 } }, { "threshold": 0.99, "public": { "n": 231, "escalation": 0.4069264069264069, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.7207207207207207 }, "accuracy": 0.8095238095238095, "ece_hard": 0.0898698698698698, "calibration": 78.43401301301303, "p50_s": 0.4255792573094368, "p95_s": 1.2149329222738743, "cost_usd_per_1000": 0.019575489955737144, "axes": { "intelligence": 81.34175842509177, "calibration": 78.43401301301303, "speed": 82.86467246926404, "cost": 61.24862077479008 }, "jevbench_score": 75.43465892532016 }, "heldout": { "n": 303, "escalation": 0.49504950495049505, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6972477064220184 }, "accuracy": 0.8514851485148515, "ece_hard": 0.08495412844036687, "calibration": 81.51158715596331, "p50_s": 0.48460932821035385, "p95_s": 1.1517731554806232, "cost_usd_per_1000": 0.023092856083361143, "axes": { "intelligence": 84.71651795065142, "calibration": 81.51158715596331, "speed": 82.53241278257939, "cost": 59.09567052976098 }, "jevbench_score": 76.18016574577516 } }, { "threshold": 1.0, "public": { "n": 231, "escalation": 0.48917748917748916, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.7117117117117117 }, "accuracy": 0.8051948051948052, "ece_hard": 0.09392392392392392, "calibration": 78.02860760760763, "p50_s": 0.5186787471175194, "p95_s": 1.2149329222738743, "cost_usd_per_1000": 0.022858479060265575, "axes": { "intelligence": 80.9663830497164, "calibration": 78.02860760760763, "speed": 82.0054924613516, "cost": 59.228580091855434 }, "jevbench_score": 74.42746283770617 }, "heldout": { "n": 303, "escalation": 0.5973597359735974, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6972477064220184 }, "accuracy": 0.8514851485148515, "ece_hard": 0.08321100917431191, "calibration": 81.68589908256881, "p50_s": 1.0143876299262047, "p95_s": 1.153977608680725, "cost_usd_per_1000": 0.027176490785033557, "axes": { "intelligence": 84.71651795065142, "calibration": 81.68589908256881, "speed": 79.31598673461828, "cost": 56.97419869200009 }, "jevbench_score": 74.78052966890709 } } ] }, { "cheap": "laya", "backend": "classifier-dev-fast", "chosen": { "threshold": 0.96, "public": { "n": 231, "escalation": 0.8874458874458875, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6846846846846847 }, "accuracy": 0.7922077922077922, "ece_hard": 0.11600782600782594, "calibration": 74.5452173992174, "p50_s": 1.5650968194007873, "p95_s": 4.825077628344298, "cost_usd_per_1000": 0.005841280298976928, "axes": { "intelligence": 79.84025692359026, "calibration": 74.5452173992174, "speed": 71.21954486242686, "cost": 77.00475859771826 }, "jevbench_score": 75.58569614404567 }, "heldout": { "n": 303, "escalation": 0.9537953795379538, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6513761467889908 }, "accuracy": 0.8481848184818482, "ece_hard": 0.14760911871003615, "calibration": 73.04858812899639, "p50_s": 3.141072319447994, "p95_s": 5.1524781987071036, "cost_usd_per_1000": 0.006062445272617149, "axes": { "intelligence": 84.10749444933184, "calibration": 73.04858812899639, "speed": 67.90905901515731, "cost": 76.52056506607882 }, "jevbench_score": 75.16888369854337 } }, "full": { "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9726027397260274, "hard": 0.6681818181818182 }, "accuracy": 0.8239700374531835, "ece_hard": 0.12748301193755732, "calibration": 74.22944880624428, "p50_s": 2.6266960330307483, "p95_s": 4.994102110341192, "cost_usd_per_1000": 0.005966772784019975, "axes": { "intelligence": 84.31999792444998, "calibration": 74.22944880624428, "speed": 68.82132764034878, "cost": 76.72781499602524 }, "jevbench_score": 75.82207666946415 }, "full_escalation": 0.9250936329588015, "backend_heldout_accuracy": 0.8481848184818482, "useful": false, "curve": [ { "threshold": 0.0, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.01, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.02, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.03, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.04, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.05, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.06, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.07, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.08, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.09, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.1, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.11, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.12, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.13, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.14, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.15, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.16, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.17, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.18, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.19, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.2, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.21, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.22, "public": { "n": 231, "escalation": 0.004329004329004329, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.34234234234234234 }, "accuracy": 0.5324675324675324, "ece_hard": 0.20223603603603607, "calibration": 64.4588963963964, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028975573552539845, "axes": { "intelligence": 53.963338338338346, "calibration": 64.4588963963964, "speed": 72.85668019555155, "cost": 86.13903876111615 }, "jevbench_score": 68.35381653239362 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.23, "public": { "n": 231, "escalation": 0.004329004329004329, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.34234234234234234 }, "accuracy": 0.5324675324675324, "ece_hard": 0.20223603603603607, "calibration": 64.4588963963964, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028975573552539845, "axes": { "intelligence": 53.963338338338346, "calibration": 64.4588963963964, "speed": 72.85668019555155, "cost": 86.13903876111615 }, "jevbench_score": 68.35381653239362 }, "heldout": { "n": 303, "escalation": 0.0033003300330033004, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2661174311926606, "calibration": 54.73700688073394, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028941284409339813, "axes": { "intelligence": 59.67720246323992, "calibration": 54.73700688073394, "speed": 68.8164811989393, "cost": 86.15446596619417 }, "jevbench_score": 66.33839190479289 } }, { "threshold": 0.24, "public": { "n": 231, "escalation": 0.004329004329004329, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.34234234234234234 }, "accuracy": 0.5324675324675324, "ece_hard": 0.20223603603603607, "calibration": 64.4588963963964, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028975573552539845, "axes": { "intelligence": 53.963338338338346, "calibration": 64.4588963963964, "speed": 72.85668019555155, "cost": 86.13903876111615 }, "jevbench_score": 68.35381653239362 }, "heldout": { "n": 303, "escalation": 0.006600660066006601, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.30275229357798167 }, "accuracy": 0.5676567656765676, "ece_hard": 0.26831467889908267, "calibration": 54.51728211009173, "p50_s": 2.7680240362882613, "p95_s": 4.776861734688282, "cost_usd_per_1000": 0.0029051295410439924, "axes": { "intelligence": 59.95243182103809, "calibration": 54.51728211009173, "speed": 68.78687472081019, "cost": 86.10503492361818 }, "jevbench_score": 66.33134161539225 } }, { "threshold": 0.25, "public": { "n": 231, "escalation": 0.008658008658008658, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.35135135135135137 }, "accuracy": 0.5367965367965368, "ece_hard": 0.20445495495495497, "calibration": 64.23700450450451, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.002911987369683999, "axes": { "intelligence": 54.33871371371373, "calibration": 64.23700450450451, "speed": 72.85668019555155, "cost": 86.07431539129652 }, "jevbench_score": 68.40051907028403 }, "heldout": { "n": 303, "escalation": 0.009900990099009901, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2651376146788991, "calibration": 54.83498853211009, "p50_s": 2.7680240362882613, "p95_s": 4.776861734688282, "cost_usd_per_1000": 0.0029161306411540034, "axes": { "intelligence": 59.67720246323992, "calibration": 54.83498853211009, "speed": 68.78687472081019, "cost": 86.05579071245948 }, "jevbench_score": 66.34191028199702 } }, { "threshold": 0.26, "public": { "n": 231, "escalation": 0.008658008658008658, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.35135135135135137 }, "accuracy": 0.5367965367965368, "ece_hard": 0.20445495495495497, "calibration": 64.23700450450451, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.002911987369683999, "axes": { "intelligence": 54.33871371371373, "calibration": 64.23700450450451, "speed": 72.85668019555155, "cost": 86.07431539129652 }, "jevbench_score": 68.40051907028403 }, "heldout": { "n": 303, "escalation": 0.013201320132013201, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2749247706422019, "calibration": 53.85627293577981, "p50_s": 2.7680240362882613, "p95_s": 4.805826105177402, "cost_usd_per_1000": 0.002927131741264014, "axes": { "intelligence": 59.67720246323992, "calibration": 53.85627293577981, "speed": 68.76062090811617, "cost": 86.00673192572431 }, "jevbench_score": 66.02816786093783 } }, { "threshold": 0.27, "public": { "n": 231, "escalation": 0.021645021645021644, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.35135135135135137 }, "accuracy": 0.5367965367965368, "ece_hard": 0.21054414414414419, "calibration": 63.62808558558559, "p50_s": 1.1657145261764525, "p95_s": 4.4528342053294185, "cost_usd_per_1000": 0.002955277412974042, "axes": { "intelligence": 54.33871371371373, "calibration": 63.62808558558559, "speed": 72.84771266552642, "cost": 85.88205236460911 }, "jevbench_score": 68.1976070477389 }, "heldout": { "n": 303, "escalation": 0.013201320132013201, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2749247706422019, "calibration": 53.85627293577981, "p50_s": 2.7680240362882613, "p95_s": 4.805826105177402, "cost_usd_per_1000": 0.002927131741264014, "axes": { "intelligence": 59.67720246323992, "calibration": 53.85627293577981, "speed": 68.76062090811617, "cost": 86.00673192572431 }, "jevbench_score": 66.02816786093783 } }, { "threshold": 0.28, "public": { "n": 231, "escalation": 0.021645021645021644, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.35135135135135137 }, "accuracy": 0.5367965367965368, "ece_hard": 0.21054414414414419, "calibration": 63.62808558558559, "p50_s": 1.1657145261764525, "p95_s": 4.4528342053294185, "cost_usd_per_1000": 0.002955277412974042, "axes": { "intelligence": 54.33871371371373, "calibration": 63.62808558558559, "speed": 72.84771266552642, "cost": 85.88205236460911 }, "jevbench_score": 68.1976070477389 }, "heldout": { "n": 303, "escalation": 0.019801980198019802, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.30275229357798167 }, "accuracy": 0.5676567656765676, "ece_hard": 0.2767513761467891, "calibration": 53.67361238532109, "p50_s": 2.7680240362882613, "p95_s": 4.807024084031581, "cost_usd_per_1000": 0.002949133941484036, "axes": { "intelligence": 59.95243182103809, "calibration": 53.67361238532109, "speed": 68.75953844953571, "cost": 85.90916507648336 }, "jevbench_score": 66.02904563034339 } }, { "threshold": 0.29, "public": { "n": 231, "escalation": 0.03896103896103896, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.22890613340613344, "calibration": 61.79188665938666, "p50_s": 1.1657145261764525, "p95_s": 4.467663718014956, "cost_usd_per_1000": 0.0030129974706941, "axes": { "intelligence": 53.58796296296297, "calibration": 61.79188665938666, "speed": 72.83327315437609, "cost": 85.63003698704512 }, "jevbench_score": 67.41222565229677 }, "heldout": { "n": 303, "escalation": 0.0231023102310231, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.30275229357798167 }, "accuracy": 0.5676567656765676, "ece_hard": 0.27010183486238537, "calibration": 54.338566513761464, "p50_s": 2.7680240362882613, "p95_s": 4.807024084031581, "cost_usd_per_1000": 0.0029601350415940472, "axes": { "intelligence": 59.95243182103809, "calibration": 54.338566513761464, "speed": 68.75953844953571, "cost": 85.86065427821823 }, "jevbench_score": 66.22325630120746 } }, { "threshold": 0.3, "public": { "n": 231, "escalation": 0.04329004329004329, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.34234234234234234 }, "accuracy": 0.5324675324675324, "ece_hard": 0.2250592865592866, "calibration": 62.17657134407135, "p50_s": 1.1657145261764525, "p95_s": 4.467663718014956, "cost_usd_per_1000": 0.0030274274851241146, "axes": { "intelligence": 53.963338338338346, "calibration": 62.17657134407135, "speed": 72.83327315437609, "cost": 85.56778751677061 }, "jevbench_score": 67.62253186509137 }, "heldout": { "n": 303, "escalation": 0.033003300330033, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6986301369863014, "hard": 0.30275229357798167 }, "accuracy": 0.570957095709571, "ece_hard": 0.26870183486238536, "calibration": 54.478566513761464, "p50_s": 2.7680240362882613, "p95_s": 4.807024084031581, "cost_usd_per_1000": 0.0029931383419240804, "axes": { "intelligence": 60.1442126429559, "calibration": 54.478566513761464, "speed": 68.75953844953571, "cost": 85.7161962921973 }, "jevbench_score": 66.29088846929578 } }, { "threshold": 0.31, "public": { "n": 231, "escalation": 0.05627705627705628, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.34234234234234234 }, "accuracy": 0.5324675324675324, "ece_hard": 0.23898721448721455, "calibration": 60.783778551278544, "p50_s": 1.1657145261764525, "p95_s": 4.467663718014956, "cost_usd_per_1000": 0.0030707175284141576, "axes": { "intelligence": 53.963338338338346, "calibration": 60.783778551278544, "speed": 72.83327315437609, "cost": 85.38280395828644 }, "jevbench_score": 67.20424181091859 }, "heldout": { "n": 303, "escalation": 0.0429042904290429, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6986301369863014, "hard": 0.3302752293577982 }, "accuracy": 0.5808580858085809, "ece_hard": 0.25499266055045877, "calibration": 55.84948394495412, "p50_s": 2.7680240362882613, "p95_s": 4.8142633035779, "cost_usd_per_1000": 0.003026141642254113, "axes": { "intelligence": 60.96990071635039, "calibration": 55.84948394495412, "speed": 68.75300303786179, "cost": 85.57332244460922 }, "jevbench_score": 66.90232022365875 } }, { "threshold": 0.32, "public": { "n": 231, "escalation": 0.06060606060606061, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2475277550277551, "calibration": 59.9297244972245, "p50_s": 1.1657145261764525, "p95_s": 4.486792547255755, "cost_usd_per_1000": 0.0030851475428441723, "axes": { "intelligence": 53.58796296296297, "calibration": 59.9297244972245, "speed": 72.81471801852942, "cost": 85.32172184833048 }, "jevbench_score": 66.83394212353933 }, "heldout": { "n": 303, "escalation": 0.0462046204620462, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6986301369863014, "hard": 0.3302752293577982 }, "accuracy": 0.5808580858085809, "ece_hard": 0.2574935779816514, "calibration": 55.59939220183486, "p50_s": 2.7680240362882613, "p95_s": 4.8142633035779, "cost_usd_per_1000": 0.0030371427423641242, "axes": { "intelligence": 60.96990071635039, "calibration": 55.59939220183486, "speed": 68.75300303786179, "cost": 85.52604388719092 }, "jevbench_score": 66.81806535426922 } }, { "threshold": 0.33, "public": { "n": 231, "escalation": 0.06493506493506493, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2535304577304578, "calibration": 59.329454226954226, "p50_s": 1.1657145261764525, "p95_s": 4.4975419580936435, "cost_usd_per_1000": 0.0030995775572741865, "axes": { "intelligence": 53.58796296296297, "calibration": 59.329454226954226, "speed": 72.80432568054275, "cost": 85.26092476929185 }, "jevbench_score": 66.6516961962081 }, "heldout": { "n": 303, "escalation": 0.0594059405940594, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6986301369863014, "hard": 0.3394495412844037 }, "accuracy": 0.5841584158415841, "ece_hard": 0.2563293577981652, "calibration": 55.94806422018348, "p50_s": 2.7680240362882613, "p95_s": 4.8142633035779, "cost_usd_per_1000": 0.0030811471428041685, "axes": { "intelligence": 61.245130074148555, "calibration": 55.94806422018348, "speed": 68.75300303786179, "cost": 85.33862683255605 }, "jevbench_score": 66.9612399690877 } }, { "threshold": 0.34, "public": { "n": 231, "escalation": 0.0735930735930736, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.34234234234234234 }, "accuracy": 0.5324675324675324, "ece_hard": 0.2521412685412686, "calibration": 59.468373145873144, "p50_s": 1.1657145261764525, "p95_s": 4.505149663984776, "cost_usd_per_1000": 0.0031284375861342154, "axes": { "intelligence": 53.963338338338346, "calibration": 59.468373145873144, "speed": 72.79698568576424, "cost": 85.14017514964502 }, "jevbench_score": 66.78181230458033 }, "heldout": { "n": 303, "escalation": 0.06930693069306931, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7054794520547946, "hard": 0.3486238532110092 }, "accuracy": 0.5907590759075908, "ece_hard": 0.25898990825688073, "calibration": 55.68200917431193, "p50_s": 2.7680240362882613, "p95_s": 4.8142633035779, "cost_usd_per_1000": 0.003114150443134201, "axes": { "intelligence": 61.71214025386453, "calibration": 55.68200917431193, "speed": 68.75300303786179, "cost": 85.19981232100605 }, "jevbench_score": 66.98135889683054 } }, { "threshold": 0.35, "public": { "n": 231, "escalation": 0.0735930735930736, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.34234234234234234 }, "accuracy": 0.5324675324675324, "ece_hard": 0.2521412685412686, "calibration": 59.468373145873144, "p50_s": 1.1657145261764525, "p95_s": 4.505149663984776, "cost_usd_per_1000": 0.0031284375861342154, "axes": { "intelligence": 53.963338338338346, "calibration": 59.468373145873144, "speed": 72.79698568576424, "cost": 85.14017514964502 }, "jevbench_score": 66.78181230458033 }, "heldout": { "n": 303, "escalation": 0.07920792079207921, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7054794520547946, "hard": 0.3669724770642202 }, "accuracy": 0.5973597359735974, "ece_hard": 0.25261376146788994, "calibration": 56.319623853211006, "p50_s": 2.7680240362882613, "p95_s": 4.8142633035779, "cost_usd_per_1000": 0.0031471537434642344, "axes": { "intelligence": 62.26259896946086, "calibration": 56.319623853211006, "speed": 68.75300303786179, "cost": 85.06246121777775 }, "jevbench_score": 67.29443510998284 } }, { "threshold": 0.36, "public": { "n": 231, "escalation": 0.08658008658008658, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5411255411255411, "ece_hard": 0.24759352079352087, "calibration": 59.92314792064792, "p50_s": 1.1657145261764525, "p95_s": 4.538626746833325, "cost_usd_per_1000": 0.003171727629424259, "axes": { "intelligence": 54.7140890890891, "calibration": 59.92314792064792, "speed": 72.76483323506824, "cost": 84.96112343745148 }, "jevbench_score": 67.09789481800553 }, "heldout": { "n": 303, "escalation": 0.08250825082508251, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7054794520547946, "hard": 0.3669724770642202 }, "accuracy": 0.5973597359735974, "ece_hard": 0.2572330275229358, "calibration": 56.94619724770642, "p50_s": 2.7680240362882613, "p95_s": 4.8142633035779, "cost_usd_per_1000": 0.0031581548435742454, "axes": { "intelligence": 62.26259896946086, "calibration": 56.94619724770642, "speed": 68.75300303786179, "cost": 85.01699741354368 }, "jevbench_score": 67.47180865447906 } }, { "threshold": 0.37, "public": { "n": 231, "escalation": 0.11255411255411256, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3963963963963964 }, "accuracy": 0.5584415584415584, "ece_hard": 0.2300394121394122, "calibration": 61.67855878605879, "p50_s": 1.169974410533905, "p95_s": 4.690320664644242, "cost_usd_per_1000": 0.0032583077160043454, "axes": { "intelligence": 56.2155905905906, "calibration": 61.67855878605879, "speed": 72.60621101680323, "cost": 84.61023709390076 }, "jevbench_score": 67.9355225216964 }, "heldout": { "n": 303, "escalation": 0.0891089108910891, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7054794520547946, "hard": 0.3761467889908257 }, "accuracy": 0.6006600660066007, "ece_hard": 0.25524122880177924, "calibration": 57.14537711982207, "p50_s": 2.7680240362882613, "p95_s": 4.8142633035779, "cost_usd_per_1000": 0.003180157043794267, "axes": { "intelligence": 62.53782832725903, "calibration": 57.14537711982207, "speed": 68.75300303786179, "cost": 84.92654298943665 }, "jevbench_score": 67.58724660825652 } }, { "threshold": 0.38, "public": { "n": 231, "escalation": 0.12554112554112554, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3963963963963964 }, "accuracy": 0.5584415584415584, "ece_hard": 0.2316340067340068, "calibration": 61.519099326599324, "p50_s": 1.2405554592609405, "p95_s": 4.690320664644242, "cost_usd_per_1000": 0.0033015977592943885, "axes": { "intelligence": 56.2155905905906, "calibration": 61.519099326599324, "speed": 72.35181280463719, "cost": 84.43827516623578 }, "jevbench_score": 67.7975311228811 }, "heldout": { "n": 303, "escalation": 0.09900990099009901, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7123287671232876, "hard": 0.3853211009174312 }, "accuracy": 0.6072607260726073, "ece_hard": 0.2572687517375591, "calibration": 56.94262482624409, "p50_s": 2.7859468325972556, "p95_s": 4.8142633035779, "cost_usd_per_1000": 0.0032131603441243003, "axes": { "intelligence": 63.004838506975005, "calibration": 56.94262482624409, "speed": 68.72497336777371, "cost": 84.79202804829248 }, "jevbench_score": 67.61923405507102 } }, { "threshold": 0.39, "public": { "n": 231, "escalation": 0.14285714285714285, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.40540540540540543 }, "accuracy": 0.5627705627705628, "ece_hard": 0.23433580853580857, "calibration": 61.24891914641914, "p50_s": 1.2405554592609405, "p95_s": 4.712589012086392, "cost_usd_per_1000": 0.0033593178170144462, "axes": { "intelligence": 56.59096596596597, "calibration": 61.24891914641914, "speed": 72.33124249432274, "cost": 84.21246719960932 }, "jevbench_score": 67.78552487404835 }, "heldout": { "n": 303, "escalation": 0.11221122112211221, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7191780821917808, "hard": 0.3944954128440367 }, "accuracy": 0.6138613861386139, "ece_hard": 0.2629008618293022, "calibration": 56.379413817069775, "p50_s": 2.79284847676754, "p95_s": 4.8142633035779, "cost_usd_per_1000": 0.003257164744564344, "axes": { "intelligence": 63.471848686690954, "calibration": 56.379413817069775, "speed": 68.71422786684701, "cost": 84.6148082392548 }, "jevbench_score": 67.53807644448501 } }, { "threshold": 0.4, "public": { "n": 231, "escalation": 0.1471861471861472, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.4144144144144144 }, "accuracy": 0.5670995670995671, "ece_hard": 0.23547905177905176, "calibration": 61.134594822094826, "p50_s": 1.2405554592609405, "p95_s": 4.712589012086392, "cost_usd_per_1000": 0.003373747831444461, "axes": { "intelligence": 56.96634134134135, "calibration": 61.134594822094826, "speed": 72.33124249432274, "cost": 84.15662144764902 }, "jevbench_score": 67.85469392573874 }, "heldout": { "n": 303, "escalation": 0.132013201320132, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7191780821917808, "hard": 0.4036697247706422 }, "accuracy": 0.6171617161716172, "ece_hard": 0.26782563247150404, "calibration": 55.820186752849594, "p50_s": 2.79284847676754, "p95_s": 4.822164891660214, "cost_usd_per_1000": 0.0033231713452244105, "axes": { "intelligence": 63.74707804448912, "calibration": 55.820186752849594, "speed": 68.7071056908665, "cost": 84.35341796777469 }, "jevbench_score": 67.38899455406859 } }, { "threshold": 0.41, "public": { "n": 231, "escalation": 0.1645021645021645, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.43243243243243246 }, "accuracy": 0.5757575757575758, "ece_hard": 0.2318159887159888, "calibration": 61.500901128401125, "p50_s": 1.2405554592609405, "p95_s": 4.732986265420914, "cost_usd_per_1000": 0.0034314678891645186, "axes": { "intelligence": 57.7170920920921, "calibration": 61.500901128401125, "speed": 72.31248571545316, "cost": 83.93560182326843 }, "jevbench_score": 68.1296809549965 }, "heldout": { "n": 303, "escalation": 0.13861386138613863, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7191780821917808, "hard": 0.4036697247706422 }, "accuracy": 0.6171617161716172, "ece_hard": 0.27266508201278844, "calibration": 54.67374179872116, "p50_s": 2.8226395934820174, "p95_s": 4.822164891660214, "cost_usd_per_1000": 0.003345173545444432, "axes": { "intelligence": 63.74707804448912, "calibration": 54.67374179872116, "speed": 68.66102514071885, "cost": 84.26744039182847 }, "jevbench_score": 67.01195552121199 } }, { "threshold": 0.42, "public": { "n": 231, "escalation": 0.16883116883116883, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.44144144144144143 }, "accuracy": 0.5800865800865801, "ece_hard": 0.22446373646373652, "calibration": 62.236126353626354, "p50_s": 1.2405554592609405, "p95_s": 4.732986265420914, "cost_usd_per_1000": 0.0034458979035945332, "axes": { "intelligence": 58.09246746746747, "calibration": 62.236126353626354, "speed": 72.31248571545316, "cost": 83.880927824304 }, "jevbench_score": 68.43207728022723 }, "heldout": { "n": 303, "escalation": 0.15841584158415842, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.726027397260274, "hard": 0.42201834862385323 }, "accuracy": 0.6270627062706271, "ece_hard": 0.2634971921045315, "calibration": 55.590530789546854, "p50_s": 2.8226395934820174, "p95_s": 4.822164891660214, "cost_usd_per_1000": 0.003411180146104498, "axes": { "intelligence": 64.48931758200327, "calibration": 55.590530789546854, "speed": 68.66102514071885, "cost": 84.01286034044097 }, "jevbench_score": 67.43512457878148 } }, { "threshold": 0.43, "public": { "n": 231, "escalation": 0.18614718614718614, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.4594594594594595 }, "accuracy": 0.5887445887445888, "ece_hard": 0.22487178997179, "calibration": 61.836821002821, "p50_s": 1.2405554592609405, "p95_s": 4.732986265420914, "cost_usd_per_1000": 0.003503617961314591, "axes": { "intelligence": 58.84321821821823, "calibration": 61.836821002821, "speed": 72.31248571545316, "cost": 83.66449767733212 }, "jevbench_score": 68.49746789419666 }, "heldout": { "n": 303, "escalation": 0.19141914191419143, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7397260273972602, "hard": 0.43119266055045874 }, "accuracy": 0.636963696369637, "ece_hard": 0.25695407283847654, "calibration": 56.24484271615235, "p50_s": 2.8282473921775817, "p95_s": 4.961904856562615, "cost_usd_per_1000": 0.003521191147204608, "axes": { "intelligence": 65.14810858363705, "calibration": 56.24484271615235, "speed": 68.52834172134803, "cost": 83.59931196084862 }, "jevbench_score": 67.68841753737458 } }, { "threshold": 0.44, "public": { "n": 231, "escalation": 0.19913419913419914, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.46846846846846846 }, "accuracy": 0.5930735930735931, "ece_hard": 0.2254844025844026, "calibration": 62.356559741559735, "p50_s": 1.2405554592609405, "p95_s": 4.732986265420914, "cost_usd_per_1000": 0.003546908004604634, "axes": { "intelligence": 59.2185935935936, "calibration": 62.356559741559735, "speed": 72.31248571545316, "cost": 83.50450226755605 }, "jevbench_score": 68.71726319447318 }, "heldout": { "n": 303, "escalation": 0.20132013201320131, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7397260273972602, "hard": 0.43119266055045874 }, "accuracy": 0.636963696369637, "ece_hard": 0.25695407283847654, "calibration": 56.24484271615235, "p50_s": 2.8282473921775817, "p95_s": 4.961904856562615, "cost_usd_per_1000": 0.0035541944475346414, "axes": { "intelligence": 65.14810858363705, "calibration": 56.24484271615235, "speed": 68.52834172134803, "cost": 83.4777644778469 }, "jevbench_score": 67.66380056941402 } }, { "threshold": 0.45, "public": { "n": 231, "escalation": 0.21212121212121213, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.4954954954954955 }, "accuracy": 0.6060606060606061, "ece_hard": 0.20620332150332155, "calibration": 64.28466784966784, "p50_s": 1.2405554592609405, "p95_s": 4.732986265420914, "cost_usd_per_1000": 0.003590198047894677, "axes": { "intelligence": 60.34471971971972, "calibration": 64.28466784966784, "speed": 72.31248571545316, "cost": 83.34644780669976 }, "jevbench_score": 69.53633006594983 }, "heldout": { "n": 303, "escalation": 0.2145214521452145, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7465753424657534, "hard": 0.43119266055045874 }, "accuracy": 0.6402640264026402, "ece_hard": 0.26133021962746733, "calibration": 55.80722803725327, "p50_s": 2.8427457749843597, "p95_s": 4.961904856562615, "cost_usd_per_1000": 0.003598198847974685, "axes": { "intelligence": 65.33988940555486, "calibration": 55.80722803725327, "speed": 68.50613544748256, "cost": 83.31744519476233 }, "jevbench_score": 67.54350056473933 } }, { "threshold": 0.46, "public": { "n": 231, "escalation": 0.22510822510822512, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.5045045045045045 }, "accuracy": 0.6103896103896104, "ece_hard": 0.20219971789971797, "calibration": 64.6850282100282, "p50_s": 1.2450210839509963, "p95_s": 4.746018932759762, "cost_usd_per_1000": 0.0036334880911847206, "axes": { "intelligence": 60.720095095095104, "calibration": 64.6850282100282, "speed": 72.28493827529797, "cost": 83.19028776626479 }, "jevbench_score": 69.7130628630743 }, "heldout": { "n": 303, "escalation": 0.21782178217821782, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7465753424657534, "hard": 0.44036697247706424 }, "accuracy": 0.6435643564356436, "ece_hard": 0.25522838476508203, "calibration": 57.655911523491795, "p50_s": 2.8427457749843597, "p95_s": 4.961904856562615, "cost_usd_per_1000": 0.0036091999480846963, "axes": { "intelligence": 65.61511876335302, "calibration": 57.655911523491795, "speed": 68.50613544748256, "cost": 83.27767172654015 }, "jevbench_score": 68.15950870284128 } }, { "threshold": 0.47, "public": { "n": 231, "escalation": 0.24242424242424243, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.4954954954954955 }, "accuracy": 0.6060606060606061, "ece_hard": 0.21457359177359187, "calibration": 63.601140822640815, "p50_s": 1.2450210839509963, "p95_s": 4.746018932759762, "cost_usd_per_1000": 0.0036912081489047783, "axes": { "intelligence": 60.34471971971972, "calibration": 63.601140822640815, "speed": 72.28493827529797, "cost": 82.98494392151916 }, "jevbench_score": 69.26882565133599 }, "heldout": { "n": 303, "escalation": 0.23432343234323433, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7671232876712328, "hard": 0.44954128440366975 }, "accuracy": 0.6567656765676567, "ece_hard": 0.2509861829302196, "calibration": 58.08013170697804, "p50_s": 2.8838920310139655, "p95_s": 4.961904856562615, "cost_usd_per_1000": 0.003664205448634751, "axes": { "intelligence": 66.46569058690461, "calibration": 58.08013170697804, "speed": 68.44372570526994, "cost": 83.08060551486128 }, "jevbench_score": 68.44860512568506 } }, { "threshold": 0.48, "public": { "n": 231, "escalation": 0.2510822510822511, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.4864864864864865 }, "accuracy": 0.6017316017316018, "ece_hard": 0.224253771953772, "calibration": 62.096122804622794, "p50_s": 1.296530620753765, "p95_s": 4.746018932759762, "cost_usd_per_1000": 0.003720068177764807, "axes": { "intelligence": 59.96934434434435, "calibration": 62.096122804622794, "speed": 72.10887755476139, "cost": 82.88347302164212 }, "jevbench_score": 68.6851127148648 }, "heldout": { "n": 303, "escalation": 0.264026402640264, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7808219178082192, "hard": 0.46788990825688076 }, "accuracy": 0.66996699669967, "ece_hard": 0.24881095357242144, "calibration": 58.297654642757855, "p50_s": 2.9360936373472213, "p95_s": 4.961904856562615, "cost_usd_per_1000": 0.00376321534962485, "axes": { "intelligence": 67.39971094633657, "calibration": 58.297654642757855, "speed": 68.36581665582412, "cost": 82.73322785531198 }, "jevbench_score": 68.66050968217877 } }, { "threshold": 0.49, "public": { "n": 231, "escalation": 0.2554112554112554, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.4954954954954955 }, "accuracy": 0.6060606060606061, "ece_hard": 0.2191231413231414, "calibration": 62.60918586768586, "p50_s": 1.296530620753765, "p95_s": 4.746018932759762, "cost_usd_per_1000": 0.003734498192194822, "axes": { "intelligence": 60.34471971971972, "calibration": 62.60918586768586, "speed": 72.10887755476139, "cost": 82.83303239378671 }, "jevbench_score": 68.92351325364011 }, "heldout": { "n": 303, "escalation": 0.26732673267326734, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7808219178082192, "hard": 0.47706422018348627 }, "accuracy": 0.6732673267326733, "ece_hard": 0.24119352237976094, "calibration": 59.059397762023906, "p50_s": 2.9360936373472213, "p95_s": 4.961904856562615, "cost_usd_per_1000": 0.0037742164497348613, "axes": { "intelligence": 67.67494030413474, "calibration": 59.059397762023906, "speed": 68.36581665582412, "cost": 82.69519590549913 }, "jevbench_score": 68.94599530364384 } }, { "threshold": 0.5, "public": { "n": 231, "escalation": 0.26406926406926406, "tiers": { "easy": 0.9583333333333334, "standard": 0.5555555555555556, "hard": 0.5045045045045045 }, "accuracy": 0.6147186147186147, "ece_hard": 0.21312224042224046, "calibration": 63.209275957775944, "p50_s": 1.3233075231313705, "p95_s": 4.746018932759762, "cost_usd_per_1000": 0.0037633582210548503, "axes": { "intelligence": 61.260218551885224, "calibration": 63.209275957775944, "speed": 72.02009752121813, "cost": 82.73273322168012 }, "jevbench_score": 69.30628317716341 }, "heldout": { "n": 303, "escalation": 0.27722772277227725, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7808219178082192, "hard": 0.46788990825688076 }, "accuracy": 0.66996699669967, "ece_hard": 0.2508036141228802, "calibration": 58.09838858771198, "p50_s": 2.9360936373472213, "p95_s": 4.990756732225417, "cost_usd_per_1000": 0.0038072197500648945, "axes": { "intelligence": 67.39971094633657, "calibration": 58.09838858771198, "speed": 68.34063696819729, "cost": 82.58176165864012 }, "jevbench_score": 68.5640276865627 } }, { "threshold": 0.51, "public": { "n": 231, "escalation": 0.2813852813852814, "tiers": { "easy": 0.9583333333333334, "standard": 0.5972222222222222, "hard": 0.5045045045045045 }, "accuracy": 0.6277056277056277, "ece_hard": 0.21751683501683503, "calibration": 62.76981649831649, "p50_s": 1.3687158465385436, "p95_s": 4.746018932759762, "cost_usd_per_1000": 0.003821078278774908, "axes": { "intelligence": 62.8805889222556, "calibration": 62.76981649831649, "speed": 71.87357238230547, "cost": 82.53442195734746 }, "jevbench_score": 69.56134188540501 }, "heldout": { "n": 303, "escalation": 0.29042904290429045, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7808219178082192, "hard": 0.47706422018348627 }, "accuracy": 0.6732673267326733, "ece_hard": 0.24922012788434805, "calibration": 59.659737211565194, "p50_s": 2.952823558449745, "p95_s": 4.990756732225417, "cost_usd_per_1000": 0.0038512241505049383, "axes": { "intelligence": 67.67494030413474, "calibration": 59.659737211565194, "speed": 68.31596101598828, "cost": 82.43203610975927 }, "jevbench_score": 69.05288956996623 } }, { "threshold": 0.52, "public": { "n": 231, "escalation": 0.2943722943722944, "tiers": { "easy": 0.9583333333333334, "standard": 0.5972222222222222, "hard": 0.5045045045045045 }, "accuracy": 0.6277056277056277, "ece_hard": 0.22147088907088913, "calibration": 62.37441109291108, "p50_s": 1.3951993823051452, "p95_s": 4.746018932759762, "cost_usd_per_1000": 0.0038643683220649515, "axes": { "intelligence": 62.8805889222556, "calibration": 62.37441109291108, "speed": 71.79034258319184, "cost": 82.38764459955163 }, "jevbench_score": 69.40053092168891 }, "heldout": { "n": 303, "escalation": 0.32673267326732675, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7945205479452054, "hard": 0.46788990825688076 }, "accuracy": 0.6765676567656765, "ece_hard": 0.26164306366416457, "calibration": 58.417443633583545, "p50_s": 2.952823558449745, "p95_s": 5.016249541938305, "cost_usd_per_1000": 0.003972236251715059, "axes": { "intelligence": 67.78327259017217, "calibration": 58.417443633583545, "speed": 68.29383369792049, "cost": 82.02894788260377 }, "jevbench_score": 68.62832965662284 } }, { "threshold": 0.53, "public": { "n": 231, "escalation": 0.30735930735930733, "tiers": { "easy": 0.9583333333333334, "standard": 0.6111111111111112, "hard": 0.5135135135135135 }, "accuracy": 0.6363636363636364, "ece_hard": 0.22094386204386204, "calibration": 62.42711379561379, "p50_s": 1.3951993823051452, "p95_s": 4.7561725653707985, "cost_usd_per_1000": 0.003907658365354994, "axes": { "intelligence": 63.79608775442109, "calibration": 62.42711379561379, "speed": 71.78106121206088, "cost": 82.24250237004165 }, "jevbench_score": 69.63352385250315 }, "heldout": { "n": 303, "escalation": 0.3465346534653465, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.8082191780821918, "hard": 0.47706422018348627 }, "accuracy": 0.6864686468646864, "ece_hard": 0.260465999443981, "calibration": 58.535150055601896, "p50_s": 2.952823558449745, "p95_s": 5.087025054544209, "cost_usd_per_1000": 0.004038242852375125, "axes": { "intelligence": 68.44206359180596, "calibration": 58.535150055601896, "speed": 68.23298626258405, "cost": 81.81422700846528 }, "jevbench_score": 68.76869143593478 } }, { "threshold": 0.54, "public": { "n": 231, "escalation": 0.3246753246753247, "tiers": { "easy": 0.9583333333333334, "standard": 0.6111111111111112, "hard": 0.5225225225225225 }, "accuracy": 0.6406926406926406, "ece_hard": 0.21950872690872694, "calibration": 62.5706273091273, "p50_s": 1.3951993823051452, "p95_s": 4.7561725653707985, "cost_usd_per_1000": 0.003965378423075053, "axes": { "intelligence": 64.17146312979646, "calibration": 62.5706273091273, "speed": 71.78106121206088, "cost": 82.05146082634765 }, "jevbench_score": 69.7352176448955 }, "heldout": { "n": 303, "escalation": 0.36303630363036304, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.815068493150685, "hard": 0.4954128440366973 }, "accuracy": 0.6963696369636964, "ece_hard": 0.24626232971921042, "calibration": 59.95551702807896, "p50_s": 2.952823558449745, "p95_s": 5.087025054544209, "cost_usd_per_1000": 0.00409324835292518, "axes": { "intelligence": 69.18430312932009, "calibration": 59.95551702807896, "speed": 68.23298626258405, "cost": 81.63795712778472 }, "jevbench_score": 69.33153373929838 } }, { "threshold": 0.55, "public": { "n": 231, "escalation": 0.3463203463203463, "tiers": { "easy": 0.9791666666666666, "standard": 0.6111111111111112, "hard": 0.5315315315315315 }, "accuracy": 0.6493506493506493, "ece_hard": 0.21895917735917736, "calibration": 63.117582264082266, "p50_s": 1.3951993823051452, "p95_s": 4.7561725653707985, "cost_usd_per_1000": 0.004037528495225124, "axes": { "intelligence": 64.95193109776444, "calibration": 63.117582264082266, "speed": 71.78106121206088, "cost": 81.81653198731479 }, "jevbench_score": 70.04841953530865 }, "heldout": { "n": 303, "escalation": 0.37293729372937295, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.815068493150685, "hard": 0.5045871559633027 }, "accuracy": 0.6996699669966997, "ece_hard": 0.2402531554072838, "calibration": 60.556434459271614, "p50_s": 2.952823558449745, "p95_s": 5.087025054544209, "cost_usd_per_1000": 0.004126251653255213, "axes": { "intelligence": 69.45953248711825, "calibration": 60.556434459271614, "speed": 68.23298626258405, "cost": 81.5333286596165 }, "jevbench_score": 69.55132788016257 } }, { "threshold": 0.56, "public": { "n": 231, "escalation": 0.354978354978355, "tiers": { "easy": 0.9791666666666666, "standard": 0.625, "hard": 0.5315315315315315 }, "accuracy": 0.6536796536796536, "ece_hard": 0.21895917735917736, "calibration": 63.117582264082266, "p50_s": 1.4203849747776984, "p95_s": 4.7561725653707985, "cost_usd_per_1000": 0.0040663885240851535, "axes": { "intelligence": 65.49205455455457, "calibration": 63.117582264082266, "speed": 71.70336326952386, "cost": 81.7237338682944 }, "jevbench_score": 70.15468425629167 }, "heldout": { "n": 303, "escalation": 0.39933993399339934, "tiers": { "easy": 0.9166666666666666, "standard": 0.7083333333333334, "judge": 0.821917808219178, "hard": 0.5137614678899083 }, "accuracy": 0.7095709570957096, "ece_hard": 0.23302746733388932, "calibration": 61.27900326661106, "p50_s": 2.9652757436037063, "p95_s": 5.087025054544209, "cost_usd_per_1000": 0.0042142604541353015, "axes": { "intelligence": 71.0932093335009, "calibration": 61.27900326661106, "speed": 68.214710383777, "cost": 81.25835881529065 }, "jevbench_score": 70.10055795433425 } }, { "threshold": 0.57, "public": { "n": 231, "escalation": 0.36363636363636365, "tiers": { "easy": 0.9791666666666666, "standard": 0.6388888888888888, "hard": 0.5315315315315315 }, "accuracy": 0.658008658008658, "ece_hard": 0.22229701519701522, "calibration": 62.46379848029848, "p50_s": 1.4203849747776984, "p95_s": 4.7561725653707985, "cost_usd_per_1000": 0.004095248552945182, "axes": { "intelligence": 66.03217801134468, "calibration": 62.46379848029848, "speed": 71.70336326952386, "cost": 81.63159203398067 }, "jevbench_score": 70.09635762229706 }, "heldout": { "n": 303, "escalation": 0.40924092409240925, "tiers": { "easy": 0.9166666666666666, "standard": 0.7083333333333334, "judge": 0.8287671232876712, "hard": 0.5229357798165137 }, "accuracy": 0.7161716171617162, "ece_hard": 0.22761278843480676, "calibration": 61.820471156519325, "p50_s": 2.984680587053299, "p95_s": 5.087025054544209, "cost_usd_per_1000": 0.004247263754465334, "axes": { "intelligence": 71.56021951321688, "calibration": 61.820471156519325, "speed": 68.18638262524165, "cost": 81.15672305641958 }, "jevbench_score": 70.34067510123123 } }, { "threshold": 0.58, "public": { "n": 231, "escalation": 0.36796536796536794, "tiers": { "easy": 0.9791666666666666, "standard": 0.6388888888888888, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.21694926744926754, "calibration": 62.99857325507324, "p50_s": 1.4203849747776984, "p95_s": 4.782240876555443, "cost_usd_per_1000": 0.004109678567375197, "axes": { "intelligence": 66.40755338672005, "calibration": 62.99857325507324, "speed": 71.67962483135251, "cost": 81.585764335501 }, "jevbench_score": 70.32983136188402 }, "heldout": { "n": 303, "escalation": 0.42244224422442245, "tiers": { "easy": 0.9166666666666666, "standard": 0.7083333333333334, "judge": 0.8356164383561644, "hard": 0.5321100917431193 }, "accuracy": 0.7227722772277227, "ece_hard": 0.22191554072838474, "calibration": 62.39019592716153, "p50_s": 2.984680587053299, "p95_s": 5.087025054544209, "cost_usd_per_1000": 0.0042912681549053784, "axes": { "intelligence": 72.02722969293285, "calibration": 62.39019592716153, "speed": 68.18638262524165, "cost": 81.02243038627357 }, "jevbench_score": 70.58769403703695 } }, { "threshold": 0.59, "public": { "n": 231, "escalation": 0.37662337662337664, "tiers": { "easy": 0.9791666666666666, "standard": 0.6388888888888888, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.21988169988170003, "calibration": 62.70533001183, "p50_s": 1.4203849747776984, "p95_s": 4.782240876555443, "cost_usd_per_1000": 0.004138538596235226, "axes": { "intelligence": 66.40755338672005, "calibration": 62.70533001183, "speed": 71.67962483135251, "cost": 81.49458970577119 }, "jevbench_score": 70.2282117233141 }, "heldout": { "n": 303, "escalation": 0.43564356435643564, "tiers": { "easy": 0.9166666666666666, "standard": 0.7083333333333334, "judge": 0.8356164383561644, "hard": 0.5412844036697247 }, "accuracy": 0.7260726072607261, "ece_hard": 0.21693572421462323, "calibration": 62.88817757853767, "p50_s": 2.9955545231699943, "p95_s": 5.087740131467579, "cost_usd_per_1000": 0.004335272555345422, "axes": { "intelligence": 72.30245905073102, "calibration": 62.88817757853767, "speed": 68.16997850676584, "cost": 80.88950780679882 }, "jevbench_score": 70.76228639471232 } }, { "threshold": 0.6, "public": { "n": 231, "escalation": 0.3939393939393939, "tiers": { "easy": 0.9791666666666666, "standard": 0.6666666666666666, "hard": 0.5495495495495496 }, "accuracy": 0.6753246753246753, "ece_hard": 0.21212584402584417, "calibration": 63.48091559741559, "p50_s": 1.4203849747776984, "p95_s": 4.782240876555443, "cost_usd_per_1000": 0.004196258653955284, "axes": { "intelligence": 67.86317567567568, "calibration": 63.48091559741559, "speed": 71.67962483135251, "cost": 81.31413250289059 }, "jevbench_score": 70.78802235354338 }, "heldout": { "n": 303, "escalation": 0.44224422442244227, "tiers": { "easy": 0.9166666666666666, "standard": 0.7083333333333334, "judge": 0.8356164383561644, "hard": 0.5504587155963303 }, "accuracy": 0.7293729372937293, "ece_hard": 0.21359719210453149, "calibration": 63.22203078954685, "p50_s": 2.9955545231699943, "p95_s": 5.087740131467579, "cost_usd_per_1000": 0.004357274755565445, "axes": { "intelligence": 72.57768840852918, "calibration": 63.22203078954685, "speed": 68.16997850676584, "cost": 80.82355162012513 }, "jevbench_score": 70.90888646606757 } }, { "threshold": 0.61, "public": { "n": 231, "escalation": 0.4199134199134199, "tiers": { "easy": 0.9791666666666666, "standard": 0.6666666666666666, "hard": 0.5765765765765766 }, "accuracy": 0.6883116883116883, "ece_hard": 0.1946708890708891, "calibration": 66.98541109291108, "p50_s": 1.442924478650093, "p95_s": 4.782240876555443, "cost_usd_per_1000": 0.00428283874053537, "axes": { "intelligence": 68.9893018018018, "calibration": 66.98541109291108, "speed": 71.61124951591216, "cost": 81.04804832668876 }, "jevbench_score": 71.96501950178167 }, "heldout": { "n": 303, "escalation": 0.46534653465346537, "tiers": { "easy": 0.9166666666666666, "standard": 0.7083333333333334, "judge": 0.8424657534246576, "hard": 0.5504587155963303 }, "accuracy": 0.7326732673267327, "ece_hard": 0.21526049485682508, "calibration": 63.05570051431749, "p50_s": 2.9955545231699943, "p95_s": 5.087740131467579, "cost_usd_per_1000": 0.004434282456335521, "axes": { "intelligence": 72.76946923044699, "calibration": 63.05570051431749, "speed": 68.16997850676584, "cost": 80.59529939536783 }, "jevbench_score": 70.85885135896798 } }, { "threshold": 0.62, "public": { "n": 231, "escalation": 0.44155844155844154, "tiers": { "easy": 0.9791666666666666, "standard": 0.6666666666666666, "hard": 0.5945945945945946 }, "accuracy": 0.696969696969697, "ece_hard": 0.18803845663845664, "calibration": 67.64865433615434, "p50_s": 1.442924478650093, "p95_s": 4.782240876555443, "cost_usd_per_1000": 0.004354988812685442, "axes": { "intelligence": 69.74005255255254, "calibration": 67.64865433615434, "speed": 71.61124951591216, "cost": 80.83038868877257 }, "jevbench_score": 72.2893530043655 }, "heldout": { "n": 303, "escalation": 0.4752475247524752, "tiers": { "easy": 0.9166666666666666, "standard": 0.75, "judge": 0.8424657534246576, "hard": 0.5504587155963303 }, "accuracy": 0.735973597359736, "ece_hard": 0.21487425632471502, "calibration": 63.0943243675285, "p50_s": 2.9955545231699943, "p95_s": 5.087740131467579, "cost_usd_per_1000": 0.0044672857566655545, "axes": { "intelligence": 73.93613589711366, "calibration": 63.0943243675285, "speed": 68.16997850676584, "cost": 80.49868799015827 }, "jevbench_score": 71.13072752438715 } }, { "threshold": 0.63, "public": { "n": 231, "escalation": 0.44155844155844154, "tiers": { "easy": 0.9791666666666666, "standard": 0.6666666666666666, "hard": 0.5945945945945946 }, "accuracy": 0.696969696969697, "ece_hard": 0.18803845663845664, "calibration": 67.64865433615434, "p50_s": 1.442924478650093, "p95_s": 4.782240876555443, "cost_usd_per_1000": 0.004354988812685442, "axes": { "intelligence": 69.74005255255254, "calibration": 67.64865433615434, "speed": 71.61124951591216, "cost": 80.83038868877257 }, "jevbench_score": 72.2893530043655 }, "heldout": { "n": 303, "escalation": 0.49504950495049505, "tiers": { "easy": 0.9166666666666666, "standard": 0.7916666666666666, "judge": 0.8424657534246576, "hard": 0.5504587155963303 }, "accuracy": 0.7392739273927392, "ece_hard": 0.21758434806783428, "calibration": 62.82331519321657, "p50_s": 2.9999807715415954, "p95_s": 5.087740131467579, "cost_usd_per_1000": 0.00453329235732562, "axes": { "intelligence": 75.10280256378033, "calibration": 62.82331519321657, "speed": 68.16356608319376, "cost": 80.3075881563732 }, "jevbench_score": 71.28882720810728 } }, { "threshold": 0.64, "public": { "n": 231, "escalation": 0.4588744588744589, "tiers": { "easy": 0.9791666666666666, "standard": 0.6666666666666666, "hard": 0.5855855855855856 }, "accuracy": 0.6926406926406926, "ece_hard": 0.1957051233051233, "calibration": 66.88198766948767, "p50_s": 1.442924478650093, "p95_s": 4.812191115319729, "cost_usd_per_1000": 0.0044127088704055, "axes": { "intelligence": 69.36467717717717, "calibration": 66.88198766948767, "speed": 71.5841353194946, "cost": 80.65884172891013 }, "jevbench_score": 71.94143168932182 }, "heldout": { "n": 303, "escalation": 0.5148514851485149, "tiers": { "easy": 0.9166666666666666, "standard": 0.7916666666666666, "judge": 0.8493150684931506, "hard": 0.5596330275229358 }, "accuracy": 0.7458745874587459, "ece_hard": 0.21496141228801777, "calibration": 63.085608771198224, "p50_s": 3.000173182785511, "p95_s": 5.099191654473543, "cost_usd_per_1000": 0.004599298957985686, "axes": { "intelligence": 75.5698127434963, "calibration": 63.085608771198224, "speed": 68.15352339880152, "cost": 80.11925080094782 }, "jevbench_score": 71.4292284714158 } }, { "threshold": 0.65, "public": { "n": 231, "escalation": 0.48917748917748916, "tiers": { "easy": 0.9791666666666666, "standard": 0.6666666666666666, "hard": 0.5945945945945946 }, "accuracy": 0.696969696969697, "ece_hard": 0.19285467285467275, "calibration": 67.16703271453272, "p50_s": 1.4510595366358756, "p95_s": 4.812191115319729, "cost_usd_per_1000": 0.004513718971415601, "axes": { "intelligence": 69.74005255255254, "calibration": 67.16703271453272, "speed": 71.55971901256028, "cost": 80.36396452054899 }, "jevbench_score": 72.0430520640829 }, "heldout": { "n": 303, "escalation": 0.528052805280528, "tiers": { "easy": 0.9583333333333334, "standard": 0.7916666666666666, "judge": 0.863013698630137, "hard": 0.5688073394495413 }, "accuracy": 0.759075907590759, "ece_hard": 0.2047513205448985, "calibration": 64.10661794551015, "p50_s": 3.000173182785511, "p95_s": 5.099191654473543, "cost_usd_per_1000": 0.00464330335842573, "axes": { "intelligence": 76.81193707846342, "calibration": 64.10661794551015, "speed": 68.15352339880152, "cost": 79.99518825641647 }, "jevbench_score": 71.98150789561214 } }, { "threshold": 0.66, "public": { "n": 231, "escalation": 0.5151515151515151, "tiers": { "easy": 0.9791666666666666, "standard": 0.6944444444444444, "hard": 0.5765765765765766 }, "accuracy": 0.696969696969697, "ece_hard": 0.214970889070889, "calibration": 63.8404110929111, "p50_s": 1.4510595366358756, "p95_s": 4.812191115319729, "cost_usd_per_1000": 0.004600299057995687, "axes": { "intelligence": 70.06954871538206, "calibration": 63.8404110929111, "speed": 71.55971901256028, "cost": 80.11641803858157 }, "jevbench_score": 71.16292433776687 }, "heldout": { "n": 303, "escalation": 0.5478547854785478, "tiers": { "easy": 0.9583333333333334, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.5779816513761468 }, "accuracy": 0.768976897689769, "ece_hard": 0.20021187100361418, "calibration": 64.56056289963858, "p50_s": 3.030290076136589, "p95_s": 5.099191654473543, "cost_usd_per_1000": 0.004709309959085796, "axes": { "intelligence": 77.47072808009719, "calibration": 64.56056289963858, "speed": 68.11014461088686, "cost": 79.81128172190334 }, "jevbench_score": 72.20965306387897 } }, { "threshold": 0.67, "public": { "n": 231, "escalation": 0.5194805194805194, "tiers": { "easy": 0.9791666666666666, "standard": 0.6944444444444444, "hard": 0.5855855855855856 }, "accuracy": 0.7012987012987013, "ece_hard": 0.2086627809627809, "calibration": 64.47122190372191, "p50_s": 1.4510595366358756, "p95_s": 4.812191115319729, "cost_usd_per_1000": 0.004614729072425701, "axes": { "intelligence": 70.44492409075743, "calibration": 64.47122190372191, "speed": 71.55971901256028, "cost": 80.07561373061152 }, "jevbench_score": 71.42432213744257 }, "heldout": { "n": 303, "escalation": 0.570957095709571, "tiers": { "easy": 0.9583333333333334, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.5963302752293578 }, "accuracy": 0.7755775577557755, "ece_hard": 0.18736875173755912, "calibration": 65.84487482624408, "p50_s": 3.030290076136589, "p95_s": 5.099191654473543, "cost_usd_per_1000": 0.004786317659855873, "axes": { "intelligence": 78.02118679569352, "calibration": 65.84487482624408, "speed": 68.11014461088686, "cost": 79.59995444114416 }, "jevbench_score": 72.64651539726569 } }, { "threshold": 0.68, "public": { "n": 231, "escalation": 0.5238095238095238, "tiers": { "easy": 0.9791666666666666, "standard": 0.6944444444444444, "hard": 0.5855855855855856 }, "accuracy": 0.7012987012987013, "ece_hard": 0.2086627809627809, "calibration": 64.47122190372191, "p50_s": 1.4510595366358756, "p95_s": 4.812191115319729, "cost_usd_per_1000": 0.004629159086855716, "axes": { "intelligence": 70.44492409075743, "calibration": 64.47122190372191, "speed": 71.55971901256028, "cost": 80.03493681659101 }, "jevbench_score": 71.41524985412649 }, "heldout": { "n": 303, "escalation": 0.5907590759075908, "tiers": { "easy": 0.9583333333333334, "standard": 0.7916666666666666, "judge": 0.8835616438356164, "hard": 0.5963302752293578 }, "accuracy": 0.7788778877887789, "ece_hard": 0.18661370586599949, "calibration": 68.05262941340006, "p50_s": 3.040104070305824, "p95_s": 5.1252806268632405, "cost_usd_per_1000": 0.004852324260515939, "axes": { "intelligence": 78.21296761761131, "calibration": 68.05262941340006, "speed": 68.07393899653368, "cost": 79.42150554261009 }, "jevbench_score": 73.24207974438326 } }, { "threshold": 0.69, "public": { "n": 231, "escalation": 0.5238095238095238, "tiers": { "easy": 0.9791666666666666, "standard": 0.6944444444444444, "hard": 0.5855855855855856 }, "accuracy": 0.7012987012987013, "ece_hard": 0.2086627809627809, "calibration": 64.47122190372191, "p50_s": 1.4510595366358756, "p95_s": 4.812191115319729, "cost_usd_per_1000": 0.004629159086855716, "axes": { "intelligence": 70.44492409075743, "calibration": 64.47122190372191, "speed": 71.55971901256028, "cost": 80.03493681659101 }, "jevbench_score": 71.41524985412649 }, "heldout": { "n": 303, "escalation": 0.594059405940594, "tiers": { "easy": 0.9583333333333334, "standard": 0.7916666666666666, "judge": 0.8904109589041096, "hard": 0.5963302752293578 }, "accuracy": 0.7821782178217822, "ece_hard": 0.18661370586599949, "calibration": 68.05262941340006, "p50_s": 3.040104070305824, "p95_s": 5.1252806268632405, "cost_usd_per_1000": 0.00486332536062595, "axes": { "intelligence": 78.40474843952914, "calibration": 68.05262941340006, "speed": 68.07393899653368, "cost": 79.39200024362864 }, "jevbench_score": 73.2801290074974 } }, { "threshold": 0.7, "public": { "n": 231, "escalation": 0.5367965367965368, "tiers": { "easy": 0.9791666666666666, "standard": 0.7083333333333334, "hard": 0.5945945945945946 }, "accuracy": 0.70995670995671, "ece_hard": 0.20207719537719532, "calibration": 65.12978046228046, "p50_s": 1.4510595366358756, "p95_s": 4.812191115319729, "cost_usd_per_1000": 0.004672449130145759, "axes": { "intelligence": 71.36042292292292, "calibration": 65.12978046228046, "speed": 71.55971901256028, "cost": 79.91366254513365 }, "jevbench_score": 71.80119508836819 }, "heldout": { "n": 303, "escalation": 0.6105610561056105, "tiers": { "easy": 0.9583333333333334, "standard": 0.7916666666666666, "judge": 0.9041095890410958, "hard": 0.5963302752293578 }, "accuracy": 0.7887788778877888, "ece_hard": 0.1872595774256325, "calibration": 67.98804225743675, "p50_s": 3.040104070305824, "p95_s": 5.1252806268632405, "cost_usd_per_1000": 0.004918330861176006, "axes": { "intelligence": 78.78831008336475, "calibration": 67.98804225743675, "speed": 68.07393899653368, "cost": 79.2454677753421 }, "jevbench_score": 73.31830376395807 } }, { "threshold": 0.71, "public": { "n": 231, "escalation": 0.5454545454545454, "tiers": { "easy": 0.9791666666666666, "standard": 0.7083333333333334, "hard": 0.5945945945945946 }, "accuracy": 0.70995670995671, "ece_hard": 0.20207719537719532, "calibration": 65.12978046228046, "p50_s": 1.4510595366358756, "p95_s": 4.812191115319729, "cost_usd_per_1000": 0.0047013091590057884, "axes": { "intelligence": 71.36042292292292, "calibration": 65.12978046228046, "speed": 71.55971901256028, "cost": 79.8334356574892 }, "jevbench_score": 71.78316764440521 }, "heldout": { "n": 303, "escalation": 0.6303630363036303, "tiers": { "easy": 0.9583333333333334, "standard": 0.7916666666666666, "judge": 0.9178082191780822, "hard": 0.6055045871559633 }, "accuracy": 0.7986798679867987, "ece_hard": 0.17992104531554076, "calibration": 68.72189546844592, "p50_s": 3.040104070305824, "p95_s": 5.1252806268632405, "cost_usd_per_1000": 0.004984337461836071, "axes": { "intelligence": 79.44710108499855, "calibration": 68.72189546844592, "speed": 68.07393899653368, "cost": 79.0717768505077 }, "jevbench_score": 73.62815043067694 } }, { "threshold": 0.72, "public": { "n": 231, "escalation": 0.5497835497835498, "tiers": { "easy": 0.9791666666666666, "standard": 0.7083333333333334, "hard": 0.5945945945945946 }, "accuracy": 0.70995670995671, "ece_hard": 0.2045222404222404, "calibration": 64.88527595777596, "p50_s": 1.4510595366358756, "p95_s": 4.812191115319729, "cost_usd_per_1000": 0.004715739173435803, "axes": { "intelligence": 71.36042292292292, "calibration": 64.88527595777596, "speed": 71.55971901256028, "cost": 79.79350670953575 }, "jevbench_score": 71.70673318408956 }, "heldout": { "n": 303, "escalation": 0.6501650165016502, "tiers": { "easy": 0.9583333333333334, "standard": 0.7916666666666666, "judge": 0.9178082191780822, "hard": 0.6146788990825688 }, "accuracy": 0.801980198019802, "ece_hard": 0.17559719210453162, "calibration": 69.15428078954685, "p50_s": 3.040104070305824, "p95_s": 5.1252806268632405, "cost_usd_per_1000": 0.005050344062496137, "axes": { "intelligence": 79.7223304427967, "calibration": 69.15428078954685, "speed": 68.07393899653368, "cost": 78.90037101671639 }, "jevbench_score": 73.76744563018828 } }, { "threshold": 0.73, "public": { "n": 231, "escalation": 0.5584415584415584, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "hard": 0.5945945945945946 }, "accuracy": 0.7142857142857143, "ece_hard": 0.2045222404222404, "calibration": 64.88527595777596, "p50_s": 1.4510595366358756, "p95_s": 4.812191115319729, "cost_usd_per_1000": 0.0047445992022958315, "axes": { "intelligence": 71.76551551551552, "calibration": 64.88527595777596, "speed": 71.55971901256028, "cost": 79.71401405505472 }, "jevbench_score": 71.79039082005781 }, "heldout": { "n": 303, "escalation": 0.6666666666666666, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6146788990825688 }, "accuracy": 0.8118811881188119, "ece_hard": 0.17559719210453162, "calibration": 69.15428078954685, "p50_s": 3.040104070305824, "p95_s": 5.1252806268632405, "cost_usd_per_1000": 0.005105349563046192, "axes": { "intelligence": 80.68922541996565, "calibration": 69.15428078954685, "speed": 68.07393899653368, "cost": 78.75923549308337 }, "jevbench_score": 73.95699336333021 } }, { "threshold": 0.74, "public": { "n": 231, "escalation": 0.5627705627705628, "tiers": { "easy": 1.0, "standard": 0.7222222222222222, "hard": 0.5945945945945946 }, "accuracy": 0.7186147186147186, "ece_hard": 0.2045222404222404, "calibration": 64.88527595777596, "p50_s": 1.4510595366358756, "p95_s": 4.812191115319729, "cost_usd_per_1000": 0.004759029216725846, "axes": { "intelligence": 72.30563897230566, "calibration": 64.88527595777596, "speed": 71.55971901256028, "cost": 79.67444886891478 }, "jevbench_score": 71.9161624482477 }, "heldout": { "n": 303, "escalation": 0.6897689768976898, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9452054794520548, "hard": 0.6146788990825688 }, "accuracy": 0.8184818481848185, "ece_hard": 0.17243572421462333, "calibration": 69.47042757853767, "p50_s": 3.040104070305824, "p95_s": 5.1252806268632405, "cost_usd_per_1000": 0.0051823572638162695, "axes": { "intelligence": 81.07278706380127, "calibration": 69.47042757853767, "speed": 68.07393899653368, "cost": 78.56417952154192 }, "jevbench_score": 74.08326833373026 } }, { "threshold": 0.75, "public": { "n": 231, "escalation": 0.5844155844155844, "tiers": { "easy": 1.0, "standard": 0.7361111111111112, "hard": 0.6036036036036037 }, "accuracy": 0.7272727272727273, "ece_hard": 0.19387359177359179, "calibration": 65.95014082264082, "p50_s": 1.4701635405421256, "p95_s": 4.820986015349627, "cost_usd_per_1000": 0.004831179288875918, "axes": { "intelligence": 73.22113780447116, "calibration": 65.95014082264082, "speed": 71.49498480090054, "cost": 79.4784053561041 }, "jevbench_score": 72.37594226713652 }, "heldout": { "n": 303, "escalation": 0.7029702970297029, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9452054794520548, "hard": 0.6146788990825688 }, "accuracy": 0.8184818481848185, "ece_hard": 0.17479719210453162, "calibration": 69.23428078954683, "p50_s": 3.040104070305824, "p95_s": 5.1252806268632405, "cost_usd_per_1000": 0.005226361664256314, "axes": { "intelligence": 81.07278706380127, "calibration": 69.23428078954683, "speed": 68.07393899653368, "cost": 78.45401621164706 }, "jevbench_score": 73.99426959185095 } }, { "threshold": 0.76, "public": { "n": 231, "escalation": 0.5974025974025974, "tiers": { "easy": 1.0, "standard": 0.7361111111111112, "hard": 0.6036036036036037 }, "accuracy": 0.7272727272727273, "ece_hard": 0.19380422240422238, "calibration": 65.95707775957777, "p50_s": 1.4701635405421256, "p95_s": 4.820986015349627, "cost_usd_per_1000": 0.004874469332165962, "axes": { "intelligence": 73.22113780447116, "calibration": 65.95707775957777, "speed": 71.49498480090054, "cost": 79.36217972915807 }, "jevbench_score": 72.35137035492953 }, "heldout": { "n": 303, "escalation": 0.7227722772277227, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9452054794520548, "hard": 0.6238532110091743 }, "accuracy": 0.8217821782178217, "ece_hard": 0.16724489852654995, "calibration": 69.989510147345, "p50_s": 3.040104070305824, "p95_s": 5.1252806268632405, "cost_usd_per_1000": 0.005292368264916379, "axes": { "intelligence": 81.34801642159944, "calibration": 69.989510147345, "speed": 68.07393899653368, "cost": 78.29049830269682 }, "jevbench_score": 74.2194044189072 } }, { "threshold": 0.77, "public": { "n": 231, "escalation": 0.6190476190476191, "tiers": { "easy": 1.0, "standard": 0.7361111111111112, "hard": 0.6216216216216216 }, "accuracy": 0.7359307359307359, "ece_hard": 0.18003305123305124, "calibration": 67.33419487669488, "p50_s": 1.4701635405421256, "p95_s": 4.820986015349627, "cost_usd_per_1000": 0.004946619404316033, "axes": { "intelligence": 73.9718885552219, "calibration": 67.33419487669488, "speed": 71.49498480090054, "cost": 79.17074509637594 }, "jevbench_score": 72.8678024350421 }, "heldout": { "n": 303, "escalation": 0.735973597359736, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.6238532110091743 }, "accuracy": 0.8250825082508251, "ece_hard": 0.16724489852654995, "calibration": 69.989510147345, "p50_s": 3.0780361533164977, "p95_s": 5.1252806268632405, "cost_usd_per_1000": 0.005336372665356423, "axes": { "intelligence": 81.53979724351724, "calibration": 69.989510147345, "speed": 68.02008633928816, "cost": 78.18261547287742 }, "jevbench_score": 74.22282648608086 } }, { "threshold": 0.78, "public": { "n": 231, "escalation": 0.6406926406926406, "tiers": { "easy": 1.0, "standard": 0.7361111111111112, "hard": 0.6126126126126126 }, "accuracy": 0.7316017316017316, "ece_hard": 0.1918006188006188, "calibration": 66.15743811993812, "p50_s": 1.470290558040142, "p95_s": 4.825077628344298, "cost_usd_per_1000": 0.0050187694764661055, "axes": { "intelligence": 73.59651317984653, "calibration": 66.15743811993812, "speed": 71.49092526834971, "cost": 78.9820825598906 }, "jevbench_score": 72.41088480555037 }, "heldout": { "n": 303, "escalation": 0.7557755775577558, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.958904109589041, "hard": 0.6146788990825688 }, "accuracy": 0.8250825082508251, "ece_hard": 0.18030728384765085, "calibration": 68.68327161523491, "p50_s": 3.095755861699581, "p95_s": 5.1252806268632405, "cost_usd_per_1000": 0.005402379266016489, "axes": { "intelligence": 81.45634870763688, "calibration": 68.68327161523491, "speed": 67.99515646576953, "cost": 78.02244790236738 }, "jevbench_score": 73.81053709116478 } }, { "threshold": 0.79, "public": { "n": 231, "escalation": 0.645021645021645, "tiers": { "easy": 1.0, "standard": 0.75, "hard": 0.6126126126126126 }, "accuracy": 0.7359307359307359, "ece_hard": 0.1918006188006188, "calibration": 66.15743811993812, "p50_s": 1.470290558040142, "p95_s": 4.825077628344298, "cost_usd_per_1000": 0.00503319949089612, "axes": { "intelligence": 74.13663663663664, "calibration": 66.15743811993812, "speed": 71.49092526834971, "cost": 78.94467567990934 }, "jevbench_score": 72.53478538343923 }, "heldout": { "n": 303, "escalation": 0.7755775577557755, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9657534246575342, "hard": 0.6146788990825688 }, "accuracy": 0.8283828382838284, "ece_hard": 0.18225865999443983, "calibration": 68.48813400055602, "p50_s": 3.095755861699581, "p95_s": 5.1252806268632405, "cost_usd_per_1000": 0.005468385866676555, "axes": { "intelligence": 81.6481295295547, "calibration": 68.48813400055602, "speed": 67.99515646576953, "cost": 77.86422543474447 }, "jevbench_score": 73.76398650954296 } }, { "threshold": 0.8, "public": { "n": 231, "escalation": 0.658008658008658, "tiers": { "easy": 1.0, "standard": 0.75, "hard": 0.6126126126126126 }, "accuracy": 0.7359307359307359, "ece_hard": 0.1918006188006188, "calibration": 66.15743811993812, "p50_s": 1.4783253163099288, "p95_s": 4.825077628344298, "cost_usd_per_1000": 0.005076489534186164, "axes": { "intelligence": 74.13663663663664, "calibration": 66.15743811993812, "speed": 71.46725680853416, "cost": 78.83309514492846 }, "jevbench_score": 72.50313945595396 }, "heldout": { "n": 303, "escalation": 0.7821782178217822, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9657534246575342, "hard": 0.6146788990825688 }, "accuracy": 0.8283828382838284, "ece_hard": 0.18369168751737558, "calibration": 68.34483124826244, "p50_s": 3.095755861699581, "p95_s": 5.1252806268632405, "cost_usd_per_1000": 0.005490388066896577, "axes": { "intelligence": 81.6481295295547, "calibration": 68.34483124826244, "speed": 67.99515646576953, "cost": 77.81190874105725 }, "jevbench_score": 73.71298367084545 } }, { "threshold": 0.81, "public": { "n": 231, "escalation": 0.6623376623376623, "tiers": { "easy": 1.0, "standard": 0.75, "hard": 0.6216216216216216 }, "accuracy": 0.7402597402597403, "ece_hard": 0.1849961142961143, "calibration": 66.83788857038857, "p50_s": 1.4783253163099288, "p95_s": 4.825077628344298, "cost_usd_per_1000": 0.005090919548616178, "axes": { "intelligence": 74.51201201201201, "calibration": 66.83788857038857, "speed": 71.46725680853416, "cost": 78.79611298085467 }, "jevbench_score": 72.77215398747518 }, "heldout": { "n": 303, "escalation": 0.7887788778877888, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9657534246575342, "hard": 0.6146788990825688 }, "accuracy": 0.8283828382838284, "ece_hard": 0.18585223797609118, "calibration": 67.82077620239087, "p50_s": 3.1247262448072433, "p95_s": 5.1252806268632405, "cost_usd_per_1000": 0.0055123902671165995, "axes": { "intelligence": 81.6481295295547, "calibration": 67.82077620239087, "speed": 67.95470375399013, "cost": 77.7598012827848 }, "jevbench_score": 73.5480081241969 } }, { "threshold": 0.82, "public": { "n": 231, "escalation": 0.670995670995671, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.6216216216216216 }, "accuracy": 0.7445887445887446, "ece_hard": 0.1849961142961143, "calibration": 66.83788857038857, "p50_s": 1.4783253163099288, "p95_s": 4.825077628344298, "cost_usd_per_1000": 0.005119779577476207, "axes": { "intelligence": 75.05213546880213, "calibration": 66.83788857038857, "speed": 71.46725680853416, "cost": 78.72246209072375 }, "jevbench_score": 72.88663312198042 }, "heldout": { "n": 303, "escalation": 0.801980198019802, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9726027397260274, "hard": 0.6146788990825688 }, "accuracy": 0.8316831683168316, "ece_hard": 0.18585223797609118, "calibration": 67.82077620239087, "p50_s": 3.139329770207405, "p95_s": 5.1252806268632405, "cost_usd_per_1000": 0.005556394667556643, "axes": { "intelligence": 81.8399103514725, "calibration": 67.82077620239087, "speed": 67.93445412029405, "cost": 77.6562074244549 }, "jevbench_score": 73.56115539300038 } }, { "threshold": 0.83, "public": { "n": 231, "escalation": 0.696969696969697, "tiers": { "easy": 1.0, "standard": 0.7777777777777778, "hard": 0.6396396396396397 }, "accuracy": 0.7575757575757576, "ece_hard": 0.17054025844025839, "calibration": 68.28347415597416, "p50_s": 1.4999325588345527, "p95_s": 4.825077628344298, "cost_usd_per_1000": 0.005206359664056293, "axes": { "intelligence": 76.34300967634302, "calibration": 68.28347415597416, "speed": 71.40423962710378, "cost": 78.5039750019807 }, "jevbench_score": 73.5233239735231 }, "heldout": { "n": 303, "escalation": 0.8250825082508251, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9726027397260274, "hard": 0.6238532110091743 }, "accuracy": 0.834983498349835, "ece_hard": 0.17832654990269672, "calibration": 68.57334500973033, "p50_s": 3.139329770207405, "p95_s": 5.1524781987071036, "cost_usd_per_1000": 0.005633402368326721, "axes": { "intelligence": 82.11513970927066, "calibration": 68.57334500973033, "speed": 67.91146898677458, "cost": 77.47687683927623 }, "jevbench_score": 73.77741770384162 } }, { "threshold": 0.84, "public": { "n": 231, "escalation": 0.7056277056277056, "tiers": { "easy": 1.0, "standard": 0.7777777777777778, "hard": 0.6396396396396397 }, "accuracy": 0.7575757575757576, "ece_hard": 0.17115827645827644, "calibration": 68.22167235417236, "p50_s": 1.4999325588345527, "p95_s": 4.825077628344298, "cost_usd_per_1000": 0.005235219692916323, "axes": { "intelligence": 76.34300967634302, "calibration": 68.22167235417236, "speed": 71.40423962710378, "cost": 78.4319526607043 }, "jevbench_score": 73.48981704171389 }, "heldout": { "n": 303, "escalation": 0.834983498349835, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9726027397260274, "hard": 0.6238532110091743 }, "accuracy": 0.834983498349835, "ece_hard": 0.17971003614122882, "calibration": 68.43499638587711, "p50_s": 3.139329770207405, "p95_s": 5.1524781987071036, "cost_usd_per_1000": 0.0056664056686567535, "axes": { "intelligence": 82.11513970927066, "calibration": 68.43499638587711, "speed": 67.91146898677458, "cost": 77.40077010227309 }, "jevbench_score": 73.72206181643844 } }, { "threshold": 0.85, "public": { "n": 231, "escalation": 0.70995670995671, "tiers": { "easy": 1.0, "standard": 0.7777777777777778, "hard": 0.6306306306306306 }, "accuracy": 0.7532467532467533, "ece_hard": 0.17476097916097916, "calibration": 67.86140208390208, "p50_s": 1.4999325588345527, "p95_s": 4.825077628344298, "cost_usd_per_1000": 0.005249649707346336, "axes": { "intelligence": 75.96763430096765, "calibration": 67.86140208390208, "speed": 71.40423962710378, "cost": 78.39609024206135 }, "jevbench_score": 73.29383706637759 }, "heldout": { "n": 303, "escalation": 0.8481848184818482, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9726027397260274, "hard": 0.6422018348623854 }, "accuracy": 0.8415841584158416, "ece_hard": 0.1616091187100361, "calibration": 70.24508812899639, "p50_s": 3.139329770207405, "p95_s": 5.1524781987071036, "cost_usd_per_1000": 0.005710410069096798, "axes": { "intelligence": 82.665598424867, "calibration": 70.24508812899639, "speed": 67.91146898677458, "cost": 77.29998110804645 }, "jevbench_score": 74.30462077894528 } }, { "threshold": 0.86, "public": { "n": 231, "escalation": 0.7229437229437229, "tiers": { "easy": 1.0, "standard": 0.7777777777777778, "hard": 0.6486486486486487 }, "accuracy": 0.7619047619047619, "ece_hard": 0.1564943124943124, "calibration": 69.68806875056876, "p50_s": 1.4999325588345527, "p95_s": 4.825077628344298, "cost_usd_per_1000": 0.005292939750636379, "axes": { "intelligence": 76.71838505171839, "calibration": 69.68806875056876, "speed": 71.40423962710378, "cost": 78.28909148622313 }, "jevbench_score": 73.93852513321383 }, "heldout": { "n": 303, "escalation": 0.8547854785478548, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9726027397260274, "hard": 0.6422018348623854 }, "accuracy": 0.8415841584158416, "ece_hard": 0.16231554072838478, "calibration": 69.78944592716152, "p50_s": 3.139329770207405, "p95_s": 5.1524781987071036, "cost_usd_per_1000": 0.00573241226931682, "axes": { "intelligence": 82.665598424867, "calibration": 69.78944592716152, "speed": 67.91146898677458, "cost": 77.24987749389265 }, "jevbench_score": 74.17180914330396 } }, { "threshold": 0.87, "public": { "n": 231, "escalation": 0.7532467532467533, "tiers": { "easy": 1.0, "standard": 0.7777777777777778, "hard": 0.6666666666666666 }, "accuracy": 0.7705627705627706, "ece_hard": 0.14092404222404215, "calibration": 71.2450957775958, "p50_s": 1.4999325588345527, "p95_s": 4.825077628344298, "cost_usd_per_1000": 0.005393949851646481, "axes": { "intelligence": 77.46913580246914, "calibration": 71.2450957775958, "speed": 71.40423962710378, "cost": 78.04279286694128 }, "jevbench_score": 74.47064612104384 }, "heldout": { "n": 303, "escalation": 0.8646864686468647, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9726027397260274, "hard": 0.6422018348623854 }, "accuracy": 0.8415841584158416, "ece_hard": 0.16231554072838478, "calibration": 69.78944592716152, "p50_s": 3.139329770207405, "p95_s": 5.1524781987071036, "cost_usd_per_1000": 0.0057654155696468526, "axes": { "intelligence": 82.665598424867, "calibration": 69.78944592716152, "speed": 67.91146898677458, "cost": 77.17508150225007 }, "jevbench_score": 74.15384869579998 } }, { "threshold": 0.88, "public": { "n": 231, "escalation": 0.7792207792207793, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "hard": 0.6666666666666666 }, "accuracy": 0.7748917748917749, "ece_hard": 0.13051233051233044, "calibration": 72.28626694876695, "p50_s": 1.4999325588345527, "p95_s": 4.825077628344298, "cost_usd_per_1000": 0.005480529938226567, "axes": { "intelligence": 78.00925925925925, "calibration": 72.28626694876695, "speed": 71.40423962710378, "cost": 77.8353233652495 }, "jevbench_score": 74.82137259007293 }, "heldout": { "n": 303, "escalation": 0.8712871287128713, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9726027397260274, "hard": 0.6422018348623854 }, "accuracy": 0.8415841584158416, "ece_hard": 0.16231554072838478, "calibration": 69.78944592716152, "p50_s": 3.139329770207405, "p95_s": 5.1524781987071036, "cost_usd_per_1000": 0.005787417769866875, "axes": { "intelligence": 82.665598424867, "calibration": 69.78944592716152, "speed": 67.91146898677458, "cost": 77.12545499721591 }, "jevbench_score": 74.14192488568392 } }, { "threshold": 0.89, "public": { "n": 231, "escalation": 0.7878787878787878, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "hard": 0.6666666666666666 }, "accuracy": 0.7748917748917749, "ece_hard": 0.13051233051233044, "calibration": 72.28626694876695, "p50_s": 1.4999325588345527, "p95_s": 4.825077628344298, "cost_usd_per_1000": 0.0055093899670865965, "axes": { "intelligence": 78.00925925925925, "calibration": 72.28626694876695, "speed": 71.40423962710378, "cost": 77.76689458578078 }, "jevbench_score": 74.80492239820533 }, "heldout": { "n": 303, "escalation": 0.8877887788778878, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6422018348623854 }, "accuracy": 0.8448844884488449, "ece_hard": 0.16231554072838478, "calibration": 69.78944592716152, "p50_s": 3.139329770207405, "p95_s": 5.1524781987071036, "cost_usd_per_1000": 0.0058424232704169295, "axes": { "intelligence": 83.83226509153367, "calibration": 69.78944592716152, "speed": 67.91146898677458, "cost": 77.00220947694675 }, "jevbench_score": 74.37240329706266 } }, { "threshold": 0.9, "public": { "n": 231, "escalation": 0.8008658008658008, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "hard": 0.6756756756756757 }, "accuracy": 0.7792207792207793, "ece_hard": 0.1223402584402584, "calibration": 73.10347415597415, "p50_s": 1.525398464500904, "p95_s": 4.825077628344298, "cost_usd_per_1000": 0.00555268001037664, "axes": { "intelligence": 78.38463463463465, "calibration": 73.10347415597415, "speed": 71.33112390227056, "cost": 77.66492059934481 }, "jevbench_score": 75.0616714894156 }, "heldout": { "n": 303, "escalation": 0.9042904290429042, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6513761467889908 }, "accuracy": 0.8481848184818482, "ece_hard": 0.15410086182930222, "calibration": 70.61091381706979, "p50_s": 3.141072319447994, "p95_s": 5.1524781987071036, "cost_usd_per_1000": 0.005897428770966984, "axes": { "intelligence": 84.10749444933184, "calibration": 70.61091381706979, "speed": 67.90905901515731, "cost": 76.88011887433655 }, "jevbench_score": 74.62117300112475 } }, { "threshold": 0.91, "public": { "n": 231, "escalation": 0.8138528138528138, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.6756756756756757 }, "accuracy": 0.7835497835497836, "ece_hard": 0.1223402584402584, "calibration": 73.10347415597415, "p50_s": 1.525398464500904, "p95_s": 4.825077628344298, "cost_usd_per_1000": 0.005595970053666683, "axes": { "intelligence": 78.92475809142476, "calibration": 73.10347415597415, "speed": 71.33112390227056, "cost": 77.56373854786222 }, "jevbench_score": 75.16614357915876 }, "heldout": { "n": 303, "escalation": 0.9108910891089109, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6513761467889908 }, "accuracy": 0.8481848184818482, "ece_hard": 0.15410086182930222, "calibration": 70.61091381706979, "p50_s": 3.141072319447994, "p95_s": 5.1524781987071036, "cost_usd_per_1000": 0.0059194309711870065, "axes": { "intelligence": 84.10749444933184, "calibration": 70.61091381706979, "speed": 67.90905901515731, "cost": 76.8316011865783 }, "jevbench_score": 74.60939718723277 } }, { "threshold": 0.92, "public": { "n": 231, "escalation": 0.8225108225108225, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.6756756756756757 }, "accuracy": 0.7835497835497836, "ece_hard": 0.12088350168350163, "calibration": 74.05764983164983, "p50_s": 1.525398464500904, "p95_s": 4.825077628344298, "cost_usd_per_1000": 0.005624830082526712, "axes": { "intelligence": 78.92475809142476, "calibration": 74.05764983164983, "speed": 71.33112390227056, "cost": 77.49671777162132 }, "jevbench_score": 75.39393144318008 }, "heldout": { "n": 303, "escalation": 0.9207920792079208, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6513761467889908 }, "accuracy": 0.8481848184818482, "ece_hard": 0.1547256324715041, "calibration": 70.5484367528496, "p50_s": 3.141072319447994, "p95_s": 5.1524781987071036, "cost_usd_per_1000": 0.005952434271517039, "axes": { "intelligence": 84.10749444933184, "calibration": 70.5484367528496, "speed": 67.90905901515731, "cost": 76.75916174501742 }, "jevbench_score": 74.57529957386045 } }, { "threshold": 0.93, "public": { "n": 231, "escalation": 0.8398268398268398, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.6756756756756757 }, "accuracy": 0.7835497835497836, "ece_hard": 0.12088350168350163, "calibration": 74.05764983164983, "p50_s": 1.525398464500904, "p95_s": 4.825077628344298, "cost_usd_per_1000": 0.005682550140246769, "axes": { "intelligence": 78.92475809142476, "calibration": 74.05764983164983, "speed": 71.33112390227056, "cost": 77.36370170705275 }, "jevbench_score": 75.36155889322913 }, "heldout": { "n": 303, "escalation": 0.9240924092409241, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6513761467889908 }, "accuracy": 0.8481848184818482, "ece_hard": 0.1547256324715041, "calibration": 70.5484367528496, "p50_s": 3.141072319447994, "p95_s": 5.1524781987071036, "cost_usd_per_1000": 0.00596343537162705, "axes": { "intelligence": 84.10749444933184, "calibration": 70.5484367528496, "speed": 67.90905901515731, "cost": 76.73510449075148 }, "jevbench_score": 74.56945568565658 } }, { "threshold": 0.94, "public": { "n": 231, "escalation": 0.8528138528138528, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.6756756756756757 }, "accuracy": 0.7835497835497836, "ece_hard": 0.12088350168350163, "calibration": 74.05764983164983, "p50_s": 1.5650968194007873, "p95_s": 4.825077628344298, "cost_usd_per_1000": 0.005725840183536813, "axes": { "intelligence": 78.92475809142476, "calibration": 74.05764983164983, "speed": 71.21954486242686, "cost": 77.26482333793086 }, "jevbench_score": 75.3079887058293 }, "heldout": { "n": 303, "escalation": 0.9405940594059405, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6513761467889908 }, "accuracy": 0.8481848184818482, "ece_hard": 0.14760911871003615, "calibration": 73.04858812899639, "p50_s": 3.141072319447994, "p95_s": 5.1524781987071036, "cost_usd_per_1000": 0.006018440872177105, "axes": { "intelligence": 84.10749444933184, "calibration": 73.04858812899639, "speed": 67.90905901515731, "cost": 76.61548005450476 }, "jevbench_score": 75.19218245971385 } }, { "threshold": 0.95, "public": { "n": 231, "escalation": 0.8614718614718615, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.6756756756756757 }, "accuracy": 0.7835497835497836, "ece_hard": 0.12088350168350163, "calibration": 74.05764983164983, "p50_s": 1.5650968194007873, "p95_s": 4.825077628344298, "cost_usd_per_1000": 0.005754700212396841, "axes": { "intelligence": 78.92475809142476, "calibration": 74.05764983164983, "speed": 71.21954486242686, "cost": 77.19931887254425 }, "jevbench_score": 75.29202225947688 }, "heldout": { "n": 303, "escalation": 0.9471947194719472, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6513761467889908 }, "accuracy": 0.8481848184818482, "ece_hard": 0.14760911871003615, "calibration": 73.04858812899639, "p50_s": 3.141072319447994, "p95_s": 5.1524781987071036, "cost_usd_per_1000": 0.006040443072397128, "axes": { "intelligence": 84.10749444933184, "calibration": 73.04858812899639, "speed": 67.90905901515731, "cost": 76.56793612858587 }, "jevbench_score": 75.18051457039336 } }, { "threshold": 0.96, "public": { "n": 231, "escalation": 0.8874458874458875, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6846846846846847 }, "accuracy": 0.7922077922077922, "ece_hard": 0.11600782600782594, "calibration": 74.5452173992174, "p50_s": 1.5650968194007873, "p95_s": 4.825077628344298, "cost_usd_per_1000": 0.005841280298976928, "axes": { "intelligence": 79.84025692359026, "calibration": 74.5452173992174, "speed": 71.21954486242686, "cost": 77.00475859771826 }, "jevbench_score": 75.58569614404567 }, "heldout": { "n": 303, "escalation": 0.9537953795379538, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6513761467889908 }, "accuracy": 0.8481848184818482, "ece_hard": 0.14760911871003615, "calibration": 73.04858812899639, "p50_s": 3.141072319447994, "p95_s": 5.1524781987071036, "cost_usd_per_1000": 0.006062445272617149, "axes": { "intelligence": 84.10749444933184, "calibration": 73.04858812899639, "speed": 67.90905901515731, "cost": 76.52056506607882 }, "jevbench_score": 75.16888369854337 } }, { "threshold": 0.97, "public": { "n": 231, "escalation": 0.8961038961038961, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6846846846846847 }, "accuracy": 0.7922077922077922, "ece_hard": 0.11600782600782594, "calibration": 74.5452173992174, "p50_s": 1.5650968194007873, "p95_s": 4.825077628344298, "cost_usd_per_1000": 0.005870140327836957, "axes": { "intelligence": 79.84025692359026, "calibration": 74.5452173992174, "speed": 71.21954486242686, "cost": 76.94054549982812 }, "jevbench_score": 75.56993377336936 }, "heldout": { "n": 303, "escalation": 0.9603960396039604, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6513761467889908 }, "accuracy": 0.8481848184818482, "ece_hard": 0.14760911871003615, "calibration": 73.04858812899639, "p50_s": 3.141072319447994, "p95_s": 5.1524781987071036, "cost_usd_per_1000": 0.006084447472837171, "axes": { "intelligence": 84.10749444933184, "calibration": 73.04858812899639, "speed": 67.90905901515731, "cost": 76.47336561451914 }, "jevbench_score": 75.15728958968928 } }, { "threshold": 0.98, "public": { "n": 231, "escalation": 0.9090909090909091, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6846846846846847 }, "accuracy": 0.7922077922077922, "ece_hard": 0.11600782600782594, "calibration": 74.5452173992174, "p50_s": 1.5650968194007873, "p95_s": 4.825077628344298, "cost_usd_per_1000": 0.005913430371127, "axes": { "intelligence": 79.84025692359026, "calibration": 74.5452173992174, "speed": 71.21954486242686, "cost": 76.8448153753134 }, "jevbench_score": 75.5464165986741 }, "heldout": { "n": 303, "escalation": 0.9702970297029703, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6513761467889908 }, "accuracy": 0.8481848184818482, "ece_hard": 0.14760911871003615, "calibration": 73.04858812899639, "p50_s": 3.141072319447994, "p95_s": 5.1524781987071036, "cost_usd_per_1000": 0.006117450773167204, "axes": { "intelligence": 84.10749444933184, "calibration": 73.04858812899639, "speed": 67.90905901515731, "cost": 76.402885501269 }, "jevbench_score": 75.13996680691841 } }, { "threshold": 0.99, "public": { "n": 231, "escalation": 0.9393939393939394, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6846846846846847 }, "accuracy": 0.7922077922077922, "ece_hard": 0.11600782600782594, "calibration": 74.5452173992174, "p50_s": 1.6205359265208243, "p95_s": 4.825077628344298, "cost_usd_per_1000": 0.006014440472137101, "axes": { "intelligence": 79.84025692359026, "calibration": 74.5452173992174, "speed": 71.06837031529986, "cost": 76.62414307549071 }, "jevbench_score": 75.45202929400934 }, "heldout": { "n": 303, "escalation": 0.9834983498349835, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6513761467889908 }, "accuracy": 0.8481848184818482, "ece_hard": 0.14760911871003615, "calibration": 73.04858812899639, "p50_s": 3.141072319447994, "p95_s": 5.1524781987071036, "cost_usd_per_1000": 0.006161455173607248, "axes": { "intelligence": 84.10749444933184, "calibration": 73.04858812899639, "speed": 67.90905901515731, "cost": 76.30950120373203 }, "jevbench_score": 75.11699610546773 } }, { "threshold": 1.0, "public": { "n": 231, "escalation": 1.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6846846846846847 }, "accuracy": 0.7922077922077922, "ece_hard": 0.11600782600782594, "calibration": 74.5452173992174, "p50_s": 1.6205359265208243, "p95_s": 4.825077628344298, "cost_usd_per_1000": 0.006216460674157303, "axes": { "intelligence": 79.84025692359026, "calibration": 74.5452173992174, "speed": 71.06837031529986, "cost": 76.19370428166211 }, "jevbench_score": 75.34584173841574 }, "heldout": { "n": 303, "escalation": 1.0, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6513761467889908 }, "accuracy": 0.8481848184818482, "ece_hard": 0.14760911871003615, "calibration": 73.04858812899639, "p50_s": 3.141072319447994, "p95_s": 5.1524781987071036, "cost_usd_per_1000": 0.006216460674157303, "axes": { "intelligence": 84.10749444933184, "calibration": 73.04858812899639, "speed": 67.90905901515731, "cost": 76.19370428166211 }, "jevbench_score": 75.0884830403791 } } ] }, { "cheap": "laya", "backend": "system-one-open", "chosen": { "threshold": 0.91, "public": { "n": 231, "escalation": 0.8138528138528138, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.4594594594594595 }, "accuracy": 0.6666666666666666, "ece_hard": 0.3137055500866056, "calibration": 50.439975401472736, "p50_s": 2.5121211051940917, "p95_s": 6.363939964771271, "cost_usd_per_1000": 0.01499401924542374, "axes": { "intelligence": 68.29537871204539, "calibration": 50.439975401472736, "speed": 67.96233353804574, "cost": 64.72245808165604 }, "jevbench_score": 62.391060436713175 }, "heldout": { "n": 303, "escalation": 0.9108910891089109, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.2841529622766471, "calibration": 53.51857296709248, "p50_s": 4.074841046333313, "p95_s": 6.448686248064041, "cost_usd_per_1000": 0.016438039641042755, "axes": { "intelligence": 74.75130911985254, "calibration": 53.51857296709248, "speed": 65.80418080342096, "cost": 63.524499297021315 }, "jevbench_score": 63.94837547956297 } }, "full": { "tiers": { "easy": 1.0, "standard": 0.7708333333333334, "judge": 0.8767123287671232, "hard": 0.4636363636363636 }, "accuracy": 0.704119850187266, "ece_hard": 0.28892783904029906, "calibration": 52.97941589841535, "p50_s": 3.5437258943915366, "p95_s": 6.403170879930256, "cost_usd_per_1000": 0.01581337913282554, "axes": { "intelligence": 74.0403694479037, "calibration": 52.97941589841535, "speed": 66.44144786074723, "cost": 64.0292594960931 }, "jevbench_score": 63.91434278007289 }, "full_escalation": 0.8689138576779026, "backend_heldout_accuracy": 0.7326732673267327, "useful": false, "curve": [ { "threshold": 0.0, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.01, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.02, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.03, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.04, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.05, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.06, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.07, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.08, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.09, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.1, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.11, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.12, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.13, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.14, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.15, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.16, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.17, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.18, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.19, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.2, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.21, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.22, "public": { "n": 231, "escalation": 0.004329004329004329, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2109514675973772, "calibration": 63.58735324026229, "p50_s": 1.1657145261764525, "p95_s": 4.4528342053294185, "cost_usd_per_1000": 0.0029475469786143945, "axes": { "intelligence": 53.58796296296297, "calibration": 63.58735324026229, "speed": 72.84771266552642, "cost": 85.91617792906855 }, "jevbench_score": 67.95668985424277 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.23, "public": { "n": 231, "escalation": 0.004329004329004329, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2109514675973772, "calibration": 63.58735324026229, "p50_s": 1.1657145261764525, "p95_s": 4.4528342053294185, "cost_usd_per_1000": 0.0029475469786143945, "axes": { "intelligence": 53.58796296296297, "calibration": 63.58735324026229, "speed": 72.84771266552642, "cost": 85.91617792906855 }, "jevbench_score": 67.95668985424277 }, "heldout": { "n": 303, "escalation": 0.0033003300330033004, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.26490004594654126, "calibration": 54.85874540534587, "p50_s": 2.7680240362882613, "p95_s": 4.776861734688282, "cost_usd_per_1000": 0.0029322393419117195, "axes": { "intelligence": 59.67720246323992, "calibration": 54.85874540534587, "speed": 68.78687472081019, "cost": 85.98401750835001 }, "jevbench_score": 66.33525602570792 } }, { "threshold": 0.24, "public": { "n": 231, "escalation": 0.004329004329004329, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2109514675973772, "calibration": 63.58735324026229, "p50_s": 1.1657145261764525, "p95_s": 4.4528342053294185, "cost_usd_per_1000": 0.0029475469786143945, "axes": { "intelligence": 53.58796296296297, "calibration": 63.58735324026229, "speed": 72.84771266552642, "cost": 85.91617792906855 }, "jevbench_score": 67.95668985424277 }, "heldout": { "n": 303, "escalation": 0.006600660066006601, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.30275229357798167 }, "accuracy": 0.5676567656765676, "ece_hard": 0.266448921982739, "calibration": 54.7038578017261, "p50_s": 2.7680240362882613, "p95_s": 4.808721804618835, "cost_usd_per_1000": 0.002981351342999469, "axes": { "intelligence": 59.95243182103809, "calibration": 54.7038578017261, "speed": 68.75800490091856, "cost": 85.76760522085276 }, "jevbench_score": 66.31592431921413 } }, { "threshold": 0.25, "public": { "n": 231, "escalation": 0.008658008658008658, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.34234234234234234 }, "accuracy": 0.5324675324675324, "ece_hard": 0.2122342461680507, "calibration": 63.459075383194936, "p50_s": 1.1657145261764525, "p95_s": 4.4528342053294185, "cost_usd_per_1000": 0.0030119666164048187, "axes": { "intelligence": 53.963338338338346, "calibration": 63.459075383194936, "speed": 72.84771266552642, "cost": 85.63449538043015 }, "jevbench_score": 67.98518800513597 }, "heldout": { "n": 303, "escalation": 0.009900990099009901, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.26275862263976985, "calibration": 55.07288773602301, "p50_s": 2.7680240362882613, "p95_s": 4.8142633035779, "cost_usd_per_1000": 0.003030463344087218, "axes": { "intelligence": 59.67720246323992, "calibration": 55.07288773602301, "speed": 68.75300303786179, "cost": 85.55472894298916 }, "jevbench_score": 66.30869783379735 } }, { "threshold": 0.26, "public": { "n": 231, "escalation": 0.008658008658008658, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.34234234234234234 }, "accuracy": 0.5324675324675324, "ece_hard": 0.2122342461680507, "calibration": 63.459075383194936, "p50_s": 1.1657145261764525, "p95_s": 4.4528342053294185, "cost_usd_per_1000": 0.0030119666164048187, "axes": { "intelligence": 53.963338338338346, "calibration": 63.459075383194936, "speed": 72.84771266552642, "cost": 85.63449538043015 }, "jevbench_score": 67.98518800513597 }, "heldout": { "n": 303, "escalation": 0.013201320132013201, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.27401382352715237, "calibration": 53.947367647284764, "p50_s": 2.7680240362882613, "p95_s": 4.822164891660214, "cost_usd_per_1000": 0.003079575345174967, "axes": { "intelligence": 59.67720246323992, "calibration": 53.947367647284764, "speed": 68.74588086188128, "cost": 85.34527497864451 }, "jevbench_score": 65.92516407893501 } }, { "threshold": 0.27, "public": { "n": 231, "escalation": 0.021645021645021644, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.34234234234234234 }, "accuracy": 0.5324675324675324, "ece_hard": 0.21176639376803566, "calibration": 63.50586062319644, "p50_s": 1.1657145261764525, "p95_s": 4.455602088570595, "cost_usd_per_1000": 0.003205225529776092, "axes": { "intelligence": 53.963338338338346, "calibration": 63.50586062319644, "speed": 72.84501392824589, "cost": 84.8242422022213 }, "jevbench_score": 67.83566837785781 }, "heldout": { "n": 303, "escalation": 0.013201320132013201, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.27401382352715237, "calibration": 53.947367647284764, "p50_s": 2.7680240362882613, "p95_s": 4.822164891660214, "cost_usd_per_1000": 0.003079575345174967, "axes": { "intelligence": 59.67720246323992, "calibration": 53.947367647284764, "speed": 68.74588086188128, "cost": 85.34527497864451 }, "jevbench_score": 65.92516407893501 } }, { "threshold": 0.28, "public": { "n": 231, "escalation": 0.021645021645021644, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.34234234234234234 }, "accuracy": 0.5324675324675324, "ece_hard": 0.21176639376803566, "calibration": 63.50586062319644, "p50_s": 1.1657145261764525, "p95_s": 4.455602088570595, "cost_usd_per_1000": 0.003205225529776092, "axes": { "intelligence": 53.963338338338346, "calibration": 63.50586062319644, "speed": 72.84501392824589, "cost": 84.8242422022213 }, "jevbench_score": 67.83566837785781 }, "heldout": { "n": 303, "escalation": 0.019801980198019802, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.27959022181537174, "calibration": 53.38972781846282, "p50_s": 2.7680240362882613, "p95_s": 4.90338304042816, "cost_usd_per_1000": 0.0031777993473504655, "axes": { "intelligence": 59.67720246323992, "calibration": 53.38972781846282, "speed": 68.6733432237271, "cost": 84.93620585473003 }, "jevbench_score": 65.6578719538466 } }, { "threshold": 0.29, "public": { "n": 231, "escalation": 0.03896103896103896, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.32432432432432434 }, "accuracy": 0.5238095238095238, "ece_hard": 0.23881953195838246, "calibration": 60.800546804161755, "p50_s": 1.1657145261764525, "p95_s": 4.538626746833325, "cost_usd_per_1000": 0.0034629040809377887, "axes": { "intelligence": 53.2125875875876, "calibration": 60.800546804161755, "speed": 72.76483323506824, "cost": 83.8167861345126 }, "jevbench_score": 66.64899444073065 }, "heldout": { "n": 303, "escalation": 0.0231023102310231, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.28440366972477066 }, "accuracy": 0.5610561056105611, "ece_hard": 0.2772077593659042, "calibration": 53.62797406340958, "p50_s": 2.7680240362882613, "p95_s": 4.969698165357111, "cost_usd_per_1000": 0.0032269113484382148, "axes": { "intelligence": 59.401973105441755, "calibration": 53.62797406340958, "speed": 68.61500130322224, "cost": 84.73638896858893 }, "jevbench_score": 65.60248981845197 } }, { "threshold": 0.3, "public": { "n": 231, "escalation": 0.04329004329004329, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.32432432432432434 }, "accuracy": 0.5238095238095238, "ece_hard": 0.24123091618005224, "calibration": 60.559408381994785, "p50_s": 1.1657145261764525, "p95_s": 4.646009536087513, "cost_usd_per_1000": 0.0035273237187282134, "axes": { "intelligence": 53.2125875875876, "calibration": 60.559408381994785, "speed": 72.66327694805801, "cost": 83.57664043901507 }, "jevbench_score": 66.51184158617701 }, "heldout": { "n": 303, "escalation": 0.033003300330033, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6986301369863014, "hard": 0.27522935779816515 }, "accuracy": 0.5610561056105611, "ece_hard": 0.2859139026064392, "calibration": 52.75735973935608, "p50_s": 2.79284847676754, "p95_s": 5.036343874037265, "cost_usd_per_1000": 0.0033742473517014626, "axes": { "intelligence": 59.31852456956139, "calibration": 52.75735973935608, "speed": 68.5183725045077, "cost": 84.15469252903787 }, "jevbench_score": 65.17629851127684 } }, { "threshold": 0.31, "public": { "n": 231, "escalation": 0.05627705627705628, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.32432432432432434 }, "accuracy": 0.5238095238095238, "ece_hard": 0.25366384120245244, "calibration": 59.31611587975476, "p50_s": 1.1657145261764525, "p95_s": 4.646009536087513, "cost_usd_per_1000": 0.003720582632099486, "axes": { "intelligence": 53.2125875875876, "calibration": 59.31611587975476, "speed": 72.66327694805801, "cost": 82.88167136731262 }, "jevbench_score": 66.02982416206478 }, "heldout": { "n": 303, "escalation": 0.0429042904290429, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6986301369863014, "hard": 0.29357798165137616 }, "accuracy": 0.5676567656765676, "ece_hard": 0.27689776303965025, "calibration": 53.65897369603498, "p50_s": 2.8226395934820174, "p95_s": 5.103181174397469, "cost_usd_per_1000": 0.00352158335496471, "axes": { "intelligence": 59.868983285157725, "calibration": 53.65897369603498, "speed": 68.41503583826945, "cost": 83.59786082536705 }, "jevbench_score": 65.470814221402 } }, { "threshold": 0.32, "public": { "n": 231, "escalation": 0.06060606060606061, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3153153153153153 }, "accuracy": 0.5194805194805194, "ece_hard": 0.26338560878693523, "calibration": 58.34393912130648, "p50_s": 1.1657145261764525, "p95_s": 4.732986265420914, "cost_usd_per_1000": 0.0037850022698899108, "axes": { "intelligence": 52.837212212212215, "calibration": 58.34393912130648, "speed": 72.58272547477219, "cost": 82.6580156714401 }, "jevbench_score": 65.57880106225045 }, "heldout": { "n": 303, "escalation": 0.0462046204620462, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6986301369863014, "hard": 0.30275229357798167 }, "accuracy": 0.570957095709571, "ece_hard": 0.27175752007020726, "calibration": 54.172997992979276, "p50_s": 2.8226395934820174, "p95_s": 5.178344944119452, "cost_usd_per_1000": 0.0035706953560524592, "axes": { "intelligence": 60.1442126429559, "calibration": 54.172997992979276, "speed": 68.35153594223539, "cost": 83.41741603856713 }, "jevbench_score": 65.65161762420618 } }, { "threshold": 0.33, "public": { "n": 231, "escalation": 0.06493506493506493, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3063063063063063 }, "accuracy": 0.5151515151515151, "ece_hard": 0.277256904916076, "calibration": 56.9568095083924, "p50_s": 1.1657145261764525, "p95_s": 5.088029319047928, "cost_usd_per_1000": 0.003849421907680335, "axes": { "intelligence": 52.461836836836845, "calibration": 56.9568095083924, "speed": 72.26858186466677, "cost": 82.43813459128005 }, "jevbench_score": 64.95561644909728 }, "heldout": { "n": 303, "escalation": 0.0594059405940594, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6986301369863014, "hard": 0.3211009174311927 }, "accuracy": 0.5775577557755776, "ece_hard": 0.26353738612997424, "calibration": 55.653158134021716, "p50_s": 2.8282473921775817, "p95_s": 5.359854796528816, "cost_usd_per_1000": 0.0037671433604034564, "axes": { "intelligence": 60.69467135855222, "calibration": 55.653158134021716, "speed": 68.1932958024029, "cost": 82.71963556607518 }, "jevbench_score": 66.06899000713626 } }, { "threshold": 0.34, "public": { "n": 231, "escalation": 0.0735930735930736, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3063063063063063 }, "accuracy": 0.5151515151515151, "ece_hard": 0.2816842482399296, "calibration": 56.51407517600704, "p50_s": 1.1657145261764525, "p95_s": 5.44004242271185, "cost_usd_per_1000": 0.0039782611832611834, "axes": { "intelligence": 52.461836836836845, "calibration": 56.51407517600704, "speed": 71.97805505234226, "cost": 82.00920123117771 }, "jevbench_score": 64.67935816812641 }, "heldout": { "n": 303, "escalation": 0.06930693069306931, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6986301369863014, "hard": 0.3211009174311927 }, "accuracy": 0.5775577557755776, "ece_hard": 0.27348758936934514, "calibration": 54.658137810084625, "p50_s": 2.8282473921775817, "p95_s": 5.40956619977951, "cost_usd_per_1000": 0.003914479363666704, "axes": { "intelligence": 60.69467135855222, "calibration": 54.658137810084625, "speed": 68.15320164791567, "cost": 82.2197797631432 }, "jevbench_score": 65.66243619569761 } }, { "threshold": 0.35, "public": { "n": 231, "escalation": 0.0735930735930736, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3063063063063063 }, "accuracy": 0.5151515151515151, "ece_hard": 0.2816842482399296, "calibration": 56.51407517600704, "p50_s": 1.1657145261764525, "p95_s": 5.44004242271185, "cost_usd_per_1000": 0.0039782611832611834, "axes": { "intelligence": 52.461836836836845, "calibration": 56.51407517600704, "speed": 71.97805505234226, "cost": 82.00920123117771 }, "jevbench_score": 64.67935816812641 }, "heldout": { "n": 303, "escalation": 0.07920792079207921, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6986301369863014, "hard": 0.3302752293577982 }, "accuracy": 0.5808580858085809, "ece_hard": 0.27494599083060517, "calibration": 54.51229766395862, "p50_s": 2.8282473921775817, "p95_s": 5.507749010622499, "cost_usd_per_1000": 0.004061815366929952, "axes": { "intelligence": 60.96990071635039, "calibration": 54.51229766395862, "speed": 68.07508463512085, "cost": 81.73839465057972 }, "jevbench_score": 65.57768289192799 } }, { "threshold": 0.36, "public": { "n": 231, "escalation": 0.08658008658008658, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3153153153153153 }, "accuracy": 0.5194805194805194, "ece_hard": 0.27860886867884044, "calibration": 56.82161313211596, "p50_s": 1.1657145261764525, "p95_s": 5.452234508097172, "cost_usd_per_1000": 0.004171520096632456, "axes": { "intelligence": 52.837212212212215, "calibration": 56.82161313211596, "speed": 71.96833264507931, "cost": 81.3911697947992 }, "jevbench_score": 64.75794319124392 }, "heldout": { "n": 303, "escalation": 0.08250825082508251, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6986301369863014, "hard": 0.3302752293577982 }, "accuracy": 0.5808580858085809, "ece_hard": 0.2760796279723491, "calibration": 53.353129969985076, "p50_s": 2.8282473921775817, "p95_s": 5.605073678493498, "cost_usd_per_1000": 0.004110927368017701, "axes": { "intelligence": 60.96990071635039, "calibration": 53.353129969985076, "speed": 67.99901278895409, "cost": 81.58180588837577 }, "jevbench_score": 65.17676910463331 } }, { "threshold": 0.37, "public": { "n": 231, "escalation": 0.11255411255411256, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3153153153153153 }, "accuracy": 0.5194805194805194, "ece_hard": 0.2928936804578111, "calibration": 55.393131954218894, "p50_s": 1.169974410533905, "p95_s": 5.9908315107226375, "cost_usd_per_1000": 0.004558037923375002, "axes": { "intelligence": 52.837212212212215, "calibration": 55.393131954218894, "speed": 71.54336531522287, "cost": 80.23666197288793 }, "jevbench_score": 64.02277710667613 }, "heldout": { "n": 303, "escalation": 0.0891089108910891, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6986301369863014, "hard": 0.3394495412844037 }, "accuracy": 0.5841584158415841, "ece_hard": 0.27525717168248026, "calibration": 53.43537559897196, "p50_s": 2.8282473921775817, "p95_s": 5.643718923628329, "cost_usd_per_1000": 0.004209151370193199, "axes": { "intelligence": 61.245130074148555, "calibration": 53.43537559897196, "speed": 67.96917228368801, "cost": 81.27416367404861 }, "jevbench_score": 65.20655100972081 } }, { "threshold": 0.38, "public": { "n": 231, "escalation": 0.12554112554112554, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3153153153153153 }, "accuracy": 0.5194805194805194, "ece_hard": 0.295288666155102, "calibration": 55.1536333844898, "p50_s": 1.2405554592609405, "p95_s": 6.054991522431374, "cost_usd_per_1000": 0.004751296836746275, "axes": { "intelligence": 52.837212212212215, "calibration": 55.1536333844898, "speed": 71.24270276317164, "cost": 79.69563508699937 }, "jevbench_score": 63.77819684264697 }, "heldout": { "n": 303, "escalation": 0.09900990099009901, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7054794520547946, "hard": 0.3486238532110092 }, "accuracy": 0.5907590759075908, "ece_hard": 0.27324755800229694, "calibration": 53.63633696699029, "p50_s": 2.8427457749843597, "p95_s": 5.655404330790043, "cost_usd_per_1000": 0.004356487373456447, "axes": { "intelligence": 61.71214025386453, "calibration": 53.63633696699029, "speed": 67.93798317187284, "cost": 80.82590621072069 }, "jevbench_score": 65.29399444492813 } }, { "threshold": 0.39, "public": { "n": 231, "escalation": 0.14285714285714285, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3153153153153153 }, "accuracy": 0.5194805194805194, "ece_hard": 0.30433094587497883, "calibration": 54.24940541250212, "p50_s": 1.2405554592609405, "p95_s": 6.1078165709972385, "cost_usd_per_1000": 0.005008975387907972, "axes": { "intelligence": 52.837212212212215, "calibration": 54.24940541250212, "speed": 71.20497829827244, "cost": 79.00753306761374 }, "jevbench_score": 63.36923064853023 }, "heldout": { "n": 303, "escalation": 0.11221122112211221, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7123287671232876, "hard": 0.3486238532110092 }, "accuracy": 0.594059405940594, "ece_hard": 0.28213187039795273, "calibration": 52.747905727424715, "p50_s": 2.9652757436037063, "p95_s": 5.655404330790043, "cost_usd_per_1000": 0.0045529353778074435, "axes": { "intelligence": 61.90392107578233, "calibration": 52.747905727424715, "speed": 67.75471254919833, "cost": 80.2512554145022 }, "jevbench_score": 64.91255155666327 } }, { "threshold": 0.4, "public": { "n": 231, "escalation": 0.1471861471861472, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3153153153153153 }, "accuracy": 0.5194805194805194, "ece_hard": 0.31595692398913267, "calibration": 53.08680760108674, "p50_s": 1.2405554592609405, "p95_s": 6.1078165709972385, "cost_usd_per_1000": 0.005073395025698396, "axes": { "intelligence": 52.837212212212215, "calibration": 53.08680760108674, "speed": 71.20497829827244, "cost": 78.84103963726719 }, "jevbench_score": 62.99372663650139 }, "heldout": { "n": 303, "escalation": 0.132013201320132, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7054794520547946, "hard": 0.3486238532110092 }, "accuracy": 0.5907590759075908, "ece_hard": 0.28990590984536974, "calibration": 52.399382261541966, "p50_s": 2.9652757436037063, "p95_s": 5.862377908825874, "cost_usd_per_1000": 0.004847607384333939, "axes": { "intelligence": 61.71214025386453, "calibration": 52.399382261541966, "speed": 67.5986110389065, "cost": 79.4341768491768 }, "jevbench_score": 64.55211659673223 } }, { "threshold": 0.41, "public": { "n": 231, "escalation": 0.1645021645021645, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.34234234234234234 }, "accuracy": 0.5324675324675324, "ece_hard": 0.30609473809766335, "calibration": 54.07302619023367, "p50_s": 1.2405554592609405, "p95_s": 6.147259482741356, "cost_usd_per_1000": 0.005331073576860094, "axes": { "intelligence": 53.963338338338346, "calibration": 54.07302619023367, "speed": 71.17702270176105, "cost": 78.19555970543382 }, "jevbench_score": 63.48196314439285 }, "heldout": { "n": 303, "escalation": 0.13861386138613863, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7054794520547946, "hard": 0.3486238532110092 }, "accuracy": 0.5907590759075908, "ece_hard": 0.2906108843610921, "calibration": 51.853992308436695, "p50_s": 2.984680587053299, "p95_s": 5.902544519305229, "cost_usd_per_1000": 0.004945831386509438, "axes": { "intelligence": 61.71214025386453, "calibration": 51.853992308436695, "speed": 67.54062867759465, "cost": 79.17282081123976 }, "jevbench_score": 64.31666340882096 } }, { "threshold": 0.42, "public": { "n": 231, "escalation": 0.16883116883116883, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.35135135135135137 }, "accuracy": 0.5367965367965368, "ece_hard": 0.3018299969939498, "calibration": 54.49950030060502, "p50_s": 1.2405554592609405, "p95_s": 6.147259482741356, "cost_usd_per_1000": 0.005395493214650518, "axes": { "intelligence": 54.33871371371373, "calibration": 54.49950030060502, "speed": 71.17702270176105, "cost": 78.03906547877112 }, "jevbench_score": 63.68518840534482 }, "heldout": { "n": 303, "escalation": 0.15841584158415842, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7123287671232876, "hard": 0.3669724770642202 }, "accuracy": 0.6006600660066007, "ece_hard": 0.28243810696820604, "calibration": 52.6712700477253, "p50_s": 3.030290076136589, "p95_s": 5.934103228151798, "cost_usd_per_1000": 0.0052405033930359324, "axes": { "intelligence": 62.45437979137867, "calibration": 52.6712700477253, "speed": 67.45160699513465, "cost": 78.41880980470246 }, "jevbench_score": 64.58583957114234 } }, { "threshold": 0.43, "public": { "n": 231, "escalation": 0.18614718614718614, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5411255411255411, "ece_hard": 0.3053170246382017, "calibration": 53.400463312536885, "p50_s": 1.2405554592609405, "p95_s": 6.147259482741356, "cost_usd_per_1000": 0.005653171765812215, "axes": { "intelligence": 54.7140890890891, "calibration": 53.400463312536885, "speed": 71.17702270176105, "cost": 77.43123456244194 }, "jevbench_score": 63.3468537933097 }, "heldout": { "n": 303, "escalation": 0.19141914191419143, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7123287671232876, "hard": 0.3761467889908257 }, "accuracy": 0.6039603960396039, "ece_hard": 0.2843230141071005, "calibration": 52.482779333835865, "p50_s": 3.0780361533164977, "p95_s": 6.025533692538739, "cost_usd_per_1000": 0.005731623403913425, "axes": { "intelligence": 62.72960914917684, "calibration": 52.482779333835865, "speed": 67.31730764227453, "cost": 77.2516705791654 }, "jevbench_score": 64.32517943630818 } }, { "threshold": 0.44, "public": { "n": 231, "escalation": 0.19913419913419914, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.36936936936936937 }, "accuracy": 0.5454545454545454, "ece_hard": 0.30317582054739595, "calibration": 55.34772487280726, "p50_s": 1.2405554592609405, "p95_s": 6.147259482741356, "cost_usd_per_1000": 0.005846430679183489, "axes": { "intelligence": 55.08946446446447, "calibration": 55.34772487280726, "speed": 71.17702270176105, "cost": 76.99327586075269 }, "jevbench_score": 63.93523045803402 }, "heldout": { "n": 303, "escalation": 0.20132013201320131, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7123287671232876, "hard": 0.3761467889908257 }, "accuracy": 0.6039603960396039, "ece_hard": 0.2843230141071005, "calibration": 52.482779333835865, "p50_s": 3.1050253003835677, "p95_s": 6.025533692538739, "cost_usd_per_1000": 0.005878959407176672, "axes": { "intelligence": 62.72960914917684, "calibration": 52.482779333835865, "speed": 67.27939337545364, "cost": 76.92098615503535 }, "jevbench_score": 64.24718141076109 } }, { "threshold": 0.45, "public": { "n": 231, "escalation": 0.21212121212121213, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3783783783783784 }, "accuracy": 0.5497835497835498, "ece_hard": 0.3040647227523563, "calibration": 55.25883465231123, "p50_s": 1.2405554592609405, "p95_s": 6.16721652597189, "cost_usd_per_1000": 0.006039689592554761, "axes": { "intelligence": 55.46483983983985, "calibration": 55.25883465231123, "speed": 71.16294619321634, "cost": 76.56956143592609 }, "jevbench_score": 63.92671547462319 }, "heldout": { "n": 303, "escalation": 0.2145214521452145, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7191780821917808, "hard": 0.3761467889908257 }, "accuracy": 0.6072607260726073, "ece_hard": 0.2906778016492861, "calibration": 51.8473005796173, "p50_s": 3.1790891528129577, "p95_s": 6.0302905783057215, "cost_usd_per_1000": 0.006075407411527669, "axes": { "intelligence": 62.921389971094634, "calibration": 51.8473005796173, "speed": 67.17359053019106, "cost": 76.49273780030634 }, "jevbench_score": 63.98612445855979 } }, { "threshold": 0.46, "public": { "n": 231, "escalation": 0.22510822510822512, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.38738738738738737 }, "accuracy": 0.5541125541125541, "ece_hard": 0.29980788021302435, "calibration": 55.68451890624442, "p50_s": 1.3233075231313705, "p95_s": 6.233743192255497, "cost_usd_per_1000": 0.006232948505926034, "axes": { "intelligence": 55.84021521521522, "calibration": 55.68451890624442, "speed": 70.83590311546257, "cost": 76.15919383170251 }, "jevbench_score": 63.99769498468546 }, "heldout": { "n": 303, "escalation": 0.21782178217821782, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7191780821917808, "hard": 0.3761467889908257 }, "accuracy": 0.6072607260726073, "ece_hard": 0.2925299263044235, "calibration": 51.2926802143066, "p50_s": 3.1790891528129577, "p95_s": 6.0302905783057215, "cost_usd_per_1000": 0.006124519412615419, "axes": { "intelligence": 62.921389971094634, "calibration": 51.2926802143066, "speed": 67.17359053019106, "cost": 76.38783953366 }, "jevbench_score": 63.79242660898158 } }, { "threshold": 0.47, "public": { "n": 231, "escalation": 0.24242424242424243, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3783783783783784 }, "accuracy": 0.5497835497835498, "ece_hard": 0.315610408542822, "calibration": 53.28977956312943, "p50_s": 1.3951993823051452, "p95_s": 6.236580775678158, "cost_usd_per_1000": 0.006490627057087731, "axes": { "intelligence": 55.46483983983985, "calibration": 53.28977956312943, "speed": 70.60417173023757, "cost": 75.63140032428835 }, "jevbench_score": 63.0301800583521 }, "heldout": { "n": 303, "escalation": 0.23432343234323433, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7397260273972602, "hard": 0.3853211009174312 }, "accuracy": 0.6204620462046204, "ece_hard": 0.2879284522782772, "calibration": 51.75282761692122, "p50_s": 3.250187510251999, "p95_s": 6.0302905783057215, "cost_usd_per_1000": 0.006370079418054165, "axes": { "intelligence": 63.77196179464624, "calibration": 51.75282761692122, "speed": 67.07753343168639, "cost": 75.87565459379655 }, "jevbench_score": 64.01928935342934 } }, { "threshold": 0.48, "public": { "n": 231, "escalation": 0.2510822510822511, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.36936936936936937 }, "accuracy": 0.5454545454545454, "ece_hard": 0.32674068338419937, "calibration": 50.834949411642114, "p50_s": 1.416904667019844, "p95_s": 6.236580775678158, "cost_usd_per_1000": 0.006619466332668579, "axes": { "intelligence": 55.08946446446447, "calibration": 50.834949411642114, "speed": 70.53712817553861, "cost": 75.3753106708256 }, "jevbench_score": 62.118285270625485 }, "heldout": { "n": 303, "escalation": 0.264026402640264, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7465753424657534, "hard": 0.3944954128440367 }, "accuracy": 0.6270627062706271, "ece_hard": 0.28272264752694226, "calibration": 52.27340809205472, "p50_s": 3.3139643013477325, "p95_s": 6.047200620174408, "cost_usd_per_1000": 0.0068120874278439075, "axes": { "intelligence": 64.23897197436219, "calibration": 52.27340809205472, "speed": 66.98097797321294, "cost": 75.00159360525322 }, "jevbench_score": 64.0877970475172 } }, { "threshold": 0.49, "public": { "n": 231, "escalation": 0.2554112554112554, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3783783783783784 }, "accuracy": 0.5497835497835498, "ece_hard": 0.3202578517299514, "calibration": 51.48323257706691, "p50_s": 1.416904667019844, "p95_s": 6.236580775678158, "cost_usd_per_1000": 0.0066838859704590045, "axes": { "intelligence": 55.46483983983985, "calibration": 51.48323257706691, "speed": 70.53712817553861, "cost": 75.24912903814781 }, "jevbench_score": 62.395131695454744 }, "heldout": { "n": 303, "escalation": 0.26732673267326734, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7465753424657534, "hard": 0.3944954128440367 }, "accuracy": 0.6270627062706271, "ece_hard": 0.28380735849634225, "calibration": 52.16493699511472, "p50_s": 3.3139643013477325, "p95_s": 6.047200620174408, "cost_usd_per_1000": 0.006861199428931658, "axes": { "intelligence": 64.23897197436219, "calibration": 52.16493699511472, "speed": 66.98097797321294, "cost": 74.90799871611578 }, "jevbench_score": 64.03453158920478 } }, { "threshold": 0.5, "public": { "n": 231, "escalation": 0.26406926406926406, "tiers": { "easy": 0.9583333333333334, "standard": 0.5555555555555556, "hard": 0.3783783783783784 }, "accuracy": 0.5541125541125541, "ece_hard": 0.32141197708018193, "calibration": 51.36782004204386, "p50_s": 1.4668009370565414, "p95_s": 6.236580775678158, "cost_usd_per_1000": 0.006812725246039852, "axes": { "intelligence": 56.00496329662997, "calibration": 51.36782004204386, "speed": 70.38682269602697, "cost": 75.00037376790299 }, "jevbench_score": 62.42637406342563 }, "heldout": { "n": 303, "escalation": 0.27722772277227725, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7465753424657534, "hard": 0.3853211009174312 }, "accuracy": 0.6237623762376238, "ece_hard": 0.2944627776994618, "calibration": 51.09939507480277, "p50_s": 3.3139643013477325, "p95_s": 6.098373472690581, "cost_usd_per_1000": 0.007008535432194905, "axes": { "intelligence": 63.96374261656403, "calibration": 51.09939507480277, "speed": 66.94438156565151, "cost": 74.63118180123978 }, "jevbench_score": 63.569093684816906 } }, { "threshold": 0.51, "public": { "n": 231, "escalation": 0.2813852813852814, "tiers": { "easy": 0.9583333333333334, "standard": 0.5833333333333334, "hard": 0.3783783783783784 }, "accuracy": 0.5627705627705628, "ece_hard": 0.3243304638067881, "calibration": 51.07597136938324, "p50_s": 1.5417469084262847, "p95_s": 6.236580775678158, "cost_usd_per_1000": 0.007070403797201551, "axes": { "intelligence": 57.085210210210214, "calibration": 51.07597136938324, "speed": 70.17040362103418, "cost": 74.51667347624013 }, "jevbench_score": 62.48660340877388 }, "heldout": { "n": 303, "escalation": 0.29042904290429045, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7465753424657534, "hard": 0.3853211009174312 }, "accuracy": 0.6237623762376238, "ece_hard": 0.30552305882655156, "calibration": 51.77213749890395, "p50_s": 3.32881940305233, "p95_s": 6.098373472690581, "cost_usd_per_1000": 0.007204983436545902, "axes": { "intelligence": 63.96374261656403, "calibration": 51.77213749890395, "speed": 66.9249574815927, "cost": 74.271010397304 }, "jevbench_score": 63.69558780333126 } }, { "threshold": 0.52, "public": { "n": 231, "escalation": 0.2943722943722944, "tiers": { "easy": 0.9583333333333334, "standard": 0.5833333333333334, "hard": 0.3783783783783784 }, "accuracy": 0.5627705627705628, "ece_hard": 0.32430627393421824, "calibration": 51.07839035664023, "p50_s": 1.7314340859651565, "p95_s": 6.236580775678158, "cost_usd_per_1000": 0.007263662710572823, "axes": { "intelligence": 57.085210210210214, "calibration": 51.07839035664023, "speed": 69.66647485311819, "cost": 74.16532991673175 }, "jevbench_score": 62.30119744848229 }, "heldout": { "n": 303, "escalation": 0.32673267326732675, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7602739726027398, "hard": 0.3761467889908257 }, "accuracy": 0.6270627062706271, "ece_hard": 0.32131235061173047, "calibration": 50.19320832038606, "p50_s": 3.32881940305233, "p95_s": 6.114326000213623, "cost_usd_per_1000": 0.007745215448511143, "axes": { "intelligence": 64.07207490260149, "calibration": 50.19320832038606, "speed": 66.91361175310519, "cost": 73.32899490926158 }, "jevbench_score": 63.026905890653204 } }, { "threshold": 0.53, "public": { "n": 231, "escalation": 0.30735930735930733, "tiers": { "easy": 0.9583333333333334, "standard": 0.5972222222222222, "hard": 0.38738738738738737 }, "accuracy": 0.5714285714285714, "ece_hard": 0.3183670681051306, "calibration": 51.67231093954899, "p50_s": 1.7968375474214553, "p95_s": 6.23870878815651, "cost_usd_per_1000": 0.007456921623944096, "axes": { "intelligence": 58.00070904237572, "calibration": 51.67231093954899, "speed": 69.50396471791564, "cost": 73.8232126462672 }, "jevbench_score": 62.62149431764669 }, "heldout": { "n": 303, "escalation": 0.3465346534653465, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7602739726027398, "hard": 0.3761467889908257 }, "accuracy": 0.6270627062706271, "ece_hard": 0.3263337626710945, "calibration": 49.69106711444966, "p50_s": 3.3391009658575057, "p95_s": 6.18123611807823, "cost_usd_per_1000": 0.008039887455037639, "axes": { "intelligence": 64.07207490260149, "calibration": 49.69106711444966, "speed": 66.85295115198079, "cost": 72.84250091813684 }, "jevbench_score": 62.74991412604973 } }, { "threshold": 0.54, "public": { "n": 231, "escalation": 0.3246753246753247, "tiers": { "easy": 0.9583333333333334, "standard": 0.5972222222222222, "hard": 0.3783783783783784 }, "accuracy": 0.5670995670995671, "ece_hard": 0.336951526488055, "calibration": 49.81386510125655, "p50_s": 1.8519584774971007, "p95_s": 6.23870878815651, "cost_usd_per_1000": 0.007714600175105793, "axes": { "intelligence": 57.62533366700035, "calibration": 49.81386510125655, "speed": 69.37274040792407, "cost": 73.38059731645612 }, "jevbench_score": 61.82775063585122 }, "heldout": { "n": 303, "escalation": 0.36303630363036304, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7671232876712328, "hard": 0.3944954128440367 }, "accuracy": 0.636963696369637, "ece_hard": 0.31112322405456405, "calibration": 51.2121209761027, "p50_s": 3.3391009658575057, "p95_s": 6.1954798161983495, "cost_usd_per_1000": 0.008285447460476384, "axes": { "intelligence": 64.81431444011562, "calibration": 51.2121209761027, "speed": 66.84295502334811, "cost": 72.45052096845016 }, "jevbench_score": 63.3191698047681 } }, { "threshold": 0.55, "public": { "n": 231, "escalation": 0.3463203463203463, "tiers": { "easy": 0.9791666666666666, "standard": 0.5972222222222222, "hard": 0.3783783783783784 }, "accuracy": 0.5714285714285714, "ece_hard": 0.3437137248288404, "calibration": 48.365108981777624, "p50_s": 1.8988913238048553, "p95_s": 6.23870878815651, "cost_usd_per_1000": 0.008036698364057914, "axes": { "intelligence": 58.030426259592936, "calibration": 48.365108981777624, "speed": 69.26405175821255, "cost": 72.84766994338337 }, "jevbench_score": 61.34481660083853 }, "heldout": { "n": 303, "escalation": 0.37293729372937295, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7534246575342466, "hard": 0.3944954128440367 }, "accuracy": 0.6303630363036303, "ece_hard": 0.3117221797995611, "calibration": 51.15222540160299, "p50_s": 3.3391009658575057, "p95_s": 6.2163633614778515, "cost_usd_per_1000": 0.008432783463739633, "axes": { "intelligence": 64.43075279628002, "calibration": 51.15222540160299, "speed": 66.82834056696761, "cost": 72.22087153913881 }, "jevbench_score": 63.15318891582388 } }, { "threshold": 0.56, "public": { "n": 231, "escalation": 0.354978354978355, "tiers": { "easy": 0.9791666666666666, "standard": 0.6111111111111112, "hard": 0.3783783783783784 }, "accuracy": 0.5757575757575758, "ece_hard": 0.3437137248288404, "calibration": 48.365108981777624, "p50_s": 2.0464984118938445, "p95_s": 6.23870878815651, "cost_usd_per_1000": 0.008165537639638764, "axes": { "intelligence": 58.570549716383056, "calibration": 48.365108981777624, "speed": 68.93893874027495, "cost": 72.64045644818209 }, "jevbench_score": 61.37106462561545 }, "heldout": { "n": 303, "escalation": 0.39933993399339934, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7602739726027398, "hard": 0.4036697247706422 }, "accuracy": 0.636963696369637, "ece_hard": 0.30700001032505564, "calibration": 51.62444234905354, "p50_s": 3.351239749789238, "p95_s": 6.3183612748980496, "cost_usd_per_1000": 0.008825679472441625, "axes": { "intelligence": 64.89776297599597, "calibration": 51.62444234905354, "speed": 66.74190046735265, "cost": 71.62755547416185 }, "jevbench_score": 63.26171331934215 } }, { "threshold": 0.57, "public": { "n": 231, "escalation": 0.36363636363636365, "tiers": { "easy": 0.9791666666666666, "standard": 0.625, "hard": 0.36936936936936937 }, "accuracy": 0.5757575757575758, "ece_hard": 0.3546035973282548, "calibration": 45.577242485347966, "p50_s": 2.0771628260612487, "p95_s": 6.23870878815651, "cost_usd_per_1000": 0.008294376915219611, "axes": { "intelligence": 58.735297797797806, "calibration": 45.577242485347966, "speed": 68.87434744348856, "cost": 72.43648699604442 }, "jevbench_score": 60.452675217351256 }, "heldout": { "n": 303, "escalation": 0.40924092409240925, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7671232876712328, "hard": 0.41284403669724773 }, "accuracy": 0.6435643564356436, "ece_hard": 0.30100346458417565, "calibration": 52.224096923141545, "p50_s": 3.4096150040626525, "p95_s": 6.3183612748980496, "cost_usd_per_1000": 0.008973015475704874, "axes": { "intelligence": 65.36477315571196, "calibration": 52.224096923141545, "speed": 66.66690201887221, "cost": 71.41184749670015 }, "jevbench_score": 63.4927020993517 } }, { "threshold": 0.58, "public": { "n": 231, "escalation": 0.36796536796536794, "tiers": { "easy": 0.9791666666666666, "standard": 0.625, "hard": 0.36936936936936937 }, "accuracy": 0.5757575757575758, "ece_hard": 0.3575112826072418, "calibration": 45.28647395744926, "p50_s": 2.0771628260612487, "p95_s": 6.246346229314804, "cost_usd_per_1000": 0.008358796553010035, "axes": { "intelligence": 58.735297797797806, "calibration": 45.28647395744926, "speed": 68.86903405246224, "cost": 72.33568735150352 }, "jevbench_score": 60.33385450997754 }, "heldout": { "n": 303, "escalation": 0.42244224422442245, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.773972602739726, "hard": 0.42201834862385323 }, "accuracy": 0.6501650165016502, "ece_hard": 0.2936802124487151, "calibration": 52.95642213668759, "p50_s": 3.4247110724449157, "p95_s": 6.3183612748980496, "cost_usd_per_1000": 0.009169463480055871, "axes": { "intelligence": 65.83178333542791, "calibration": 52.95642213668759, "speed": 66.6477160749003, "cost": 71.12968224549759 }, "jevbench_score": 63.75989582890591 } }, { "threshold": 0.59, "public": { "n": 231, "escalation": 0.37662337662337664, "tiers": { "easy": 0.9791666666666666, "standard": 0.625, "hard": 0.36936936936936937 }, "accuracy": 0.5757575757575758, "ece_hard": 0.35963759628502096, "calibration": 45.073842589671344, "p50_s": 2.0771628260612487, "p95_s": 6.246346229314804, "cost_usd_per_1000": 0.008487635828590885, "axes": { "intelligence": 58.735297797797806, "calibration": 45.073842589671344, "speed": 68.86903405246224, "cost": 72.13639787777154 }, "jevbench_score": 60.22135882250014 }, "heldout": { "n": 303, "escalation": 0.43564356435643564, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.773972602739726, "hard": 0.42201834862385323 }, "accuracy": 0.6501650165016502, "ece_hard": 0.3026325542668684, "calibration": 52.06118795487227, "p50_s": 3.501546213030815, "p95_s": 6.339170089364052, "cost_usd_per_1000": 0.009365911484406868, "axes": { "intelligence": 65.83178333542791, "calibration": 52.06118795487227, "speed": 66.53707732810898, "cost": 70.85349852927452 }, "jevbench_score": 63.40064532029043 } }, { "threshold": 0.6, "public": { "n": 231, "escalation": 0.3939393939393939, "tiers": { "easy": 0.9791666666666666, "standard": 0.6388888888888888, "hard": 0.36936936936936937 }, "accuracy": 0.5800865800865801, "ece_hard": 0.363347066364632, "calibration": 44.702895581710244, "p50_s": 2.0771628260612487, "p95_s": 6.246346229314804, "cost_usd_per_1000": 0.008745314379752582, "axes": { "intelligence": 59.27542125458792, "calibration": 44.702895581710244, "speed": 68.86903405246224, "cost": 71.74673721184503 }, "jevbench_score": 60.153252242969224 }, "heldout": { "n": 303, "escalation": 0.44224422442244227, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.773972602739726, "hard": 0.43119266055045874 }, "accuracy": 0.6534653465346535, "ece_hard": 0.3058974596684132, "calibration": 51.734697414717786, "p50_s": 3.501546213030815, "p95_s": 6.339170089364052, "cost_usd_per_1000": 0.009464135486582367, "axes": { "intelligence": 66.10701269322608, "calibration": 51.734697414717786, "speed": 66.53707732810898, "cost": 70.71757153200211 }, "jevbench_score": 63.33665541789147 } }, { "threshold": 0.61, "public": { "n": 231, "escalation": 0.4199134199134199, "tiers": { "easy": 0.9791666666666666, "standard": 0.6388888888888888, "hard": 0.3963963963963964 }, "accuracy": 0.5930735930735931, "ece_hard": 0.34073503158414686, "calibration": 48.65258203983125, "p50_s": 2.145356276631355, "p95_s": 6.246346229314804, "cost_usd_per_1000": 0.009131832206495128, "axes": { "intelligence": 60.40154738071405, "calibration": 48.65258203983125, "speed": 68.72874521637758, "cost": 71.18326231250649 }, "jevbench_score": 61.57686371325283 }, "heldout": { "n": 303, "escalation": 0.46534653465346537, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7808219178082192, "hard": 0.42201834862385323 }, "accuracy": 0.6534653465346535, "ece_hard": 0.3185968976089851, "calibration": 50.4647536206606, "p50_s": 3.52294003367424, "p95_s": 6.339170089364052, "cost_usd_per_1000": 0.00980791949419661, "axes": { "intelligence": 66.02356415734573, "calibration": 50.4647536206606, "speed": 66.51062344478233, "cost": 70.25269322819253 }, "jevbench_score": 62.814553902999165 } }, { "threshold": 0.62, "public": { "n": 231, "escalation": 0.44155844155844154, "tiers": { "easy": 0.9791666666666666, "standard": 0.6388888888888888, "hard": 0.3963963963963964 }, "accuracy": 0.5930735930735931, "ece_hard": 0.3517162511028685, "calibration": 47.554460087959086, "p50_s": 2.1577178329229354, "p95_s": 6.294179813563824, "cost_usd_per_1000": 0.009453930395447249, "axes": { "intelligence": 60.40154738071405, "calibration": 47.554460087959086, "speed": 68.67066203317347, "cost": 70.73162798542023 }, "jevbench_score": 61.11615899723226 }, "heldout": { "n": 303, "escalation": 0.4752475247524752, "tiers": { "easy": 0.9166666666666666, "standard": 0.7083333333333334, "judge": 0.7808219178082192, "hard": 0.41284403669724773 }, "accuracy": 0.6534653465346535, "ece_hard": 0.3289502971823199, "calibration": 49.42941366332712, "p50_s": 3.52294003367424, "p95_s": 6.339170089364052, "cost_usd_per_1000": 0.009955255497459858, "axes": { "intelligence": 66.91500146621424, "calibration": 49.42941366332712, "speed": 66.51062344478233, "cost": 70.05842768523057 }, "jevbench_score": 62.656349588295804 } }, { "threshold": 0.63, "public": { "n": 231, "escalation": 0.44155844155844154, "tiers": { "easy": 0.9791666666666666, "standard": 0.6388888888888888, "hard": 0.3963963963963964 }, "accuracy": 0.5930735930735931, "ece_hard": 0.3517162511028685, "calibration": 47.554460087959086, "p50_s": 2.1577178329229354, "p95_s": 6.294179813563824, "cost_usd_per_1000": 0.009453930395447249, "axes": { "intelligence": 60.40154738071405, "calibration": 47.554460087959086, "speed": 68.67066203317347, "cost": 70.73162798542023 }, "jevbench_score": 61.11615899723226 }, "heldout": { "n": 303, "escalation": 0.49504950495049505, "tiers": { "easy": 0.9166666666666666, "standard": 0.75, "judge": 0.773972602739726, "hard": 0.41284403669724773 }, "accuracy": 0.6534653465346535, "ece_hard": 0.3364336794108563, "calibration": 48.68107544047348, "p50_s": 3.574419969320297, "p95_s": 6.339170089364052, "cost_usd_per_1000": 0.010249927503986353, "axes": { "intelligence": 67.8898873109631, "calibration": 48.68107544047348, "speed": 66.4476201757623, "cost": 69.67837618867615 }, "jevbench_score": 62.54400330456679 } }, { "threshold": 0.64, "public": { "n": 231, "escalation": 0.4588744588744589, "tiers": { "easy": 0.9791666666666666, "standard": 0.6388888888888888, "hard": 0.3783783783783784 }, "accuracy": 0.5844155844155844, "ece_hard": 0.37540302008787796, "calibration": 45.18578318945814, "p50_s": 2.1577178329229354, "p95_s": 6.294179813563824, "cost_usd_per_1000": 0.009711608946608946, "axes": { "intelligence": 59.6507966299633, "calibration": 45.18578318945814, "speed": 68.67066203317347, "cost": 70.38126440425822 }, "jevbench_score": 60.07746378173476 }, "heldout": { "n": 303, "escalation": 0.5148514851485149, "tiers": { "easy": 0.9166666666666666, "standard": 0.75, "judge": 0.7671232876712328, "hard": 0.42201834862385323 }, "accuracy": 0.6534653465346535, "ece_hard": 0.33423061710289853, "calibration": 48.90138167126925, "p50_s": 3.601574608683586, "p95_s": 6.347663775086403, "cost_usd_per_1000": 0.010544599510512849, "axes": { "intelligence": 67.97333584684347, "calibration": 48.90138167126925, "speed": 66.408936692801, "cost": 69.3090973103871 }, "jevbench_score": 62.54161917296283 } }, { "threshold": 0.65, "public": { "n": 231, "escalation": 0.48917748917748916, "tiers": { "easy": 0.9791666666666666, "standard": 0.6527777777777778, "hard": 0.3783783783783784 }, "accuracy": 0.5887445887445888, "ece_hard": 0.3745425249273713, "calibration": 45.271832705508814, "p50_s": 2.2619401454925536, "p95_s": 6.294179813563824, "cost_usd_per_1000": 0.010162546411141915, "axes": { "intelligence": 60.19092008675343, "calibration": 45.271832705508814, "speed": 68.46579745597985, "cost": 69.78992374066155 }, "jevbench_score": 60.06982451435983 }, "heldout": { "n": 303, "escalation": 0.528052805280528, "tiers": { "easy": 0.9583333333333334, "standard": 0.75, "judge": 0.7808219178082192, "hard": 0.43119266055045874 }, "accuracy": 0.6666666666666666, "ece_hard": 0.3226590773605425, "calibration": 50.05853564550486, "p50_s": 3.601574608683586, "p95_s": 6.347663775086403, "cost_usd_per_1000": 0.010741047514863846, "axes": { "intelligence": 69.21546018181057, "calibration": 50.05853564550486, "speed": 66.408936692801, "cost": 69.06860086708004 }, "jevbench_score": 63.138914223935 } }, { "threshold": 0.66, "public": { "n": 231, "escalation": 0.5151515151515151, "tiers": { "easy": 0.9791666666666666, "standard": 0.6666666666666666, "hard": 0.3783783783783784 }, "accuracy": 0.5930735930735931, "ece_hard": 0.37860938338325895, "calibration": 43.9495920718074, "p50_s": 2.278751590847969, "p95_s": 6.294179813563824, "cost_usd_per_1000": 0.010549064237884462, "axes": { "intelligence": 60.73104354354354, "calibration": 43.9495920718074, "speed": 68.43363869367877, "cost": 69.30358189159753 }, "jevbench_score": 59.648252735013784 }, "heldout": { "n": 303, "escalation": 0.5478547854785478, "tiers": { "easy": 0.9583333333333334, "standard": 0.75, "judge": 0.7876712328767124, "hard": 0.43119266055045874 }, "accuracy": 0.66996699669967, "ece_hard": 0.314047865585559, "calibration": 50.91965682300321, "p50_s": 3.6074275016784667, "p95_s": 6.347663775086403, "cost_usd_per_1000": 0.011035719521390342, "axes": { "intelligence": 69.40724100372839, "calibration": 50.91965682300321, "speed": 66.401884732903, "cost": 68.71598037625057 }, "jevbench_score": 63.36976559305422 } }, { "threshold": 0.67, "public": { "n": 231, "escalation": 0.5194805194805194, "tiers": { "easy": 0.9791666666666666, "standard": 0.6666666666666666, "hard": 0.38738738738738737 }, "accuracy": 0.5974025974025974, "ece_hard": 0.36897048698497237, "calibration": 44.91348171163606, "p50_s": 2.278751590847969, "p95_s": 6.294179813563824, "cost_usd_per_1000": 0.010613483875674887, "axes": { "intelligence": 61.10641891891892, "calibration": 44.91348171163606, "speed": 68.43363869367877, "cost": 69.22426106647993 }, "jevbench_score": 60.047907671667474 }, "heldout": { "n": 303, "escalation": 0.570957095709571, "tiers": { "easy": 0.9583333333333334, "standard": 0.75, "judge": 0.7876712328767124, "hard": 0.43119266055045874 }, "accuracy": 0.66996699669967, "ece_hard": 0.30234978797556067, "calibration": 52.08946458400304, "p50_s": 3.6074275016784667, "p95_s": 6.347663775086403, "cost_usd_per_1000": 0.011379503529004586, "axes": { "intelligence": 69.40724100372839, "calibration": 52.08946458400304, "speed": 66.401884732903, "cost": 68.31630055470595 }, "jevbench_score": 63.63775574776625 } }, { "threshold": 0.68, "public": { "n": 231, "escalation": 0.5238095238095238, "tiers": { "easy": 0.9791666666666666, "standard": 0.6666666666666666, "hard": 0.38738738738738737 }, "accuracy": 0.5974025974025974, "ece_hard": 0.36897048698497237, "calibration": 44.91348171163606, "p50_s": 2.278751590847969, "p95_s": 6.294179813563824, "cost_usd_per_1000": 0.010677903513465311, "axes": { "intelligence": 61.10641891891892, "calibration": 44.91348171163606, "speed": 68.43363869367877, "cost": 69.14542023357794 }, "jevbench_score": 60.030802937397375 }, "heldout": { "n": 303, "escalation": 0.5907590759075908, "tiers": { "easy": 0.9583333333333334, "standard": 0.75, "judge": 0.7945205479452054, "hard": 0.43119266055045874 }, "accuracy": 0.6732673267326733, "ece_hard": 0.3012427970078013, "calibration": 52.84134688972138, "p50_s": 3.6765943259000777, "p95_s": 6.354333575069905, "cost_usd_per_1000": 0.011674175535531081, "axes": { "intelligence": 69.59902182564619, "calibration": 52.84134688972138, "speed": 66.31484277710793, "cost": 67.98321342481537 }, "jevbench_score": 63.81126614940574 } }, { "threshold": 0.69, "public": { "n": 231, "escalation": 0.5238095238095238, "tiers": { "easy": 0.9791666666666666, "standard": 0.6666666666666666, "hard": 0.38738738738738737 }, "accuracy": 0.5974025974025974, "ece_hard": 0.36897048698497237, "calibration": 44.91348171163606, "p50_s": 2.278751590847969, "p95_s": 6.294179813563824, "cost_usd_per_1000": 0.010677903513465311, "axes": { "intelligence": 61.10641891891892, "calibration": 44.91348171163606, "speed": 68.43363869367877, "cost": 69.14542023357794 }, "jevbench_score": 60.030802937397375 }, "heldout": { "n": 303, "escalation": 0.594059405940594, "tiers": { "easy": 0.9583333333333334, "standard": 0.75, "judge": 0.8013698630136986, "hard": 0.43119266055045874 }, "accuracy": 0.6765676567656765, "ece_hard": 0.3012427970078013, "calibration": 52.84134688972138, "p50_s": 3.770245724916458, "p95_s": 6.354333575069905, "cost_usd_per_1000": 0.01172328753661883, "axes": { "intelligence": 69.79080264756399, "calibration": 52.84134688972138, "speed": 66.20560334152425, "cost": 67.92851748878141 }, "jevbench_score": 63.816023509236025 } }, { "threshold": 0.7, "public": { "n": 231, "escalation": 0.5367965367965368, "tiers": { "easy": 0.9791666666666666, "standard": 0.6805555555555556, "hard": 0.3963963963963964 }, "accuracy": 0.6060606060606061, "ece_hard": 0.3623994737362539, "calibration": 45.570583036507905, "p50_s": 2.278751590847969, "p95_s": 6.306573109328747, "cost_usd_per_1000": 0.010871162426836585, "axes": { "intelligence": 62.02191775108443, "calibration": 45.570583036507905, "speed": 68.42509580464386, "cost": 68.91172046148834 }, "jevbench_score": 60.42053558440923 }, "heldout": { "n": 303, "escalation": 0.6105610561056105, "tiers": { "easy": 0.9583333333333334, "standard": 0.75, "judge": 0.815068493150685, "hard": 0.44036697247706424 }, "accuracy": 0.6864686468646864, "ece_hard": 0.29900027540594065, "calibration": 53.06559904990745, "p50_s": 3.7860111236572265, "p95_s": 6.354333575069905, "cost_usd_per_1000": 0.011968847542057575, "axes": { "intelligence": 70.44959364919778, "calibration": 53.06559904990745, "speed": 66.18748104698007, "cost": 67.65842994951356 }, "jevbench_score": 63.96572595157775 } }, { "threshold": 0.71, "public": { "n": 231, "escalation": 0.5454545454545454, "tiers": { "easy": 0.9791666666666666, "standard": 0.6805555555555556, "hard": 0.3963963963963964 }, "accuracy": 0.6060606060606061, "ece_hard": 0.3623994737362539, "calibration": 45.570583036507905, "p50_s": 2.278751590847969, "p95_s": 6.306573109328747, "cost_usd_per_1000": 0.011000001702417432, "axes": { "intelligence": 62.02191775108443, "calibration": 45.570583036507905, "speed": 68.42509580464386, "cost": 68.75821742884295 }, "jevbench_score": 60.38686027896508 }, "heldout": { "n": 303, "escalation": 0.6303630363036303, "tiers": { "easy": 0.9583333333333334, "standard": 0.75, "judge": 0.8287671232876712, "hard": 0.44954128440366975 }, "accuracy": 0.6963696369636964, "ece_hard": 0.2936481179759043, "calibration": 53.60081479291109, "p50_s": 3.7860111236572265, "p95_s": 6.354333575069905, "cost_usd_per_1000": 0.012263519548584071, "axes": { "intelligence": 71.10838465083157, "calibration": 53.60081479291109, "speed": 66.18748104698007, "cost": 67.34154616898917 }, "jevbench_score": 64.20040759236846 } }, { "threshold": 0.72, "public": { "n": 231, "escalation": 0.5497835497835498, "tiers": { "easy": 0.9791666666666666, "standard": 0.6805555555555556, "hard": 0.3963963963963964 }, "accuracy": 0.6060606060606061, "ece_hard": 0.36448008532948317, "calibration": 45.362521877184975, "p50_s": 2.2860569298267364, "p95_s": 6.306573109328747, "cost_usd_per_1000": 0.011064421340207858, "axes": { "intelligence": 62.02191775108443, "calibration": 45.362521877184975, "speed": 68.41119523860212, "cost": 68.68213883120552 }, "jevbench_score": 60.29806042756017 }, "heldout": { "n": 303, "escalation": 0.6501650165016502, "tiers": { "easy": 0.9583333333333334, "standard": 0.75, "judge": 0.8287671232876712, "hard": 0.45871559633027525 }, "accuracy": 0.6996699669966997, "ece_hard": 0.286780234977849, "calibration": 54.287603092716616, "p50_s": 3.7948069512844085, "p95_s": 6.354333575069905, "cost_usd_per_1000": 0.012558191555110567, "axes": { "intelligence": 71.38361400862973, "calibration": 54.287603092716616, "speed": 66.17740302880658, "cost": 67.03218690279971 }, "jevbench_score": 64.3906896892996 } }, { "threshold": 0.73, "public": { "n": 231, "escalation": 0.5584415584415584, "tiers": { "easy": 1.0, "standard": 0.6805555555555556, "hard": 0.3963963963963964 }, "accuracy": 0.6103896103896104, "ece_hard": 0.36448008532948317, "calibration": 45.362521877184975, "p50_s": 2.303595685958862, "p95_s": 6.306573109328747, "cost_usd_per_1000": 0.011193260615788705, "axes": { "intelligence": 62.42701034367702, "calibration": 45.362521877184975, "speed": 68.37800308781343, "cost": 68.53130152969055 }, "jevbench_score": 60.35576796415074 }, "heldout": { "n": 303, "escalation": 0.6666666666666666, "tiers": { "easy": 1.0, "standard": 0.75, "judge": 0.8424657534246576, "hard": 0.45871559633027525 }, "accuracy": 0.7095709570957096, "ece_hard": 0.286780234977849, "calibration": 54.287603092716616, "p50_s": 3.7948069512844085, "p95_s": 6.354333575069905, "cost_usd_per_1000": 0.012803751560549312, "axes": { "intelligence": 72.35050898579867, "calibration": 54.287603092716616, "speed": 66.17740302880658, "cost": 66.77988284026382 }, "jevbench_score": 64.54675413000612 } }, { "threshold": 0.74, "public": { "n": 231, "escalation": 0.5627705627705628, "tiers": { "easy": 1.0, "standard": 0.6944444444444444, "hard": 0.3963963963963964 }, "accuracy": 0.6147186147186147, "ece_hard": 0.36448008532948317, "calibration": 45.362521877184975, "p50_s": 2.303595685958862, "p95_s": 6.306573109328747, "cost_usd_per_1000": 0.01125768025357913, "axes": { "intelligence": 62.96713380046714, "calibration": 45.362521877184975, "speed": 68.37800308781343, "cost": 68.45653271667231 }, "jevbench_score": 60.469392700202526 }, "heldout": { "n": 303, "escalation": 0.6897689768976898, "tiers": { "easy": 1.0, "standard": 0.75, "judge": 0.8561643835616438, "hard": 0.45871559633027525 }, "accuracy": 0.7161716171617162, "ece_hard": 0.28937399147374365, "calibration": 54.02822744312715, "p50_s": 3.8325090944766997, "p95_s": 6.439382283389567, "cost_usd_per_1000": 0.013147535568163558, "axes": { "intelligence": 72.73407062963429, "calibration": 54.02822744312715, "speed": 66.07672597511777, "cost": 66.43466936834726 }, "jevbench_score": 64.44666923595815 } }, { "threshold": 0.75, "public": { "n": 231, "escalation": 0.5844155844155844, "tiers": { "easy": 1.0, "standard": 0.6944444444444444, "hard": 0.3963963963963964 }, "accuracy": 0.6147186147186147, "ece_hard": 0.3663017153563263, "calibration": 45.18035887450067, "p50_s": 2.329591715335846, "p95_s": 6.306573109328747, "cost_usd_per_1000": 0.011579778442531252, "axes": { "intelligence": 62.96713380046714, "calibration": 45.18035887450067, "speed": 68.32926752152372, "cost": 68.08899249832639 }, "jevbench_score": 60.316595269875044 }, "heldout": { "n": 303, "escalation": 0.7029702970297029, "tiers": { "easy": 1.0, "standard": 0.75, "judge": 0.8561643835616438, "hard": 0.45871559633027525 }, "accuracy": 0.7161716171617162, "ece_hard": 0.2916962453815573, "calibration": 53.796002052345784, "p50_s": 3.8417299330234527, "p95_s": 6.439382283389567, "cost_usd_per_1000": 0.013343983572514553, "axes": { "intelligence": 72.73407062963429, "calibration": 53.796002052345784, "speed": 66.06628960165551, "cost": 66.24143504069251 }, "jevbench_score": 64.32790179954836 } }, { "threshold": 0.76, "public": { "n": 231, "escalation": 0.5974025974025974, "tiers": { "easy": 1.0, "standard": 0.6944444444444444, "hard": 0.38738738738738737 }, "accuracy": 0.6103896103896104, "ece_hard": 0.3748387374340414, "calibration": 44.32665666672916, "p50_s": 2.348108598589897, "p95_s": 6.33516194075346, "cost_usd_per_1000": 0.011773037355902525, "axes": { "intelligence": 62.591758425091754, "calibration": 44.32665666672916, "speed": 68.27524100216114, "cost": 67.8733443388256 }, "jevbench_score": 59.880601584115546 }, "heldout": { "n": 303, "escalation": 0.7227722772277227, "tiers": { "easy": 1.0, "standard": 0.75, "judge": 0.8561643835616438, "hard": 0.45871559633027525 }, "accuracy": 0.7161716171617162, "ece_hard": 0.2951042370515884, "calibration": 53.45520288534267, "p50_s": 3.8417299330234527, "p95_s": 6.439382283389567, "cost_usd_per_1000": 0.013638655579041049, "axes": { "intelligence": 72.73407062963429, "calibration": 53.45520288534267, "speed": 66.06628960165551, "cost": 65.95685313535775 }, "jevbench_score": 64.15668710482123 } }, { "threshold": 0.77, "public": { "n": 231, "escalation": 0.6190476190476191, "tiers": { "easy": 1.0, "standard": 0.6944444444444444, "hard": 0.40540540540540543 }, "accuracy": 0.6190476190476191, "ece_hard": 0.3620531703444751, "calibration": 45.605213375685786, "p50_s": 2.354624429345131, "p95_s": 6.33516194075346, "cost_usd_per_1000": 0.012095135544854646, "axes": { "intelligence": 63.34250917584251, "calibration": 45.605213375685786, "speed": 68.26320633578305, "cost": 67.52167780983797 }, "jevbench_score": 60.40667286512095 }, "heldout": { "n": 303, "escalation": 0.735973597359736, "tiers": { "easy": 1.0, "standard": 0.75, "judge": 0.8561643835616438, "hard": 0.45871559633027525 }, "accuracy": 0.7161716171617162, "ece_hard": 0.2951042370515884, "calibration": 53.45520288534267, "p50_s": 3.901866951584816, "p95_s": 6.439382283389567, "cost_usd_per_1000": 0.013835103583392048, "axes": { "intelligence": 72.73407062963429, "calibration": 53.45520288534267, "speed": 65.9988333515933, "cost": 65.77052754846895 }, "jevbench_score": 64.094957669654 } }, { "threshold": 0.78, "public": { "n": 231, "escalation": 0.6406926406926406, "tiers": { "easy": 1.0, "standard": 0.6944444444444444, "hard": 0.40540540540540543 }, "accuracy": 0.6190476190476191, "ece_hard": 0.3620848134082179, "calibration": 45.602049069311505, "p50_s": 2.373017871379852, "p95_s": 6.363939964771271, "cost_usd_per_1000": 0.012417233733806767, "axes": { "intelligence": 63.34250917584251, "calibration": 45.602049069311505, "speed": 68.20972917056687, "cost": 67.17925431790428 }, "jevbench_score": 60.31707615927486 }, "heldout": { "n": 303, "escalation": 0.7557755775577558, "tiers": { "easy": 1.0, "standard": 0.75, "judge": 0.863013698630137, "hard": 0.45871559633027525 }, "accuracy": 0.7194719471947195, "ece_hard": 0.2990730517728271, "calibration": 53.058321413218806, "p50_s": 3.9898528546094894, "p95_s": 6.439382283389567, "cost_usd_per_1000": 0.014129775589918544, "axes": { "intelligence": 72.9258514515521, "calibration": 53.058321413218806, "speed": 65.90198912369321, "cost": 65.49594206775765 }, "jevbench_score": 63.9273913487069 } }, { "threshold": 0.79, "public": { "n": 231, "escalation": 0.645021645021645, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "hard": 0.40540540540540543 }, "accuracy": 0.6233766233766234, "ece_hard": 0.3620848134082179, "calibration": 45.602049069311505, "p50_s": 2.373017871379852, "p95_s": 6.363939964771271, "cost_usd_per_1000": 0.012481653371597193, "axes": { "intelligence": 63.882632632632635, "calibration": 45.602049069311505, "speed": 68.20972917056687, "cost": 67.11183647181663 }, "jevbench_score": 60.43007771956622 }, "heldout": { "n": 303, "escalation": 0.7755775577557755, "tiers": { "easy": 1.0, "standard": 0.75, "judge": 0.8698630136986302, "hard": 0.44954128440366975 }, "accuracy": 0.7194719471947195, "ece_hard": 0.29634878147891763, "calibration": 53.33074844260975, "p50_s": 3.9898528546094894, "p95_s": 6.439382283389567, "cost_usd_per_1000": 0.014424447596445037, "axes": { "intelligence": 72.84240291567174, "calibration": 53.33074844260975, "speed": 65.90198912369321, "cost": 65.22702429229221 }, "jevbench_score": 63.925187114292015 } }, { "threshold": 0.8, "public": { "n": 231, "escalation": 0.658008658008658, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "hard": 0.40540540540540543 }, "accuracy": 0.6233766233766234, "ece_hard": 0.3620848134082179, "calibration": 45.602049069311505, "p50_s": 2.3767098128795623, "p95_s": 6.363939964771271, "cost_usd_per_1000": 0.012674912284968465, "axes": { "intelligence": 63.882632632632635, "calibration": 45.602049069311505, "speed": 68.20297766711094, "cost": 66.9116511238293 }, "jevbench_score": 60.38346927428058 }, "heldout": { "n": 303, "escalation": 0.7821782178217822, "tiers": { "easy": 1.0, "standard": 0.75, "judge": 0.8698630136986302, "hard": 0.44036697247706424 }, "accuracy": 0.7161716171617162, "ece_hard": 0.3047291753619841, "calibration": 52.492709054303106, "p50_s": 3.9898528546094894, "p95_s": 6.439382283389567, "cost_usd_per_1000": 0.014522671598620536, "axes": { "intelligence": 72.56717355787357, "calibration": 52.492709054303106, "speed": 65.90198912369321, "cost": 65.13860449707128 }, "jevbench_score": 63.590763878139526 } }, { "threshold": 0.81, "public": { "n": 231, "escalation": 0.6623376623376623, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "hard": 0.4144144144144144 }, "accuracy": 0.6277056277056277, "ece_hard": 0.3527412921006078, "calibration": 46.53640120007252, "p50_s": 2.3767098128795623, "p95_s": 6.363939964771271, "cost_usd_per_1000": 0.01273933192275889, "axes": { "intelligence": 64.25800800800802, "calibration": 46.53640120007252, "speed": 68.20297766711094, "cost": 66.8456004014651 }, "jevbench_score": 60.76437767728607 }, "heldout": { "n": 303, "escalation": 0.7887788778877888, "tiers": { "easy": 1.0, "standard": 0.75, "judge": 0.8698630136986302, "hard": 0.44036697247706424 }, "accuracy": 0.7161716171617162, "ece_hard": 0.3055715283078543, "calibration": 51.97922894770437, "p50_s": 3.9908935219049453, "p95_s": 6.439382283389567, "cost_usd_per_1000": 0.014620895600796035, "axes": { "intelligence": 72.56717355787357, "calibration": 51.97922894770437, "speed": 65.90085650765668, "cost": 65.05078071757433 }, "jevbench_score": 63.41301513076184 } }, { "threshold": 0.82, "public": { "n": 231, "escalation": 0.670995670995671, "tiers": { "easy": 1.0, "standard": 0.7222222222222222, "hard": 0.4144144144144144 }, "accuracy": 0.6320346320346321, "ece_hard": 0.3527412921006078, "calibration": 46.53640120007252, "p50_s": 2.3767098128795623, "p95_s": 6.363939964771271, "cost_usd_per_1000": 0.012868171198339738, "axes": { "intelligence": 64.79813146479813, "calibration": 46.53640120007252, "speed": 68.20297766711094, "cost": 66.71449509614288 }, "jevbench_score": 60.8617876259806 }, "heldout": { "n": 303, "escalation": 0.801980198019802, "tiers": { "easy": 1.0, "standard": 0.75, "judge": 0.8767123287671232, "hard": 0.44036697247706424 }, "accuracy": 0.7194719471947195, "ece_hard": 0.3055715283078543, "calibration": 51.97922894770437, "p50_s": 4.0156786233186725, "p95_s": 6.439382283389567, "cost_usd_per_1000": 0.014817343605147032, "axes": { "intelligence": 72.75895437979139, "calibration": 51.97922894770437, "speed": 65.87396842880045, "cost": 64.8768894393481 }, "jevbench_score": 63.4059526679077 } }, { "threshold": 0.83, "public": { "n": 231, "escalation": 0.696969696969697, "tiers": { "easy": 1.0, "standard": 0.7361111111111112, "hard": 0.43243243243243246 }, "accuracy": 0.645021645021645, "ece_hard": 0.344508459972221, "calibration": 47.35968441291119, "p50_s": 2.452158460021019, "p95_s": 6.363939964771271, "cost_usd_per_1000": 0.013254689025082284, "axes": { "intelligence": 66.08900567233901, "calibration": 47.35968441291119, "speed": 68.06725394855856, "cost": 66.32891371037113 }, "jevbench_score": 61.311901185297984 }, "heldout": { "n": 303, "escalation": 0.8250825082508251, "tiers": { "easy": 1.0, "standard": 0.75, "judge": 0.8767123287671232, "hard": 0.44954128440366975 }, "accuracy": 0.7227722772277227, "ece_hard": 0.2979171328644482, "calibration": 52.74466849204498, "p50_s": 4.0156786233186725, "p95_s": 6.439382283389567, "cost_usd_per_1000": 0.015161127612761276, "axes": { "intelligence": 73.03418373758956, "calibration": 52.74466849204498, "speed": 65.87396842880045, "cost": 64.5780549021715 }, "jevbench_score": 63.62471991284887 } }, { "threshold": 0.84, "public": { "n": 231, "escalation": 0.7056277056277056, "tiers": { "easy": 1.0, "standard": 0.7361111111111112, "hard": 0.43243243243243246 }, "accuracy": 0.645021645021645, "ece_hard": 0.3436231911774825, "calibration": 47.44821129238504, "p50_s": 2.454006576538086, "p95_s": 6.363939964771271, "cost_usd_per_1000": 0.013383528300663132, "axes": { "intelligence": 66.08900567233901, "calibration": 47.44821129238504, "speed": 68.06398203725, "cost": 66.20288135151287 }, "jevbench_score": 61.31063681387088 }, "heldout": { "n": 303, "escalation": 0.834983498349835, "tiers": { "easy": 1.0, "standard": 0.75, "judge": 0.8767123287671232, "hard": 0.44954128440366975 }, "accuracy": 0.7227722772277227, "ece_hard": 0.29816734076807394, "calibration": 52.719647701682405, "p50_s": 4.045683214068413, "p95_s": 6.439382283389567, "cost_usd_per_1000": 0.015308463616024524, "axes": { "intelligence": 73.03418373758956, "calibration": 52.719647701682405, "speed": 65.84163918063668, "cost": 64.45205180987716 }, "jevbench_score": 63.578315228125355 } }, { "threshold": 0.85, "public": { "n": 231, "escalation": 0.70995670995671, "tiers": { "easy": 1.0, "standard": 0.7361111111111112, "hard": 0.42342342342342343 }, "accuracy": 0.6406926406926406, "ece_hard": 0.353057868549805, "calibration": 46.5047435551528, "p50_s": 2.455227532982826, "p95_s": 6.363939964771271, "cost_usd_per_1000": 0.013447947938453558, "axes": { "intelligence": 65.71363029696364, "calibration": 46.5047435551528, "speed": 68.0618218035121, "cost": 66.1403194259766 }, "jevbench_score": 60.90187210255751 }, "heldout": { "n": 303, "escalation": 0.8481848184818482, "tiers": { "easy": 1.0, "standard": 0.75, "judge": 0.8767123287671232, "hard": 0.45871559633027525 }, "accuracy": 0.7260726072607261, "ece_hard": 0.29130706688478036, "calibration": 53.40567509001176, "p50_s": 4.045683214068413, "p95_s": 6.439382283389567, "cost_usd_per_1000": 0.015504911620375521, "axes": { "intelligence": 73.30941309538773, "calibration": 53.40567509001176, "speed": 65.84163918063668, "cost": 64.28592114831261 }, "jevbench_score": 63.802972789940426 } }, { "threshold": 0.86, "public": { "n": 231, "escalation": 0.7229437229437229, "tiers": { "easy": 1.0, "standard": 0.7361111111111112, "hard": 0.43243243243243246 }, "accuracy": 0.645021645021645, "ece_hard": 0.34592149106346004, "calibration": 47.2183813037873, "p50_s": 2.455227532982826, "p95_s": 6.363939964771271, "cost_usd_per_1000": 0.013641206851824829, "axes": { "intelligence": 66.08900567233901, "calibration": 47.2183813037873, "speed": 68.0618218035121, "cost": 65.95441616479488 }, "jevbench_score": 61.17823461798865 }, "heldout": { "n": 303, "escalation": 0.8547854785478548, "tiers": { "easy": 1.0, "standard": 0.75, "judge": 0.8767123287671232, "hard": 0.45871559633027525 }, "accuracy": 0.7260726072607261, "ece_hard": 0.292412594561354, "calibration": 52.6926097386218, "p50_s": 4.045683214068413, "p95_s": 6.439382283389567, "cost_usd_per_1000": 0.01560313562255102, "axes": { "intelligence": 73.30941309538773, "calibration": 52.6926097386218, "speed": 65.84163918063668, "cost": 64.20364349796357 }, "jevbench_score": 63.56856973149936 } }, { "threshold": 0.87, "public": { "n": 231, "escalation": 0.7532467532467533, "tiers": { "easy": 1.0, "standard": 0.7361111111111112, "hard": 0.45045045045045046 }, "accuracy": 0.6536796536796536, "ece_hard": 0.3296631188227249, "calibration": 48.844218527860804, "p50_s": 2.5032336354255675, "p95_s": 6.363939964771271, "cost_usd_per_1000": 0.0140921443163578, "axes": { "intelligence": 66.83975642308977, "calibration": 48.844218527860804, "speed": 67.97772540290194, "cost": 65.53068753694161 }, "jevbench_score": 61.75396457473737 }, "heldout": { "n": 303, "escalation": 0.8646864686468647, "tiers": { "easy": 1.0, "standard": 0.75, "judge": 0.8767123287671232, "hard": 0.45871559633027525 }, "accuracy": 0.7260726072607261, "ece_hard": 0.292412594561354, "calibration": 52.6926097386218, "p50_s": 4.045683214068413, "p95_s": 6.439382283389567, "cost_usd_per_1000": 0.015750471625814267, "axes": { "intelligence": 73.30941309538773, "calibration": 52.6926097386218, "speed": 65.84163918063668, "cost": 64.08119312018945 }, "jevbench_score": 63.53823824124748 } }, { "threshold": 0.88, "public": { "n": 231, "escalation": 0.7792207792207793, "tiers": { "easy": 1.0, "standard": 0.75, "hard": 0.45045045045045046 }, "accuracy": 0.658008658008658, "ece_hard": 0.32237931463632497, "calibration": 49.572598946500804, "p50_s": 2.5032336354255675, "p95_s": 6.363939964771271, "cost_usd_per_1000": 0.014478662143100346, "axes": { "intelligence": 67.37987987987988, "calibration": 49.572598946500804, "speed": 67.97772540290194, "cost": 65.17814697851452 }, "jevbench_score": 62.02405265355474 }, "heldout": { "n": 303, "escalation": 0.8712871287128713, "tiers": { "easy": 1.0, "standard": 0.75, "judge": 0.8767123287671232, "hard": 0.45871559633027525 }, "accuracy": 0.7260726072607261, "ece_hard": 0.292412594561354, "calibration": 52.6926097386218, "p50_s": 4.045683214068413, "p95_s": 6.439382283389567, "cost_usd_per_1000": 0.015848695627989764, "axes": { "intelligence": 73.30941309538773, "calibration": 52.6926097386218, "speed": 65.84163918063668, "cost": 64.00019425236528 }, "jevbench_score": 63.51815057444979 } }, { "threshold": 0.89, "public": { "n": 231, "escalation": 0.7878787878787878, "tiers": { "easy": 1.0, "standard": 0.75, "hard": 0.45045045045045046 }, "accuracy": 0.658008658008658, "ece_hard": 0.32237931463632497, "calibration": 49.572598946500804, "p50_s": 2.5032336354255675, "p95_s": 6.363939964771271, "cost_usd_per_1000": 0.014607501418681194, "axes": { "intelligence": 67.37987987987988, "calibration": 49.572598946500804, "speed": 67.97772540290194, "cost": 65.0627218851698 }, "jevbench_score": 61.99657453530272 }, "heldout": { "n": 303, "escalation": 0.8877887788778878, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.45871559633027525 }, "accuracy": 0.7293729372937293, "ece_hard": 0.292412594561354, "calibration": 52.6926097386218, "p50_s": 4.048538202047348, "p95_s": 6.439382283389567, "cost_usd_per_1000": 0.01609425563342851, "axes": { "intelligence": 74.47607976205437, "calibration": 52.6926097386218, "speed": 65.83857549973933, "cost": 63.799873144877935 }, "jevbench_score": 63.71866813949218 } }, { "threshold": 0.9, "public": { "n": 231, "escalation": 0.8008658008658008, "tiers": { "easy": 1.0, "standard": 0.75, "hard": 0.4594594594594595 }, "accuracy": 0.6623376623376623, "ece_hard": 0.3137055500866056, "calibration": 50.439975401472736, "p50_s": 2.5121211051940917, "p95_s": 6.363939964771271, "cost_usd_per_1000": 0.014800760332052467, "axes": { "intelligence": 67.75525525525525, "calibration": 50.439975401472736, "speed": 67.96233353804574, "cost": 64.89147921477343 }, "jevbench_score": 62.3079484698159 }, "heldout": { "n": 303, "escalation": 0.9042904290429042, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.2841529622766471, "calibration": 53.51857296709248, "p50_s": 4.074841046333313, "p95_s": 6.448686248064041, "cost_usd_per_1000": 0.016339815638867255, "axes": { "intelligence": 74.75130911985254, "calibration": 53.51857296709248, "speed": 65.80418080342096, "cost": 63.60258543681494 }, "jevbench_score": 63.96801822249045 } }, { "threshold": 0.91, "public": { "n": 231, "escalation": 0.8138528138528138, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.4594594594594595 }, "accuracy": 0.6666666666666666, "ece_hard": 0.3137055500866056, "calibration": 50.439975401472736, "p50_s": 2.5121211051940917, "p95_s": 6.363939964771271, "cost_usd_per_1000": 0.01499401924542374, "axes": { "intelligence": 68.29537871204539, "calibration": 50.439975401472736, "speed": 67.96233353804574, "cost": 64.72245808165604 }, "jevbench_score": 62.391060436713175 }, "heldout": { "n": 303, "escalation": 0.9108910891089109, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.2841529622766471, "calibration": 53.51857296709248, "p50_s": 4.074841046333313, "p95_s": 6.448686248064041, "cost_usd_per_1000": 0.016438039641042755, "axes": { "intelligence": 74.75130911985254, "calibration": 53.51857296709248, "speed": 65.80418080342096, "cost": 63.524499297021315 }, "jevbench_score": 63.94837547956297 } }, { "threshold": 0.92, "public": { "n": 231, "escalation": 0.8225108225108225, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.4594594594594595 }, "accuracy": 0.6666666666666666, "ece_hard": 0.3140483466940957, "calibration": 50.21544351879295, "p50_s": 2.5121211051940917, "p95_s": 6.363939964771271, "cost_usd_per_1000": 0.015122858521004588, "axes": { "intelligence": 68.29537871204539, "calibration": 50.21544351879295, "speed": 67.96233353804574, "cost": 64.61098332345134 }, "jevbench_score": 62.294659073623656 }, "heldout": { "n": 303, "escalation": 0.9207920792079208, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.2831849941810952, "calibration": 53.61536977664767, "p50_s": 4.074841046333313, "p95_s": 6.448686248064041, "cost_usd_per_1000": 0.016585375644306004, "axes": { "intelligence": 74.75130911985254, "calibration": 53.61536977664767, "speed": 65.80418080342096, "cost": 63.40824062924588 }, "jevbench_score": 63.947979141555464 } }, { "threshold": 0.93, "public": { "n": 231, "escalation": 0.8398268398268398, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.4594594594594595 }, "accuracy": 0.6666666666666666, "ece_hard": 0.3140483466940957, "calibration": 50.21544351879295, "p50_s": 2.5121211051940917, "p95_s": 6.363939964771271, "cost_usd_per_1000": 0.015380537072166285, "axes": { "intelligence": 68.29537871204539, "calibration": 50.21544351879295, "speed": 67.96233353804574, "cost": 64.39085497490802 }, "jevbench_score": 62.24153197989953 }, "heldout": { "n": 303, "escalation": 0.9240924092409241, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.2831849941810952, "calibration": 53.61536977664767, "p50_s": 4.074841046333313, "p95_s": 6.448686248064041, "cost_usd_per_1000": 0.016634487645393752, "axes": { "intelligence": 74.75130911985254, "calibration": 53.61536977664767, "speed": 65.80418080342096, "cost": 63.36971713547787 }, "jevbench_score": 63.938264075303806 } }, { "threshold": 0.94, "public": { "n": 231, "escalation": 0.8528138528138528, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.4594594594594595 }, "accuracy": 0.6666666666666666, "ece_hard": 0.3140483466940957, "calibration": 50.21544351879295, "p50_s": 2.5972673535346984, "p95_s": 6.380025783181191, "cost_usd_per_1000": 0.015573795985537557, "axes": { "intelligence": 68.29537871204539, "calibration": 50.21544351879295, "speed": 67.80660909517864, "cost": 64.22816556391476 }, "jevbench_score": 62.16651777969932 }, "heldout": { "n": 303, "escalation": 0.9405940594059405, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28023325293453055, "calibration": 55.17142992768658, "p50_s": 4.078099781274796, "p95_s": 6.448686248064041, "cost_usd_per_1000": 0.016880047650832498, "axes": { "intelligence": 74.75130911985254, "calibration": 55.17142992768658, "speed": 65.80070904836137, "cost": 63.178789952049534 }, "jevbench_score": 64.34780395963587 } }, { "threshold": 0.95, "public": { "n": 231, "escalation": 0.8614718614718615, "tiers": { "easy": 1.0, "standard": 0.75, "hard": 0.4594594594594595 }, "accuracy": 0.6623376623376623, "ece_hard": 0.3140483466940957, "calibration": 50.21544351879295, "p50_s": 2.612508574128151, "p95_s": 6.380025783181191, "cost_usd_per_1000": 0.015702635261118408, "axes": { "intelligence": 67.75525525525525, "calibration": 50.21544351879295, "speed": 67.78119841734227, "cost": 64.12082370799975 }, "jevbench_score": 62.01148850389215 }, "heldout": { "n": 303, "escalation": 0.9471947194719472, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28023325293453055, "calibration": 55.17142992768658, "p50_s": 4.078099781274796, "p95_s": 6.448686248064041, "cost_usd_per_1000": 0.016978271653007998, "axes": { "intelligence": 74.75130911985254, "calibration": 55.17142992768658, "speed": 65.80070904836137, "cost": 63.10319565909655 }, "jevbench_score": 64.32854705846903 } }, { "threshold": 0.96, "public": { "n": 231, "escalation": 0.8874458874458875, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 2.612508574128151, "p95_s": 6.380025783181191, "cost_usd_per_1000": 0.016089153087860953, "axes": { "intelligence": 68.67075408742076, "calibration": 50.471910537865476, "speed": 67.78119841734227, "cost": 63.8040044799172 }, "jevbench_score": 62.22210297988559 }, "heldout": { "n": 303, "escalation": 0.9537953795379538, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28023325293453055, "calibration": 55.17142992768658, "p50_s": 4.078099781274796, "p95_s": 6.448686248064041, "cost_usd_per_1000": 0.017076495655183495, "axes": { "intelligence": 74.75130911985254, "calibration": 55.17142992768658, "speed": 65.80070904836137, "cost": 63.02803744108303 }, "jevbench_score": 64.30938408331077 } }, { "threshold": 0.97, "public": { "n": 231, "escalation": 0.8961038961038961, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 2.612508574128151, "p95_s": 6.380025783181191, "cost_usd_per_1000": 0.0162179923634418, "axes": { "intelligence": 68.67075408742076, "calibration": 50.471910537865476, "speed": 67.78119841734227, "cost": 63.700087252408444 }, "jevbench_score": 62.19675229309993 }, "heldout": { "n": 303, "escalation": 0.9603960396039604, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28023325293453055, "calibration": 55.17142992768658, "p50_s": 4.078099781274796, "p95_s": 6.448686248064041, "cost_usd_per_1000": 0.01717471965735899, "axes": { "intelligence": 74.75130911985254, "calibration": 55.17142992768658, "speed": 65.80070904836137, "cost": 62.9533102957479 }, "jevbench_score": 64.29031401944931 } }, { "threshold": 0.98, "public": { "n": 231, "escalation": 0.9090909090909091, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 2.612508574128151, "p95_s": 6.380025783181191, "cost_usd_per_1000": 0.016411251276813074, "axes": { "intelligence": 68.67075408742076, "calibration": 50.471910537865476, "speed": 67.78119841734227, "cost": 63.545749146290056 }, "jevbench_score": 62.15904409087975 }, "heldout": { "n": 303, "escalation": 0.9702970297029703, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28023325293453055, "calibration": 55.17142992768658, "p50_s": 4.078099781274796, "p95_s": 6.448686248064041, "cost_usd_per_1000": 0.01732205566062224, "axes": { "intelligence": 74.75130911985254, "calibration": 55.17142992768658, "speed": 65.80070904836137, "cost": 62.842017106703196 }, "jevbench_score": 64.2618809537461 } }, { "threshold": 0.99, "public": { "n": 231, "escalation": 0.9393939393939394, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 2.612508574128151, "p95_s": 6.380025783181191, "cost_usd_per_1000": 0.016862188741346044, "axes": { "intelligence": 68.67075408742076, "calibration": 50.471910537865476, "speed": 67.78119841734227, "cost": 63.192581616259694 }, "jevbench_score": 62.072498345019405 }, "heldout": { "n": 303, "escalation": 0.9834983498349835, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28023325293453055, "calibration": 55.17142992768658, "p50_s": 4.078099781274796, "p95_s": 6.448686248064041, "cost_usd_per_1000": 0.017518503664973237, "axes": { "intelligence": 74.75130911985254, "calibration": 55.17142992768658, "speed": 65.80070904836137, "cost": 62.695089749486314 }, "jevbench_score": 64.2242862107767 } }, { "threshold": 1.0, "public": { "n": 231, "escalation": 1.0, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 2.612508574128151, "p95_s": 6.380025783181191, "cost_usd_per_1000": 0.017764063670411983, "axes": { "intelligence": 68.67075408742076, "calibration": 50.471910537865476, "speed": 67.78119841734227, "cost": 62.513730366585 }, "jevbench_score": 61.90511787422022 }, "heldout": { "n": 303, "escalation": 1.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28023325293453055, "calibration": 55.17142992768658, "p50_s": 4.078099781274796, "p95_s": 6.448686248064041, "cost_usd_per_1000": 0.017764063670411983, "axes": { "intelligence": 74.75130911985254, "calibration": 55.17142992768658, "speed": 65.80070904836137, "cost": 62.513730366585 }, "jevbench_score": 64.17779001235962 } } ] }, { "cheap": "openjev-verdict-1.4", "backend": "system-one-open", "chosen": { "threshold": 0.72, "public": { "n": 231, "escalation": 0.8225108225108225, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.4594594594594595 }, "accuracy": 0.6666666666666666, "ece_hard": 0.3121943049912075, "calibration": 50.40084768908178, "p50_s": 1.9963175445795058, "p95_s": 3.618648448586464, "cost_usd_per_1000": 0.016112652528495225, "axes": { "intelligence": 68.29537871204539, "calibration": 50.40084768908178, "speed": 71.41223982947865, "cost": 63.784988752671616 }, "jevbench_score": 62.92599249634102 }, "heldout": { "n": 303, "escalation": 0.9207920792079208, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.2819824452550945, "calibration": 54.996510695630185, "p50_s": 2.5588221192359923, "p95_s": 3.646907018125056, "cost_usd_per_1000": 0.01757516965179664, "axes": { "intelligence": 74.75130911985254, "calibration": 54.996510695630185, "speed": 70.30035213121674, "cost": 62.65301422227372 }, "jevbench_score": 65.23239926177158 } }, "full": { "tiers": { "easy": 1.0, "standard": 0.7708333333333334, "judge": 0.8767123287671232, "hard": 0.4636363636363636 }, "accuracy": 0.704119850187266, "ece_hard": 0.2851988323204039, "calibration": 53.8876334726307, "p50_s": 2.3431415051221847, "p95_s": 3.632843355834483, "cost_usd_per_1000": 0.01694250775014378, "axes": { "intelligence": 74.0403694479037, "calibration": 53.8876334726307, "speed": 70.69954811075735, "cost": 63.13066921075514 }, "jevbench_score": 64.96114631290207 }, "full_escalation": 0.8782771535580525, "backend_heldout_accuracy": 0.7326732673267327, "useful": false, "curve": [ { "threshold": 0.0, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.01, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.02, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.03, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.04, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.05, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.06, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.07, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.08, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.09, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.1, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.11, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.12, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.13, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.14, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.15, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.16, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.17, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.18, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.19, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.2, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.21, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.22, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.23, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.24, "public": { "n": 231, "escalation": 0.004329004329004329, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12789820038745092, "calibration": 73.60171553017571, "p50_s": 0.6819422334432602, "p95_s": 1.6338174432516097, "cost_usd_per_1000": 0.003937340986105031, "axes": { "intelligence": 53.09371871871872, "calibration": 73.60171553017571, "speed": 79.53048883940582, "cost": 82.14390916941598 }, "jevbench_score": 71.08206322195157 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.25, "public": { "n": 231, "escalation": 0.008658008658008658, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.13448113286469693, "calibration": 72.94342228245111, "p50_s": 0.6819422334432602, "p95_s": 1.6590973153710364, "cost_usd_per_1000": 0.004001760623895455, "axes": { "intelligence": 53.09371871871872, "calibration": 72.94342228245111, "speed": 79.46380551799933, "cost": 81.93246680255761 }, "jevbench_score": 70.86204273237155 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.26, "public": { "n": 231, "escalation": 0.008658008658008658, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.13448113286469693, "calibration": 72.94342228245111, "p50_s": 0.6819422334432602, "p95_s": 1.6590973153710364, "cost_usd_per_1000": 0.004001760623895455, "axes": { "intelligence": 53.09371871871872, "calibration": 72.94342228245111, "speed": 79.46380551799933, "cost": 81.93246680255761 }, "jevbench_score": 70.86204273237155 }, "heldout": { "n": 303, "escalation": 0.0033003300330033004, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5684931506849316, "hard": 0.3577981651376147 }, "accuracy": 0.504950495049505, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9941574066877361, "cost_usd_per_1000": 0.003922033349402356, "axes": { "intelligence": 51.151753173306524, "calibration": 70.2289302105776, "speed": 76.12358067087503, "cost": 82.19466153422471 }, "jevbench_score": 68.85489953862414 } }, { "threshold": 0.27, "public": { "n": 231, "escalation": 0.008658008658008658, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.13448113286469693, "calibration": 72.94342228245111, "p50_s": 0.6819422334432602, "p95_s": 1.6590973153710364, "cost_usd_per_1000": 0.004001760623895455, "axes": { "intelligence": 53.09371871871872, "calibration": 72.94342228245111, "speed": 79.46380551799933, "cost": 81.93246680255761 }, "jevbench_score": 70.86204273237155 }, "heldout": { "n": 303, "escalation": 0.006600660066006601, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5684931506849316, "hard": 0.3669724770642202 }, "accuracy": 0.5082508250825083, "ece_hard": 0.1447057595981996, "calibration": 70.54044148223348, "p50_s": 1.2242849111557006, "p95_s": 1.9994657054543494, "cost_usd_per_1000": 0.003971145350490105, "axes": { "intelligence": 51.42698253110469, "calibration": 70.54044148223348, "speed": 76.11203543410345, "cost": 82.03252650245867 }, "jevbench_score": 68.98698420900178 } }, { "threshold": 0.28, "public": { "n": 231, "escalation": 0.008658008658008658, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.13448113286469693, "calibration": 72.94342228245111, "p50_s": 0.6819422334432602, "p95_s": 1.6590973153710364, "cost_usd_per_1000": 0.004001760623895455, "axes": { "intelligence": 53.09371871871872, "calibration": 72.94342228245111, "speed": 79.46380551799933, "cost": 81.93246680255761 }, "jevbench_score": 70.86204273237155 }, "heldout": { "n": 303, "escalation": 0.013201320132013201, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5684931506849316, "hard": 0.3761467889908257 }, "accuracy": 0.5115511551155115, "ece_hard": 0.14626770082509608, "calibration": 70.38424735954382, "p50_s": 1.234055870771408, "p95_s": 2.052325816452502, "cost_usd_per_1000": 0.004069369352665604, "axes": { "intelligence": 51.70221188890286, "calibration": 70.38424735954382, "speed": 75.96418869344052, "cost": 81.71418670023807 }, "jevbench_score": 68.94023192956884 } }, { "threshold": 0.29, "public": { "n": 231, "escalation": 0.017316017316017316, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36936936936936937 }, "accuracy": 0.5281385281385281, "ece_hard": 0.13943728824951312, "calibration": 72.4478067439695, "p50_s": 0.6819422334432602, "p95_s": 1.6796640172600745, "cost_usd_per_1000": 0.004130599899476304, "axes": { "intelligence": 53.46909409409409, "calibration": 72.4478067439695, "speed": 79.41029993775524, "cost": 81.51960609598103 }, "jevbench_score": 70.76471213324649 }, "heldout": { "n": 303, "escalation": 0.019801980198019802, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5684931506849316, "hard": 0.3853211009174312 }, "accuracy": 0.5148514851485149, "ece_hard": 0.14364276185220282, "calibration": 71.36763890847877, "p50_s": 1.234055870771408, "p95_s": 2.0718681916594504, "cost_usd_per_1000": 0.0041675933548411025, "axes": { "intelligence": 51.97744124670103, "calibration": 71.36763890847877, "speed": 75.92303054607518, "cost": 81.40343989349357 }, "jevbench_score": 69.19634087316626 } }, { "threshold": 0.3, "public": { "n": 231, "escalation": 0.030303030303030304, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36936936936936937 }, "accuracy": 0.5281385281385281, "ece_hard": 0.13731067773900893, "calibration": 72.66046779501991, "p50_s": 0.6859634757041931, "p95_s": 1.6896898761391639, "cost_usd_per_1000": 0.004323858812847577, "axes": { "intelligence": 53.46909409409409, "calibration": 72.66046779501991, "speed": 79.35892005809262, "cost": 80.92385486572203 }, "jevbench_score": 70.67540922677235 }, "heldout": { "n": 303, "escalation": 0.033003300330033, "tiers": { "easy": 0.875, "standard": 0.4583333333333333, "judge": 0.5821917808219178, "hard": 0.3944954128440367 }, "accuracy": 0.528052805280528, "ece_hard": 0.13722799849790074, "calibration": 72.00911524390898, "p50_s": 1.2723965734243392, "p95_s": 2.1154292389750475, "cost_usd_per_1000": 0.004364041359192099, "axes": { "intelligence": 53.219565581668135, "calibration": 72.00911524390898, "speed": 75.69979006345216, "cost": 80.80333426416219 }, "jevbench_score": 69.5818048074503 } }, { "threshold": 0.31, "public": { "n": 231, "escalation": 0.030303030303030304, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36936936936936937 }, "accuracy": 0.5281385281385281, "ece_hard": 0.13731067773900893, "calibration": 72.66046779501991, "p50_s": 0.6859634757041931, "p95_s": 1.6896898761391639, "cost_usd_per_1000": 0.004323858812847577, "axes": { "intelligence": 53.46909409409409, "calibration": 72.66046779501991, "speed": 79.35892005809262, "cost": 80.92385486572203 }, "jevbench_score": 70.67540922677235 }, "heldout": { "n": 303, "escalation": 0.0462046204620462, "tiers": { "easy": 0.875, "standard": 0.4583333333333333, "judge": 0.5958904109589042, "hard": 0.3944954128440367 }, "accuracy": 0.5346534653465347, "ece_hard": 0.13731915431670194, "calibration": 71.99999966202884, "p50_s": 1.2723965734243392, "p95_s": 2.1620058521628374, "cost_usd_per_1000": 0.004560489363543096, "axes": { "intelligence": 53.603127225503755, "calibration": 71.99999966202884, "speed": 75.60520643945824, "cost": 80.2296565853081 }, "jevbench_score": 69.55883740042624 } }, { "threshold": 0.32, "public": { "n": 231, "escalation": 0.05194805194805195, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.3783783783783784 }, "accuracy": 0.5324675324675324, "ece_hard": 0.14257085450590068, "calibration": 72.13445011833073, "p50_s": 0.6935107052326203, "p95_s": 1.8279631301760673, "cost_usd_per_1000": 0.0046459570017996984, "axes": { "intelligence": 53.84446946946947, "calibration": 72.13445011833073, "speed": 78.96979398831061, "cost": 79.98774441700894 }, "jevbench_score": 70.37883454460362 }, "heldout": { "n": 303, "escalation": 0.0627062706270627, "tiers": { "easy": 0.875, "standard": 0.4583333333333333, "judge": 0.6164383561643836, "hard": 0.4036697247706422 }, "accuracy": 0.5478547854785478, "ece_hard": 0.1363535109831694, "calibration": 72.0965639953821, "p50_s": 1.3009985327720641, "p95_s": 2.336846156418322, "cost_usd_per_1000": 0.004806049368981842, "axes": { "intelligence": 54.453699049055345, "calibration": 72.0965639953821, "speed": 75.17093071167591, "cost": 79.54635316971968 }, "jevbench_score": 69.60701962839715 } }, { "threshold": 0.33, "public": { "n": 231, "escalation": 0.05627705627705628, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.3783783783783784 }, "accuracy": 0.5324675324675324, "ece_hard": 0.14317429834767997, "calibration": 72.07410573415281, "p50_s": 0.6935107052326203, "p95_s": 1.8427960887551307, "cost_usd_per_1000": 0.004710376639590123, "axes": { "intelligence": 53.84446946946947, "calibration": 72.07410573415281, "speed": 78.93469548830473, "cost": 79.80833096475801 }, "jevbench_score": 70.3168055510151 }, "heldout": { "n": 303, "escalation": 0.10561056105610561, "tiers": { "easy": 0.875, "standard": 0.4583333333333333, "judge": 0.6712328767123288, "hard": 0.4036697247706422 }, "accuracy": 0.5742574257425742, "ece_hard": 0.15095106445288714, "calibration": 70.63680864841032, "p50_s": 1.340339982509613, "p95_s": 2.9473727598786352, "cost_usd_per_1000": 0.005444505383122582, "axes": { "intelligence": 55.9879456243978, "calibration": 70.63680864841032, "speed": 74.03349961779071, "cost": 77.92124705376197 }, "jevbench_score": 69.11183814017465 } }, { "threshold": 0.34, "public": { "n": 231, "escalation": 0.06926406926406926, "tiers": { "easy": 0.875, "standard": 0.5555555555555556, "hard": 0.3963963963963964 }, "accuracy": 0.5454545454545454, "ece_hard": 0.130966062937369, "calibration": 73.2949292751839, "p50_s": 0.6951739430427551, "p95_s": 2.020040327310562, "cost_usd_per_1000": 0.004903635552961395, "axes": { "intelligence": 55.135343677010354, "calibration": 73.2949292751839, "speed": 78.52546475137976, "cost": 79.28445444518309 }, "jevbench_score": 70.82321596040734 }, "heldout": { "n": 303, "escalation": 0.1353135313531353, "tiers": { "easy": 0.875, "standard": 0.4583333333333333, "judge": 0.7123287671232876, "hard": 0.43119266055045874 }, "accuracy": 0.6039603960396039, "ece_hard": 0.13016822470809425, "calibration": 72.71509262288961, "p50_s": 1.3770495146512984, "p95_s": 3.0740213140845287, "cost_usd_per_1000": 0.005886513392912325, "axes": { "intelligence": 57.96431862929915, "calibration": 72.71509262288961, "speed": 73.73343569104304, "cost": 76.90425590630892 }, "jevbench_score": 69.91977716125558 } }, { "threshold": 0.35, "public": { "n": 231, "escalation": 0.08225108225108226, "tiers": { "easy": 0.875, "standard": 0.5555555555555556, "hard": 0.3963963963963964 }, "accuracy": 0.5454545454545454, "ece_hard": 0.14454462091937395, "calibration": 71.93707347698341, "p50_s": 0.6951739430427551, "p95_s": 2.360152767598629, "cost_usd_per_1000": 0.005096894466332669, "axes": { "intelligence": 55.135343677010354, "calibration": 71.93707347698341, "speed": 77.84966399934541, "cost": 78.78083075787065 }, "jevbench_score": 70.22876604012954 }, "heldout": { "n": 303, "escalation": 0.16501650165016502, "tiers": { "easy": 0.875, "standard": 0.4583333333333333, "judge": 0.7534246575342466, "hard": 0.43119266055045874 }, "accuracy": 0.6237623762376238, "ece_hard": 0.13049791246130557, "calibration": 72.06645313827693, "p50_s": 1.3770495146512984, "p95_s": 3.1613638833165165, "cost_usd_per_1000": 0.006328521402702068, "axes": { "intelligence": 59.115003560806, "calibration": 72.06645313827693, "speed": 73.61175956078624, "cost": 75.96093240401612 }, "jevbench_score": 69.86217278555115 } }, { "threshold": 0.36, "public": { "n": 231, "escalation": 0.1038961038961039, "tiers": { "easy": 0.875, "standard": 0.5694444444444444, "hard": 0.40540540540540543 }, "accuracy": 0.5541125541125541, "ece_hard": 0.15095830093613588, "calibration": 71.29570547530722, "p50_s": 0.7169283866882324, "p95_s": 2.713510499894619, "cost_usd_per_1000": 0.00541899265528479, "axes": { "intelligence": 56.050842509175844, "calibration": 71.29570547530722, "speed": 77.1099271702302, "cost": 77.98244312844788 }, "jevbench_score": 70.01452736555859 }, "heldout": { "n": 303, "escalation": 0.1782178217821782, "tiers": { "easy": 0.875, "standard": 0.4583333333333333, "judge": 0.7671232876712328, "hard": 0.44036697247706424 }, "accuracy": 0.6336633663366337, "ece_hard": 0.12943786721476536, "calibration": 72.17245766293095, "p50_s": 1.3870477318763732, "p95_s": 3.1613638833165165, "cost_usd_per_1000": 0.006524969407053065, "axes": { "intelligence": 59.77379456243979, "calibration": 72.17245766293095, "speed": 73.5803410604937, "cost": 75.5626456064913 }, "jevbench_score": 69.98223659095444 } }, { "threshold": 0.37, "public": { "n": 231, "escalation": 0.12554112554112554, "tiers": { "easy": 0.875, "standard": 0.5694444444444444, "hard": 0.3963963963963964 }, "accuracy": 0.5497835497835498, "ece_hard": 0.1645148996637238, "calibration": 71.27085574457308, "p50_s": 0.7518308401107788, "p95_s": 2.724019208550453, "cost_usd_per_1000": 0.005741090844236912, "axes": { "intelligence": 55.675467133800474, "calibration": 71.27085574457308, "speed": 76.88669697791903, "cost": 77.23016743038323 }, "jevbench_score": 69.67123450615318 }, "heldout": { "n": 303, "escalation": 0.2079207920792079, "tiers": { "easy": 0.875, "standard": 0.5, "judge": 0.8082191780821918, "hard": 0.44036697247706424 }, "accuracy": 0.6567656765676567, "ece_hard": 0.132005225809375, "calibration": 71.91572180346998, "p50_s": 1.4307569205760955, "p95_s": 3.214877389371394, "cost_usd_per_1000": 0.006966977416842808, "axes": { "intelligence": 62.0911461606133, "calibration": 71.91572180346998, "speed": 73.37269730498306, "cost": 74.70866791630016 }, "jevbench_score": 70.33795694214098 } }, { "threshold": 0.38, "public": { "n": 231, "escalation": 0.12987012987012986, "tiers": { "easy": 0.875, "standard": 0.5694444444444444, "hard": 0.38738738738738737 }, "accuracy": 0.5454545454545454, "ece_hard": 0.17385718386104104, "calibration": 70.33662732484137, "p50_s": 0.7518308401107788, "p95_s": 2.7465455770492553, "cost_usd_per_1000": 0.005805510482027336, "axes": { "intelligence": 55.30009175842509, "calibration": 70.33662732484137, "speed": 76.85093053253871, "cost": 77.08478759414356 }, "jevbench_score": 69.28373637943447 }, "heldout": { "n": 303, "escalation": 0.22772277227722773, "tiers": { "easy": 0.875, "standard": 0.5, "judge": 0.8493150684931506, "hard": 0.44036697247706424 }, "accuracy": 0.6765676567656765, "ece_hard": 0.132005225809375, "calibration": 71.91572180346998, "p50_s": 1.4789225697517394, "p95_s": 3.214877389371394, "cost_usd_per_1000": 0.007261649423369303, "axes": { "intelligence": 63.24183109212015, "calibration": 71.91572180346998, "speed": 73.22890149032361, "cost": 74.16894165128387 }, "jevbench_score": 70.499042771 } }, { "threshold": 0.39, "public": { "n": 231, "escalation": 0.1645021645021645, "tiers": { "easy": 0.875, "standard": 0.5694444444444444, "hard": 0.36936936936936937 }, "accuracy": 0.5367965367965368, "ece_hard": 0.20967395208336723, "calibration": 65.26940151230502, "p50_s": 0.7781706541776657, "p95_s": 2.824376957118511, "cost_usd_per_1000": 0.00632086758435073, "axes": { "intelligence": 54.54934100767434, "calibration": 65.26940151230502, "speed": 76.58002491415776, "cost": 75.97669922793717 }, "jevbench_score": 67.46426077153608 }, "heldout": { "n": 303, "escalation": 0.24422442244224424, "tiers": { "easy": 0.875, "standard": 0.5, "judge": 0.8767123287671232, "hard": 0.44036697247706424 }, "accuracy": 0.6897689768976898, "ece_hard": 0.132005225809375, "calibration": 71.91572180346998, "p50_s": 1.4937828123569488, "p95_s": 3.214877389371394, "cost_usd_per_1000": 0.007507209428808049, "axes": { "intelligence": 64.00895437979138, "calibration": 71.91572180346998, "speed": 73.18548127680307, "cost": 73.73564405321981 }, "jevbench_score": 70.59789417656592 } }, { "threshold": 0.4, "public": { "n": 231, "escalation": 0.18614718614718614, "tiers": { "easy": 0.875, "standard": 0.5694444444444444, "hard": 0.3783783783783784 }, "accuracy": 0.5411255411255411, "ece_hard": 0.225792949653982, "calibration": 62.8153229366811, "p50_s": 0.8036727398633957, "p95_s": 2.951724798977375, "cost_usd_per_1000": 0.006642965773302851, "axes": { "intelligence": 54.92471638304972, "calibration": 62.8153229366811, "speed": 76.24844898412309, "cost": 75.32913955532794 }, "jevbench_score": 66.72012635754147 }, "heldout": { "n": 303, "escalation": 0.2739273927392739, "tiers": { "easy": 0.875, "standard": 0.5, "judge": 0.9246575342465754, "hard": 0.45871559633027525 }, "accuracy": 0.7194719471947195, "ece_hard": 0.12012147438789127, "calibration": 73.10409694561835, "p50_s": 1.5044434458017348, "p95_s": 3.2503889426589003, "cost_usd_per_1000": 0.007949217438597792, "axes": { "intelligence": 65.90187884881237, "calibration": 73.10409694561835, "speed": 73.1068880145735, "cost": 72.99026870191743 }, "jevbench_score": 71.2058529766564 } }, { "threshold": 0.41, "public": { "n": 231, "escalation": 0.21645021645021645, "tiers": { "easy": 0.875, "standard": 0.5694444444444444, "hard": 0.36036036036036034 }, "accuracy": 0.5324675324675324, "ece_hard": 0.261067977335561, "calibration": 60.3187366797964, "p50_s": 0.8354266524314881, "p95_s": 2.951724798977375, "cost_usd_per_1000": 0.007093903237835823, "axes": { "intelligence": 54.17396563229897, "calibration": 60.3187366797964, "speed": 76.0801580960963, "cost": 74.47344219044396 }, "jevbench_score": 65.59623676102382 }, "heldout": { "n": 303, "escalation": 0.29042904290429045, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "judge": 0.9315068493150684, "hard": 0.45871559633027525 }, "accuracy": 0.7260726072607261, "ece_hard": 0.12642060111251663, "calibration": 71.23402984267446, "p50_s": 1.5230037212371825, "p95_s": 3.273900851607322, "cost_usd_per_1000": 0.008194777444036539, "axes": { "intelligence": 67.26032633739685, "calibration": 71.23402984267446, "speed": 73.02233512636029, "cost": 72.5938850985799 }, "jevbench_score": 70.99055314591853 } }, { "threshold": 0.42, "public": { "n": 231, "escalation": 0.22943722943722944, "tiers": { "easy": 0.875, "standard": 0.5833333333333334, "hard": 0.36036036036036034 }, "accuracy": 0.5367965367965368, "ece_hard": 0.2615310674205897, "calibration": 60.27242767129353, "p50_s": 0.8431772083044052, "p95_s": 3.016723120212555, "cost_usd_per_1000": 0.007287162151207095, "axes": { "intelligence": 54.7140890890891, "calibration": 60.27242767129353, "speed": 75.94545689524372, "cost": 74.12324699827855 }, "jevbench_score": 65.63999255233271 }, "heldout": { "n": 303, "escalation": 0.3102310231023102, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "judge": 0.9383561643835616, "hard": 0.44954128440366975 }, "accuracy": 0.7260726072607261, "ece_hard": 0.1417258007065354, "calibration": 69.70350988327259, "p50_s": 1.5274034827947616, "p95_s": 3.2784052431583404, "cost_usd_per_1000": 0.008489449450563034, "axes": { "intelligence": 67.17687780151648, "calibration": 69.70350988327259, "speed": 73.00383587825877, "cost": 72.1336141984598 }, "jevbench_score": 70.46759648845133 } }, { "threshold": 0.43, "public": { "n": 231, "escalation": 0.24242424242424243, "tiers": { "easy": 0.875, "standard": 0.5972222222222222, "hard": 0.36036036036036034 }, "accuracy": 0.5411255411255411, "ece_hard": 0.26359356009835394, "calibration": 60.0661784035171, "p50_s": 0.8988664388656616, "p95_s": 3.12084940969944, "cost_usd_per_1000": 0.007480421064578368, "axes": { "intelligence": 55.25421254587922, "calibration": 60.0661784035171, "speed": 75.5203202143352, "cost": 73.78221866483025 }, "jevbench_score": 65.57717929787749 }, "heldout": { "n": 303, "escalation": 0.3234323432343234, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "judge": 0.9315068493150684, "hard": 0.45871559633027525 }, "accuracy": 0.7260726072607261, "ece_hard": 0.1382621010330352, "calibration": 70.45623528027515, "p50_s": 1.5354039013385772, "p95_s": 3.2784052431583404, "cost_usd_per_1000": 0.00868589745491403, "axes": { "intelligence": 67.26032633739685, "calibration": 70.45623528027515, "speed": 72.98114724594726, "cost": 71.83555906579625 }, "jevbench_score": 70.60039672075138 } }, { "threshold": 0.44, "public": { "n": 231, "escalation": 0.2597402597402597, "tiers": { "easy": 0.875, "standard": 0.5972222222222222, "hard": 0.36036036036036034 }, "accuracy": 0.5411255411255411, "ece_hard": 0.2646610551894109, "calibration": 58.36743958307534, "p50_s": 0.935780817270279, "p95_s": 3.3824233576655387, "cost_usd_per_1000": 0.007738099615740065, "axes": { "intelligence": 55.25421254587922, "calibration": 58.36743958307534, "speed": 74.9959789702683, "cost": 73.34097051181715 }, "jevbench_score": 64.89782872186606 }, "heldout": { "n": 303, "escalation": 0.33003300330033003, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "judge": 0.9315068493150684, "hard": 0.47706422018348627 }, "accuracy": 0.7326732673267327, "ece_hard": 0.12907506571956623, "calibration": 71.37493881162206, "p50_s": 1.5415462344884872, "p95_s": 3.2824722230434418, "cost_usd_per_1000": 0.008784121457089528, "axes": { "intelligence": 67.81078505299317, "calibration": 71.37493881162206, "speed": 72.95842385572176, "cost": 71.68905003730073 }, "jevbench_score": 70.9321620359039 } }, { "threshold": 0.45, "public": { "n": 231, "escalation": 0.27705627705627706, "tiers": { "easy": 0.875, "standard": 0.5972222222222222, "hard": 0.36036036036036034 }, "accuracy": 0.5411255411255411, "ece_hard": 0.2800965304344345, "calibration": 56.82389205857298, "p50_s": 0.9514630824327469, "p95_s": 3.3915603771805762, "cost_usd_per_1000": 0.007995778166901762, "axes": { "intelligence": 55.25421254587922, "calibration": 56.82389205857298, "speed": 74.91208505180984, "cost": 72.91417790070048 }, "jevbench_score": 64.35244614475192 }, "heldout": { "n": 303, "escalation": 0.35313531353135313, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "judge": 0.9315068493150684, "hard": 0.47706422018348627 }, "accuracy": 0.7326732673267327, "ece_hard": 0.14485175643612513, "calibration": 71.47382629109656, "p50_s": 1.5707663983106612, "p95_s": 3.3748548001050924, "cost_usd_per_1000": 0.009127905464703774, "axes": { "intelligence": 67.81078505299317, "calibration": 71.47382629109656, "speed": 72.75633305224666, "cost": 71.1888659931798 }, "jevbench_score": 70.78352237018908 } }, { "threshold": 0.46, "public": { "n": 231, "escalation": 0.30735930735930733, "tiers": { "easy": 0.875, "standard": 0.5972222222222222, "hard": 0.36036036036036034 }, "accuracy": 0.5411255411255411, "ece_hard": 0.29446893347069647, "calibration": 55.38665175494678, "p50_s": 1.0029189229011535, "p95_s": 3.4126361846923827, "cost_usd_per_1000": 0.008446715631434731, "axes": { "intelligence": 55.25421254587922, "calibration": 55.38665175494678, "speed": 74.65644183062736, "cost": 72.1993637980944 }, "jevbench_score": 63.729834545532256 }, "heldout": { "n": 303, "escalation": 0.3795379537953795, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "judge": 0.9246575342465754, "hard": 0.46788990825688076 }, "accuracy": 0.7260726072607261, "ece_hard": 0.16363066292627151, "calibration": 69.59593564208193, "p50_s": 1.6103754729032516, "p95_s": 3.4659880295395844, "cost_usd_per_1000": 0.009520801473405768, "axes": { "intelligence": 67.34377487327721, "calibration": 69.59593564208193, "speed": 72.53245794204341, "cost": 70.63979471813053 }, "jevbench_score": 70.00286092812776 } }, { "threshold": 0.47, "public": { "n": 231, "escalation": 0.3203463203463203, "tiers": { "easy": 0.875, "standard": 0.5972222222222222, "hard": 0.3783783783783784 }, "accuracy": 0.5497835497835498, "ece_hard": 0.28069547240842774, "calibration": 56.76399786117366, "p50_s": 1.1182467132806777, "p95_s": 3.421720692515373, "cost_usd_per_1000": 0.008639974544806004, "axes": { "intelligence": 56.00496329662997, "calibration": 56.76399786117366, "speed": 74.17217813464748, "cost": 71.90462611128103 }, "jevbench_score": 64.16886227627757 }, "heldout": { "n": 303, "escalation": 0.38943894389438943, "tiers": { "easy": 0.875, "standard": 0.5833333333333334, "judge": 0.9178082191780822, "hard": 0.46788990825688076 }, "accuracy": 0.7260726072607261, "ece_hard": 0.16498630038897227, "calibration": 69.46037189581185, "p50_s": 1.6408134341239928, "p95_s": 3.478417921066284, "cost_usd_per_1000": 0.009668137476669015, "axes": { "intelligence": 68.31866071802605, "calibration": 69.46037189581185, "speed": 72.4355903691935, "cost": 70.43971548231023 }, "jevbench_score": 70.14738913896721 } }, { "threshold": 0.48, "public": { "n": 231, "escalation": 0.354978354978355, "tiers": { "easy": 0.875, "standard": 0.5833333333333334, "hard": 0.40540540540540543 }, "accuracy": 0.5584415584415584, "ece_hard": 0.26286011167832457, "calibration": 58.547533934183974, "p50_s": 1.162949138879776, "p95_s": 3.4577306374907493, "cost_usd_per_1000": 0.009155331647129401, "axes": { "intelligence": 56.59096596596597, "calibration": 58.547533934183974, "speed": 73.95648120502739, "cost": 71.14977756956503 }, "jevbench_score": 64.61768205528071 }, "heldout": { "n": 303, "escalation": 0.41254125412541254, "tiers": { "easy": 0.875, "standard": 0.625, "judge": 0.910958904109589, "hard": 0.47706422018348627 }, "accuracy": 0.7293729372937293, "ece_hard": 0.17161284924686837, "calibration": 68.79771701002224, "p50_s": 1.680403822660446, "p95_s": 3.478417921066284, "cost_usd_per_1000": 0.01001192148428326, "axes": { "intelligence": 69.56877592057309, "calibration": 68.79771701002224, "speed": 72.33204579250479, "cost": 69.98447694653358 }, "jevbench_score": 70.15848698344598 } }, { "threshold": 0.49, "public": { "n": 231, "escalation": 0.36363636363636365, "tiers": { "easy": 0.875, "standard": 0.5833333333333334, "hard": 0.4144144144144144 }, "accuracy": 0.5627705627705628, "ece_hard": 0.25486465506149, "calibration": 59.34707959586743, "p50_s": 1.1630701512098311, "p95_s": 3.4593338459730147, "cost_usd_per_1000": 0.009284170922710248, "axes": { "intelligence": 56.96634134134135, "calibration": 59.34707959586743, "speed": 73.95401613775783, "cost": 70.96770618185653 }, "jevbench_score": 64.90229485163454 }, "heldout": { "n": 303, "escalation": 0.42244224422442245, "tiers": { "easy": 0.875, "standard": 0.625, "judge": 0.9041095890410958, "hard": 0.47706422018348627 }, "accuracy": 0.7260726072607261, "ece_hard": 0.1666334489053553, "calibration": 69.29565704417354, "p50_s": 1.702413910627365, "p95_s": 3.516429756581782, "cost_usd_per_1000": 0.010159257487546508, "axes": { "intelligence": 69.37699509865529, "calibration": 69.29565704417354, "speed": 72.22832894232317, "cost": 69.79414096878307 }, "jevbench_score": 70.16362342656666 } }, { "threshold": 0.5, "public": { "n": 231, "escalation": 0.36363636363636365, "tiers": { "easy": 0.875, "standard": 0.5833333333333334, "hard": 0.4144144144144144 }, "accuracy": 0.5627705627705628, "ece_hard": 0.25486465506149, "calibration": 59.34707959586743, "p50_s": 1.1630701512098311, "p95_s": 3.4593338459730147, "cost_usd_per_1000": 0.009284170922710248, "axes": { "intelligence": 56.96634134134135, "calibration": 59.34707959586743, "speed": 73.95401613775783, "cost": 70.96770618185653 }, "jevbench_score": 64.90229485163454 }, "heldout": { "n": 303, "escalation": 0.43234323432343236, "tiers": { "easy": 0.875, "standard": 0.625, "judge": 0.8972602739726028, "hard": 0.46788990825688076 }, "accuracy": 0.7194719471947195, "ece_hard": 0.17544680785270417, "calibration": 68.41432114943865, "p50_s": 1.7202474415302276, "p95_s": 3.5270464122295375, "cost_usd_per_1000": 0.010306593490809755, "axes": { "intelligence": 68.9099849189393, "calibration": 68.41432114943865, "speed": 72.1699790482885, "cost": 69.60654558738791 }, "jevbench_score": 69.76039954884311 } }, { "threshold": 0.51, "public": { "n": 231, "escalation": 0.38095238095238093, "tiers": { "easy": 0.875, "standard": 0.5833333333333334, "hard": 0.4144144144144144 }, "accuracy": 0.5627705627705628, "ece_hard": 0.262792691564266, "calibration": 58.55427594558983, "p50_s": 1.2381848335266112, "p95_s": 3.486275540292263, "cost_usd_per_1000": 0.009541849473871945, "axes": { "intelligence": 56.96634134134135, "calibration": 58.55427594558983, "speed": 73.6485281103827, "cost": 70.61102316626949 }, "jevbench_score": 64.53619850881405 }, "heldout": { "n": 303, "escalation": 0.45544554455445546, "tiers": { "easy": 0.875, "standard": 0.625, "judge": 0.8972602739726028, "hard": 0.44954128440366975 }, "accuracy": 0.7128712871287128, "ece_hard": 0.20544526486457615, "calibration": 64.13096106565092, "p50_s": 1.787559726834297, "p95_s": 3.5310502648353577, "cost_usd_per_1000": 0.010650377498423999, "axes": { "intelligence": 68.35952620334298, "calibration": 64.13096106565092, "speed": 71.9983553859108, "cost": 69.17904995668141 }, "jevbench_score": 68.35825856786606 } }, { "threshold": 0.52, "public": { "n": 231, "escalation": 0.4025974025974026, "tiers": { "easy": 0.8958333333333334, "standard": 0.5833333333333334, "hard": 0.3963963963963964 }, "accuracy": 0.5584415584415584, "ece_hard": 0.29046612185407245, "calibration": 54.011203097677495, "p50_s": 1.2882852345705031, "p95_s": 3.5139913126826285, "cost_usd_per_1000": 0.009863947662824068, "axes": { "intelligence": 56.620683183183196, "calibration": 54.011203097677495, "speed": 73.44187287266391, "cost": 70.1784772219531 }, "jevbench_score": 63.00886550780149 }, "heldout": { "n": 303, "escalation": 0.46864686468646866, "tiers": { "easy": 0.875, "standard": 0.625, "judge": 0.8972602739726028, "hard": 0.45871559633027525 }, "accuracy": 0.7161716171617162, "ece_hard": 0.2020625176686618, "calibration": 63.617711517098925, "p50_s": 1.7915094584226607, "p95_s": 3.5310502648353577, "cost_usd_per_1000": 0.010846825502774996, "axes": { "intelligence": 68.63475556114113, "calibration": 63.617711517098925, "speed": 71.98876994767875, "cost": 68.94092039368677 }, "jevbench_score": 68.2285263171068 } }, { "threshold": 0.53, "public": { "n": 231, "escalation": 0.42424242424242425, "tiers": { "easy": 0.8958333333333334, "standard": 0.5833333333333334, "hard": 0.4144144144144144 }, "accuracy": 0.5670995670995671, "ece_hard": 0.2845619140216916, "calibration": 54.60162388091558, "p50_s": 1.3699952155351638, "p95_s": 3.5139913126826285, "cost_usd_per_1000": 0.010186045851776189, "axes": { "intelligence": 57.37143393393395, "calibration": 54.60162388091558, "speed": 73.17480266042928, "cost": 69.75983119627205 }, "jevbench_score": 63.236388825934114 }, "heldout": { "n": 303, "escalation": 0.4884488448844885, "tiers": { "easy": 0.875, "standard": 0.625, "judge": 0.8972602739726028, "hard": 0.45871559633027525 }, "accuracy": 0.7161716171617162, "ece_hard": 0.21733156282929994, "calibration": 62.0908070010351, "p50_s": 1.8467198252677917, "p95_s": 3.5310502648353577, "cost_usd_per_1000": 0.011141497509301492, "axes": { "intelligence": 68.63475556114113, "calibration": 62.0908070010351, "speed": 71.85695088633102, "cost": 68.59169297419348 }, "jevbench_score": 67.69832534902406 } }, { "threshold": 0.54, "public": { "n": 231, "escalation": 0.4458874458874459, "tiers": { "easy": 0.8958333333333334, "standard": 0.5833333333333334, "hard": 0.40540540540540543 }, "accuracy": 0.5627705627705628, "ece_hard": 0.30514824014584296, "calibration": 52.542991268500444, "p50_s": 1.3880736619234084, "p95_s": 3.530861423909664, "cost_usd_per_1000": 0.01050814404072831, "axes": { "intelligence": 56.996058558558566, "calibration": 52.542991268500444, "speed": 73.09706813149438, "cost": 69.3542194821803 }, "jevbench_score": 62.42134410887289 }, "heldout": { "n": 303, "escalation": 0.5082508250825083, "tiers": { "easy": 0.875, "standard": 0.625, "judge": 0.8972602739726028, "hard": 0.48623853211009177 }, "accuracy": 0.7260726072607261, "ece_hard": 0.24088495898508022, "calibration": 59.73546738545708, "p50_s": 1.9161066651344298, "p95_s": 3.5310502648353577, "cost_usd_per_1000": 0.011436169515827987, "axes": { "intelligence": 69.46044363453564, "calibration": 59.73546738545708, "speed": 71.69676418605795, "cost": 68.25158247443966 }, "jevbench_score": 67.12673515937746 } }, { "threshold": 0.55, "public": { "n": 231, "escalation": 0.4675324675324675, "tiers": { "easy": 0.8958333333333334, "standard": 0.5833333333333334, "hard": 0.40540540540540543 }, "accuracy": 0.5627705627705628, "ece_hard": 0.3176371803204616, "calibration": 51.29409725103858, "p50_s": 1.400191181898117, "p95_s": 3.530861423909664, "cost_usd_per_1000": 0.010830242229680431, "axes": { "intelligence": 56.996058558558566, "calibration": 51.29409725103858, "speed": 73.05931988672104, "cost": 68.96085489448939 }, "jevbench_score": 61.95089987468362 }, "heldout": { "n": 303, "escalation": 0.528052805280528, "tiers": { "easy": 0.875, "standard": 0.7083333333333334, "judge": 0.8972602739726028, "hard": 0.47706422018348627 }, "accuracy": 0.7293729372937293, "ece_hard": 0.2348297456562791, "calibration": 60.87531989668874, "p50_s": 1.9443440377712249, "p95_s": 3.5359846144914626, "cost_usd_per_1000": 0.011730841522354481, "axes": { "intelligence": 71.5185476100708, "calibration": 60.87531989668874, "speed": 71.62716520632021, "cost": 67.9201249679765 }, "jevbench_score": 67.83971981950276 } }, { "threshold": 0.56, "public": { "n": 231, "escalation": 0.48484848484848486, "tiers": { "easy": 0.8958333333333334, "standard": 0.5833333333333334, "hard": 0.4144144144144144 }, "accuracy": 0.5670995670995671, "ece_hard": 0.3178605644581039, "calibration": 51.27175883727435, "p50_s": 1.4294556319713592, "p95_s": 3.5474014446139335, "cost_usd_per_1000": 0.011087920780842128, "axes": { "intelligence": 57.37143393393395, "calibration": 51.27175883727435, "speed": 72.94918981359848, "cost": 68.6544965679151 }, "jevbench_score": 61.95349980782889 }, "heldout": { "n": 303, "escalation": 0.5445544554455446, "tiers": { "easy": 0.875, "standard": 0.7083333333333334, "judge": 0.8972602739726028, "hard": 0.47706422018348627 }, "accuracy": 0.7293729372937293, "ece_hard": 0.2632788344513057, "calibration": 58.03041101718608, "p50_s": 1.9711510777473449, "p95_s": 3.5359846144914626, "cost_usd_per_1000": 0.01197640152779323, "axes": { "intelligence": 71.5185476100708, "calibration": 58.03041101718608, "speed": 71.56769721516753, "cost": 67.6502095603567 }, "jevbench_score": 66.95224292852109 } }, { "threshold": 0.57, "public": { "n": 231, "escalation": 0.5108225108225108, "tiers": { "easy": 0.8958333333333334, "standard": 0.5833333333333334, "hard": 0.4144144144144144 }, "accuracy": 0.5670995670995671, "ece_hard": 0.32853686559472584, "calibration": 50.20412872361216, "p50_s": 1.6976624965667724, "p95_s": 3.5474014446139335, "cost_usd_per_1000": 0.011474438607584675, "axes": { "intelligence": 57.37143393393395, "calibration": 50.20412872361216, "speed": 72.20238306786615, "cost": 68.20805660010616 }, "jevbench_score": 61.369924172304444 }, "heldout": { "n": 303, "escalation": 0.5742574257425742, "tiers": { "easy": 0.875, "standard": 0.7083333333333334, "judge": 0.9041095890410958, "hard": 0.47706422018348627 }, "accuracy": 0.7326732673267327, "ece_hard": 0.2522568998644805, "calibration": 59.1326044758686, "p50_s": 1.9947936326265334, "p95_s": 3.5520695313811297, "cost_usd_per_1000": 0.01241840953758297, "axes": { "intelligence": 71.71032843198861, "calibration": 59.1326044758686, "speed": 71.49620568259948, "cost": 67.17802065927918 }, "jevbench_score": 67.17841134684245 } }, { "threshold": 0.58, "public": { "n": 231, "escalation": 0.5238095238095238, "tiers": { "easy": 0.8958333333333334, "standard": 0.5833333333333334, "hard": 0.4144144144144144 }, "accuracy": 0.5670995670995671, "ece_hard": 0.3335843069054609, "calibration": 49.14013633780527, "p50_s": 1.7188292682170867, "p95_s": 3.5588197648525237, "cost_usd_per_1000": 0.011667697520955948, "axes": { "intelligence": 57.37143393393395, "calibration": 49.14013633780527, "speed": 72.13461265887787, "cost": 67.99044514794853 }, "jevbench_score": 60.9790862917456 }, "heldout": { "n": 303, "escalation": 0.5841584158415841, "tiers": { "easy": 0.875, "standard": 0.7083333333333334, "judge": 0.9041095890410958, "hard": 0.47706422018348627 }, "accuracy": 0.7326732673267327, "ece_hard": 0.2664550862811552, "calibration": 57.71278583420113, "p50_s": 1.9949055552482604, "p95_s": 3.5627083748579023, "cost_usd_per_1000": 0.012565745540846218, "axes": { "intelligence": 71.71032843198861, "calibration": 57.71278583420113, "speed": 71.48297385606756, "cost": 67.02435217273882 }, "jevbench_score": 66.73017237791791 } }, { "threshold": 0.59, "public": { "n": 231, "escalation": 0.5584415584415584, "tiers": { "easy": 0.9166666666666666, "standard": 0.5833333333333334, "hard": 0.40540540540540543 }, "accuracy": 0.5670995670995671, "ece_hard": 0.34987345114362284, "calibration": 46.560424881434685, "p50_s": 1.755893924832344, "p95_s": 3.575243228673935, "cost_usd_per_1000": 0.01218305462327934, "axes": { "intelligence": 57.40115115115116, "calibration": 46.560424881434685, "speed": 72.02196130978537, "cost": 67.4273142582064 }, "jevbench_score": 60.02187879637964 }, "heldout": { "n": 303, "escalation": 0.6171617161716172, "tiers": { "easy": 0.875, "standard": 0.7083333333333334, "judge": 0.9041095890410958, "hard": 0.47706422018348627 }, "accuracy": 0.7326732673267327, "ece_hard": 0.2766352185263727, "calibration": 56.69477260967938, "p50_s": 2.0749642491340636, "p95_s": 3.597061644494533, "cost_usd_per_1000": 0.013056865551723713, "axes": { "intelligence": 71.71032843198861, "calibration": 56.69477260967938, "speed": 71.27041501034658, "cost": 66.52483203556118 }, "jevbench_score": 66.26046066842082 } }, { "threshold": 0.6, "public": { "n": 231, "escalation": 0.5670995670995671, "tiers": { "easy": 0.9166666666666666, "standard": 0.5972222222222222, "hard": 0.40540540540540543 }, "accuracy": 0.5714285714285714, "ece_hard": 0.3525363084287998, "calibration": 46.29413915291699, "p50_s": 1.7581063002347945, "p95_s": 3.575243228673935, "cost_usd_per_1000": 0.012311893898860192, "axes": { "intelligence": 57.94127460794128, "calibration": 46.29413915291699, "speed": 72.01649276969093, "cost": 67.29025407438624 }, "jevbench_score": 60.04468187266936 }, "heldout": { "n": 303, "escalation": 0.6303630363036303, "tiers": { "easy": 0.875, "standard": 0.7083333333333334, "judge": 0.9041095890410958, "hard": 0.46788990825688076 }, "accuracy": 0.7293729372937293, "ece_hard": 0.26866915961476306, "calibration": 57.51980954330942, "p50_s": 2.111688548326492, "p95_s": 3.597061644494533, "cost_usd_per_1000": 0.013253313556074706, "axes": { "intelligence": 71.43509907419045, "calibration": 57.51980954330942, "speed": 71.19422254660972, "cost": 66.33026581209677 }, "jevbench_score": 66.36993542091048 } }, { "threshold": 0.61, "public": { "n": 231, "escalation": 0.5887445887445888, "tiers": { "easy": 0.9166666666666666, "standard": 0.5972222222222222, "hard": 0.42342342342342343 }, "accuracy": 0.5800865800865801, "ece_hard": 0.3373580038480321, "calibration": 47.811969610993756, "p50_s": 1.7840687870979308, "p95_s": 3.575243228673935, "cost_usd_per_1000": 0.012633992087812313, "axes": { "intelligence": 58.692025358692035, "calibration": 47.811969610993756, "speed": 71.95282812058953, "cost": 66.95378198270939 }, "jevbench_score": 60.63657721124578 }, "heldout": { "n": 303, "escalation": 0.66996699669967, "tiers": { "easy": 0.875, "standard": 0.7083333333333334, "judge": 0.8972602739726028, "hard": 0.46788990825688076 }, "accuracy": 0.7260726072607261, "ece_hard": 0.2684410255680504, "calibration": 57.54262294798069, "p50_s": 2.217090719938278, "p95_s": 3.606947714090347, "cost_usd_per_1000": 0.013842657569127698, "axes": { "intelligence": 71.24331825227263, "calibration": 57.54262294798069, "speed": 70.970766889806, "cost": 65.7634157281492 }, "jevbench_score": 66.13774958390056 } }, { "threshold": 0.62, "public": { "n": 231, "escalation": 0.6017316017316018, "tiers": { "easy": 0.9375, "standard": 0.5972222222222222, "hard": 0.42342342342342343 }, "accuracy": 0.5844155844155844, "ece_hard": 0.3406630149318666, "calibration": 47.481468502610305, "p50_s": 1.789837846159935, "p95_s": 3.575243228673935, "cost_usd_per_1000": 0.012827251001183586, "axes": { "intelligence": 59.09711795128463, "calibration": 47.481468502610305, "speed": 71.93880720202802, "cost": 66.75599220961769 }, "jevbench_score": 60.587911585824415 }, "heldout": { "n": 303, "escalation": 0.7227722772277227, "tiers": { "easy": 0.9166666666666666, "standard": 0.7083333333333334, "judge": 0.8835616438356164, "hard": 0.46788990825688076 }, "accuracy": 0.7227722772277227, "ece_hard": 0.2684410255680504, "calibration": 57.54262294798069, "p50_s": 2.347588562965393, "p95_s": 3.606947714090347, "cost_usd_per_1000": 0.014628449586531686, "axes": { "intelligence": 71.44308994177035, "calibration": 57.54262294798069, "speed": 70.7223816807903, "cost": 65.04405101944269 }, "jevbench_score": 65.94450093619272 } }, { "threshold": 0.63, "public": { "n": 231, "escalation": 0.6190476190476191, "tiers": { "easy": 0.9375, "standard": 0.625, "hard": 0.42342342342342343 }, "accuracy": 0.5930735930735931, "ece_hard": 0.3394471524236485, "calibration": 47.67556294583768, "p50_s": 1.80832582116127, "p95_s": 3.590818943083286, "cost_usd_per_1000": 0.013084929552345283, "axes": { "intelligence": 60.17736486486488, "calibration": 47.67556294583768, "speed": 71.87529808743597, "cost": 66.4968583361798 }, "jevbench_score": 60.852362763914414 }, "heldout": { "n": 303, "escalation": 0.768976897689769, "tiers": { "easy": 0.9583333333333334, "standard": 0.7083333333333334, "judge": 0.8698630136986302, "hard": 0.45871559633027525 }, "accuracy": 0.7161716171617162, "ece_hard": 0.28404471205925985, "calibration": 55.98225429885974, "p50_s": 2.405140319466591, "p95_s": 3.606947714090347, "cost_usd_per_1000": 0.015316017601760174, "axes": { "intelligence": 71.36763227346992, "calibration": 55.98225429885974, "speed": 70.61719734284559, "cost": 64.4456242964811 }, "jevbench_score": 65.3001074556567 } }, { "threshold": 0.64, "public": { "n": 231, "escalation": 0.6536796536796536, "tiers": { "easy": 0.9375, "standard": 0.6666666666666666, "hard": 0.42342342342342343 }, "accuracy": 0.6060606060606061, "ece_hard": 0.3420502364126381, "calibration": 47.41525454693871, "p50_s": 1.8429425597190856, "p95_s": 3.590818943083286, "cost_usd_per_1000": 0.013600286654668677, "axes": { "intelligence": 61.79773523523523, "calibration": 47.41525454693871, "speed": 71.79294692926257, "cost": 65.99355813588869 }, "jevbench_score": 61.04055766897029 }, "heldout": { "n": 303, "escalation": 0.8085808580858086, "tiers": { "easy": 0.9583333333333334, "standard": 0.7083333333333334, "judge": 0.8698630136986302, "hard": 0.45871559633027525 }, "accuracy": 0.7161716171617162, "ece_hard": 0.2860358195102267, "calibration": 55.783143553763054, "p50_s": 2.4738951712846755, "p95_s": 3.606947714090347, "cost_usd_per_1000": 0.015905361614813165, "axes": { "intelligence": 71.36763227346992, "calibration": 55.783143553763054, "speed": 70.49478860210678, "cost": 63.95369357752346 }, "jevbench_score": 65.08886962102838 } }, { "threshold": 0.65, "public": { "n": 231, "escalation": 0.6926406926406926, "tiers": { "easy": 0.9583333333333334, "standard": 0.7222222222222222, "hard": 0.43243243243243246 }, "accuracy": 0.6320346320346321, "ece_hard": 0.3467592039019373, "calibration": 46.944357798008795, "p50_s": 1.8830789119005202, "p95_s": 3.590818943083286, "cost_usd_per_1000": 0.014180063394782495, "axes": { "intelligence": 64.73869703036371, "calibration": 46.944357798008795, "speed": 71.69937972546816, "cost": 65.4496548264729 }, "jevbench_score": 61.45292074001007 }, "heldout": { "n": 303, "escalation": 0.8283828382838284, "tiers": { "easy": 0.9583333333333334, "standard": 0.7083333333333334, "judge": 0.8698630136986302, "hard": 0.45871559633027525 }, "accuracy": 0.7161716171617162, "ece_hard": 0.2860358195102267, "calibration": 55.783143553763054, "p50_s": 2.4738951712846755, "p95_s": 3.606947714090347, "cost_usd_per_1000": 0.016200033621339663, "axes": { "intelligence": 71.36763227346992, "calibration": 55.783143553763054, "speed": 70.49478860210678, "cost": 63.714522523818964 }, "jevbench_score": 65.02793000905378 } }, { "threshold": 0.66, "public": { "n": 231, "escalation": 0.7056277056277056, "tiers": { "easy": 0.9583333333333334, "standard": 0.7222222222222222, "hard": 0.44144144144144143 }, "accuracy": 0.6363636363636364, "ece_hard": 0.3375303378959703, "calibration": 47.86724439860549, "p50_s": 1.904844531416893, "p95_s": 3.590818943083286, "cost_usd_per_1000": 0.014373322308153767, "axes": { "intelligence": 65.11407240573908, "calibration": 47.86724439860549, "speed": 71.64946956953014, "cost": 65.27328507010427 }, "jevbench_score": 61.78960711040713 }, "heldout": { "n": 303, "escalation": 0.8646864686468647, "tiers": { "easy": 1.0, "standard": 0.75, "judge": 0.8767123287671232, "hard": 0.45871559633027525 }, "accuracy": 0.7260726072607261, "ece_hard": 0.29114197939413, "calibration": 54.08055728172664, "p50_s": 2.5353830844163894, "p95_s": 3.606947714090347, "cost_usd_per_1000": 0.016740265633304902, "axes": { "intelligence": 73.30941309538773, "calibration": 54.08055728172664, "speed": 70.38816564646612, "cost": 63.28712964805474 }, "jevbench_score": 64.82671080232944 } }, { "threshold": 0.67, "public": { "n": 231, "escalation": 0.7272727272727273, "tiers": { "easy": 0.9791666666666666, "standard": 0.7222222222222222, "hard": 0.43243243243243246 }, "accuracy": 0.6363636363636364, "ece_hard": 0.3378174060112733, "calibration": 47.838537587075194, "p50_s": 1.93047876060009, "p95_s": 3.6071916311979293, "cost_usd_per_1000": 0.014695420497105888, "axes": { "intelligence": 65.14378962295629, "calibration": 47.838537587075194, "speed": 71.5716575746052, "cost": 64.98453947326001 }, "jevbench_score": 61.7021800608241 }, "heldout": { "n": 303, "escalation": 0.8811881188118812, "tiers": { "easy": 1.0, "standard": 0.75, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7293729372937293, "ece_hard": 0.2863967046797736, "calibration": 54.55508475316228, "p50_s": 2.5353830844163894, "p95_s": 3.606947714090347, "cost_usd_per_1000": 0.016985825638743648, "axes": { "intelligence": 73.58464245318588, "calibration": 54.55508475316228, "speed": 70.38816564646612, "cost": 63.09740014918392 }, "jevbench_score": 64.98054998273098 } }, { "threshold": 0.68, "public": { "n": 231, "escalation": 0.7402597402597403, "tiers": { "easy": 1.0, "standard": 0.7361111111111112, "hard": 0.43243243243243246 }, "accuracy": 0.645021645021645, "ece_hard": 0.3378174060112733, "calibration": 47.838537587075194, "p50_s": 1.9433941483497619, "p95_s": 3.618648448586464, "cost_usd_per_1000": 0.014888679410477165, "axes": { "intelligence": 66.08900567233901, "calibration": 47.838537587075194, "speed": 71.52892715802508, "cost": 64.8143146420022 }, "jevbench_score": 61.87496115104693 }, "heldout": { "n": 303, "escalation": 0.900990099009901, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28921610862808994, "calibration": 54.27314435833064, "p50_s": 2.5353830844163894, "p95_s": 3.606947714090347, "cost_usd_per_1000": 0.017280497645270145, "axes": { "intelligence": 74.75130911985254, "calibration": 54.27314435833064, "speed": 70.38816564646612, "cost": 62.87331264480621 }, "jevbench_score": 65.09422251656115 } }, { "threshold": 0.69, "public": { "n": 231, "escalation": 0.7619047619047619, "tiers": { "easy": 1.0, "standard": 0.7361111111111112, "hard": 0.44144144144144143 }, "accuracy": 0.6493506493506493, "ece_hard": 0.3286320553146608, "calibration": 48.75707265673645, "p50_s": 1.9622834265232085, "p95_s": 3.618648448586464, "cost_usd_per_1000": 0.015210777599429282, "axes": { "intelligence": 66.46438104771438, "calibration": 48.75707265673645, "speed": 71.48691870708328, "cost": 64.53545750640008 }, "jevbench_score": 62.18174275434698 }, "heldout": { "n": 303, "escalation": 0.9075907590759076, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28472555544401457, "calibration": 54.722199676738185, "p50_s": 2.5478042304515838, "p95_s": 3.646907018125056, "cost_usd_per_1000": 0.017378721647445642, "axes": { "intelligence": 74.75130911985254, "calibration": 54.722199676738185, "speed": 70.31909254963995, "cost": 62.799465182786754 }, "jevbench_score": 65.19328844430422 } }, { "threshold": 0.7, "public": { "n": 231, "escalation": 0.7965367965367965, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.45045045045045046 }, "accuracy": 0.6623376623376623, "ece_hard": 0.319220906849006, "calibration": 49.698187503301924, "p50_s": 1.9807270854711532, "p95_s": 3.618648448586464, "cost_usd_per_1000": 0.01572613470175268, "axes": { "intelligence": 67.92000333667002, "calibration": 49.698187503301924, "speed": 71.44628966132328, "cost": 64.10134026150942 }, "jevbench_score": 62.70414634316038 }, "heldout": { "n": 303, "escalation": 0.9141914191419142, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28472555544401457, "calibration": 54.722199676738185, "p50_s": 2.5478042304515838, "p95_s": 3.646907018125056, "cost_usd_per_1000": 0.01747694564962114, "axes": { "intelligence": 74.75130911985254, "calibration": 54.722199676738185, "speed": 70.31909254963995, "cost": 62.726033930373546 }, "jevbench_score": 65.17422249616271 } }, { "threshold": 0.71, "public": { "n": 231, "escalation": 0.8051948051948052, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.45045045045045046 }, "accuracy": 0.6623376623376623, "ece_hard": 0.319220906849006, "calibration": 49.698187503301924, "p50_s": 1.9828382074832915, "p95_s": 3.618648448586464, "cost_usd_per_1000": 0.015854973977333528, "axes": { "intelligence": 67.92000333667002, "calibration": 49.698187503301924, "speed": 71.44166327745202, "cost": 63.99503399323114 }, "jevbench_score": 62.67711819371619 }, "heldout": { "n": 303, "escalation": 0.9174917491749175, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28532514282196864, "calibration": 54.66224093894277, "p50_s": 2.5478042304515838, "p95_s": 3.646907018125056, "cost_usd_per_1000": 0.01752605765070889, "axes": { "intelligence": 74.75130911985254, "calibration": 54.66224093894277, "speed": 70.31909254963995, "cost": 62.68947292179777 }, "jevbench_score": 65.14686595085205 } }, { "threshold": 0.72, "public": { "n": 231, "escalation": 0.8225108225108225, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.4594594594594595 }, "accuracy": 0.6666666666666666, "ece_hard": 0.3121943049912075, "calibration": 50.40084768908178, "p50_s": 1.9963175445795058, "p95_s": 3.618648448586464, "cost_usd_per_1000": 0.016112652528495225, "axes": { "intelligence": 68.29537871204539, "calibration": 50.40084768908178, "speed": 71.41223982947865, "cost": 63.784988752671616 }, "jevbench_score": 62.92599249634102 }, "heldout": { "n": 303, "escalation": 0.9207920792079208, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.2819824452550945, "calibration": 54.996510695630185, "p50_s": 2.5588221192359923, "p95_s": 3.646907018125056, "cost_usd_per_1000": 0.01757516965179664, "axes": { "intelligence": 74.75130911985254, "calibration": 54.996510695630185, "speed": 70.30035213121674, "cost": 62.65301422227372 }, "jevbench_score": 65.23239926177158 } }, { "threshold": 0.73, "public": { "n": 231, "escalation": 0.8225108225108225, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.4594594594594595 }, "accuracy": 0.6666666666666666, "ece_hard": 0.3121943049912075, "calibration": 50.40084768908178, "p50_s": 1.9963175445795058, "p95_s": 3.618648448586464, "cost_usd_per_1000": 0.016112652528495225, "axes": { "intelligence": 68.29537871204539, "calibration": 50.40084768908178, "speed": 71.41223982947865, "cost": 63.784988752671616 }, "jevbench_score": 62.92599249634102 }, "heldout": { "n": 303, "escalation": 0.9273927392739274, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.2828116708290094, "calibration": 54.9135881382387, "p50_s": 2.5588221192359923, "p95_s": 3.646907018125056, "cost_usd_per_1000": 0.01767339365397214, "axes": { "intelligence": 74.75130911985254, "calibration": 54.9135881382387, "speed": 70.30035213121674, "cost": 62.58040147119728 }, "jevbench_score": 65.18889471249292 } }, { "threshold": 0.74, "public": { "n": 231, "escalation": 0.8268398268398268, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.4594594594594595 }, "accuracy": 0.6666666666666666, "ece_hard": 0.3121943049912075, "calibration": 50.40084768908178, "p50_s": 1.9963175445795058, "p95_s": 3.618648448586464, "cost_usd_per_1000": 0.01617707216628565, "axes": { "intelligence": 68.29537871204539, "calibration": 50.40084768908178, "speed": 71.41223982947865, "cost": 63.733002313143515 }, "jevbench_score": 62.91316699199435 }, "heldout": { "n": 303, "escalation": 0.9273927392739274, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.2828116708290094, "calibration": 54.9135881382387, "p50_s": 2.5588221192359923, "p95_s": 3.646907018125056, "cost_usd_per_1000": 0.01767339365397214, "axes": { "intelligence": 74.75130911985254, "calibration": 54.9135881382387, "speed": 70.30035213121674, "cost": 62.58040147119728 }, "jevbench_score": 65.18889471249292 } }, { "threshold": 0.75, "public": { "n": 231, "escalation": 0.8311688311688312, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 2.003491559624672, "p95_s": 3.618648448586464, "cost_usd_per_1000": 0.016241491804076073, "axes": { "intelligence": 68.67075408742076, "calibration": 50.471910537865476, "speed": 71.39666089354944, "cost": 63.68122248094329 }, "jevbench_score": 63.005391353985154 }, "heldout": { "n": 303, "escalation": 0.933993399339934, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.47706422018348627 }, "accuracy": 0.735973597359736, "ece_hard": 0.2754005055623087, "calibration": 55.65470466490876, "p50_s": 2.5588221192359923, "p95_s": 3.646907018125056, "cost_usd_per_1000": 0.017771617656147636, "axes": { "intelligence": 75.0265384776507, "calibration": 55.65470466490876, "speed": 70.30035213121674, "cost": 62.508191166127915 }, "jevbench_score": 65.44896846518574 } }, { "threshold": 0.76, "public": { "n": 231, "escalation": 0.8311688311688312, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 2.003491559624672, "p95_s": 3.618648448586464, "cost_usd_per_1000": 0.016241491804076073, "axes": { "intelligence": 68.67075408742076, "calibration": 50.471910537865476, "speed": 71.39666089354944, "cost": 63.68122248094329 }, "jevbench_score": 63.005391353985154 }, "heldout": { "n": 303, "escalation": 0.933993399339934, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.47706422018348627 }, "accuracy": 0.735973597359736, "ece_hard": 0.2754005055623087, "calibration": 55.65470466490876, "p50_s": 2.5588221192359923, "p95_s": 3.646907018125056, "cost_usd_per_1000": 0.017771617656147636, "axes": { "intelligence": 75.0265384776507, "calibration": 55.65470466490876, "speed": 70.30035213121674, "cost": 62.508191166127915 }, "jevbench_score": 65.44896846518574 } }, { "threshold": 0.77, "public": { "n": 231, "escalation": 0.8441558441558441, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 2.003491559624672, "p95_s": 3.618648448586464, "cost_usd_per_1000": 0.016434750717447346, "axes": { "intelligence": 68.67075408742076, "calibration": 50.471910537865476, "speed": 71.39666089354944, "cost": 63.52710636753418 }, "jevbench_score": 62.96723657887374 }, "heldout": { "n": 303, "escalation": 0.9372937293729373, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28538890361769753, "calibration": 54.655864859369885, "p50_s": 2.569989839196205, "p95_s": 3.6649639233946796, "cost_usd_per_1000": 0.017820729657235385, "axes": { "intelligence": 74.75130911985254, "calibration": 54.655864859369885, "speed": 70.25998889750214, "cost": 62.47223554159456 }, "jevbench_score": 65.07477489010674 } }, { "threshold": 0.78, "public": { "n": 231, "escalation": 0.8441558441558441, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 2.003491559624672, "p95_s": 3.618648448586464, "cost_usd_per_1000": 0.016434750717447346, "axes": { "intelligence": 68.67075408742076, "calibration": 50.471910537865476, "speed": 71.39666089354944, "cost": 63.52710636753418 }, "jevbench_score": 62.96723657887374 }, "heldout": { "n": 303, "escalation": 0.9405940594059405, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28538890361769753, "calibration": 54.655864859369885, "p50_s": 2.569989839196205, "p95_s": 3.6649639233946796, "cost_usd_per_1000": 0.017869841658323133, "axes": { "intelligence": 74.75130911985254, "calibration": 54.655864859369885, "speed": 70.25998889750214, "cost": 62.43637887064128 }, "jevbench_score": 65.06543527242765 } }, { "threshold": 0.79, "public": { "n": 231, "escalation": 0.8441558441558441, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 2.003491559624672, "p95_s": 3.618648448586464, "cost_usd_per_1000": 0.016434750717447346, "axes": { "intelligence": 68.67075408742076, "calibration": 50.471910537865476, "speed": 71.39666089354944, "cost": 63.52710636753418 }, "jevbench_score": 62.96723657887374 }, "heldout": { "n": 303, "escalation": 0.9438943894389439, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28538890361769753, "calibration": 54.655864859369885, "p50_s": 2.569989839196205, "p95_s": 3.6649639233946796, "cost_usd_per_1000": 0.017918953659410885, "axes": { "intelligence": 74.75130911985254, "calibration": 54.655864859369885, "speed": 70.25998889750214, "cost": 62.400620610101704 }, "jevbench_score": 65.05611728073312 } }, { "threshold": 0.8, "public": { "n": 231, "escalation": 0.8484848484848485, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 2.003491559624672, "p95_s": 3.618648448586464, "cost_usd_per_1000": 0.01649917035523777, "axes": { "intelligence": 68.67075408742076, "calibration": 50.471910537865476, "speed": 71.39666089354944, "cost": 63.47613679940272 }, "jevbench_score": 62.95460268479238 }, "heldout": { "n": 303, "escalation": 0.9438943894389439, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28538890361769753, "calibration": 54.655864859369885, "p50_s": 2.569989839196205, "p95_s": 3.6649639233946796, "cost_usd_per_1000": 0.017918953659410885, "axes": { "intelligence": 74.75130911985254, "calibration": 54.655864859369885, "speed": 70.25998889750214, "cost": 62.400620610101704 }, "jevbench_score": 65.05611728073312 } }, { "threshold": 0.81, "public": { "n": 231, "escalation": 0.8484848484848485, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 2.003491559624672, "p95_s": 3.618648448586464, "cost_usd_per_1000": 0.01649917035523777, "axes": { "intelligence": 68.67075408742076, "calibration": 50.471910537865476, "speed": 71.39666089354944, "cost": 63.47613679940272 }, "jevbench_score": 62.95460268479238 }, "heldout": { "n": 303, "escalation": 0.9471947194719472, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28538890361769753, "calibration": 54.655864859369885, "p50_s": 2.569989839196205, "p95_s": 3.6649639233946796, "cost_usd_per_1000": 0.017968065660498633, "axes": { "intelligence": 74.75130911985254, "calibration": 54.655864859369885, "speed": 70.25998889750214, "cost": 62.364960221269484 }, "jevbench_score": 65.04682080359612 } }, { "threshold": 0.82, "public": { "n": 231, "escalation": 0.8484848484848485, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 2.003491559624672, "p95_s": 3.618648448586464, "cost_usd_per_1000": 0.01649917035523777, "axes": { "intelligence": 68.67075408742076, "calibration": 50.471910537865476, "speed": 71.39666089354944, "cost": 63.47613679940272 }, "jevbench_score": 62.95460268479238 }, "heldout": { "n": 303, "escalation": 0.9504950495049505, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28623177577145353, "calibration": 54.57157764399429, "p50_s": 2.570494332909584, "p95_s": 3.6649639233946796, "cost_usd_per_1000": 0.01801717766158638, "axes": { "intelligence": 74.75130911985254, "calibration": 54.57157764399429, "speed": 70.25913645314381, "cost": 62.329397169849564 }, "jevbench_score": 65.01225970592476 } }, { "threshold": 0.83, "public": { "n": 231, "escalation": 0.8528138528138528, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 2.003491559624672, "p95_s": 3.618648448586464, "cost_usd_per_1000": 0.016563589993028194, "axes": { "intelligence": 68.67075408742076, "calibration": 50.471910537865476, "speed": 71.39666089354944, "cost": 63.42536585046753 }, "jevbench_score": 62.942010456364905 }, "heldout": { "n": 303, "escalation": 0.9504950495049505, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28623177577145353, "calibration": 54.57157764399429, "p50_s": 2.570494332909584, "p95_s": 3.6649639233946796, "cost_usd_per_1000": 0.01801717766158638, "axes": { "intelligence": 74.75130911985254, "calibration": 54.57157764399429, "speed": 70.25913645314381, "cost": 62.329397169849564 }, "jevbench_score": 65.01225970592476 } }, { "threshold": 0.84, "public": { "n": 231, "escalation": 0.8571428571428571, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 2.003491559624672, "p95_s": 3.618648448586464, "cost_usd_per_1000": 0.01662800963081862, "axes": { "intelligence": 68.67075408742076, "calibration": 50.471910537865476, "speed": 71.39666089354944, "cost": 63.37479197876918 }, "jevbench_score": 62.92945958891728 }, "heldout": { "n": 303, "escalation": 0.9504950495049505, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28623177577145353, "calibration": 54.57157764399429, "p50_s": 2.570494332909584, "p95_s": 3.6649639233946796, "cost_usd_per_1000": 0.01801717766158638, "axes": { "intelligence": 74.75130911985254, "calibration": 54.57157764399429, "speed": 70.25913645314381, "cost": 62.329397169849564 }, "jevbench_score": 65.01225970592476 } }, { "threshold": 0.85, "public": { "n": 231, "escalation": 0.8571428571428571, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 2.003491559624672, "p95_s": 3.618648448586464, "cost_usd_per_1000": 0.01662800963081862, "axes": { "intelligence": 68.67075408742076, "calibration": 50.471910537865476, "speed": 71.39666089354944, "cost": 63.37479197876918 }, "jevbench_score": 62.92945958891728 }, "heldout": { "n": 303, "escalation": 0.9537953795379538, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28623177577145353, "calibration": 54.57157764399429, "p50_s": 2.570494332909584, "p95_s": 3.6649639233946796, "cost_usd_per_1000": 0.01806628966267413, "axes": { "intelligence": 74.75130911985254, "calibration": 54.57157764399429, "speed": 70.25913645314381, "cost": 62.29393092591014 }, "jevbench_score": 65.00300952491625 } }, { "threshold": 0.86, "public": { "n": 231, "escalation": 0.8658008658008658, "tiers": { "easy": 1.0, "standard": 0.75, "hard": 0.46846846846846846 }, "accuracy": 0.6666666666666666, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 2.003491559624672, "p95_s": 3.618648448586464, "cost_usd_per_1000": 0.016756848906399467, "axes": { "intelligence": 68.13063063063062, "calibration": 50.471910537865476, "speed": 71.39666089354944, "cost": 63.27422938840394 }, "jevbench_score": 62.78042187629496 }, "heldout": { "n": 303, "escalation": 0.9537953795379538, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28623177577145353, "calibration": 54.57157764399429, "p50_s": 2.570494332909584, "p95_s": 3.6649639233946796, "cost_usd_per_1000": 0.01806628966267413, "axes": { "intelligence": 74.75130911985254, "calibration": 54.57157764399429, "speed": 70.25913645314381, "cost": 62.29393092591014 }, "jevbench_score": 65.00300952491625 } }, { "threshold": 0.87, "public": { "n": 231, "escalation": 0.8701298701298701, "tiers": { "easy": 1.0, "standard": 0.75, "hard": 0.46846846846846846 }, "accuracy": 0.6666666666666666, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 2.003491559624672, "p95_s": 3.618648448586464, "cost_usd_per_1000": 0.01682126854418989, "axes": { "intelligence": 68.13063063063062, "calibration": 50.471910537865476, "speed": 71.39666089354944, "cost": 63.22423767415538 }, "jevbench_score": 62.76801780893119 }, "heldout": { "n": 303, "escalation": 0.9537953795379538, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28623177577145353, "calibration": 54.57157764399429, "p50_s": 2.570494332909584, "p95_s": 3.6649639233946796, "cost_usd_per_1000": 0.01806628966267413, "axes": { "intelligence": 74.75130911985254, "calibration": 54.57157764399429, "speed": 70.25913645314381, "cost": 62.29393092591014 }, "jevbench_score": 65.00300952491625 } }, { "threshold": 0.88, "public": { "n": 231, "escalation": 0.8831168831168831, "tiers": { "easy": 1.0, "standard": 0.75, "hard": 0.46846846846846846 }, "accuracy": 0.6666666666666666, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 2.003491559624672, "p95_s": 3.618648448586464, "cost_usd_per_1000": 0.017014527457561164, "axes": { "intelligence": 68.13063063063062, "calibration": 50.471910537865476, "speed": 71.39666089354944, "cost": 63.075403240283656 }, "jevbench_score": 62.73104503937097 }, "heldout": { "n": 303, "escalation": 0.9537953795379538, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28623177577145353, "calibration": 54.57157764399429, "p50_s": 2.570494332909584, "p95_s": 3.6649639233946796, "cost_usd_per_1000": 0.01806628966267413, "axes": { "intelligence": 74.75130911985254, "calibration": 54.57157764399429, "speed": 70.25913645314381, "cost": 62.29393092591014 }, "jevbench_score": 65.00300952491625 } }, { "threshold": 0.89, "public": { "n": 231, "escalation": 0.8961038961038961, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 2.003491559624672, "p95_s": 3.618648448586464, "cost_usd_per_1000": 0.017207786370932437, "axes": { "intelligence": 68.67075408742076, "calibration": 50.471910537865476, "speed": 71.39666089354944, "cost": 62.92824982635338 }, "jevbench_score": 62.818314401528426 }, "heldout": { "n": 303, "escalation": 0.9570957095709571, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28623177577145353, "calibration": 54.57157764399429, "p50_s": 2.570494332909584, "p95_s": 3.6649639233946796, "cost_usd_per_1000": 0.018115401663761882, "axes": { "intelligence": 74.75130911985254, "calibration": 54.57157764399429, "speed": 70.25913645314381, "cost": 62.25856096383537 }, "jevbench_score": 64.99378052118479 } }, { "threshold": 0.9, "public": { "n": 231, "escalation": 0.8961038961038961, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 2.003491559624672, "p95_s": 3.618648448586464, "cost_usd_per_1000": 0.017207786370932437, "axes": { "intelligence": 68.67075408742076, "calibration": 50.471910537865476, "speed": 71.39666089354944, "cost": 62.92824982635338 }, "jevbench_score": 62.818314401528426 }, "heldout": { "n": 303, "escalation": 0.9570957095709571, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28623177577145353, "calibration": 54.57157764399429, "p50_s": 2.570494332909584, "p95_s": 3.6649639233946796, "cost_usd_per_1000": 0.018115401663761882, "axes": { "intelligence": 74.75130911985254, "calibration": 54.57157764399429, "speed": 70.25913645314381, "cost": 62.25856096383537 }, "jevbench_score": 64.99378052118479 } }, { "threshold": 0.91, "public": { "n": 231, "escalation": 0.9090909090909091, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 2.003491559624672, "p95_s": 3.618648448586464, "cost_usd_per_1000": 0.01740104528430371, "axes": { "intelligence": 68.67075408742076, "calibration": 50.471910537865476, "speed": 71.39666089354944, "cost": 62.78273988329774 }, "jevbench_score": 62.78196893608742 }, "heldout": { "n": 303, "escalation": 0.9603960396039604, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28623177577145353, "calibration": 54.57157764399429, "p50_s": 2.570494332909584, "p95_s": 3.6649639233946796, "cost_usd_per_1000": 0.01816451366484963, "axes": { "intelligence": 74.75130911985254, "calibration": 54.57157764399429, "speed": 70.25913645314381, "cost": 62.223286762278505 }, "jevbench_score": 64.9845725868065 } }, { "threshold": 0.92, "public": { "n": 231, "escalation": 0.922077922077922, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 2.003491559624672, "p95_s": 3.618648448586464, "cost_usd_per_1000": 0.017594304197674982, "axes": { "intelligence": 68.67075408742076, "calibration": 50.471910537865476, "speed": 71.39666089354944, "cost": 62.638837106271005 }, "jevbench_score": 62.74596272044161 }, "heldout": { "n": 303, "escalation": 0.9636963696369637, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28623177577145353, "calibration": 54.57157764399429, "p50_s": 2.570494332909584, "p95_s": 3.6649639233946796, "cost_usd_per_1000": 0.01821362566593738, "axes": { "intelligence": 74.75130911985254, "calibration": 54.57157764399429, "speed": 70.25913645314381, "cost": 62.188107804115894 }, "jevbench_score": 64.97538561469963 } }, { "threshold": 0.93, "public": { "n": 231, "escalation": 0.922077922077922, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 2.003491559624672, "p95_s": 3.618648448586464, "cost_usd_per_1000": 0.017594304197674982, "axes": { "intelligence": 68.67075408742076, "calibration": 50.471910537865476, "speed": 71.39666089354944, "cost": 62.638837106271005 }, "jevbench_score": 62.74596272044161 }, "heldout": { "n": 303, "escalation": 0.9702970297029703, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28623177577145353, "calibration": 54.57157764399429, "p50_s": 2.570494332909584, "p95_s": 3.6649639233946796, "cost_usd_per_1000": 0.01831184966811288, "axes": { "intelligence": 74.75130911985254, "calibration": 54.57157764399429, "speed": 70.25913645314381, "cost": 62.11803357032163 }, "jevbench_score": 64.95707413313218 } }, { "threshold": 0.94, "public": { "n": 231, "escalation": 0.9264069264069265, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 2.003491559624672, "p95_s": 3.618648448586464, "cost_usd_per_1000": 0.01765872383546541, "axes": { "intelligence": 68.67075408742076, "calibration": 50.471910537865476, "speed": 71.39666089354944, "cost": 62.591220559403915 }, "jevbench_score": 62.734034824365175 }, "heldout": { "n": 303, "escalation": 0.9735973597359736, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28623177577145353, "calibration": 54.57157764399429, "p50_s": 2.570494332909584, "p95_s": 3.6649639233946796, "cost_usd_per_1000": 0.018360961669200627, "axes": { "intelligence": 74.75130911985254, "calibration": 54.57157764399429, "speed": 70.25913645314381, "cost": 62.08313728115154 }, "jevbench_score": 64.9479494136406 } }, { "threshold": 0.95, "public": { "n": 231, "escalation": 0.935064935064935, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 2.003491559624672, "p95_s": 3.618648448586464, "cost_usd_per_1000": 0.017787563111046255, "axes": { "intelligence": 68.67075408742076, "calibration": 50.471910537865476, "speed": 71.39666089354944, "cost": 62.4965063802774 }, "jevbench_score": 62.710288773183706 }, "heldout": { "n": 303, "escalation": 0.976897689768977, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28623177577145353, "calibration": 54.57157764399429, "p50_s": 2.570494332909584, "p95_s": 3.6649639233946796, "cost_usd_per_1000": 0.018410073670288376, "axes": { "intelligence": 74.75130911985254, "calibration": 54.57157764399429, "speed": 70.25913645314381, "cost": 62.04833420821083 }, "jevbench_score": 64.93884523634166 } }, { "threshold": 0.96, "public": { "n": 231, "escalation": 0.9523809523809523, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 2.003491559624672, "p95_s": 3.618648448586464, "cost_usd_per_1000": 0.01804524166220795, "axes": { "intelligence": 68.67075408742076, "calibration": 50.471910537865476, "speed": 71.39666089354944, "cost": 62.3091189245754 }, "jevbench_score": 62.66322871289558 }, "heldout": { "n": 303, "escalation": 0.976897689768977, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28623177577145353, "calibration": 54.57157764399429, "p50_s": 2.570494332909584, "p95_s": 3.6649639233946796, "cost_usd_per_1000": 0.018410073670288376, "axes": { "intelligence": 74.75130911985254, "calibration": 54.57157764399429, "speed": 70.25913645314381, "cost": 62.04833420821083 }, "jevbench_score": 64.93884523634166 } }, { "threshold": 0.97, "public": { "n": 231, "escalation": 0.9696969696969697, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 2.010362187027931, "p95_s": 3.6374172061681747, "cost_usd_per_1000": 0.018302920213369652, "axes": { "intelligence": 68.67075408742076, "calibration": 50.471910537865476, "speed": 71.35932576176762, "cost": 62.12438840419466 }, "jevbench_score": 62.608544318957314 }, "heldout": { "n": 303, "escalation": 0.9933993399339934, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28023325293453055, "calibration": 55.17142992768658, "p50_s": 2.5800421327352523, "p95_s": 3.6649639233946796, "cost_usd_per_1000": 0.01865563367572712, "axes": { "intelligence": 74.75130911985254, "calibration": 55.17142992768658, "speed": 70.24303497883724, "cost": 61.8756998408854 }, "jevbench_score": 65.06749819436108 } }, { "threshold": 0.98, "public": { "n": 231, "escalation": 0.9826839826839827, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 2.010362187027931, "p95_s": 3.6374172061681747, "cost_usd_per_1000": 0.018496179126740925, "axes": { "intelligence": 68.67075408742076, "calibration": 50.471910537865476, "speed": 71.35932576176762, "cost": 61.98753931908057 }, "jevbench_score": 62.57403690187173 }, "heldout": { "n": 303, "escalation": 0.9966996699669967, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28023325293453055, "calibration": 55.17142992768658, "p50_s": 2.5800421327352523, "p95_s": 3.6649639233946796, "cost_usd_per_1000": 0.01870474567681487, "axes": { "intelligence": 74.75130911985254, "calibration": 55.17142992768658, "speed": 70.24303497883724, "cost": 61.84144577217987 }, "jevbench_score": 65.05849106632472 } }, { "threshold": 0.99, "public": { "n": 231, "escalation": 1.0, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 2.010362187027931, "p95_s": 3.6374172061681747, "cost_usd_per_1000": 0.01875385767790262, "axes": { "intelligence": 68.67075408742076, "calibration": 50.471910537865476, "speed": 71.35932576176762, "cost": 61.80728152464433 }, "jevbench_score": 62.528496382656066 }, "heldout": { "n": 303, "escalation": 1.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28023325293453055, "calibration": 55.17142992768658, "p50_s": 2.5800421327352523, "p95_s": 3.6649639233946796, "cost_usd_per_1000": 0.01875385767790262, "axes": { "intelligence": 74.75130911985254, "calibration": 55.17142992768658, "speed": 70.24303497883724, "cost": 61.80728152464433 }, "jevbench_score": 65.04950382895649 } }, { "threshold": 1.0, "public": { "n": 231, "escalation": 1.0, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 2.010362187027931, "p95_s": 3.6374172061681747, "cost_usd_per_1000": 0.01875385767790262, "axes": { "intelligence": 68.67075408742076, "calibration": 50.471910537865476, "speed": 71.35932576176762, "cost": 61.80728152464433 }, "jevbench_score": 62.528496382656066 }, "heldout": { "n": 303, "escalation": 1.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28023325293453055, "calibration": 55.17142992768658, "p50_s": 2.5800421327352523, "p95_s": 3.6649639233946796, "cost_usd_per_1000": 0.01875385767790262, "axes": { "intelligence": 74.75130911985254, "calibration": 55.17142992768658, "speed": 70.24303497883724, "cost": 61.80728152464433 }, "jevbench_score": 65.04950382895649 } } ] }, { "cheap": "laya", "backend": "open-alternative-jev", "chosen": { "threshold": 0.88, "public": { "n": 231, "escalation": 0.7792207792207793, "tiers": { "easy": 1.0, "standard": 0.6388888888888888, "hard": 0.5315315315315315 }, "accuracy": 0.6623376623376623, "ece_hard": 0.24959788792149712, "calibration": 56.326111706783166, "p50_s": 1.611314061284065, "p95_s": 5.476784098148347, "cost_usd_per_1000": 0.020159546427355415, "axes": { "intelligence": 66.43727060393728, "calibration": 56.326111706783166, "speed": 70.54294182155672, "cost": 60.8655773011892 }, "jevbench_score": 63.312099687996884 }, "heldout": { "n": 303, "escalation": 0.8712871287128713, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5273972602739726, "hard": 0.4954128440366973 }, "accuracy": 0.5676567656765676, "ece_hard": 0.29825105368372024, "calibration": 54.73603344486585, "p50_s": 3.3082447081804274, "p95_s": 5.587395147979261, "cost_usd_per_1000": 0.022200786702265732, "axes": { "intelligence": 63.4628419421055, "calibration": 54.73603344486585, "speed": 67.33192987193615, "cost": 59.6089490713466 }, "jevbench_score": 61.10564625120703 } }, "full": { "tiers": { "easy": 1.0, "standard": 0.65625, "judge": 0.5273972602739726, "hard": 0.5136363636363637 }, "accuracy": 0.6086142322097379, "ece_hard": 0.2737033200491441, "calibration": 55.553187651170965, "p50_s": 2.796613584458828, "p95_s": 5.542336564511061, "cost_usd_per_1000": 0.02131777826873711, "axes": { "intelligence": 62.55121419676215, "calibration": 55.553187651170965, "speed": 68.09674641848201, "cost": 60.13774177910038 }, "jevbench_score": 61.41925604145468 }, "full_escalation": 0.8314606741573034, "backend_heldout_accuracy": 0.5643564356435643, "useful": false, "curve": [ { "threshold": 0.0, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.01, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.02, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.03, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.04, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.05, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.06, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.07, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.08, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.09, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.1, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.11, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.12, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.13, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.14, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.15, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.16, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.17, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.18, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.19, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.2, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.21, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.22, "public": { "n": 231, "escalation": 0.004329004329004329, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.21278612221122534, "calibration": 63.403887778877476, "p50_s": 1.1657145261764525, "p95_s": 4.4528342053294185, "cost_usd_per_1000": 0.002979107446860256, "axes": { "intelligence": 53.58796296296297, "calibration": 63.403887778877476, "speed": 72.84771266552642, "cost": 85.77741498676124 }, "jevbench_score": 67.88018273891647 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.23, "public": { "n": 231, "escalation": 0.004329004329004329, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.21278612221122534, "calibration": 63.403887778877476, "p50_s": 1.1657145261764525, "p95_s": 4.4528342053294185, "cost_usd_per_1000": 0.002979107446860256, "axes": { "intelligence": 53.58796296296297, "calibration": 63.403887778877476, "speed": 72.84771266552642, "cost": 85.77741498676124 }, "jevbench_score": 67.88018273891647 }, "heldout": { "n": 303, "escalation": 0.0033003300330033004, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2648896103782679, "calibration": 54.85978896217321, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0029563002929506436, "axes": { "intelligence": 59.67720246323992, "calibration": 54.85978896217321, "speed": 68.8164811989393, "cost": 85.87754360749533 }, "jevbench_score": 66.32216062606311 } }, { "threshold": 0.24, "public": { "n": 231, "escalation": 0.004329004329004329, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.21278612221122534, "calibration": 63.403887778877476, "p50_s": 1.1657145261764525, "p95_s": 4.4528342053294185, "cost_usd_per_1000": 0.002979107446860256, "axes": { "intelligence": 53.58796296296297, "calibration": 63.403887778877476, "speed": 72.84771266552642, "cost": 85.77741498676124 }, "jevbench_score": 67.88018273891647 }, "heldout": { "n": 303, "escalation": 0.006600660066006601, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.30275229357798167 }, "accuracy": 0.5676567656765676, "ece_hard": 0.2670575358532289, "calibration": 54.642996414677114, "p50_s": 2.7680240362882613, "p95_s": 4.776861734688282, "cost_usd_per_1000": 0.003029473245077317, "axes": { "intelligence": 59.95243182103809, "calibration": 54.642996414677114, "speed": 68.78687472081019, "cost": 85.55898635922483 }, "jevbench_score": 66.2640734526458 } }, { "threshold": 0.25, "public": { "n": 231, "escalation": 0.008658008658008658, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.34234234234234234 }, "accuracy": 0.5324675324675324, "ece_hard": 0.21476462782221095, "calibration": 63.20603721777891, "p50_s": 1.1657145261764525, "p95_s": 4.4528342053294185, "cost_usd_per_1000": 0.0030750875528965417, "axes": { "intelligence": 53.963338338338346, "calibration": 63.20603721777891, "speed": 72.84771266552642, "cost": 85.36427543862241 }, "jevbench_score": 67.86367316210656 }, "heldout": { "n": 303, "escalation": 0.009900990099009901, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2647147042301286, "calibration": 54.877279576987135, "p50_s": 2.7680240362882613, "p95_s": 4.808721804618835, "cost_usd_per_1000": 0.0031026461972039903, "axes": { "intelligence": 59.67720246323992, "calibration": 54.877279576987135, "speed": 68.75800490091856, "cost": 85.2480323595695 }, "jevbench_score": 66.19149118182833 } }, { "threshold": 0.26, "public": { "n": 231, "escalation": 0.008658008658008658, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.34234234234234234 }, "accuracy": 0.5324675324675324, "ece_hard": 0.21476462782221095, "calibration": 63.20603721777891, "p50_s": 1.1657145261764525, "p95_s": 4.4528342053294185, "cost_usd_per_1000": 0.0030750875528965417, "axes": { "intelligence": 53.963338338338346, "calibration": 63.20603721777891, "speed": 72.84771266552642, "cost": 85.36427543862241 }, "jevbench_score": 67.86367316210656 }, "heldout": { "n": 303, "escalation": 0.013201320132013201, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2760406043167706, "calibration": 53.74468956832294, "p50_s": 2.7680240362882613, "p95_s": 4.8142633035779, "cost_usd_per_1000": 0.0031758191493306636, "axes": { "intelligence": 59.67720246323992, "calibration": 53.74468956832294, "speed": 68.75300303786179, "cost": 84.9443271081673 }, "jevbench_score": 65.78736879855717 } }, { "threshold": 0.27, "public": { "n": 231, "escalation": 0.021645021645021644, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.35135135135135137 }, "accuracy": 0.5367965367965368, "ece_hard": 0.21289556104103685, "calibration": 63.39294389589632, "p50_s": 1.1657145261764525, "p95_s": 4.455602088570595, "cost_usd_per_1000": 0.003363027871005399, "axes": { "intelligence": 54.33871371371373, "calibration": 63.39294389589632, "speed": 72.84501392824589, "cost": 84.19808600398919 }, "jevbench_score": 67.79740652745981 }, "heldout": { "n": 303, "escalation": 0.013201320132013201, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2760406043167706, "calibration": 53.74468956832294, "p50_s": 2.7680240362882613, "p95_s": 4.8142633035779, "cost_usd_per_1000": 0.0031758191493306636, "axes": { "intelligence": 59.67720246323992, "calibration": 53.74468956832294, "speed": 68.75300303786179, "cost": 84.9443271081673 }, "jevbench_score": 65.78736879855717 } }, { "threshold": 0.28, "public": { "n": 231, "escalation": 0.021645021645021644, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.35135135135135137 }, "accuracy": 0.5367965367965368, "ece_hard": 0.21289556104103685, "calibration": 63.39294389589632, "p50_s": 1.1657145261764525, "p95_s": 4.455602088570595, "cost_usd_per_1000": 0.003363027871005399, "axes": { "intelligence": 54.33871371371373, "calibration": 63.39294389589632, "speed": 72.84501392824589, "cost": 84.19808600398919 }, "jevbench_score": 67.79740652745981 }, "heldout": { "n": 303, "escalation": 0.019801980198019802, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.28736438993076613, "calibration": 52.612311006923385, "p50_s": 2.7680240362882613, "p95_s": 4.822164891660214, "cost_usd_per_1000": 0.0033221650535840102, "axes": { "intelligence": 59.67720246323992, "calibration": 52.612311006923385, "speed": 68.74588086188128, "cost": 84.35736383492474 }, "jevbench_score": 65.32303867274688 } }, { "threshold": 0.29, "public": { "n": 231, "escalation": 0.03896103896103896, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.23268607048730366, "calibration": 61.41389295126964, "p50_s": 1.1657145261764525, "p95_s": 4.538626746833325, "cost_usd_per_1000": 0.003746948295150543, "axes": { "intelligence": 53.58796296296297, "calibration": 61.41389295126964, "speed": 72.76483323506824, "cost": 82.78966899330085 }, "jevbench_score": 66.72796650528528 }, "heldout": { "n": 303, "escalation": 0.0231023102310231, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.28440366972477066 }, "accuracy": 0.5610561056105611, "ece_hard": 0.2887852197807373, "calibration": 52.470228021926275, "p50_s": 2.7680240362882613, "p95_s": 4.840475116670132, "cost_usd_per_1000": 0.0033953380057106836, "axes": { "intelligence": 59.401973105441755, "calibration": 52.470228021926275, "speed": 68.72942151082292, "cost": 84.07350955719136 }, "jevbench_score": 65.14467502660727 } }, { "threshold": 0.3, "public": { "n": 231, "escalation": 0.04329004329004329, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.34234234234234234 }, "accuracy": 0.5324675324675324, "ece_hard": 0.22913615028856388, "calibration": 61.76888497114362, "p50_s": 1.1657145261764525, "p95_s": 4.646009536087513, "cost_usd_per_1000": 0.0038429284011868283, "axes": { "intelligence": 53.963338338338346, "calibration": 61.76888497114362, "speed": 72.66327694805801, "cost": 82.46013120813933 }, "jevbench_score": 66.85084372528357 }, "heldout": { "n": 303, "escalation": 0.033003300330033, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6986301369863014, "hard": 0.29357798165137616 }, "accuracy": 0.5676567656765676, "ece_hard": 0.2770174134890014, "calibration": 53.64700865109986, "p50_s": 2.79284847676754, "p95_s": 4.870198597609996, "cost_usd_per_1000": 0.0036148568620907036, "axes": { "intelligence": 59.868983285157725, "calibration": 53.64700865109986, "speed": 68.66405954583303, "cost": 83.25726684514908 }, "jevbench_score": 65.4598121848748 } }, { "threshold": 0.31, "public": { "n": 231, "escalation": 0.05627705627705628, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.34234234234234234 }, "accuracy": 0.5324675324675324, "ece_hard": 0.2401238959310118, "calibration": 60.67011040689883, "p50_s": 1.1657145261764525, "p95_s": 4.646009536087513, "cost_usd_per_1000": 0.004130868719295686, "axes": { "intelligence": 53.963338338338346, "calibration": 60.67011040689883, "speed": 72.66327694805801, "cost": 81.5187582058326 }, "jevbench_score": 66.36078766666212 }, "heldout": { "n": 303, "escalation": 0.0429042904290429, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6986301369863014, "hard": 0.3119266055045872 }, "accuracy": 0.5742574257425742, "ece_hard": 0.27268558215513644, "calibration": 54.08019178448636, "p50_s": 2.79284847676754, "p95_s": 4.969698165357111, "cost_usd_per_1000": 0.0038343757184707236, "axes": { "intelligence": 60.41944200075407, "calibration": 54.08019178448636, "speed": 68.57622613220747, "cost": 82.48916002649342 }, "jevbench_score": 65.56866614649235 } }, { "threshold": 0.32, "public": { "n": 231, "escalation": 0.06060606060606061, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2533230379310382, "calibration": 59.350196206896186, "p50_s": 1.1657145261764525, "p95_s": 4.732986265420914, "cost_usd_per_1000": 0.004226848825331971, "axes": { "intelligence": 53.58796296296297, "calibration": 59.350196206896186, "speed": 72.58272547477219, "cost": 81.21949853743737 }, "jevbench_score": 65.80300787617668 }, "heldout": { "n": 303, "escalation": 0.0462046204620462, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6986301369863014, "hard": 0.3119266055045872 }, "accuracy": 0.5742574257425742, "ece_hard": 0.2768947787401091, "calibration": 53.65927212598909, "p50_s": 2.79284847676754, "p95_s": 5.079665326178072, "cost_usd_per_1000": 0.003907548670597397, "axes": { "intelligence": 60.41944200075407, "calibration": 53.65927212598909, "speed": 68.48117526149129, "cost": 82.24286811718846 }, "jevbench_score": 65.36913451169521 } }, { "threshold": 0.33, "public": { "n": 231, "escalation": 0.06493506493506493, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2593583172819543, "calibration": 58.74666827180457, "p50_s": 1.1657145261764525, "p95_s": 4.7531688347458845, "cost_usd_per_1000": 0.004322828931368258, "axes": { "intelligence": 53.58796296296297, "calibration": 58.74666827180457, "speed": 72.5642455076592, "cost": 80.9269585178406 }, "jevbench_score": 65.5717233805552 }, "heldout": { "n": 303, "escalation": 0.0594059405940594, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6986301369863014, "hard": 0.3394495412844037 }, "accuracy": 0.5841584158415841, "ece_hard": 0.26201954399599703, "calibration": 56.973025795959586, "p50_s": 2.8090254992246626, "p95_s": 5.091547676324844, "cost_usd_per_1000": 0.004200240479104091, "axes": { "intelligence": 61.245130074148555, "calibration": 56.973025795959586, "speed": 68.44594503649601, "cost": 81.30177531836219 }, "jevbench_score": 66.38151347161771 } }, { "threshold": 0.34, "public": { "n": 231, "escalation": 0.0735930735930736, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.34234234234234234 }, "accuracy": 0.5324675324675324, "ece_hard": 0.258280453178424, "calibration": 58.8544546821576, "p50_s": 1.1657145261764525, "p95_s": 4.792922566831113, "cost_usd_per_1000": 0.0045147891434408295, "axes": { "intelligence": 53.963338338338346, "calibration": 58.8544546821576, "speed": 72.52807378971121, "cost": 80.3608758387767 }, "jevbench_score": 65.5929613226464 }, "heldout": { "n": 303, "escalation": 0.06930693069306931, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7054794520547946, "hard": 0.3486238532110092 }, "accuracy": 0.5907590759075908, "ece_hard": 0.2646549185070961, "calibration": 56.709488344849674, "p50_s": 2.8090254992246626, "p95_s": 5.095480171442032, "cost_usd_per_1000": 0.004419759335484111, "axes": { "intelligence": 61.71214025386453, "calibration": 56.709488344849674, "speed": 68.4425920251175, "cost": 80.63804134565956 }, "jevbench_score": 66.29384203554106 } }, { "threshold": 0.35, "public": { "n": 231, "escalation": 0.0735930735930736, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.34234234234234234 }, "accuracy": 0.5324675324675324, "ece_hard": 0.258280453178424, "calibration": 58.8544546821576, "p50_s": 1.1657145261764525, "p95_s": 4.792922566831113, "cost_usd_per_1000": 0.0045147891434408295, "axes": { "intelligence": 53.963338338338346, "calibration": 58.8544546821576, "speed": 72.52807378971121, "cost": 80.3608758387767 }, "jevbench_score": 65.5929613226464 }, "heldout": { "n": 303, "escalation": 0.07920792079207921, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7054794520547946, "hard": 0.3669724770642202 }, "accuracy": 0.5973597359735974, "ece_hard": 0.25743368141545464, "calibration": 57.431612054013826, "p50_s": 2.8090254992246626, "p95_s": 5.103613194227219, "cost_usd_per_1000": 0.004639278191864131, "axes": { "intelligence": 62.26259896946086, "calibration": 57.431612054013826, "speed": 68.4356656689428, "cost": 80.00648753384186 }, "jevbench_score": 66.51911890231804 } }, { "threshold": 0.36, "public": { "n": 231, "escalation": 0.08658008658008658, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.34234234234234234 }, "accuracy": 0.5324675324675324, "ece_hard": 0.26771101608809017, "calibration": 57.911398391190986, "p50_s": 1.1657145261764525, "p95_s": 4.911704379320145, "cost_usd_per_1000": 0.004802729461549686, "axes": { "intelligence": 53.963338338338346, "calibration": 57.911398391190986, "speed": 72.42175571630612, "cost": 79.55535629630434 }, "jevbench_score": 65.14038690161945 }, "heldout": { "n": 303, "escalation": 0.08250825082508251, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7054794520547946, "hard": 0.3669724770642202 }, "accuracy": 0.5973597359735974, "ece_hard": 0.2602705873374481, "calibration": 56.342593620241715, "p50_s": 2.8090254992246626, "p95_s": 5.103613194227219, "cost_usd_per_1000": 0.004712451143990804, "axes": { "intelligence": 62.26259896946086, "calibration": 56.342593620241715, "speed": 68.4356656689428, "cost": 79.80259417841862 }, "jevbench_score": 66.15929921539619 } }, { "threshold": 0.37, "public": { "n": 231, "escalation": 0.11255411255411256, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.36936936936936937 }, "accuracy": 0.5454545454545454, "ece_hard": 0.25375619308992103, "calibration": 59.306880691007905, "p50_s": 1.169974410533905, "p95_s": 5.184776361286641, "cost_usd_per_1000": 0.005378610097767402, "axes": { "intelligence": 55.08946446446447, "calibration": 59.306880691007905, "speed": 72.17093608568683, "cost": 78.07989811396243 }, "jevbench_score": 65.50414801230171 }, "heldout": { "n": 303, "escalation": 0.0891089108910891, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7054794520547946, "hard": 0.3669724770642202 }, "accuracy": 0.5973597359735974, "ece_hard": 0.2663938324593925, "calibration": 55.73026910804728, "p50_s": 2.8090254992246626, "p95_s": 5.105773428678513, "cost_usd_per_1000": 0.004858797048244151, "axes": { "intelligence": 62.26259896946086, "calibration": 55.73026910804728, "speed": 68.43382779571796, "cost": 79.40413723063163 }, "jevbench_score": 65.8958534740291 } }, { "threshold": 0.38, "public": { "n": 231, "escalation": 0.12554112554112554, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.36936936936936937 }, "accuracy": 0.5454545454545454, "ece_hard": 0.2556788935351704, "calibration": 59.11461064648296, "p50_s": 1.2405554592609405, "p95_s": 5.2804211929440505, "cost_usd_per_1000": 0.005666550415876259, "axes": { "intelligence": 55.08946446446447, "calibration": 59.11461064648296, "speed": 71.83715250315785, "cost": 77.40043728750753 }, "jevbench_score": 65.23249308721184 }, "heldout": { "n": 303, "escalation": 0.09900990099009901, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7123287671232876, "hard": 0.3761467889908257 }, "accuracy": 0.6039603960396039, "ece_hard": 0.2685941884096286, "calibration": 55.510233513023664, "p50_s": 2.8226395934820174, "p95_s": 5.137652306258678, "cost_usd_per_1000": 0.005078315904624171, "axes": { "intelligence": 62.72960914917684, "calibration": 55.510233513023664, "speed": 68.38579861080868, "cost": 78.82840859953002 }, "jevbench_score": 65.82237970358563 } }, { "threshold": 0.39, "public": { "n": 231, "escalation": 0.14285714285714285, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3783783783783784 }, "accuracy": 0.5497835497835498, "ece_hard": 0.2579128867420719, "calibration": 58.89121132579281, "p50_s": 1.2405554592609405, "p95_s": 5.368320199847222, "cost_usd_per_1000": 0.006050470840021402, "axes": { "intelligence": 55.46483983983985, "calibration": 58.89121132579281, "speed": 71.76545403798013, "cost": 76.54632483348395 }, "jevbench_score": 65.08441533816875 }, "heldout": { "n": 303, "escalation": 0.11221122112211221, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7191780821917808, "hard": 0.3761467889908257 }, "accuracy": 0.6072607260726073, "ece_hard": 0.281485895684578, "calibration": 54.22106278552873, "p50_s": 2.8282473921775817, "p95_s": 5.137652306258678, "cost_usd_per_1000": 0.005371007713130864, "axes": { "intelligence": 62.921389971094634, "calibration": 54.22106278552873, "speed": 68.37717894867504, "cost": 78.09832671843283 }, "jevbench_score": 65.33258023428402 } }, { "threshold": 0.4, "public": { "n": 231, "escalation": 0.1471861471861472, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.38738738738738737 }, "accuracy": 0.5541125541125541, "ece_hard": 0.25284916766273274, "calibration": 59.397583233726735, "p50_s": 1.2405554592609405, "p95_s": 5.368320199847222, "cost_usd_per_1000": 0.006146450946057688, "axes": { "intelligence": 55.84021521521522, "calibration": 59.397583233726735, "speed": 71.76545403798013, "cost": 76.34126740211536 }, "jevbench_score": 65.29014982472255 }, "heldout": { "n": 303, "escalation": 0.132013201320132, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7123287671232876, "hard": 0.3853211009174312 }, "accuracy": 0.6072607260726073, "ece_hard": 0.27785685085023554, "calibration": 54.568425692667745, "p50_s": 2.8282473921775817, "p95_s": 5.21769452303648, "cost_usd_per_1000": 0.005810045425890904, "axes": { "intelligence": 63.004838506975005, "calibration": 54.568425692667745, "speed": 68.3100395549713, "cost": 77.07461416183587 }, "jevbench_score": 65.22706029554915 } }, { "threshold": 0.41, "public": { "n": 231, "escalation": 0.1645021645021645, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.40540540540540543 }, "accuracy": 0.5627705627705628, "ece_hard": 0.25102933032000224, "calibration": 59.57956696799978, "p50_s": 1.2405554592609405, "p95_s": 5.42692114263773, "cost_usd_per_1000": 0.006530371370202832, "axes": { "intelligence": 56.59096596596597, "calibration": 59.57956696799978, "speed": 71.71830304752201, "cost": 75.55186361483914 }, "jevbench_score": 65.37774079940104 }, "heldout": { "n": 303, "escalation": 0.13861386138613863, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7054794520547946, "hard": 0.3761467889908257 }, "accuracy": 0.6006600660066007, "ece_hard": 0.28665652482355924, "calibration": 53.27179713760467, "p50_s": 2.8427457749843597, "p95_s": 5.261130438745021, "cost_usd_per_1000": 0.005956391330144251, "axes": { "intelligence": 62.53782832725903, "calibration": 53.27179713760467, "speed": 68.25182907907248, "cost": 76.75050331547736 }, "jevbench_score": 64.63368505302392 } }, { "threshold": 0.42, "public": { "n": 231, "escalation": 0.16883116883116883, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.4144144144144144 }, "accuracy": 0.5670995670995671, "ece_hard": 0.2461933120056879, "calibration": 60.06316879943121, "p50_s": 1.2405554592609405, "p95_s": 5.42692114263773, "cost_usd_per_1000": 0.006626351476239117, "axes": { "intelligence": 56.96634134134135, "calibration": 60.06316879943121, "speed": 71.71830304752201, "cost": 75.36176595729152 }, "jevbench_score": 65.5770551628415 }, "heldout": { "n": 303, "escalation": 0.15841584158415842, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7054794520547946, "hard": 0.3944954128440367 }, "accuracy": 0.6072607260726073, "ece_hard": 0.2768209807180487, "calibration": 54.25535154815572, "p50_s": 2.8427457749843597, "p95_s": 5.261130438745021, "cost_usd_per_1000": 0.006395429042904291, "axes": { "intelligence": 63.08828704285535, "calibration": 54.25535154815572, "speed": 68.25182907907248, "cost": 75.82390945558043 }, "jevbench_score": 64.87508594564562 } }, { "threshold": 0.43, "public": { "n": 231, "escalation": 0.18614718614718614, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.42342342342342343 }, "accuracy": 0.5714285714285714, "ece_hard": 0.25128331440431334, "calibration": 58.94446477913179, "p50_s": 1.2405554592609405, "p95_s": 5.42692114263773, "cost_usd_per_1000": 0.00701027190038426, "axes": { "intelligence": 57.34171671671673, "calibration": 58.94446477913179, "speed": 71.71830304752201, "cost": 74.62795411486701 }, "jevbench_score": 65.21707475668221 }, "heldout": { "n": 303, "escalation": 0.19141914191419143, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7191780821917808, "hard": 0.3944954128440367 }, "accuracy": 0.6138613861386139, "ece_hard": 0.2898307829492494, "calibration": 52.95437132503565, "p50_s": 2.8707323759794234, "p95_s": 5.309325644373894, "cost_usd_per_1000": 0.007127158564171024, "axes": { "intelligence": 63.471848686690954, "calibration": 52.95437132503565, "speed": 68.16967929363335, "cost": 74.41250736891672 }, "jevbench_score": 64.25841934868457 } }, { "threshold": 0.44, "public": { "n": 231, "escalation": 0.19913419913419914, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.43243243243243246 }, "accuracy": 0.5757575757575758, "ece_hard": 0.2504380302233182, "calibration": 59.2595517355728, "p50_s": 1.2405554592609405, "p95_s": 5.42692114263773, "cost_usd_per_1000": 0.007298212218493118, "axes": { "intelligence": 57.7170920920921, "calibration": 59.2595517355728, "speed": 71.71830304752201, "cost": 74.10350536925961 }, "jevbench_score": 65.29544565476395 }, "heldout": { "n": 303, "escalation": 0.20132013201320131, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6986301369863014, "hard": 0.3944954128440367 }, "accuracy": 0.6039603960396039, "ece_hard": 0.2898307829492494, "calibration": 52.95437132503565, "p50_s": 2.8707323759794234, "p95_s": 5.309325644373894, "cost_usd_per_1000": 0.007346677420551045, "axes": { "intelligence": 62.89650622093754, "calibration": 52.95437132503565, "speed": 68.16967929363335, "cost": 74.01727086489844 }, "jevbench_score": 64.02700161431696 } }, { "threshold": 0.45, "public": { "n": 231, "escalation": 0.21212121212121213, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.45045045045045046 }, "accuracy": 0.5844155844155844, "ece_hard": 0.24398896174864224, "calibration": 59.90445858304039, "p50_s": 1.2405554592609405, "p95_s": 5.42692114263773, "cost_usd_per_1000": 0.007586152536601975, "axes": { "intelligence": 58.46784284284285, "calibration": 59.90445858304039, "speed": 71.71830304752201, "cost": 73.59935287244065 }, "jevbench_score": 65.57224528620748 }, "heldout": { "n": 303, "escalation": 0.2145214521452145, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.3944954128440367 }, "accuracy": 0.6006600660066007, "ece_hard": 0.2934753492187509, "calibration": 52.58991469808551, "p50_s": 2.916802155971527, "p95_s": 5.309325644373894, "cost_usd_per_1000": 0.007639369229057738, "axes": { "intelligence": 62.704725399019736, "calibration": 52.58991469808551, "speed": 68.10053664322287, "cost": 73.50827496909844 }, "jevbench_score": 63.74151412157903 } }, { "threshold": 0.46, "public": { "n": 231, "escalation": 0.22510822510822512, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.4594594594594595 }, "accuracy": 0.5887445887445888, "ece_hard": 0.2414994314067719, "calibration": 60.15341161722743, "p50_s": 1.3233075231313705, "p95_s": 5.447764056921006, "cost_usd_per_1000": 0.007874092854710833, "axes": { "intelligence": 58.84321821821823, "calibration": 60.15341161722743, "speed": 71.42120928504016, "cost": 73.11398404381299 }, "jevbench_score": 65.56862588896237 }, "heldout": { "n": 303, "escalation": 0.21782178217821782, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.3944954128440367 }, "accuracy": 0.6006600660066007, "ece_hard": 0.293761746083148, "calibration": 53.0451887205494, "p50_s": 2.916802155971527, "p95_s": 5.309325644373894, "cost_usd_per_1000": 0.0077125421811844105, "axes": { "intelligence": 62.704725399019736, "calibration": 53.0451887205494, "speed": 68.10053664322287, "cost": 73.38407343168929 }, "jevbench_score": 63.85202200116319 } }, { "threshold": 0.47, "public": { "n": 231, "escalation": 0.24242424242424243, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.4594594594594595 }, "accuracy": 0.5887445887445888, "ece_hard": 0.24834261772141336, "calibration": 58.50256061822395, "p50_s": 1.3233075231313705, "p95_s": 5.453733111917973, "cost_usd_per_1000": 0.008258013278855976, "axes": { "intelligence": 58.84321821821823, "calibration": 58.50256061822395, "speed": 71.41645337332221, "cost": 72.49373269350319 }, "jevbench_score": 64.97443448884579 }, "heldout": { "n": 303, "escalation": 0.23432343234323433, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6986301369863014, "hard": 0.4036697247706422 }, "accuracy": 0.6072607260726073, "ece_hard": 0.2894569691649818, "calibration": 53.47566641236601, "p50_s": 2.984680587053299, "p95_s": 5.309325644373894, "cost_usd_per_1000": 0.008078406941817778, "axes": { "intelligence": 63.17173557873571, "calibration": 53.47566641236601, "speed": 68.00062775786493, "cost": 72.78022820370656 }, "jevbench_score": 63.94422635517044 } }, { "threshold": 0.48, "public": { "n": 231, "escalation": 0.2510822510822511, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.45045045045045046 }, "accuracy": 0.5844155844155844, "ece_hard": 0.2613286162358931, "calibration": 54.83223159123969, "p50_s": 1.3951993823051452, "p95_s": 5.453733111917973, "cost_usd_per_1000": 0.008449973490928547, "axes": { "intelligence": 58.46784284284285, "calibration": 54.83223159123969, "speed": 71.18669843529592, "cost": 72.19433960521873 }, "jevbench_score": 63.71090114623391 }, "heldout": { "n": 303, "escalation": 0.264026402640264, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6986301369863014, "hard": 0.42201834862385323 }, "accuracy": 0.6138613861386139, "ece_hard": 0.2744823974776322, "calibration": 54.97312358110098, "p50_s": 3.0606545895338058, "p95_s": 5.335074929893017, "cost_usd_per_1000": 0.008736963510957837, "axes": { "intelligence": 63.722194294332034, "calibration": 54.97312358110098, "speed": 67.87045156613647, "cost": 71.7591843421395 }, "jevbench_score": 64.26874839200548 } }, { "threshold": 0.49, "public": { "n": 231, "escalation": 0.2554112554112554, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.45045045045045046 }, "accuracy": 0.5844155844155844, "ece_hard": 0.26272632309528215, "calibration": 54.69246090530079, "p50_s": 1.3951993823051452, "p95_s": 5.453733111917973, "cost_usd_per_1000": 0.008545953596964834, "axes": { "intelligence": 58.46784284284285, "calibration": 54.69246090530079, "speed": 71.18669843529592, "cost": 72.04718408966774 }, "jevbench_score": 63.637791541324816 }, "heldout": { "n": 303, "escalation": 0.26732673267326734, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6986301369863014, "hard": 0.42201834862385323 }, "accuracy": 0.6138613861386139, "ece_hard": 0.27614343193549207, "calibration": 54.807020135314986, "p50_s": 3.0606545895338058, "p95_s": 5.335074929893017, "cost_usd_per_1000": 0.008810136463084511, "axes": { "intelligence": 63.722194294332034, "calibration": 54.807020135314986, "speed": 67.87045156613647, "cost": 71.6505209381984 }, "jevbench_score": 64.19582003423983 } }, { "threshold": 0.5, "public": { "n": 231, "escalation": 0.26406926406926406, "tiers": { "easy": 0.9583333333333334, "standard": 0.5555555555555556, "hard": 0.45045045045045046 }, "accuracy": 0.5887445887445888, "ece_hard": 0.2668289093578137, "calibration": 54.28220227904763, "p50_s": 1.416904667019844, "p95_s": 5.453733111917973, "cost_usd_per_1000": 0.008737913809037405, "axes": { "intelligence": 59.00796629963298, "calibration": 54.28220227904763, "speed": 71.11965488059697, "cost": 71.75776730485231 }, "jevbench_score": 63.58529141231047 }, "heldout": { "n": 303, "escalation": 0.27722772277227725, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6986301369863014, "hard": 0.41284403669724773 }, "accuracy": 0.6105610561056105, "ece_hard": 0.2888504529700348, "calibration": 53.536318031860716, "p50_s": 3.0606545895338058, "p95_s": 5.339561374485493, "cost_usd_per_1000": 0.009029655319464532, "axes": { "intelligence": 63.44696493653388, "calibration": 53.536318031860716, "speed": 67.86680097198217, "cost": 71.32986481854027 }, "jevbench_score": 63.67911699280819 } }, { "threshold": 0.51, "public": { "n": 231, "escalation": 0.2813852813852814, "tiers": { "easy": 0.9583333333333334, "standard": 0.5972222222222222, "hard": 0.44144144144144143 }, "accuracy": 0.5974025974025974, "ece_hard": 0.27718165588633675, "calibration": 53.24692762619533, "p50_s": 1.4668009370565414, "p95_s": 5.453733111917973, "cost_usd_per_1000": 0.009121834233182548, "axes": { "intelligence": 60.252961294627966, "calibration": 53.24692762619533, "speed": 70.96934940108532, "cost": 71.19753472760215 }, "jevbench_score": 63.45300330022145 }, "heldout": { "n": 303, "escalation": 0.29042904290429045, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6986301369863014, "hard": 0.41284403669724773 }, "accuracy": 0.6105610561056105, "ece_hard": 0.2938054406339443, "calibration": 53.4676154850329, "p50_s": 3.069469991326332, "p95_s": 5.339561374485493, "cost_usd_per_1000": 0.009322347127971226, "axes": { "intelligence": 63.44696493653388, "calibration": 53.4676154850329, "speed": 67.85431025358704, "cost": 70.91424188991805 }, "jevbench_score": 63.5628180228746 } }, { "threshold": 0.52, "public": { "n": 231, "escalation": 0.2943722943722944, "tiers": { "easy": 0.9583333333333334, "standard": 0.5972222222222222, "hard": 0.43243243243243246 }, "accuracy": 0.5930735930735931, "ece_hard": 0.2899117623726318, "calibration": 51.97391697756582, "p50_s": 1.4921533554792403, "p95_s": 5.453733111917973, "cost_usd_per_1000": 0.009409774551291406, "axes": { "intelligence": 59.877585919252596, "calibration": 51.97391697756582, "speed": 70.89492658967006, "cost": 70.79262345117702 }, "jevbench_score": 62.865622992741706 }, "heldout": { "n": 303, "escalation": 0.32673267326732675, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6986301369863014, "hard": 0.42201834862385323 }, "accuracy": 0.6138613861386139, "ece_hard": 0.2902820627608366, "calibration": 53.81995327234367, "p50_s": 3.069469991326332, "p95_s": 5.341946345567704, "cost_usd_per_1000": 0.010127249601364631, "axes": { "intelligence": 63.722194294332034, "calibration": 53.81995327234367, "speed": 67.85237086483247, "cost": 69.83525458135418 }, "jevbench_score": 63.49191799735225 } }, { "threshold": 0.53, "public": { "n": 231, "escalation": 0.30735930735930733, "tiers": { "easy": 0.9583333333333334, "standard": 0.6111111111111112, "hard": 0.43243243243243246 }, "accuracy": 0.5974025974025974, "ece_hard": 0.29289334252662497, "calibration": 51.67575896216651, "p50_s": 1.4921533554792403, "p95_s": 5.453733111917973, "cost_usd_per_1000": 0.009697714869400263, "axes": { "intelligence": 60.417709376042716, "calibration": 51.67575896216651, "speed": 70.89492658967006, "cost": 70.39991767260729 }, "jevbench_score": 62.82892181546202 }, "heldout": { "n": 303, "escalation": 0.3465346534653465, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.684931506849315, "hard": 0.43119266055045874 }, "accuracy": 0.6105610561056105, "ece_hard": 0.28724414602736914, "calibration": 54.12374494569042, "p50_s": 3.069469991326332, "p95_s": 5.390882144272328, "cost_usd_per_1000": 0.010566287314124671, "axes": { "intelligence": 63.61386200829459, "calibration": 54.12374494569042, "speed": 67.81276770777643, "cost": 69.28232752996519 }, "jevbench_score": 63.418853304040695 } }, { "threshold": 0.54, "public": { "n": 231, "escalation": 0.3246753246753247, "tiers": { "easy": 0.9583333333333334, "standard": 0.5972222222222222, "hard": 0.43243243243243246 }, "accuracy": 0.5930735930735931, "ece_hard": 0.2982198540397449, "calibration": 51.14310781085452, "p50_s": 1.4921533554792403, "p95_s": 5.453733111917973, "cost_usd_per_1000": 0.010081635293545407, "axes": { "intelligence": 59.877585919252596, "calibration": 51.14310781085452, "speed": 70.89492658967006, "cost": 69.89407052075954 }, "jevbench_score": 62.413239126074664 }, "heldout": { "n": 303, "escalation": 0.36303630363036304, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.678082191780822, "hard": 0.45871559633027525 }, "accuracy": 0.6171617161716172, "ece_hard": 0.26758937405509664, "calibration": 56.08922214291767, "p50_s": 3.069469991326332, "p95_s": 5.408535666167736, "cost_usd_per_1000": 0.010932152074758039, "axes": { "intelligence": 64.24776925977127, "calibration": 56.08922214291767, "speed": 67.7985691016869, "cost": 68.83883006687614 }, "jevbench_score": 64.0394907661869 } }, { "threshold": 0.55, "public": { "n": 231, "escalation": 0.3463203463203463, "tiers": { "easy": 0.9791666666666666, "standard": 0.5972222222222222, "hard": 0.45045045045045046 }, "accuracy": 0.6060606060606061, "ece_hard": 0.28397477166378315, "calibration": 53.903687027790234, "p50_s": 1.4921533554792403, "p95_s": 5.453733111917973, "cost_usd_per_1000": 0.010561535823726835, "axes": { "intelligence": 61.03342926259594, "calibration": 53.903687027790234, "speed": 70.89492658967006, "cost": 69.28818770613739 }, "jevbench_score": 63.40378007362155 }, "heldout": { "n": 303, "escalation": 0.37293729372937295, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.678082191780822, "hard": 0.46788990825688076 }, "accuracy": 0.6204620462046204, "ece_hard": 0.2608313566383929, "calibration": 56.76502388458805, "p50_s": 3.069469991326332, "p95_s": 5.496108880639076, "cost_usd_per_1000": 0.01115167093113806, "axes": { "intelligence": 64.52299861756944, "calibration": 56.76502388458805, "speed": 67.72881280417471, "cost": 68.57980163281451 }, "jevbench_score": 64.22309984488572 } }, { "threshold": 0.56, "public": { "n": 231, "escalation": 0.354978354978355, "tiers": { "easy": 0.9791666666666666, "standard": 0.5972222222222222, "hard": 0.45045045045045046 }, "accuracy": 0.6060606060606061, "ece_hard": 0.28397477166378315, "calibration": 53.903687027790234, "p50_s": 1.506838920712471, "p95_s": 5.453733111917973, "cost_usd_per_1000": 0.010753496035799408, "axes": { "intelligence": 61.03342926259594, "calibration": 53.903687027790234, "speed": 70.85239289610925, "cost": 69.05350961973568 }, "jevbench_score": 63.340520915583454 }, "heldout": { "n": 303, "escalation": 0.39933993399339934, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.678082191780822, "hard": 0.46788990825688076 }, "accuracy": 0.6204620462046204, "ece_hard": 0.2673778293616893, "calibration": 56.110376612258406, "p50_s": 3.0780361533164977, "p95_s": 5.527056209743023, "cost_usd_per_1000": 0.011737054548151446, "axes": { "intelligence": 64.52299861756944, "calibration": 56.110376612258406, "speed": 67.69232402088258, "cost": 67.9132263106589 }, "jevbench_score": 63.87234754173886 } }, { "threshold": 0.57, "public": { "n": 231, "escalation": 0.36363636363636365, "tiers": { "easy": 0.9791666666666666, "standard": 0.6111111111111112, "hard": 0.44144144144144143 }, "accuracy": 0.6060606060606061, "ece_hard": 0.2946949116311485, "calibration": 51.22699534916593, "p50_s": 1.528649601340294, "p95_s": 5.453733111917973, "cost_usd_per_1000": 0.010945456247871979, "axes": { "intelligence": 61.19817734401067, "calibration": 51.22699534916593, "speed": 70.78998171458655, "cost": 68.82298392063373 }, "jevbench_score": 62.515206121389234 }, "heldout": { "n": 303, "escalation": 0.40924092409240925, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.678082191780822, "hard": 0.46788990825688076 }, "accuracy": 0.6204620462046204, "ece_hard": 0.26665542396492525, "calibration": 56.18261715193481, "p50_s": 3.086716505885124, "p95_s": 5.527056209743023, "cost_usd_per_1000": 0.011956573404531465, "axes": { "intelligence": 64.52299861756944, "calibration": 56.18261715193481, "speed": 67.68009374362302, "cost": 67.67179796676331 }, "jevbench_score": 63.83315258594925 } }, { "threshold": 0.58, "public": { "n": 231, "escalation": 0.36796536796536794, "tiers": { "easy": 0.9791666666666666, "standard": 0.6111111111111112, "hard": 0.45045045045045046 }, "accuracy": 0.6103896103896104, "ece_hard": 0.2874697267709405, "calibration": 51.949513835186735, "p50_s": 1.528649601340294, "p95_s": 5.460318829119206, "cost_usd_per_1000": 0.011041436353908264, "axes": { "intelligence": 61.57355271938606, "calibration": 51.949513835186735, "speed": 70.78474050627554, "cost": 68.70923279820377 }, "jevbench_score": 62.80332119896781 }, "heldout": { "n": 303, "escalation": 0.42244224422442245, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6712328767123288, "hard": 0.46788990825688076 }, "accuracy": 0.6171617161716172, "ece_hard": 0.26937134836678, "calibration": 55.911024711749334, "p50_s": 3.086716505885124, "p95_s": 5.527056209743023, "cost_usd_per_1000": 0.012249265213038159, "axes": { "intelligence": 64.33121779565163, "calibration": 55.911024711749334, "speed": 67.68009374362302, "cost": 67.35669886590736 }, "jevbench_score": 63.634149308518424 } }, { "threshold": 0.59, "public": { "n": 231, "escalation": 0.37662337662337664, "tiers": { "easy": 0.9791666666666666, "standard": 0.6111111111111112, "hard": 0.44144144144144143 }, "accuracy": 0.6060606060606061, "ece_hard": 0.2984038418857119, "calibration": 50.85610232370959, "p50_s": 1.528649601340294, "p95_s": 5.460318829119206, "cost_usd_per_1000": 0.011233396565980837, "axes": { "intelligence": 61.19817734401067, "calibration": 50.85610232370959, "speed": 70.78474050627554, "cost": 68.48466727609858 }, "jevbench_score": 62.32375887014813 }, "heldout": { "n": 303, "escalation": 0.43564356435643564, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6712328767123288, "hard": 0.46788990825688076 }, "accuracy": 0.6171617161716172, "ece_hard": 0.27936432364516256, "calibration": 54.911727183911076, "p50_s": 3.107663083076477, "p95_s": 5.538463139533997, "cost_usd_per_1000": 0.012541957021544852, "axes": { "intelligence": 64.33121779565163, "calibration": 54.911727183911076, "speed": 67.64176801632246, "cost": 67.04904075361777 }, "jevbench_score": 63.26646938209415 } }, { "threshold": 0.6, "public": { "n": 231, "escalation": 0.3939393939393939, "tiers": { "easy": 0.9791666666666666, "standard": 0.6388888888888888, "hard": 0.44144144144144143 }, "accuracy": 0.6147186147186147, "ece_hard": 0.29899486021649896, "calibration": 50.79700049063089, "p50_s": 1.528649601340294, "p95_s": 5.460318829119206, "cost_usd_per_1000": 0.01161731699012598, "axes": { "intelligence": 62.27842425759092, "calibration": 50.79700049063089, "speed": 70.78474050627554, "cost": 68.04682480964183 }, "jevbench_score": 62.47852935060943 }, "heldout": { "n": 303, "escalation": 0.44224422442244227, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6712328767123288, "hard": 0.47706422018348627 }, "accuracy": 0.6204620462046204, "ece_hard": 0.2765421725016815, "calibration": 55.19394229825918, "p50_s": 3.107663083076477, "p95_s": 5.538463139533997, "cost_usd_per_1000": 0.012688302925798199, "axes": { "intelligence": 64.60644715344979, "calibration": 55.19394229825918, "speed": 67.64176801632246, "cost": 66.89789384129196 }, "jevbench_score": 63.3794793819838 } }, { "threshold": 0.61, "public": { "n": 231, "escalation": 0.4199134199134199, "tiers": { "easy": 0.9791666666666666, "standard": 0.6388888888888888, "hard": 0.4594594594594595 }, "accuracy": 0.6233766233766234, "ece_hard": 0.2862477063060211, "calibration": 53.36901237275976, "p50_s": 1.528649601340294, "p95_s": 5.460318829119206, "cost_usd_per_1000": 0.012193197626343695, "axes": { "intelligence": 63.02917500834168, "calibration": 53.36901237275976, "speed": 70.78474050627554, "cost": 67.41647161397239 }, "jevbench_score": 63.29714372750024 }, "heldout": { "n": 303, "escalation": 0.46534653465346537, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6643835616438356, "hard": 0.46788990825688076 }, "accuracy": 0.6138613861386139, "ece_hard": 0.28925216269566095, "calibration": 53.92294327886124, "p50_s": 3.107663083076477, "p95_s": 5.538463139533997, "cost_usd_per_1000": 0.013200513590684913, "axes": { "intelligence": 64.13943697373382, "calibration": 53.92294327886124, "speed": 67.64176801632246, "cost": 66.38227514277615 }, "jevbench_score": 62.7756840888193 } }, { "threshold": 0.62, "public": { "n": 231, "escalation": 0.44155844155844154, "tiers": { "easy": 0.9791666666666666, "standard": 0.6388888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.6277056277056277, "ece_hard": 0.28406139192146235, "calibration": 53.587643811215635, "p50_s": 1.528649601340294, "p95_s": 5.46553184390068, "cost_usd_per_1000": 0.012673098156525123, "axes": { "intelligence": 63.40455038371705, "calibration": 53.587643811215635, "speed": 70.78059623633587, "cost": 66.91351604173022 }, "jevbench_score": 63.33638753060577 }, "heldout": { "n": 303, "escalation": 0.4752475247524752, "tiers": { "easy": 0.9166666666666666, "standard": 0.7083333333333334, "judge": 0.6643835616438356, "hard": 0.46788990825688076 }, "accuracy": 0.6171617161716172, "ece_hard": 0.29260007673318356, "calibration": 53.58815187510898, "p50_s": 3.107663083076477, "p95_s": 5.538463139533997, "cost_usd_per_1000": 0.013420032447064932, "axes": { "intelligence": 65.30610364040048, "calibration": 53.58815187510898, "speed": 67.64176801632246, "cost": 66.16739302374951 }, "jevbench_score": 62.91010091207164 } }, { "threshold": 0.63, "public": { "n": 231, "escalation": 0.44155844155844154, "tiers": { "easy": 0.9791666666666666, "standard": 0.6388888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.6277056277056277, "ece_hard": 0.28406139192146235, "calibration": 53.587643811215635, "p50_s": 1.528649601340294, "p95_s": 5.46553184390068, "cost_usd_per_1000": 0.012673098156525123, "axes": { "intelligence": 63.40455038371705, "calibration": 53.587643811215635, "speed": 70.78059623633587, "cost": 66.91351604173022 }, "jevbench_score": 63.33638753060577 }, "heldout": { "n": 303, "escalation": 0.49504950495049505, "tiers": { "easy": 0.9166666666666666, "standard": 0.7083333333333334, "judge": 0.6506849315068494, "hard": 0.45871559633027525 }, "accuracy": 0.6072607260726073, "ece_hard": 0.30316886806019333, "calibration": 52.531272742408, "p50_s": 3.1176133781671522, "p95_s": 5.538463139533997, "cost_usd_per_1000": 0.013859070159824972, "axes": { "intelligence": 64.6473126387667, "calibration": 52.531272742408, "speed": 67.62788473914385, "cost": 65.7479771999592 }, "jevbench_score": 62.336749091063126 } }, { "threshold": 0.64, "public": { "n": 231, "escalation": 0.4588744588744589, "tiers": { "easy": 0.9791666666666666, "standard": 0.6388888888888888, "hard": 0.45045045045045046 }, "accuracy": 0.6190476190476191, "ece_hard": 0.30769399448225243, "calibration": 51.224383555136626, "p50_s": 1.528649601340294, "p95_s": 5.46553184390068, "cost_usd_per_1000": 0.013057018580670267, "axes": { "intelligence": 62.653799632966305, "calibration": 51.224383555136626, "speed": 70.78059623633587, "cost": 66.52467933603921 }, "jevbench_score": 62.34911455660343 }, "heldout": { "n": 303, "escalation": 0.5148514851485149, "tiers": { "easy": 0.9166666666666666, "standard": 0.7083333333333334, "judge": 0.6438356164383562, "hard": 0.45871559633027525 }, "accuracy": 0.6039603960396039, "ece_hard": 0.30659319390531575, "calibration": 52.188840157895754, "p50_s": 3.123221132159233, "p95_s": 5.538463139533997, "cost_usd_per_1000": 0.014298107872585012, "axes": { "intelligence": 64.4555318168489, "calibration": 52.188840157895754, "speed": 67.6200799583375, "cost": 65.34164292121977 }, "jevbench_score": 62.090605179831584 } }, { "threshold": 0.65, "public": { "n": 231, "escalation": 0.48917748917748916, "tiers": { "easy": 0.9791666666666666, "standard": 0.6111111111111112, "hard": 0.4594594594594595 }, "accuracy": 0.6147186147186147, "ece_hard": 0.30455147122767334, "calibration": 51.53863588059453, "p50_s": 1.5353544890880584, "p95_s": 5.46553184390068, "cost_usd_per_1000": 0.013728879322924268, "axes": { "intelligence": 61.948928094761435, "calibration": 51.53863588059453, "speed": 70.76158907895206, "cost": 65.87094737248378 }, "jevbench_score": 62.110432099532474 }, "heldout": { "n": 303, "escalation": 0.528052805280528, "tiers": { "easy": 0.9583333333333334, "standard": 0.7083333333333334, "judge": 0.6438356164383562, "hard": 0.46788990825688076 }, "accuracy": 0.6105610561056105, "ece_hard": 0.30008491992583836, "calibration": 52.8396675558435, "p50_s": 3.123221132159233, "p95_s": 5.538463139533997, "cost_usd_per_1000": 0.014590799681091705, "axes": { "intelligence": 65.3140945079804, "calibration": 52.8396675558435, "speed": 67.6200799583375, "cost": 65.0776271494993 }, "jevbench_score": 62.42644359075932 } }, { "threshold": 0.66, "public": { "n": 231, "escalation": 0.5151515151515151, "tiers": { "easy": 0.9791666666666666, "standard": 0.625, "hard": 0.44144144144144143 }, "accuracy": 0.6103896103896104, "ece_hard": 0.3282894247027264, "calibration": 48.45695802866024, "p50_s": 1.5393483668565748, "p95_s": 5.46553184390068, "cost_usd_per_1000": 0.014304759959141983, "axes": { "intelligence": 61.73830080080081, "calibration": 48.45695802866024, "speed": 70.75030655746585, "cost": 65.3355827641322 }, "jevbench_score": 60.98137725959559 }, "heldout": { "n": 303, "escalation": 0.5478547854785478, "tiers": { "easy": 0.9583333333333334, "standard": 0.7083333333333334, "judge": 0.6438356164383562, "hard": 0.47706422018348627 }, "accuracy": 0.6138613861386139, "ece_hard": 0.2934193880200762, "calibration": 53.506220746419714, "p50_s": 3.156125175952911, "p95_s": 5.538463139533997, "cost_usd_per_1000": 0.015029837393851745, "axes": { "intelligence": 65.58932386577857, "calibration": 53.506220746419714, "speed": 67.57456510776193, "cost": 64.69137153914909 }, "jevbench_score": 62.584496449490885 } }, { "threshold": 0.67, "public": { "n": 231, "escalation": 0.5194805194805194, "tiers": { "easy": 0.9791666666666666, "standard": 0.625, "hard": 0.44144144144144143 }, "accuracy": 0.6103896103896104, "ece_hard": 0.3311436826314649, "calibration": 48.17153223578639, "p50_s": 1.5393483668565748, "p95_s": 5.46553184390068, "cost_usd_per_1000": 0.014400740065178268, "axes": { "intelligence": 61.73830080080081, "calibration": 48.17153223578639, "speed": 70.75030655746585, "cost": 65.24845565805006 }, "jevbench_score": 60.871068326663355 }, "heldout": { "n": 303, "escalation": 0.570957095709571, "tiers": { "easy": 0.9583333333333334, "standard": 0.7083333333333334, "judge": 0.6438356164383562, "hard": 0.48623853211009177 }, "accuracy": 0.6171617161716172, "ece_hard": 0.28832036493780305, "calibration": 54.016123054647025, "p50_s": 3.156125175952911, "p95_s": 5.538463139533997, "cost_usd_per_1000": 0.01554204805873846, "axes": { "intelligence": 65.86455322357673, "calibration": 54.016123054647025, "speed": 67.57456510776193, "cost": 64.25475257377192 }, "jevbench_score": 62.692547661942505 } }, { "threshold": 0.68, "public": { "n": 231, "escalation": 0.5238095238095238, "tiers": { "easy": 0.9791666666666666, "standard": 0.6111111111111112, "hard": 0.44144144144144143 }, "accuracy": 0.6060606060606061, "ece_hard": 0.3311436826314649, "calibration": 48.17153223578639, "p50_s": 1.5393483668565748, "p95_s": 5.46553184390068, "cost_usd_per_1000": 0.014496720171214556, "axes": { "intelligence": 61.19817734401067, "calibration": 48.17153223578639, "speed": 70.75030655746585, "cost": 65.16190732467113 }, "jevbench_score": 60.717343983345714 }, "heldout": { "n": 303, "escalation": 0.5907590759075908, "tiers": { "easy": 0.9583333333333334, "standard": 0.7083333333333334, "judge": 0.6438356164383562, "hard": 0.48623853211009177 }, "accuracy": 0.6171617161716172, "ece_hard": 0.2905641930139436, "calibration": 55.98088581794663, "p50_s": 3.15659946501255, "p95_s": 5.5818229272961615, "cost_usd_per_1000": 0.0159810857714985, "axes": { "intelligence": 65.86455322357673, "calibration": 55.98088581794663, "speed": 67.54004466218728, "cost": 63.891811528098685 }, "jevbench_score": 63.1574396084216 } }, { "threshold": 0.69, "public": { "n": 231, "escalation": 0.5238095238095238, "tiers": { "easy": 0.9791666666666666, "standard": 0.6111111111111112, "hard": 0.44144144144144143 }, "accuracy": 0.6060606060606061, "ece_hard": 0.3311436826314649, "calibration": 48.17153223578639, "p50_s": 1.5393483668565748, "p95_s": 5.46553184390068, "cost_usd_per_1000": 0.014496720171214556, "axes": { "intelligence": 61.19817734401067, "calibration": 48.17153223578639, "speed": 70.75030655746585, "cost": 65.16190732467113 }, "jevbench_score": 60.717343983345714 }, "heldout": { "n": 303, "escalation": 0.594059405940594, "tiers": { "easy": 0.9583333333333334, "standard": 0.7083333333333334, "judge": 0.6438356164383562, "hard": 0.48623853211009177 }, "accuracy": 0.6171617161716172, "ece_hard": 0.2905641930139436, "calibration": 55.98088581794663, "p50_s": 3.15659946501255, "p95_s": 5.5818229272961615, "cost_usd_per_1000": 0.016054258723625172, "axes": { "intelligence": 65.86455322357673, "calibration": 55.98088581794663, "speed": 67.54004466218728, "cost": 63.83229227191476 }, "jevbench_score": 63.142725681722425 } }, { "threshold": 0.7, "public": { "n": 231, "escalation": 0.5367965367965368, "tiers": { "easy": 0.9791666666666666, "standard": 0.6111111111111112, "hard": 0.45045045045045046 }, "accuracy": 0.6103896103896104, "ece_hard": 0.32291214984925487, "calibration": 48.99468551400739, "p50_s": 1.5393483668565748, "p95_s": 5.46553184390068, "cost_usd_per_1000": 0.014784660489323413, "axes": { "intelligence": 61.57355271938606, "calibration": 48.99468551400739, "speed": 70.75030655746585, "cost": 64.90565932109502 }, "jevbench_score": 61.008243273990416 }, "heldout": { "n": 303, "escalation": 0.6105610561056105, "tiers": { "easy": 0.9583333333333334, "standard": 0.7083333333333334, "judge": 0.6438356164383562, "hard": 0.4954128440366973 }, "accuracy": 0.6204620462046204, "ece_hard": 0.2813012578524459, "calibration": 56.9071793340964, "p50_s": 3.15659946501255, "p95_s": 5.5818229272961615, "cost_usd_per_1000": 0.016420123484258538, "axes": { "intelligence": 66.1397825813749, "calibration": 56.9071793340964, "speed": 67.54004466218728, "cost": 63.53870743538205 }, "jevbench_score": 63.39534731789673 } }, { "threshold": 0.71, "public": { "n": 231, "escalation": 0.5454545454545454, "tiers": { "easy": 0.9791666666666666, "standard": 0.6111111111111112, "hard": 0.45045045045045046 }, "accuracy": 0.6103896103896104, "ece_hard": 0.32291214984925487, "calibration": 48.99468551400739, "p50_s": 1.5505498081445692, "p95_s": 5.46553184390068, "cost_usd_per_1000": 0.014976620701395982, "axes": { "intelligence": 61.57355271938606, "calibration": 48.99468551400739, "speed": 70.71881849426103, "cost": 64.73758507098951 }, "jevbench_score": 60.961924604814826 }, "heldout": { "n": 303, "escalation": 0.6303630363036303, "tiers": { "easy": 0.9583333333333334, "standard": 0.7083333333333334, "judge": 0.6301369863013698, "hard": 0.5137614678899083 }, "accuracy": 0.6204620462046204, "ece_hard": 0.26815928263116806, "calibration": 58.22137685622418, "p50_s": 3.15659946501255, "p95_s": 5.5818229272961615, "cost_usd_per_1000": 0.01685916119701858, "axes": { "intelligence": 66.30667965313562, "calibration": 58.22137685622418, "speed": 67.54004466218728, "cost": 63.19492110587278 }, "jevbench_score": 63.71193833704664 } }, { "threshold": 0.72, "public": { "n": 231, "escalation": 0.5497835497835498, "tiers": { "easy": 0.9791666666666666, "standard": 0.6111111111111112, "hard": 0.44144144144144143 }, "accuracy": 0.6060606060606061, "ece_hard": 0.3329741636970336, "calibration": 47.98848412922952, "p50_s": 1.5505498081445692, "p95_s": 5.46553184390068, "cost_usd_per_1000": 0.01507260080743227, "axes": { "intelligence": 61.19817734401067, "calibration": 47.98848412922952, "speed": 70.71881849426103, "cost": 64.65435408511863 }, "jevbench_score": 60.5343764552391 }, "heldout": { "n": 303, "escalation": 0.6501650165016502, "tiers": { "easy": 0.9583333333333334, "standard": 0.7083333333333334, "judge": 0.6164383561643836, "hard": 0.5137614678899083 }, "accuracy": 0.6138613861386139, "ece_hard": 0.2720126602673384, "calibration": 57.83603909260715, "p50_s": 3.15659946501255, "p95_s": 5.5818229272961615, "cost_usd_per_1000": 0.017298198909778618, "axes": { "intelligence": 65.9231180093, "calibration": 57.83603909260715, "speed": 67.54004466218728, "cost": 62.85997339908071 }, "jevbench_score": 63.429743446856456 } }, { "threshold": 0.73, "public": { "n": 231, "escalation": 0.5584415584415584, "tiers": { "easy": 1.0, "standard": 0.6111111111111112, "hard": 0.44144144144144143 }, "accuracy": 0.6103896103896104, "ece_hard": 0.3329741636970336, "calibration": 47.98848412922952, "p50_s": 1.5631504535675047, "p95_s": 5.46553184390068, "cost_usd_per_1000": 0.01526456101950484, "axes": { "intelligence": 61.603269936603276, "calibration": 47.98848412922952, "speed": 70.68366790050091, "cost": 64.48947042071327 }, "jevbench_score": 60.5880773672489 }, "heldout": { "n": 303, "escalation": 0.6666666666666666, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.6095890410958904, "hard": 0.5137614678899083 }, "accuracy": 0.6138613861386139, "ece_hard": 0.2720126602673384, "calibration": 57.83603909260715, "p50_s": 3.15659946501255, "p95_s": 5.5818229272961615, "cost_usd_per_1000": 0.017664063670411984, "axes": { "intelligence": 66.31467052071551, "calibration": 57.83603909260715, "speed": 67.54004466218728, "cost": 62.587281355791205 }, "jevbench_score": 63.45471492962431 } }, { "threshold": 0.74, "public": { "n": 231, "escalation": 0.5627705627705628, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.44144144144144143 }, "accuracy": 0.6147186147186147, "ece_hard": 0.3329741636970336, "calibration": 47.98848412922952, "p50_s": 1.5631504535675047, "p95_s": 5.46553184390068, "cost_usd_per_1000": 0.015360541125541128, "axes": { "intelligence": 62.1433933933934, "calibration": 47.98848412922952, "speed": 70.68366790050091, "cost": 64.40780453753946 }, "jevbench_score": 60.701216167147365 }, "heldout": { "n": 303, "escalation": 0.6897689768976898, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.6027397260273972, "hard": 0.5045871559633027 }, "accuracy": 0.6072607260726073, "ece_hard": 0.2836445449571239, "calibration": 56.672850623628605, "p50_s": 3.15659946501255, "p95_s": 5.587395147979261, "cost_usd_per_1000": 0.018176274335298698, "axes": { "intelligence": 65.84766034099955, "calibration": 56.672850623628605, "speed": 67.53571135097592, "cost": 62.21485393282341 }, "jevbench_score": 62.92681383763969 } }, { "threshold": 0.75, "public": { "n": 231, "escalation": 0.5844155844155844, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.45045045045045046 }, "accuracy": 0.6190476190476191, "ece_hard": 0.32614617818611624, "calibration": 48.671282680321255, "p50_s": 1.5720471918582914, "p95_s": 5.466112996637822, "cost_usd_per_1000": 0.015840441655722553, "axes": { "intelligence": 62.51876876876879, "calibration": 48.671282680321255, "speed": 70.6585581558006, "cost": 64.00698141351467 }, "jevbench_score": 60.90722757762386 }, "heldout": { "n": 303, "escalation": 0.7029702970297029, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.589041095890411, "hard": 0.5045871559633027 }, "accuracy": 0.6006600660066007, "ece_hard": 0.2859922405327502, "calibration": 56.438081066065976, "p50_s": 3.15659946501255, "p95_s": 5.587395147979261, "cost_usd_per_1000": 0.01846896614380539, "axes": { "intelligence": 65.46409869716393, "calibration": 56.438081066065976, "speed": 67.53571135097592, "cost": 62.00672244476327 }, "jevbench_score": 62.71723762591678 } }, { "threshold": 0.76, "public": { "n": 231, "escalation": 0.5974025974025974, "tiers": { "easy": 1.0, "standard": 0.6111111111111112, "hard": 0.44144144144144143 }, "accuracy": 0.6103896103896104, "ece_hard": 0.33624100305385773, "calibration": 47.66180019354711, "p50_s": 1.5720471918582914, "p95_s": 5.472383971512318, "cost_usd_per_1000": 0.016128381973831412, "axes": { "intelligence": 61.603269936603276, "calibration": 47.66180019354711, "speed": 70.65357858668713, "cost": 63.77227598716579 }, "jevbench_score": 60.30940889897862 }, "heldout": { "n": 303, "escalation": 0.7227722772277227, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5684931506849316, "hard": 0.5137614678899083 }, "accuracy": 0.594059405940594, "ece_hard": 0.27628598496192536, "calibration": 57.40870662314846, "p50_s": 3.15659946501255, "p95_s": 5.587395147979261, "cost_usd_per_1000": 0.01890800385656543, "axes": { "intelligence": 65.16398558920868, "calibration": 57.40870662314846, "speed": 67.53571135097592, "cost": 61.70062953365106 }, "jevbench_score": 62.835072259794416 } }, { "threshold": 0.77, "public": { "n": 231, "escalation": 0.6190476190476191, "tiers": { "easy": 1.0, "standard": 0.6111111111111112, "hard": 0.4594594594594595 }, "accuracy": 0.6190476190476191, "ece_hard": 0.3216111046954726, "calibration": 49.12479002938562, "p50_s": 1.5720471918582914, "p95_s": 5.472383971512318, "cost_usd_per_1000": 0.01660828250401284, "axes": { "intelligence": 62.35402068735402, "calibration": 49.12479002938562, "speed": 70.65357858668713, "cost": 63.39025829484356 }, "jevbench_score": 60.85979199696939 }, "heldout": { "n": 303, "escalation": 0.735973597359736, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5684931506849316, "hard": 0.5137614678899083 }, "accuracy": 0.594059405940594, "ece_hard": 0.27628598496192536, "calibration": 57.40870662314846, "p50_s": 3.165240454673767, "p95_s": 5.587395147979261, "cost_usd_per_1000": 0.019200695665072126, "axes": { "intelligence": 65.16398558920868, "calibration": 57.40870662314846, "speed": 67.52383905900786, "cost": 61.50049107947342 }, "jevbench_score": 62.78129617146511 } }, { "threshold": 0.78, "public": { "n": 231, "escalation": 0.6406926406926406, "tiers": { "easy": 1.0, "standard": 0.6111111111111112, "hard": 0.46846846846846846 }, "accuracy": 0.6233766233766234, "ece_hard": 0.3121383762591704, "calibration": 50.07206287301584, "p50_s": 1.5874714136123655, "p95_s": 5.476784098148347, "cost_usd_per_1000": 0.01708818303419427, "axes": { "intelligence": 62.72939606272939, "calibration": 50.07206287301584, "speed": 70.60768465701516, "cost": 63.019123384920334 }, "jevbench_score": 61.14313880106215 }, "heldout": { "n": 303, "escalation": 0.7557755775577558, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5616438356164384, "hard": 0.5045871559633027 }, "accuracy": 0.5874587458745875, "ece_hard": 0.28990204470824305, "calibration": 56.04710064851669, "p50_s": 3.279486957192421, "p95_s": 5.587395147979261, "cost_usd_per_1000": 0.019639733377832164, "axes": { "intelligence": 64.6969754094927, "calibration": 56.04710064851669, "speed": 67.36984705659012, "cost": 61.20593237018218 }, "jevbench_score": 62.18334098922328 } }, { "threshold": 0.79, "public": { "n": 231, "escalation": 0.645021645021645, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.46846846846846846 }, "accuracy": 0.6277056277056277, "ece_hard": 0.3121383762591704, "calibration": 50.07206287301584, "p50_s": 1.5874714136123655, "p95_s": 5.476784098148347, "cost_usd_per_1000": 0.01718416314023056, "axes": { "intelligence": 63.269519519519534, "calibration": 50.07206287301584, "speed": 70.60768465701516, "cost": 62.94614838778628 }, "jevbench_score": 61.25658589803654 }, "heldout": { "n": 303, "escalation": 0.7755775577557755, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5616438356164384, "hard": 0.5045871559633027 }, "accuracy": 0.5874587458745875, "ece_hard": 0.2898818676180946, "calibration": 56.04911835753153, "p50_s": 3.279486957192421, "p95_s": 5.587395147979261, "cost_usd_per_1000": 0.020078771090592203, "axes": { "intelligence": 64.6969754094927, "calibration": 56.04911835753153, "speed": 67.36984705659012, "cost": 60.91788614357981 }, "jevbench_score": 62.11060898120867 } }, { "threshold": 0.8, "public": { "n": 231, "escalation": 0.658008658008658, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.46846846846846846 }, "accuracy": 0.6277056277056277, "ece_hard": 0.3121383762591704, "calibration": 50.07206287301584, "p50_s": 1.5964648753404616, "p95_s": 5.476784098148347, "cost_usd_per_1000": 0.017472103458339414, "axes": { "intelligence": 63.269519519519534, "calibration": 50.07206287301584, "speed": 70.58315011338803, "cost": 62.729644220818244 }, "jevbench_score": 61.198527174915725 }, "heldout": { "n": 303, "escalation": 0.7821782178217822, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5547945205479452, "hard": 0.4954128440366973 }, "accuracy": 0.5808580858085809, "ece_hard": 0.2986994363049536, "calibration": 55.16736148884563, "p50_s": 3.279486957192421, "p95_s": 5.587395147979261, "cost_usd_per_1000": 0.02022511699484555, "axes": { "intelligence": 64.22996522977672, "calibration": 55.16736148884563, "speed": 67.36984705659012, "cost": 60.823268724248635 }, "jevbench_score": 61.72893786961168 } }, { "threshold": 0.81, "public": { "n": 231, "escalation": 0.6623376623376623, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.4774774774774775 }, "accuracy": 0.6320346320346321, "ece_hard": 0.30402441711593514, "calibration": 50.883458787339364, "p50_s": 1.5964648753404616, "p95_s": 5.476784098148347, "cost_usd_per_1000": 0.0175680835643757, "axes": { "intelligence": 63.644894894894904, "calibration": 50.883458787339364, "speed": 70.58315011338803, "cost": 62.658268343776875 }, "jevbench_score": 61.51838219278035 }, "heldout": { "n": 303, "escalation": 0.7887788778877888, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5547945205479452, "hard": 0.48623853211009177 }, "accuracy": 0.5775577557755776, "ece_hard": 0.3053505955899122, "calibration": 54.65774625447631, "p50_s": 3.283590021729469, "p95_s": 5.587395147979261, "cost_usd_per_1000": 0.0203714628990989, "axes": { "intelligence": 63.95473587197856, "calibration": 54.65774625447631, "speed": 67.3644168642422, "cost": 60.729333480225094 }, "jevbench_score": 61.494797365959776 } }, { "threshold": 0.82, "public": { "n": 231, "escalation": 0.670995670995671, "tiers": { "easy": 1.0, "standard": 0.6388888888888888, "hard": 0.4774774774774775 }, "accuracy": 0.6363636363636364, "ece_hard": 0.30402441711593514, "calibration": 50.883458787339364, "p50_s": 1.5964648753404616, "p95_s": 5.476784098148347, "cost_usd_per_1000": 0.017760043776448273, "axes": { "intelligence": 64.18501835168502, "calibration": 50.883458787339364, "speed": 70.58315011338803, "cost": 62.51667904211698 }, "jevbench_score": 61.61363163548056 }, "heldout": { "n": 303, "escalation": 0.801980198019802, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.547945205479452, "hard": 0.48623853211009177 }, "accuracy": 0.5742574257425742, "ece_hard": 0.3053505955899122, "calibration": 54.65774625447631, "p50_s": 3.3082447081804274, "p95_s": 5.587395147979261, "cost_usd_per_1000": 0.020664154707605593, "axes": { "intelligence": 63.762955050060754, "calibration": 54.65774625447631, "speed": 67.33192987193615, "cost": 60.54347066089597 }, "jevbench_score": 61.394170073473724 } }, { "threshold": 0.83, "public": { "n": 231, "escalation": 0.696969696969697, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.4954954954954955 }, "accuracy": 0.6406926406926406, "ece_hard": 0.28578029864051535, "calibration": 52.707870634881345, "p50_s": 1.611314061284065, "p95_s": 5.476784098148347, "cost_usd_per_1000": 0.018335924412665988, "axes": { "intelligence": 64.39564564564566, "calibration": 52.707870634881345, "speed": 70.54294182155672, "cost": 62.10091570085384 }, "jevbench_score": 62.09703814643963 }, "heldout": { "n": 303, "escalation": 0.8250825082508251, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5273972602739726, "hard": 0.4954128440366973 }, "accuracy": 0.5676567656765676, "ece_hard": 0.29779193868549547, "calibration": 55.41361194491798, "p50_s": 3.3082447081804274, "p95_s": 5.587395147979261, "cost_usd_per_1000": 0.021176365372492303, "axes": { "intelligence": 63.4628419421055, "calibration": 55.41361194491798, "speed": 67.33192987193615, "cost": 60.224457352530585 }, "jevbench_score": 61.451499306303695 } }, { "threshold": 0.84, "public": { "n": 231, "escalation": 0.7056277056277056, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.4864864864864865 }, "accuracy": 0.6363636363636364, "ece_hard": 0.29410025347716073, "calibration": 51.8758751512168, "p50_s": 1.611314061284065, "p95_s": 5.476784098148347, "cost_usd_per_1000": 0.018527884624738557, "axes": { "intelligence": 64.02027027027029, "calibration": 51.8758751512168, "speed": 70.54294182155672, "cost": 61.96522487037281 }, "jevbench_score": 61.726426339215955 }, "heldout": { "n": 303, "escalation": 0.834983498349835, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5273972602739726, "hard": 0.48623853211009177 }, "accuracy": 0.5643564356435643, "ece_hard": 0.30800638021597637, "calibration": 54.392167791869895, "p50_s": 3.3082447081804274, "p95_s": 5.587395147979261, "cost_usd_per_1000": 0.021395884228872324, "axes": { "intelligence": 63.18761258430733, "calibration": 54.392167791869895, "speed": 67.33192987193615, "cost": 60.090092820924106 }, "jevbench_score": 61.06580168477617 } }, { "threshold": 0.85, "public": { "n": 231, "escalation": 0.70995670995671, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.4864864864864865 }, "accuracy": 0.6363636363636364, "ece_hard": 0.294342917979883, "calibration": 51.85160870094458, "p50_s": 1.611314061284065, "p95_s": 5.476784098148347, "cost_usd_per_1000": 0.018623864730774844, "axes": { "intelligence": 64.02027027027029, "calibration": 51.85160870094458, "speed": 70.54294182155672, "cost": 61.89790574185237 }, "jevbench_score": 61.70243669253875 }, "heldout": { "n": 303, "escalation": 0.8481848184818482, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5273972602739726, "hard": 0.4954128440366973 }, "accuracy": 0.5676567656765676, "ece_hard": 0.30063943350065536, "calibration": 55.128862463401994, "p50_s": 3.3082447081804274, "p95_s": 5.587395147979261, "cost_usd_per_1000": 0.02168857603737902, "axes": { "intelligence": 63.4628419421055, "calibration": 55.128862463401994, "speed": 67.33192987193615, "cost": 59.91306881911698 }, "jevbench_score": 61.2929174837917 } }, { "threshold": 0.86, "public": { "n": 231, "escalation": 0.7229437229437229, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5045045045045045 }, "accuracy": 0.645021645021645, "ece_hard": 0.27505561386057015, "calibration": 53.780339112875865, "p50_s": 1.611314061284065, "p95_s": 5.476784098148347, "cost_usd_per_1000": 0.0189118050488837, "axes": { "intelligence": 64.77102102102104, "calibration": 53.780339112875865, "speed": 70.54294182155672, "cost": 61.69801053016827 }, "jevbench_score": 62.39966407256425 }, "heldout": { "n": 303, "escalation": 0.8547854785478548, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5273972602739726, "hard": 0.4954128440366973 }, "accuracy": 0.5676567656765676, "ece_hard": 0.29825105368372024, "calibration": 54.73603344486585, "p50_s": 3.3082447081804274, "p95_s": 5.587395147979261, "cost_usd_per_1000": 0.021834921941632366, "axes": { "intelligence": 63.4628419421055, "calibration": 54.73603344486585, "speed": 67.33192987193615, "cost": 59.82545068939706 }, "jevbench_score": 61.161055258045955 } }, { "threshold": 0.87, "public": { "n": 231, "escalation": 0.7532467532467533, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5225225225225225 }, "accuracy": 0.6536796536796536, "ece_hard": 0.25945808462787956, "calibration": 55.340092036144924, "p50_s": 1.611314061284065, "p95_s": 5.476784098148347, "cost_usd_per_1000": 0.0195836657911377, "axes": { "intelligence": 65.52177177177178, "calibration": 55.340092036144924, "speed": 70.54294182155672, "cost": 61.24318033025044 }, "jevbench_score": 62.912103020930886 }, "heldout": { "n": 303, "escalation": 0.8646864686468647, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5273972602739726, "hard": 0.4954128440366973 }, "accuracy": 0.5676567656765676, "ece_hard": 0.29825105368372024, "calibration": 54.73603344486585, "p50_s": 3.3082447081804274, "p95_s": 5.587395147979261, "cost_usd_per_1000": 0.022054440798012387, "axes": { "intelligence": 63.4628419421055, "calibration": 54.73603344486585, "speed": 67.33192987193615, "cost": 59.69511848535392 }, "jevbench_score": 61.12771754194965 } }, { "threshold": 0.88, "public": { "n": 231, "escalation": 0.7792207792207793, "tiers": { "easy": 1.0, "standard": 0.6388888888888888, "hard": 0.5315315315315315 }, "accuracy": 0.6623376623376623, "ece_hard": 0.24959788792149712, "calibration": 56.326111706783166, "p50_s": 1.611314061284065, "p95_s": 5.476784098148347, "cost_usd_per_1000": 0.020159546427355415, "axes": { "intelligence": 66.43727060393728, "calibration": 56.326111706783166, "speed": 70.54294182155672, "cost": 60.8655773011892 }, "jevbench_score": 63.312099687996884 }, "heldout": { "n": 303, "escalation": 0.8712871287128713, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5273972602739726, "hard": 0.4954128440366973 }, "accuracy": 0.5676567656765676, "ece_hard": 0.29825105368372024, "calibration": 54.73603344486585, "p50_s": 3.3082447081804274, "p95_s": 5.587395147979261, "cost_usd_per_1000": 0.022200786702265732, "axes": { "intelligence": 63.4628419421055, "calibration": 54.73603344486585, "speed": 67.33192987193615, "cost": 59.6089490713466 }, "jevbench_score": 61.10564625120703 } }, { "threshold": 0.89, "public": { "n": 231, "escalation": 0.7878787878787878, "tiers": { "easy": 1.0, "standard": 0.6388888888888888, "hard": 0.5315315315315315 }, "accuracy": 0.6623376623376623, "ece_hard": 0.24959788792149712, "calibration": 56.326111706783166, "p50_s": 1.611314061284065, "p95_s": 5.476784098148347, "cost_usd_per_1000": 0.020351506639427988, "axes": { "intelligence": 66.43727060393728, "calibration": 56.326111706783166, "speed": 70.54294182155672, "cost": 60.74210302170967 }, "jevbench_score": 63.27996588469051 }, "heldout": { "n": 303, "escalation": 0.8877887788778878, "tiers": { "easy": 1.0, "standard": 0.75, "judge": 0.5273972602739726, "hard": 0.4954128440366973 }, "accuracy": 0.570957095709571, "ece_hard": 0.29825105368372024, "calibration": 54.73603344486585, "p50_s": 3.3082447081804274, "p95_s": 5.587395147979261, "cost_usd_per_1000": 0.022566651462899098, "axes": { "intelligence": 64.62950860877216, "calibration": 54.73603344486585, "speed": 67.33192987193615, "cost": 59.39598635891313 }, "jevbench_score": 61.329663854356596 } }, { "threshold": 0.9, "public": { "n": 231, "escalation": 0.8008658008658008, "tiers": { "easy": 1.0, "standard": 0.6388888888888888, "hard": 0.5405405405405406 }, "accuracy": 0.6666666666666666, "ece_hard": 0.24083249891229674, "calibration": 57.202650607703205, "p50_s": 1.68062547147274, "p95_s": 5.476784098148347, "cost_usd_per_1000": 0.020639446957536844, "axes": { "intelligence": 66.81264597931265, "calibration": 57.202650607703205, "speed": 70.36003437777309, "cost": 60.55905831964113 }, "jevbench_score": 63.525047030878355 }, "heldout": { "n": 303, "escalation": 0.9042904290429042, "tiers": { "easy": 1.0, "standard": 0.75, "judge": 0.5136986301369864, "hard": 0.5045871559633027 }, "accuracy": 0.5676567656765676, "ece_hard": 0.2892716176108673, "calibration": 55.63397705215115, "p50_s": 3.343346673250198, "p95_s": 5.616659300029278, "cost_usd_per_1000": 0.022932516223532463, "axes": { "intelligence": 64.5211763227347, "calibration": 55.63397705215115, "speed": 67.26340507984729, "cost": 59.18644871740203 }, "jevbench_score": 61.48382513617758 } }, { "threshold": 0.91, "public": { "n": 231, "escalation": 0.8138528138528138, "tiers": { "easy": 1.0, "standard": 0.6388888888888888, "hard": 0.5405405405405406 }, "accuracy": 0.6666666666666666, "ece_hard": 0.24083249891229674, "calibration": 57.202650607703205, "p50_s": 1.68062547147274, "p95_s": 5.476784098148347, "cost_usd_per_1000": 0.020927387275645703, "axes": { "intelligence": 66.81264597931265, "calibration": 57.202650607703205, "speed": 70.36003437777309, "cost": 60.378549660969405 }, "jevbench_score": 63.47765667804056 }, "heldout": { "n": 303, "escalation": 0.9108910891089109, "tiers": { "easy": 1.0, "standard": 0.75, "judge": 0.5136986301369864, "hard": 0.5045871559633027 }, "accuracy": 0.5676567656765676, "ece_hard": 0.2892716176108673, "calibration": 55.63397705215115, "p50_s": 3.343346673250198, "p95_s": 5.616659300029278, "cost_usd_per_1000": 0.023078862127785812, "axes": { "intelligence": 64.5211763227347, "calibration": 55.63397705215115, "speed": 67.26340507984729, "cost": 59.103568218789576 }, "jevbench_score": 61.462289428632765 } }, { "threshold": 0.92, "public": { "n": 231, "escalation": 0.8225108225108225, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.2408793246870926, "calibration": 57.171979725211926, "p50_s": 1.68062547147274, "p95_s": 5.476784098148347, "cost_usd_per_1000": 0.021119347487718276, "axes": { "intelligence": 66.27252252252254, "calibration": 57.171979725211926, "speed": 70.36003437777309, "cost": 60.25958512230164 }, "jevbench_score": 63.309258679737134 }, "heldout": { "n": 303, "escalation": 0.9207920792079208, "tiers": { "easy": 1.0, "standard": 0.75, "judge": 0.5136986301369864, "hard": 0.5045871559633027 }, "accuracy": 0.5676567656765676, "ece_hard": 0.2857790430546633, "calibration": 55.983234507771556, "p50_s": 3.343346673250198, "p95_s": 5.616659300029278, "cost_usd_per_1000": 0.023298380984165833, "axes": { "intelligence": 64.5211763227347, "calibration": 55.983234507771556, "speed": 67.26340507984729, "cost": 58.98022771781094 }, "jevbench_score": 61.52638366642757 } }, { "threshold": 0.93, "public": { "n": 231, "escalation": 0.8398268398268398, "tiers": { "easy": 1.0, "standard": 0.6111111111111112, "hard": 0.5405405405405406 }, "accuracy": 0.658008658008658, "ece_hard": 0.2408793246870926, "calibration": 57.171979725211926, "p50_s": 1.68062547147274, "p95_s": 5.476784098148347, "cost_usd_per_1000": 0.021503267911863418, "axes": { "intelligence": 65.73239906573241, "calibration": 57.171979725211926, "speed": 70.36003437777309, "cost": 60.02486602359018 }, "jevbench_score": 63.118255801272014 }, "heldout": { "n": 303, "escalation": 0.9240924092409241, "tiers": { "easy": 1.0, "standard": 0.75, "judge": 0.5136986301369864, "hard": 0.5045871559633027 }, "accuracy": 0.5676567656765676, "ece_hard": 0.2857790430546633, "calibration": 55.983234507771556, "p50_s": 3.343346673250198, "p95_s": 5.616659300029278, "cost_usd_per_1000": 0.023371553936292505, "axes": { "intelligence": 64.5211763227347, "calibration": 55.983234507771556, "speed": 67.26340507984729, "cost": 58.939372333597504 }, "jevbench_score": 61.5157261233722 } }, { "threshold": 0.94, "public": { "n": 231, "escalation": 0.8528138528138528, "tiers": { "easy": 1.0, "standard": 0.6111111111111112, "hard": 0.5405405405405406 }, "accuracy": 0.658008658008658, "ece_hard": 0.2408793246870926, "calibration": 57.171979725211926, "p50_s": 1.6873538970947264, "p95_s": 5.481477122008801, "cost_usd_per_1000": 0.021791208229972273, "axes": { "intelligence": 65.73239906573241, "calibration": 57.171979725211926, "speed": 70.3389621528618, "cost": 59.85156067903371 }, "jevbench_score": 63.06792426655594 }, "heldout": { "n": 303, "escalation": 0.9405940594059405, "tiers": { "easy": 1.0, "standard": 0.75, "judge": 0.5136986301369864, "hard": 0.5137614678899083 }, "accuracy": 0.570957095709571, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 3.3457510352134703, "p95_s": 5.616659300029278, "cost_usd_per_1000": 0.02373741869692587, "axes": { "intelligence": 64.79640568053287, "calibration": 59.97952523584675, "speed": 67.26028298110205, "cost": 58.73699529265495 }, "jevbench_score": 62.59737030425177 } }, { "threshold": 0.95, "public": { "n": 231, "escalation": 0.8614718614718615, "tiers": { "easy": 1.0, "standard": 0.6111111111111112, "hard": 0.5405405405405406 }, "accuracy": 0.658008658008658, "ece_hard": 0.2408793246870926, "calibration": 57.171979725211926, "p50_s": 1.6873538970947264, "p95_s": 5.481477122008801, "cost_usd_per_1000": 0.021983168442044846, "axes": { "intelligence": 65.73239906573241, "calibration": 57.171979725211926, "speed": 70.3389621528618, "cost": 59.73729137139697 }, "jevbench_score": 63.03780018373738 }, "heldout": { "n": 303, "escalation": 0.9471947194719472, "tiers": { "easy": 1.0, "standard": 0.75, "judge": 0.5136986301369864, "hard": 0.5137614678899083 }, "accuracy": 0.570957095709571, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 3.3457510352134703, "p95_s": 5.616659300029278, "cost_usd_per_1000": 0.02388376460117922, "axes": { "intelligence": 64.79640568053287, "calibration": 59.97952523584675, "speed": 67.26028298110205, "cost": 58.65691653650565 }, "jevbench_score": 62.57602394349142 } }, { "threshold": 0.96, "public": { "n": 231, "escalation": 0.8874458874458875, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 1.6873538970947264, "p95_s": 5.481477122008801, "cost_usd_per_1000": 0.02255904907826256, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 70.3389621528618, "cost": 59.400376327851355 }, "jevbench_score": 63.220387476266 }, "heldout": { "n": 303, "escalation": 0.9537953795379538, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5136986301369864, "hard": 0.5137614678899083 }, "accuracy": 0.5676567656765676, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 3.3457510352134703, "p95_s": 5.616659300029278, "cost_usd_per_1000": 0.024030110505432564, "axes": { "intelligence": 63.62973901386621, "calibration": 59.97952523584675, "speed": 67.26028298110205, "cost": 58.57732696104001 }, "jevbench_score": 62.27128840070513 } }, { "threshold": 0.97, "public": { "n": 231, "escalation": 0.8961038961038961, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 1.6873538970947264, "p95_s": 5.481477122008801, "cost_usd_per_1000": 0.022751009290335134, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 70.3389621528618, "cost": 59.28997996646932 }, "jevbench_score": 63.19099300680114 }, "heldout": { "n": 303, "escalation": 0.9603960396039604, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5136986301369864, "hard": 0.5137614678899083 }, "accuracy": 0.5676567656765676, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 3.3457510352134703, "p95_s": 5.616659300029278, "cost_usd_per_1000": 0.024176456409685913, "axes": { "intelligence": 63.62973901386621, "calibration": 59.97952523584675, "speed": 67.26028298110205, "cost": 58.4982206259675 }, "jevbench_score": 62.250254024895526 } }, { "threshold": 0.98, "public": { "n": 231, "escalation": 0.9090909090909091, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 1.6873538970947264, "p95_s": 5.481477122008801, "cost_usd_per_1000": 0.02303894960844399, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 70.3389621528618, "cost": 59.12611975441205 }, "jevbench_score": 63.147287315934484 }, "heldout": { "n": 303, "escalation": 0.9702970297029703, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 3.3457510352134703, "p95_s": 5.616659300029278, "cost_usd_per_1000": 0.024395975266065933, "axes": { "intelligence": 63.4379581919484, "calibration": 59.97952523584675, "speed": 67.26028298110205, "cost": 58.380454468747025 }, "jevbench_score": 62.171965230453225 } }, { "threshold": 0.99, "public": { "n": 231, "escalation": 0.9393939393939394, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 1.7710533022880552, "p95_s": 5.481477122008801, "cost_usd_per_1000": 0.023710810350697994, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 70.12870762085296, "cost": 58.75160809407999 }, "jevbench_score": 62.99988600196496 }, "heldout": { "n": 303, "escalation": 0.9834983498349835, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 3.3457510352134703, "p95_s": 5.616659300029278, "cost_usd_per_1000": 0.024688667074572627, "axes": { "intelligence": 63.4379581919484, "calibration": 59.97952523584675, "speed": 67.26028298110205, "cost": 58.22507070153699 }, "jevbench_score": 62.130555090215886 } }, { "threshold": 1.0, "public": { "n": 231, "escalation": 1.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 1.7710533022880552, "p95_s": 5.481477122008801, "cost_usd_per_1000": 0.025054531835205993, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 70.12870762085296, "cost": 58.033411239993484 }, "jevbench_score": 62.80646480428468 }, "heldout": { "n": 303, "escalation": 1.0, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 3.3457510352134703, "p95_s": 5.616659300029278, "cost_usd_per_1000": 0.025054531835205993, "axes": { "intelligence": 63.4379581919484, "calibration": 59.97952523584675, "speed": 67.26028298110205, "cost": 58.033411239993484 }, "jevbench_score": 62.07936306995668 } } ] }, { "cheap": "semif-qwen3.5-4b", "backend": "djev", "chosen": { "threshold": 0.39, "public": { "n": 231, "escalation": 0.04329004329004329, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6396396396396397 }, "accuracy": 0.7705627705627706, "ece_hard": 0.12391053900720686, "calibration": 72.7681195209995, "p50_s": 0.538740599155426, "p95_s": 1.4027332119643687, "cost_usd_per_1000": 0.02356803768017251, "axes": { "intelligence": 77.96338004671338, "calibration": 72.7681195209995, "speed": 81.21645215327925, "cost": 58.83029728475067 }, "jevbench_score": 72.15546288396726 }, "heldout": { "n": 303, "escalation": 0.039603960396039604, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7986798679867987, "ece_hard": 0.2009330336707937, "calibration": 64.1961225283875, "p50_s": 0.5654387563467026, "p95_s": 1.1195790365338323, "cost_usd_per_1000": 0.023472379204212555, "axes": { "intelligence": 79.33796238113192, "calibration": 64.1961225283875, "speed": 81.98559669421805, "cost": 58.8832866181031 }, "jevbench_score": 70.41737978046176 } }, "full": { "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.952054794520548, "hard": 0.5954545454545455 }, "accuracy": 0.7865168539325843, "ece_hard": 0.1452392451215172, "calibration": 70.20037514644181, "p50_s": 0.5586275175213814, "p95_s": 1.2210772372782228, "cost_usd_per_1000": 0.02351375955617276, "axes": { "intelligence": 81.27117061021171, "calibration": 70.20037514644181, "speed": 81.66134539714201, "cost": 58.86033781196246 }, "jevbench_score": 72.36504013792714 }, "full_escalation": 0.04119850187265917, "backend_heldout_accuracy": 0.834983498349835, "useful": false, "curve": [ { "threshold": 0.0, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6036036036036037 }, "accuracy": 0.7532467532467533, "ece_hard": 0.12914387221294593, "calibration": 72.24478620042561, "p50_s": 0.5384091228246689, "p95_s": 1.2262836933135985, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 76.46187854521189, "calibration": 72.24478620042561, "speed": 81.80296636812625, "cost": 59.46663998783557 }, "jevbench_score": 71.99850550126745 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7986798679867987, "ece_hard": 0.17685813516393573, "calibration": 66.51802582183663, "p50_s": 0.5651323288679123, "p95_s": 1.115284267067909, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 79.33796238113192, "calibration": 66.51802582183663, "speed": 82.00464272011534, "cost": 59.46663998783557 }, "jevbench_score": 71.22509854109671 } }, { "threshold": 0.01, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6036036036036037 }, "accuracy": 0.7532467532467533, "ece_hard": 0.12914387221294593, "calibration": 72.24478620042561, "p50_s": 0.5384091228246689, "p95_s": 1.2262836933135985, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 76.46187854521189, "calibration": 72.24478620042561, "speed": 81.80296636812625, "cost": 59.46663998783557 }, "jevbench_score": 71.99850550126745 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7986798679867987, "ece_hard": 0.17685813516393573, "calibration": 66.51802582183663, "p50_s": 0.5651323288679123, "p95_s": 1.115284267067909, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 79.33796238113192, "calibration": 66.51802582183663, "speed": 82.00464272011534, "cost": 59.46663998783557 }, "jevbench_score": 71.22509854109671 } }, { "threshold": 0.02, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6036036036036037 }, "accuracy": 0.7532467532467533, "ece_hard": 0.12914387221294593, "calibration": 72.24478620042561, "p50_s": 0.5384091228246689, "p95_s": 1.2262836933135985, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 76.46187854521189, "calibration": 72.24478620042561, "speed": 81.80296636812625, "cost": 59.46663998783557 }, "jevbench_score": 71.99850550126745 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7986798679867987, "ece_hard": 0.17685813516393573, "calibration": 66.51802582183663, "p50_s": 0.5651323288679123, "p95_s": 1.115284267067909, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 79.33796238113192, "calibration": 66.51802582183663, "speed": 82.00464272011534, "cost": 59.46663998783557 }, "jevbench_score": 71.22509854109671 } }, { "threshold": 0.03, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6036036036036037 }, "accuracy": 0.7532467532467533, "ece_hard": 0.12914387221294593, "calibration": 72.24478620042561, "p50_s": 0.5384091228246689, "p95_s": 1.2262836933135985, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 76.46187854521189, "calibration": 72.24478620042561, "speed": 81.80296636812625, "cost": 59.46663998783557 }, "jevbench_score": 71.99850550126745 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7986798679867987, "ece_hard": 0.17685813516393573, "calibration": 66.51802582183663, "p50_s": 0.5651323288679123, "p95_s": 1.115284267067909, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 79.33796238113192, "calibration": 66.51802582183663, "speed": 82.00464272011534, "cost": 59.46663998783557 }, "jevbench_score": 71.22509854109671 } }, { "threshold": 0.04, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6036036036036037 }, "accuracy": 0.7532467532467533, "ece_hard": 0.12914387221294593, "calibration": 72.24478620042561, "p50_s": 0.5384091228246689, "p95_s": 1.2262836933135985, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 76.46187854521189, "calibration": 72.24478620042561, "speed": 81.80296636812625, "cost": 59.46663998783557 }, "jevbench_score": 71.99850550126745 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7986798679867987, "ece_hard": 0.17685813516393573, "calibration": 66.51802582183663, "p50_s": 0.5651323288679123, "p95_s": 1.115284267067909, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 79.33796238113192, "calibration": 66.51802582183663, "speed": 82.00464272011534, "cost": 59.46663998783557 }, "jevbench_score": 71.22509854109671 } }, { "threshold": 0.05, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6036036036036037 }, "accuracy": 0.7532467532467533, "ece_hard": 0.12914387221294593, "calibration": 72.24478620042561, "p50_s": 0.5384091228246689, "p95_s": 1.2262836933135985, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 76.46187854521189, "calibration": 72.24478620042561, "speed": 81.80296636812625, "cost": 59.46663998783557 }, "jevbench_score": 71.99850550126745 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7986798679867987, "ece_hard": 0.17685813516393573, "calibration": 66.51802582183663, "p50_s": 0.5651323288679123, "p95_s": 1.115284267067909, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 79.33796238113192, "calibration": 66.51802582183663, "speed": 82.00464272011534, "cost": 59.46663998783557 }, "jevbench_score": 71.22509854109671 } }, { "threshold": 0.06, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6036036036036037 }, "accuracy": 0.7532467532467533, "ece_hard": 0.12914387221294593, "calibration": 72.24478620042561, "p50_s": 0.5384091228246689, "p95_s": 1.2262836933135985, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 76.46187854521189, "calibration": 72.24478620042561, "speed": 81.80296636812625, "cost": 59.46663998783557 }, "jevbench_score": 71.99850550126745 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7986798679867987, "ece_hard": 0.17685813516393573, "calibration": 66.51802582183663, "p50_s": 0.5651323288679123, "p95_s": 1.115284267067909, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 79.33796238113192, "calibration": 66.51802582183663, "speed": 82.00464272011534, "cost": 59.46663998783557 }, "jevbench_score": 71.22509854109671 } }, { "threshold": 0.07, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6036036036036037 }, "accuracy": 0.7532467532467533, "ece_hard": 0.12914387221294593, "calibration": 72.24478620042561, "p50_s": 0.5384091228246689, "p95_s": 1.2262836933135985, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 76.46187854521189, "calibration": 72.24478620042561, "speed": 81.80296636812625, "cost": 59.46663998783557 }, "jevbench_score": 71.99850550126745 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7986798679867987, "ece_hard": 0.17685813516393573, "calibration": 66.51802582183663, "p50_s": 0.5651323288679123, "p95_s": 1.115284267067909, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 79.33796238113192, "calibration": 66.51802582183663, "speed": 82.00464272011534, "cost": 59.46663998783557 }, "jevbench_score": 71.22509854109671 } }, { "threshold": 0.08, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6036036036036037 }, "accuracy": 0.7532467532467533, "ece_hard": 0.12914387221294593, "calibration": 72.24478620042561, "p50_s": 0.5384091228246689, "p95_s": 1.2262836933135985, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 76.46187854521189, "calibration": 72.24478620042561, "speed": 81.80296636812625, "cost": 59.46663998783557 }, "jevbench_score": 71.99850550126745 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7986798679867987, "ece_hard": 0.17685813516393573, "calibration": 66.51802582183663, "p50_s": 0.5651323288679123, "p95_s": 1.115284267067909, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 79.33796238113192, "calibration": 66.51802582183663, "speed": 82.00464272011534, "cost": 59.46663998783557 }, "jevbench_score": 71.22509854109671 } }, { "threshold": 0.09, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6036036036036037 }, "accuracy": 0.7532467532467533, "ece_hard": 0.12914387221294593, "calibration": 72.24478620042561, "p50_s": 0.5384091228246689, "p95_s": 1.2262836933135985, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 76.46187854521189, "calibration": 72.24478620042561, "speed": 81.80296636812625, "cost": 59.46663998783557 }, "jevbench_score": 71.99850550126745 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7986798679867987, "ece_hard": 0.17685813516393573, "calibration": 66.51802582183663, "p50_s": 0.5651323288679123, "p95_s": 1.115284267067909, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 79.33796238113192, "calibration": 66.51802582183663, "speed": 82.00464272011534, "cost": 59.46663998783557 }, "jevbench_score": 71.22509854109671 } }, { "threshold": 0.1, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6036036036036037 }, "accuracy": 0.7532467532467533, "ece_hard": 0.12914387221294593, "calibration": 72.24478620042561, "p50_s": 0.5384091228246689, "p95_s": 1.2262836933135985, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 76.46187854521189, "calibration": 72.24478620042561, "speed": 81.80296636812625, "cost": 59.46663998783557 }, "jevbench_score": 71.99850550126745 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7986798679867987, "ece_hard": 0.17685813516393573, "calibration": 66.51802582183663, "p50_s": 0.5651323288679123, "p95_s": 1.115284267067909, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 79.33796238113192, "calibration": 66.51802582183663, "speed": 82.00464272011534, "cost": 59.46663998783557 }, "jevbench_score": 71.22509854109671 } }, { "threshold": 0.11, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6036036036036037 }, "accuracy": 0.7532467532467533, "ece_hard": 0.12914387221294593, "calibration": 72.24478620042561, "p50_s": 0.5384091228246689, "p95_s": 1.2262836933135985, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 76.46187854521189, "calibration": 72.24478620042561, "speed": 81.80296636812625, "cost": 59.46663998783557 }, "jevbench_score": 71.99850550126745 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7986798679867987, "ece_hard": 0.17685813516393573, "calibration": 66.51802582183663, "p50_s": 0.5651323288679123, "p95_s": 1.115284267067909, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 79.33796238113192, "calibration": 66.51802582183663, "speed": 82.00464272011534, "cost": 59.46663998783557 }, "jevbench_score": 71.22509854109671 } }, { "threshold": 0.12, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6036036036036037 }, "accuracy": 0.7532467532467533, "ece_hard": 0.12914387221294593, "calibration": 72.24478620042561, "p50_s": 0.5384091228246689, "p95_s": 1.2262836933135985, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 76.46187854521189, "calibration": 72.24478620042561, "speed": 81.80296636812625, "cost": 59.46663998783557 }, "jevbench_score": 71.99850550126745 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7986798679867987, "ece_hard": 0.17685813516393573, "calibration": 66.51802582183663, "p50_s": 0.5651323288679123, "p95_s": 1.115284267067909, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 79.33796238113192, "calibration": 66.51802582183663, "speed": 82.00464272011534, "cost": 59.46663998783557 }, "jevbench_score": 71.22509854109671 } }, { "threshold": 0.13, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6036036036036037 }, "accuracy": 0.7532467532467533, "ece_hard": 0.12914387221294593, "calibration": 72.24478620042561, "p50_s": 0.5384091228246689, "p95_s": 1.2262836933135985, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 76.46187854521189, "calibration": 72.24478620042561, "speed": 81.80296636812625, "cost": 59.46663998783557 }, "jevbench_score": 71.99850550126745 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7986798679867987, "ece_hard": 0.17685813516393573, "calibration": 66.51802582183663, "p50_s": 0.5651323288679123, "p95_s": 1.115284267067909, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 79.33796238113192, "calibration": 66.51802582183663, "speed": 82.00464272011534, "cost": 59.46663998783557 }, "jevbench_score": 71.22509854109671 } }, { "threshold": 0.14, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6036036036036037 }, "accuracy": 0.7532467532467533, "ece_hard": 0.12914387221294593, "calibration": 72.24478620042561, "p50_s": 0.5384091228246689, "p95_s": 1.2262836933135985, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 76.46187854521189, "calibration": 72.24478620042561, "speed": 81.80296636812625, "cost": 59.46663998783557 }, "jevbench_score": 71.99850550126745 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7986798679867987, "ece_hard": 0.17685813516393573, "calibration": 66.51802582183663, "p50_s": 0.5651323288679123, "p95_s": 1.115284267067909, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 79.33796238113192, "calibration": 66.51802582183663, "speed": 82.00464272011534, "cost": 59.46663998783557 }, "jevbench_score": 71.22509854109671 } }, { "threshold": 0.15, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6036036036036037 }, "accuracy": 0.7532467532467533, "ece_hard": 0.12914387221294593, "calibration": 72.24478620042561, "p50_s": 0.5384091228246689, "p95_s": 1.2262836933135985, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 76.46187854521189, "calibration": 72.24478620042561, "speed": 81.80296636812625, "cost": 59.46663998783557 }, "jevbench_score": 71.99850550126745 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7986798679867987, "ece_hard": 0.17685813516393573, "calibration": 66.51802582183663, "p50_s": 0.5651323288679123, "p95_s": 1.115284267067909, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 79.33796238113192, "calibration": 66.51802582183663, "speed": 82.00464272011534, "cost": 59.46663998783557 }, "jevbench_score": 71.22509854109671 } }, { "threshold": 0.16, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6036036036036037 }, "accuracy": 0.7532467532467533, "ece_hard": 0.12914387221294593, "calibration": 72.24478620042561, "p50_s": 0.5384091228246689, "p95_s": 1.2262836933135985, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 76.46187854521189, "calibration": 72.24478620042561, "speed": 81.80296636812625, "cost": 59.46663998783557 }, "jevbench_score": 71.99850550126745 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7986798679867987, "ece_hard": 0.17685813516393573, "calibration": 66.51802582183663, "p50_s": 0.5651323288679123, "p95_s": 1.115284267067909, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 79.33796238113192, "calibration": 66.51802582183663, "speed": 82.00464272011534, "cost": 59.46663998783557 }, "jevbench_score": 71.22509854109671 } }, { "threshold": 0.17, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6036036036036037 }, "accuracy": 0.7532467532467533, "ece_hard": 0.12914387221294593, "calibration": 72.24478620042561, "p50_s": 0.5384091228246689, "p95_s": 1.2262836933135985, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 76.46187854521189, "calibration": 72.24478620042561, "speed": 81.80296636812625, "cost": 59.46663998783557 }, "jevbench_score": 71.99850550126745 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7986798679867987, "ece_hard": 0.17685813516393573, "calibration": 66.51802582183663, "p50_s": 0.5651323288679123, "p95_s": 1.115284267067909, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 79.33796238113192, "calibration": 66.51802582183663, "speed": 82.00464272011534, "cost": 59.46663998783557 }, "jevbench_score": 71.22509854109671 } }, { "threshold": 0.18, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6036036036036037 }, "accuracy": 0.7532467532467533, "ece_hard": 0.12914387221294593, "calibration": 72.24478620042561, "p50_s": 0.5384091228246689, "p95_s": 1.2262836933135985, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 76.46187854521189, "calibration": 72.24478620042561, "speed": 81.80296636812625, "cost": 59.46663998783557 }, "jevbench_score": 71.99850550126745 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7986798679867987, "ece_hard": 0.17685813516393573, "calibration": 66.51802582183663, "p50_s": 0.5651323288679123, "p95_s": 1.115284267067909, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 79.33796238113192, "calibration": 66.51802582183663, "speed": 82.00464272011534, "cost": 59.46663998783557 }, "jevbench_score": 71.22509854109671 } }, { "threshold": 0.19, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6036036036036037 }, "accuracy": 0.7532467532467533, "ece_hard": 0.12914387221294593, "calibration": 72.24478620042561, "p50_s": 0.5384091228246689, "p95_s": 1.2262836933135985, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 76.46187854521189, "calibration": 72.24478620042561, "speed": 81.80296636812625, "cost": 59.46663998783557 }, "jevbench_score": 71.99850550126745 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7986798679867987, "ece_hard": 0.17685813516393573, "calibration": 66.51802582183663, "p50_s": 0.5651323288679123, "p95_s": 1.115284267067909, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 79.33796238113192, "calibration": 66.51802582183663, "speed": 82.00464272011534, "cost": 59.46663998783557 }, "jevbench_score": 71.22509854109671 } }, { "threshold": 0.2, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6036036036036037 }, "accuracy": 0.7532467532467533, "ece_hard": 0.12914387221294593, "calibration": 72.24478620042561, "p50_s": 0.5384091228246689, "p95_s": 1.2262836933135985, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 76.46187854521189, "calibration": 72.24478620042561, "speed": 81.80296636812625, "cost": 59.46663998783557 }, "jevbench_score": 71.99850550126745 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7986798679867987, "ece_hard": 0.17685813516393573, "calibration": 66.51802582183663, "p50_s": 0.5651323288679123, "p95_s": 1.115284267067909, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 79.33796238113192, "calibration": 66.51802582183663, "speed": 82.00464272011534, "cost": 59.46663998783557 }, "jevbench_score": 71.22509854109671 } }, { "threshold": 0.21, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6036036036036037 }, "accuracy": 0.7532467532467533, "ece_hard": 0.12914387221294593, "calibration": 72.24478620042561, "p50_s": 0.5384091228246689, "p95_s": 1.2262836933135985, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 76.46187854521189, "calibration": 72.24478620042561, "speed": 81.80296636812625, "cost": 59.46663998783557 }, "jevbench_score": 71.99850550126745 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7986798679867987, "ece_hard": 0.17685813516393573, "calibration": 66.51802582183663, "p50_s": 0.5651323288679123, "p95_s": 1.115284267067909, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 79.33796238113192, "calibration": 66.51802582183663, "speed": 82.00464272011534, "cost": 59.46663998783557 }, "jevbench_score": 71.22509854109671 } }, { "threshold": 0.22, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6036036036036037 }, "accuracy": 0.7532467532467533, "ece_hard": 0.12914387221294593, "calibration": 72.24478620042561, "p50_s": 0.5384091228246689, "p95_s": 1.2262836933135985, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 76.46187854521189, "calibration": 72.24478620042561, "speed": 81.80296636812625, "cost": 59.46663998783557 }, "jevbench_score": 71.99850550126745 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7986798679867987, "ece_hard": 0.17685813516393573, "calibration": 66.51802582183663, "p50_s": 0.5651323288679123, "p95_s": 1.115284267067909, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 79.33796238113192, "calibration": 66.51802582183663, "speed": 82.00464272011534, "cost": 59.46663998783557 }, "jevbench_score": 71.22509854109671 } }, { "threshold": 0.23, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6036036036036037 }, "accuracy": 0.7532467532467533, "ece_hard": 0.12914387221294593, "calibration": 72.24478620042561, "p50_s": 0.5384091228246689, "p95_s": 1.2262836933135985, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 76.46187854521189, "calibration": 72.24478620042561, "speed": 81.80296636812625, "cost": 59.46663998783557 }, "jevbench_score": 71.99850550126745 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7986798679867987, "ece_hard": 0.17685813516393573, "calibration": 66.51802582183663, "p50_s": 0.5651323288679123, "p95_s": 1.115284267067909, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 79.33796238113192, "calibration": 66.51802582183663, "speed": 82.00464272011534, "cost": 59.46663998783557 }, "jevbench_score": 71.22509854109671 } }, { "threshold": 0.24, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6036036036036037 }, "accuracy": 0.7532467532467533, "ece_hard": 0.12914387221294593, "calibration": 72.24478620042561, "p50_s": 0.5384091228246689, "p95_s": 1.2262836933135985, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 76.46187854521189, "calibration": 72.24478620042561, "speed": 81.80296636812625, "cost": 59.46663998783557 }, "jevbench_score": 71.99850550126745 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7986798679867987, "ece_hard": 0.17685813516393573, "calibration": 66.51802582183663, "p50_s": 0.5651323288679123, "p95_s": 1.115284267067909, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 79.33796238113192, "calibration": 66.51802582183663, "speed": 82.00464272011534, "cost": 59.46663998783557 }, "jevbench_score": 71.22509854109671 } }, { "threshold": 0.25, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6036036036036037 }, "accuracy": 0.7532467532467533, "ece_hard": 0.12914387221294593, "calibration": 72.24478620042561, "p50_s": 0.5384091228246689, "p95_s": 1.2262836933135985, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 76.46187854521189, "calibration": 72.24478620042561, "speed": 81.80296636812625, "cost": 59.46663998783557 }, "jevbench_score": 71.99850550126745 }, "heldout": { "n": 303, "escalation": 0.0033003300330033004, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5412844036697247 }, "accuracy": 0.7953795379537953, "ece_hard": 0.1742074216723155, "calibration": 66.78309717099864, "p50_s": 0.5651323288679123, "p95_s": 1.115284267067909, "cost_usd_per_1000": 0.02253025444679299, "axes": { "intelligence": 79.06273302333375, "calibration": 66.78309717099864, "speed": 82.00464272011534, "cost": 59.41701710552324 }, "jevbench_score": 71.21917135670965 } }, { "threshold": 0.26, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6036036036036037 }, "accuracy": 0.7532467532467533, "ece_hard": 0.12914387221294593, "calibration": 72.24478620042561, "p50_s": 0.5384091228246689, "p95_s": 1.2262836933135985, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 76.46187854521189, "calibration": 72.24478620042561, "speed": 81.80296636812625, "cost": 59.46663998783557 }, "jevbench_score": 71.99850550126745 }, "heldout": { "n": 303, "escalation": 0.0033003300330033004, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5412844036697247 }, "accuracy": 0.7953795379537953, "ece_hard": 0.1742074216723155, "calibration": 66.78309717099864, "p50_s": 0.5651323288679123, "p95_s": 1.115284267067909, "cost_usd_per_1000": 0.02253025444679299, "axes": { "intelligence": 79.06273302333375, "calibration": 66.78309717099864, "speed": 82.00464272011534, "cost": 59.41701710552324 }, "jevbench_score": 71.21917135670965 } }, { "threshold": 0.27, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6036036036036037 }, "accuracy": 0.7532467532467533, "ece_hard": 0.12914387221294593, "calibration": 72.24478620042561, "p50_s": 0.5384091228246689, "p95_s": 1.2262836933135985, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 76.46187854521189, "calibration": 72.24478620042561, "speed": 81.80296636812625, "cost": 59.46663998783557 }, "jevbench_score": 71.99850550126745 }, "heldout": { "n": 303, "escalation": 0.0033003300330033004, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5412844036697247 }, "accuracy": 0.7953795379537953, "ece_hard": 0.1742074216723155, "calibration": 66.78309717099864, "p50_s": 0.5651323288679123, "p95_s": 1.115284267067909, "cost_usd_per_1000": 0.02253025444679299, "axes": { "intelligence": 79.06273302333375, "calibration": 66.78309717099864, "speed": 82.00464272011534, "cost": 59.41701710552324 }, "jevbench_score": 71.21917135670965 } }, { "threshold": 0.28, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6036036036036037 }, "accuracy": 0.7532467532467533, "ece_hard": 0.12914387221294593, "calibration": 72.24478620042561, "p50_s": 0.5384091228246689, "p95_s": 1.2262836933135985, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 76.46187854521189, "calibration": 72.24478620042561, "speed": 81.80296636812625, "cost": 59.46663998783557 }, "jevbench_score": 71.99850550126745 }, "heldout": { "n": 303, "escalation": 0.0033003300330033004, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5412844036697247 }, "accuracy": 0.7953795379537953, "ece_hard": 0.1742074216723155, "calibration": 66.78309717099864, "p50_s": 0.5651323288679123, "p95_s": 1.115284267067909, "cost_usd_per_1000": 0.02253025444679299, "axes": { "intelligence": 79.06273302333375, "calibration": 66.78309717099864, "speed": 82.00464272011534, "cost": 59.41701710552324 }, "jevbench_score": 71.21917135670965 } }, { "threshold": 0.29, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6036036036036037 }, "accuracy": 0.7532467532467533, "ece_hard": 0.12914387221294593, "calibration": 72.24478620042561, "p50_s": 0.5384091228246689, "p95_s": 1.2262836933135985, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 76.46187854521189, "calibration": 72.24478620042561, "speed": 81.80296636812625, "cost": 59.46663998783557 }, "jevbench_score": 71.99850550126745 }, "heldout": { "n": 303, "escalation": 0.0033003300330033004, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5412844036697247 }, "accuracy": 0.7953795379537953, "ece_hard": 0.1742074216723155, "calibration": 66.78309717099864, "p50_s": 0.5651323288679123, "p95_s": 1.115284267067909, "cost_usd_per_1000": 0.02253025444679299, "axes": { "intelligence": 79.06273302333375, "calibration": 66.78309717099864, "speed": 82.00464272011534, "cost": 59.41701710552324 }, "jevbench_score": 71.21917135670965 } }, { "threshold": 0.3, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6036036036036037 }, "accuracy": 0.7532467532467533, "ece_hard": 0.12914387221294593, "calibration": 72.24478620042561, "p50_s": 0.5384091228246689, "p95_s": 1.2262836933135985, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 76.46187854521189, "calibration": 72.24478620042561, "speed": 81.80296636812625, "cost": 59.46663998783557 }, "jevbench_score": 71.99850550126745 }, "heldout": { "n": 303, "escalation": 0.0033003300330033004, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5412844036697247 }, "accuracy": 0.7953795379537953, "ece_hard": 0.1742074216723155, "calibration": 66.78309717099864, "p50_s": 0.5651323288679123, "p95_s": 1.115284267067909, "cost_usd_per_1000": 0.02253025444679299, "axes": { "intelligence": 79.06273302333375, "calibration": 66.78309717099864, "speed": 82.00464272011534, "cost": 59.41701710552324 }, "jevbench_score": 71.21917135670965 } }, { "threshold": 0.31, "public": { "n": 231, "escalation": 0.004329004329004329, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6126126126126126 }, "accuracy": 0.7575757575757576, "ece_hard": 0.12129612598042858, "calibration": 73.02956082367734, "p50_s": 0.5384091228246689, "p95_s": 1.2262836933135985, "cost_usd_per_1000": 0.02255694983543298, "axes": { "intelligence": 76.83725392058726, "calibration": 73.02956082367734, "speed": 81.80296636812625, "cost": 59.40158878853006 }, "jevbench_score": 72.26190549673386 }, "heldout": { "n": 303, "escalation": 0.009900990099009901, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5321100917431193 }, "accuracy": 0.7920792079207921, "ece_hard": 0.18583028194404527, "calibration": 65.62081114382568, "p50_s": 0.5651323288679123, "p95_s": 1.115284267067909, "cost_usd_per_1000": 0.02270154985723291, "axes": { "intelligence": 78.7875036655356, "calibration": 65.62081114382568, "speed": 82.00464272011534, "cost": 59.31833476244369 }, "jevbench_score": 70.81603086763108 } }, { "threshold": 0.32, "public": { "n": 231, "escalation": 0.004329004329004329, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6126126126126126 }, "accuracy": 0.7575757575757576, "ece_hard": 0.12129612598042858, "calibration": 73.02956082367734, "p50_s": 0.5384091228246689, "p95_s": 1.2262836933135985, "cost_usd_per_1000": 0.02255694983543298, "axes": { "intelligence": 76.83725392058726, "calibration": 73.02956082367734, "speed": 81.80296636812625, "cost": 59.40158878853006 }, "jevbench_score": 72.26190549673386 }, "heldout": { "n": 303, "escalation": 0.009900990099009901, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5321100917431193 }, "accuracy": 0.7920792079207921, "ece_hard": 0.18583028194404527, "calibration": 65.62081114382568, "p50_s": 0.5651323288679123, "p95_s": 1.115284267067909, "cost_usd_per_1000": 0.02270154985723291, "axes": { "intelligence": 78.7875036655356, "calibration": 65.62081114382568, "speed": 82.00464272011534, "cost": 59.31833476244369 }, "jevbench_score": 70.81603086763108 } }, { "threshold": 0.33, "public": { "n": 231, "escalation": 0.008658008658008658, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6126126126126126 }, "accuracy": 0.7575757575757576, "ece_hard": 0.1227029771638957, "calibration": 72.88887570533063, "p50_s": 0.5384091228246689, "p95_s": 1.2262836933135985, "cost_usd_per_1000": 0.022669292929292928, "axes": { "intelligence": 76.83725392058726, "calibration": 72.88887570533063, "speed": 81.80296636812625, "cost": 59.33686076809947 }, "jevbench_score": 72.20739482105681 }, "heldout": { "n": 303, "escalation": 0.013201320132013201, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5412844036697247 }, "accuracy": 0.7953795379537953, "ece_hard": 0.1784936531106821, "calibration": 66.35447402716198, "p50_s": 0.5651678681373596, "p95_s": 1.115284267067909, "cost_usd_per_1000": 0.02278719756245287, "axes": { "intelligence": 79.06273302333375, "calibration": 66.35447402716198, "speed": 82.00436961552492, "cost": 59.269272471003546 }, "jevbench_score": 71.06031955435729 } }, { "threshold": 0.34, "public": { "n": 231, "escalation": 0.008658008658008658, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6126126126126126 }, "accuracy": 0.7575757575757576, "ece_hard": 0.1227029771638957, "calibration": 72.88887570533063, "p50_s": 0.5384091228246689, "p95_s": 1.2262836933135985, "cost_usd_per_1000": 0.022669292929292928, "axes": { "intelligence": 76.83725392058726, "calibration": 72.88887570533063, "speed": 81.80296636812625, "cost": 59.33686076809947 }, "jevbench_score": 72.20739482105681 }, "heldout": { "n": 303, "escalation": 0.0165016501650165, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5412844036697247 }, "accuracy": 0.7953795379537953, "ece_hard": 0.18446923703728185, "calibration": 65.75691563450202, "p50_s": 0.5651678681373596, "p95_s": 1.115284267067909, "cost_usd_per_1000": 0.02287284526767283, "axes": { "intelligence": 79.06273302333375, "calibration": 65.75691563450202, "speed": 82.00436961552492, "cost": 59.22039423907319 }, "jevbench_score": 70.88517019392344 } }, { "threshold": 0.35, "public": { "n": 231, "escalation": 0.017316017316017316, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6126126126126126 }, "accuracy": 0.7575757575757576, "ece_hard": 0.12540138239278134, "calibration": 72.61903518244208, "p50_s": 0.5384091228246689, "p95_s": 1.2262836933135985, "cost_usd_per_1000": 0.02289397911701282, "axes": { "intelligence": 76.83725392058726, "calibration": 72.61903518244208, "speed": 81.80296636812625, "cost": 59.20836152988853 }, "jevbench_score": 72.10138402910133 }, "heldout": { "n": 303, "escalation": 0.019801980198019802, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5412844036697247 }, "accuracy": 0.7953795379537953, "ece_hard": 0.18894729586327258, "calibration": 66.48036643027022, "p50_s": 0.5651678681373596, "p95_s": 1.115284267067909, "cost_usd_per_1000": 0.02295849297289279, "axes": { "intelligence": 79.06273302333375, "calibration": 66.48036643027022, "speed": 82.00436961552492, "cost": 59.17169869079666 }, "jevbench_score": 71.06472253912516 } }, { "threshold": 0.36, "public": { "n": 231, "escalation": 0.017316017316017316, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6126126126126126 }, "accuracy": 0.7575757575757576, "ece_hard": 0.12540138239278134, "calibration": 72.61903518244208, "p50_s": 0.5384091228246689, "p95_s": 1.2262836933135985, "cost_usd_per_1000": 0.02289397911701282, "axes": { "intelligence": 76.83725392058726, "calibration": 72.61903518244208, "speed": 81.80296636812625, "cost": 59.20836152988853 }, "jevbench_score": 72.10138402910133 }, "heldout": { "n": 303, "escalation": 0.0297029702970297, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7986798679867987, "ece_hard": 0.19801159728583317, "calibration": 64.2108855461405, "p50_s": 0.5654387563467026, "p95_s": 1.115284267067909, "cost_usd_per_1000": 0.02321543608855267, "axes": { "intelligence": 79.33796238113192, "calibration": 64.2108855461405, "speed": 82.00228851579965, "cost": 59.026694618508536 }, "jevbench_score": 70.46785219080458 } }, { "threshold": 0.37, "public": { "n": 231, "escalation": 0.021645021645021644, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6126126126126126 }, "accuracy": 0.7575757575757576, "ece_hard": 0.129274702262339, "calibration": 72.23170319548629, "p50_s": 0.538740599155426, "p95_s": 1.2262836933135985, "cost_usd_per_1000": 0.02300632221087277, "axes": { "intelligence": 76.83725392058726, "calibration": 72.23170319548629, "speed": 81.80029341852722, "cost": 59.144584062084604 }, "jevbench_score": 71.98506208274257 }, "heldout": { "n": 303, "escalation": 0.033003300330033, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7986798679867987, "ece_hard": 0.19217855263975825, "calibration": 64.794190010748, "p50_s": 0.5654387563467026, "p95_s": 1.1195790365338323, "cost_usd_per_1000": 0.023301083793772635, "axes": { "intelligence": 79.33796238113192, "calibration": 64.794190010748, "speed": 81.98559669421805, "cost": 58.97871635028157 }, "jevbench_score": 70.60939597470144 } }, { "threshold": 0.38, "public": { "n": 231, "escalation": 0.030303030303030304, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6306306306306306 }, "accuracy": 0.7662337662337663, "ece_hard": 0.12046967633812916, "calibration": 73.11220578790727, "p50_s": 0.538740599155426, "p95_s": 1.2382635936141013, "cost_usd_per_1000": 0.023231008398592667, "axes": { "intelligence": 77.58800467133801, "calibration": 73.11220578790727, "speed": 81.75807190709418, "cost": 59.01795814537402 }, "jevbench_score": 72.33105885103498 }, "heldout": { "n": 303, "escalation": 0.033003300330033, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7986798679867987, "ece_hard": 0.19217855263975825, "calibration": 64.794190010748, "p50_s": 0.5654387563467026, "p95_s": 1.1195790365338323, "cost_usd_per_1000": 0.023301083793772635, "axes": { "intelligence": 79.33796238113192, "calibration": 64.794190010748, "speed": 81.98559669421805, "cost": 58.97871635028157 }, "jevbench_score": 70.60939597470144 } }, { "threshold": 0.39, "public": { "n": 231, "escalation": 0.04329004329004329, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6396396396396397 }, "accuracy": 0.7705627705627706, "ece_hard": 0.12391053900720686, "calibration": 72.7681195209995, "p50_s": 0.538740599155426, "p95_s": 1.4027332119643687, "cost_usd_per_1000": 0.02356803768017251, "axes": { "intelligence": 77.96338004671338, "calibration": 72.7681195209995, "speed": 81.21645215327925, "cost": 58.83029728475067 }, "jevbench_score": 72.15546288396726 }, "heldout": { "n": 303, "escalation": 0.039603960396039604, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7986798679867987, "ece_hard": 0.2009330336707937, "calibration": 64.1961225283875, "p50_s": 0.5654387563467026, "p95_s": 1.1195790365338323, "cost_usd_per_1000": 0.023472379204212555, "axes": { "intelligence": 79.33796238113192, "calibration": 64.1961225283875, "speed": 81.98559669421805, "cost": 58.8832866181031 }, "jevbench_score": 70.41737978046176 } }, { "threshold": 0.4, "public": { "n": 231, "escalation": 0.04329004329004329, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6396396396396397 }, "accuracy": 0.7705627705627706, "ece_hard": 0.12391053900720686, "calibration": 72.7681195209995, "p50_s": 0.538740599155426, "p95_s": 1.4027332119643687, "cost_usd_per_1000": 0.02356803768017251, "axes": { "intelligence": 77.96338004671338, "calibration": 72.7681195209995, "speed": 81.21645215327925, "cost": 58.83029728475067 }, "jevbench_score": 72.15546288396726 }, "heldout": { "n": 303, "escalation": 0.0462046204620462, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7986798679867987, "ece_hard": 0.21368912688586947, "calibration": 62.92051320687993, "p50_s": 0.5671136170625687, "p95_s": 1.1195790365338323, "cost_usd_per_1000": 0.023643674614652475, "axes": { "intelligence": 79.33796238113192, "calibration": 62.92051320687993, "speed": 81.97275166710604, "cost": 58.78855078188529 }, "jevbench_score": 70.0339935269736 } }, { "threshold": 0.41, "public": { "n": 231, "escalation": 0.04329004329004329, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6396396396396397 }, "accuracy": 0.7705627705627706, "ece_hard": 0.12391053900720686, "calibration": 72.7681195209995, "p50_s": 0.538740599155426, "p95_s": 1.4027332119643687, "cost_usd_per_1000": 0.02356803768017251, "axes": { "intelligence": 77.96338004671338, "calibration": 72.7681195209995, "speed": 81.21645215327925, "cost": 58.83029728475067 }, "jevbench_score": 72.15546288396726 }, "heldout": { "n": 303, "escalation": 0.0462046204620462, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7986798679867987, "ece_hard": 0.21368912688586947, "calibration": 62.92051320687993, "p50_s": 0.5671136170625687, "p95_s": 1.1195790365338323, "cost_usd_per_1000": 0.023643674614652475, "axes": { "intelligence": 79.33796238113192, "calibration": 62.92051320687993, "speed": 81.97275166710604, "cost": 58.78855078188529 }, "jevbench_score": 70.0339935269736 } }, { "threshold": 0.42, "public": { "n": 231, "escalation": 0.05194805194805195, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6576576576576577 }, "accuracy": 0.7792207792207793, "ece_hard": 0.1231255323773067, "calibration": 72.84662018398953, "p50_s": 0.538740599155426, "p95_s": 1.4027332119643687, "cost_usd_per_1000": 0.023792723867892406, "axes": { "intelligence": 78.71413079746414, "calibration": 72.84662018398953, "speed": 81.21645215327925, "cost": 58.70667507043569 }, "jevbench_score": 72.31000678593934 }, "heldout": { "n": 303, "escalation": 0.04950495049504951, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5412844036697247 }, "accuracy": 0.7953795379537953, "ece_hard": 0.22412657477249198, "calibration": 61.87676841821768, "p50_s": 0.5671136170625687, "p95_s": 1.1195790365338323, "cost_usd_per_1000": 0.023729322319872435, "axes": { "intelligence": 79.06273302333375, "calibration": 61.87676841821768, "speed": 81.97275166710604, "cost": 58.74143993558155 }, "jevbench_score": 69.66720499216278 } }, { "threshold": 0.43, "public": { "n": 231, "escalation": 0.05627705627705628, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6576576576576577 }, "accuracy": 0.7792207792207793, "ece_hard": 0.12247222294314211, "calibration": 72.91195112740598, "p50_s": 0.538740599155426, "p95_s": 1.4027332119643687, "cost_usd_per_1000": 0.023905066961752354, "axes": { "intelligence": 78.71413079746414, "calibration": 72.91195112740598, "speed": 81.21645215327925, "cost": 58.64530106323465 }, "jevbench_score": 72.30730325731786 }, "heldout": { "n": 303, "escalation": 0.052805280528052806, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7986798679867987, "ece_hard": 0.21692446519889075, "calibration": 62.5969793755778, "p50_s": 0.5671136170625687, "p95_s": 1.1195790365338323, "cost_usd_per_1000": 0.023814970025092395, "axes": { "intelligence": 79.33796238113192, "calibration": 62.5969793755778, "speed": 81.97275166710604, "cost": 58.69449882341089 }, "jevbench_score": 69.91580024749757 } }, { "threshold": 0.44, "public": { "n": 231, "escalation": 0.06493506493506493, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6576576576576577 }, "accuracy": 0.7792207792207793, "ece_hard": 0.12616663225577013, "calibration": 72.54251019614318, "p50_s": 0.538740599155426, "p95_s": 1.4518735907971858, "cost_usd_per_1000": 0.024129753149472247, "axes": { "intelligence": 78.71413079746414, "calibration": 72.54251019614318, "speed": 81.06691489445589, "cost": 58.52341362873363 }, "jevbench_score": 72.14473539324058 }, "heldout": { "n": 303, "escalation": 0.056105610561056105, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5596330275229358 }, "accuracy": 0.801980198019802, "ece_hard": 0.21262751068559171, "calibration": 63.02667482690771, "p50_s": 0.5671136170625687, "p95_s": 1.1252075582742689, "cost_usd_per_1000": 0.023900617730312355, "axes": { "intelligence": 79.61319173893008, "calibration": 63.02667482690771, "speed": 81.95097283911872, "cost": 58.64772622670355 }, "jevbench_score": 70.07751302379275 } }, { "threshold": 0.45, "public": { "n": 231, "escalation": 0.06493506493506493, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6576576576576577 }, "accuracy": 0.7792207792207793, "ece_hard": 0.12616663225577013, "calibration": 72.54251019614318, "p50_s": 0.538740599155426, "p95_s": 1.4518735907971858, "cost_usd_per_1000": 0.024129753149472247, "axes": { "intelligence": 78.71413079746414, "calibration": 72.54251019614318, "speed": 81.06691489445589, "cost": 58.52341362873363 }, "jevbench_score": 72.14473539324058 }, "heldout": { "n": 303, "escalation": 0.0627062706270627, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5596330275229358 }, "accuracy": 0.801980198019802, "ece_hard": 0.21849873000447706, "calibration": 62.439552895019176, "p50_s": 0.5671136170625687, "p95_s": 1.1252075582742689, "cost_usd_per_1000": 0.024071913140752275, "axes": { "intelligence": 79.61319173893008, "calibration": 62.439552895019176, "speed": 81.95097283911872, "cost": 58.55468177020284 }, "jevbench_score": 69.88599287750458 } }, { "threshold": 0.46, "public": { "n": 231, "escalation": 0.06493506493506493, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6576576576576577 }, "accuracy": 0.7792207792207793, "ece_hard": 0.12616663225577013, "calibration": 72.54251019614318, "p50_s": 0.538740599155426, "p95_s": 1.4518735907971858, "cost_usd_per_1000": 0.024129753149472247, "axes": { "intelligence": 78.71413079746414, "calibration": 72.54251019614318, "speed": 81.06691489445589, "cost": 58.52341362873363 }, "jevbench_score": 72.14473539324058 }, "heldout": { "n": 303, "escalation": 0.066006600660066, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5596330275229358 }, "accuracy": 0.801980198019802, "ece_hard": 0.2226297534986995, "calibration": 61.46513804453349, "p50_s": 0.5671136170625687, "p95_s": 1.1252075582742689, "cost_usd_per_1000": 0.024157560845972235, "axes": { "intelligence": 79.61319173893008, "calibration": 61.46513804453349, "speed": 81.95097283911872, "cost": 58.508407537716984 }, "jevbench_score": 69.59796991761759 } }, { "threshold": 0.47, "public": { "n": 231, "escalation": 0.08225108225108226, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6486486486486487 }, "accuracy": 0.7748917748917749, "ece_hard": 0.14036846211361534, "calibration": 71.02194866020676, "p50_s": 0.5388893276453018, "p95_s": 1.4563486777245997, "cost_usd_per_1000": 0.024579125524912038, "axes": { "intelligence": 78.33875542208877, "calibration": 71.02194866020676, "speed": 81.05235051409602, "cost": 58.2830071745623 }, "jevbench_score": 71.6010191971694 }, "heldout": { "n": 303, "escalation": 0.06930693069306931, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5596330275229358 }, "accuracy": 0.801980198019802, "ece_hard": 0.22432466285423955, "calibration": 60.377289084902586, "p50_s": 0.5671136170625687, "p95_s": 1.1252075582742689, "cost_usd_per_1000": 0.024243208551192195, "axes": { "intelligence": 79.61319173893008, "calibration": 60.377289084902586, "speed": 81.95097283911872, "cost": 58.46229707494639 }, "jevbench_score": 69.27430197920576 } }, { "threshold": 0.48, "public": { "n": 231, "escalation": 0.08225108225108226, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6486486486486487 }, "accuracy": 0.7748917748917749, "ece_hard": 0.14036846211361534, "calibration": 71.02194866020676, "p50_s": 0.5388893276453018, "p95_s": 1.4563486777245997, "cost_usd_per_1000": 0.024579125524912038, "axes": { "intelligence": 78.33875542208877, "calibration": 71.02194866020676, "speed": 81.05235051409602, "cost": 58.2830071745623 }, "jevbench_score": 71.6010191971694 }, "heldout": { "n": 303, "escalation": 0.06930693069306931, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5596330275229358 }, "accuracy": 0.801980198019802, "ece_hard": 0.22432466285423955, "calibration": 60.377289084902586, "p50_s": 0.5671136170625687, "p95_s": 1.1252075582742689, "cost_usd_per_1000": 0.024243208551192195, "axes": { "intelligence": 79.61319173893008, "calibration": 60.377289084902586, "speed": 81.95097283911872, "cost": 58.46229707494639 }, "jevbench_score": 69.27430197920576 } }, { "threshold": 0.49, "public": { "n": 231, "escalation": 0.09090909090909091, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6396396396396397 }, "accuracy": 0.7705627705627706, "ece_hard": 0.1393271964640658, "calibration": 70.44998457663763, "p50_s": 0.5388893276453018, "p95_s": 1.4563486777245997, "cost_usd_per_1000": 0.024803811712631935, "axes": { "intelligence": 77.96338004671338, "calibration": 70.44998457663763, "speed": 81.05235051409602, "cost": 58.16444722211755 }, "jevbench_score": 71.33434784370577 }, "heldout": { "n": 303, "escalation": 0.07260726072607261, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5596330275229358 }, "accuracy": 0.801980198019802, "ece_hard": 0.21646957795469188, "calibration": 61.16279757485735, "p50_s": 0.5671136170625687, "p95_s": 1.1252075582742689, "cost_usd_per_1000": 0.024328856256412155, "axes": { "intelligence": 79.61319173893008, "calibration": 61.16279757485735, "speed": 81.95097283911872, "cost": 58.41634922677905 }, "jevbench_score": 69.48486620789448 } }, { "threshold": 0.5, "public": { "n": 231, "escalation": 0.09956709956709957, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6396396396396397 }, "accuracy": 0.7705627705627706, "ece_hard": 0.14913039921983365, "calibration": 69.46966430106085, "p50_s": 0.5388893276453018, "p95_s": 1.4563486777245997, "cost_usd_per_1000": 0.02502849790035183, "axes": { "intelligence": 77.96338004671338, "calibration": 69.46966430106085, "speed": 81.05235051409602, "cost": 58.046956421258415 }, "jevbench_score": 71.04896106341569 }, "heldout": { "n": 303, "escalation": 0.07590759075907591, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7986798679867987, "ece_hard": 0.22848051862244445, "calibration": 59.961703508082095, "p50_s": 0.5671136170625687, "p95_s": 1.1293273240327832, "cost_usd_per_1000": 0.024414503961632115, "axes": { "intelligence": 79.33796238113192, "calibration": 59.961703508082095, "speed": 81.93510088662741, "cost": 58.370562850280955 }, "jevbench_score": 69.0644757792367 } }, { "threshold": 0.51, "public": { "n": 231, "escalation": 0.11255411255411256, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6396396396396397 }, "accuracy": 0.7705627705627706, "ece_hard": 0.153439852872532, "calibration": 69.038718935791, "p50_s": 0.5397706717252732, "p95_s": 1.4563486777245997, "cost_usd_per_1000": 0.025365527181931673, "axes": { "intelligence": 77.96338004671338, "calibration": 69.038718935791, "speed": 81.04525350565979, "cost": 57.87268321408783 }, "jevbench_score": 70.88366211969267 }, "heldout": { "n": 303, "escalation": 0.08250825082508251, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.958904109589041, "hard": 0.5504587155963303 }, "accuracy": 0.801980198019802, "ece_hard": 0.22298194277474082, "calibration": 60.51156109285246, "p50_s": 0.567900562286377, "p95_s": 1.1293273240327832, "cost_usd_per_1000": 0.024585799372072035, "axes": { "intelligence": 79.52974320304973, "calibration": 60.51156109285246, "speed": 81.92907865308634, "cost": 58.27947000042623 }, "jevbench_score": 69.2357498995429 } }, { "threshold": 0.52, "public": { "n": 231, "escalation": 0.12121212121212122, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6396396396396397 }, "accuracy": 0.7705627705627706, "ece_hard": 0.1427495309986048, "calibration": 70.10775112318373, "p50_s": 0.5397706717252732, "p95_s": 1.4563486777245997, "cost_usd_per_1000": 0.02559021336965157, "axes": { "intelligence": 77.96338004671338, "calibration": 70.10775112318373, "speed": 81.04525350565979, "cost": 57.75778278929772 }, "jevbench_score": 71.12113822245146 }, "heldout": { "n": 303, "escalation": 0.0858085808580858, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.958904109589041, "hard": 0.5504587155963303 }, "accuracy": 0.801980198019802, "ece_hard": 0.22327981763225013, "calibration": 60.48177360710153, "p50_s": 0.567900562286377, "p95_s": 1.1293273240327832, "cost_usd_per_1000": 0.024671447077292, "axes": { "intelligence": 79.52974320304973, "calibration": 60.48177360710153, "speed": 81.92907865308634, "cost": 58.23416130057137 }, "jevbench_score": 69.21376890883711 } }, { "threshold": 0.53, "public": { "n": 231, "escalation": 0.12121212121212122, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6396396396396397 }, "accuracy": 0.7705627705627706, "ece_hard": 0.1427495309986048, "calibration": 70.10775112318373, "p50_s": 0.5397706717252732, "p95_s": 1.4563486777245997, "cost_usd_per_1000": 0.02559021336965157, "axes": { "intelligence": 77.96338004671338, "calibration": 70.10775112318373, "speed": 81.04525350565979, "cost": 57.75778278929772 }, "jevbench_score": 71.12113822245146 }, "heldout": { "n": 303, "escalation": 0.09570957095709572, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.958904109589041, "hard": 0.5504587155963303 }, "accuracy": 0.801980198019802, "ece_hard": 0.23443847711355084, "calibration": 59.36590765897146, "p50_s": 0.567900562286377, "p95_s": 1.1970025122165662, "cost_usd_per_1000": 0.02492839019295188, "axes": { "intelligence": 79.52974320304973, "calibration": 59.36590765897146, "speed": 81.67632639429573, "cost": 58.09917298410629 }, "jevbench_score": 68.79917171152499 } }, { "threshold": 0.54, "public": { "n": 231, "escalation": 0.1341991341991342, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6486486486486487 }, "accuracy": 0.7748917748917749, "ece_hard": 0.13701921719922372, "calibration": 70.68078250312183, "p50_s": 0.5404984593391419, "p95_s": 1.4563486777245997, "cost_usd_per_1000": 0.025927242651231412, "axes": { "intelligence": 78.33875542208877, "calibration": 70.68078250312183, "speed": 81.0394017381061, "cost": 57.587310033012145 }, "jevbench_score": 71.29765710489401 }, "heldout": { "n": 303, "escalation": 0.11551155115511551, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9657534246575342, "hard": 0.5504587155963303 }, "accuracy": 0.8052805280528053, "ece_hard": 0.2544928154811612, "calibration": 57.36047382221042, "p50_s": 0.5697284132242203, "p95_s": 1.2169031620025612, "cost_usd_per_1000": 0.02544227642427164, "axes": { "intelligence": 79.72152402496754, "calibration": 57.36047382221042, "speed": 81.59076101498934, "cost": 57.83332099557951 }, "jevbench_score": 68.15564930111181 } }, { "threshold": 0.55, "public": { "n": 231, "escalation": 0.13852813852813853, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6486486486486487 }, "accuracy": 0.7748917748917749, "ece_hard": 0.1392202534475306, "calibration": 70.46067887829113, "p50_s": 0.5404984593391419, "p95_s": 1.4563486777245997, "cost_usd_per_1000": 0.02603958574509136, "axes": { "intelligence": 78.33875542208877, "calibration": 70.46067887829113, "speed": 81.0394017381061, "cost": 57.530977872760495 }, "jevbench_score": 71.22465729223394 }, "heldout": { "n": 303, "escalation": 0.12211221122112212, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9657534246575342, "hard": 0.5504587155963303 }, "accuracy": 0.8052805280528053, "ece_hard": 0.26276964885951437, "calibration": 56.532790484375106, "p50_s": 0.5702764481306076, "p95_s": 1.2169031620025612, "cost_usd_per_1000": 0.02561357183471156, "axes": { "intelligence": 79.72152402496754, "calibration": 56.532790484375106, "speed": 81.58658544412754, "cost": 57.745895637036035 }, "jevbench_score": 67.88189749868832 } }, { "threshold": 0.56, "public": { "n": 231, "escalation": 0.13852813852813853, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6486486486486487 }, "accuracy": 0.7748917748917749, "ece_hard": 0.1392202534475306, "calibration": 70.46067887829113, "p50_s": 0.5404984593391419, "p95_s": 1.4563486777245997, "cost_usd_per_1000": 0.02603958574509136, "axes": { "intelligence": 78.33875542208877, "calibration": 70.46067887829113, "speed": 81.0394017381061, "cost": 57.530977872760495 }, "jevbench_score": 71.22465729223394 }, "heldout": { "n": 303, "escalation": 0.12211221122112212, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9657534246575342, "hard": 0.5504587155963303 }, "accuracy": 0.8052805280528053, "ece_hard": 0.26276964885951437, "calibration": 56.532790484375106, "p50_s": 0.5702764481306076, "p95_s": 1.2169031620025612, "cost_usd_per_1000": 0.02561357183471156, "axes": { "intelligence": 79.72152402496754, "calibration": 56.532790484375106, "speed": 81.58658544412754, "cost": 57.745895637036035 }, "jevbench_score": 67.88189749868832 } }, { "threshold": 0.57, "public": { "n": 231, "escalation": 0.15584415584415584, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6396396396396397 }, "accuracy": 0.7705627705627706, "ece_hard": 0.15786862847180116, "calibration": 67.86584274415871, "p50_s": 0.5404984593391419, "p95_s": 1.4563486777245997, "cost_usd_per_1000": 0.02648895812053115, "axes": { "intelligence": 77.96338004671338, "calibration": 67.86584274415871, "speed": 81.0394017381061, "cost": 57.30805369887771 }, "jevbench_score": 70.40661223441107 }, "heldout": { "n": 303, "escalation": 0.13861386138613863, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9657534246575342, "hard": 0.5596330275229358 }, "accuracy": 0.8085808580858086, "ece_hard": 0.263875743077182, "calibration": 56.37901632639406, "p50_s": 0.5712133586406708, "p95_s": 1.2563446298241583, "cost_usd_per_1000": 0.02604181036081136, "axes": { "intelligence": 79.9967533827657, "calibration": 56.37901632639406, "speed": 81.4409285678617, "cost": 57.52986484000288 }, "jevbench_score": 67.80027909531597 } }, { "threshold": 0.58, "public": { "n": 231, "escalation": 0.16017316017316016, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6306306306306306 }, "accuracy": 0.7662337662337663, "ece_hard": 0.16875499206730582, "calibration": 66.77720638460825, "p50_s": 0.5404984593391419, "p95_s": 1.4830813348293304, "cost_usd_per_1000": 0.0266013012143911, "axes": { "intelligence": 77.58800467133801, "calibration": 66.77720638460825, "speed": 80.96040570474639, "cost": 57.252913574291725 }, "jevbench_score": 70.00406744296846 }, "heldout": { "n": 303, "escalation": 0.13861386138613863, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9657534246575342, "hard": 0.5596330275229358 }, "accuracy": 0.8085808580858086, "ece_hard": 0.263875743077182, "calibration": 56.37901632639406, "p50_s": 0.5712133586406708, "p95_s": 1.2563446298241583, "cost_usd_per_1000": 0.02604181036081136, "axes": { "intelligence": 79.9967533827657, "calibration": 56.37901632639406, "speed": 81.4409285678617, "cost": 57.52986484000288 }, "jevbench_score": 67.80027909531597 } }, { "threshold": 0.59, "public": { "n": 231, "escalation": 0.1645021645021645, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6396396396396397 }, "accuracy": 0.7705627705627706, "ece_hard": 0.16300575649739346, "calibration": 67.86796391863538, "p50_s": 0.5404984593391419, "p95_s": 1.4830813348293304, "cost_usd_per_1000": 0.026713644308251048, "axes": { "intelligence": 77.96338004671338, "calibration": 67.86796391863538, "speed": 80.96040570474639, "cost": 57.19800582846956 }, "jevbench_score": 70.35618153044237 }, "heldout": { "n": 303, "escalation": 0.1419141914191419, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9657534246575342, "hard": 0.5596330275229358 }, "accuracy": 0.8085808580858086, "ece_hard": 0.26340855933260754, "calibration": 56.4257347008515, "p50_s": 0.5712133586406708, "p95_s": 1.2563446298241583, "cost_usd_per_1000": 0.02612745806603132, "axes": { "intelligence": 79.9967533827657, "calibration": 56.4257347008515, "speed": 81.4409285678617, "cost": 57.48708521862298 }, "jevbench_score": 67.80171005777173 } }, { "threshold": 0.6, "public": { "n": 231, "escalation": 0.19913419913419914, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6576576576576577 }, "accuracy": 0.7792207792207793, "ece_hard": 0.16972725284553997, "calibration": 67.11069755096435, "p50_s": 0.5404984593391419, "p95_s": 1.4830813348293304, "cost_usd_per_1000": 0.027612389059130632, "axes": { "intelligence": 78.71413079746414, "calibration": 67.11069755096435, "speed": 80.96040570474639, "cost": 56.76688048067276 }, "jevbench_score": 70.19449282200452 }, "heldout": { "n": 303, "escalation": 0.15841584158415842, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9657534246575342, "hard": 0.5596330275229358 }, "accuracy": 0.8085808580858086, "ece_hard": 0.26813436915672983, "calibration": 55.95315371843927, "p50_s": 0.5717611402273178, "p95_s": 1.2563446298241583, "cost_usd_per_1000": 0.026555696592131122, "axes": { "intelligence": 79.9967533827657, "calibration": 55.95315371843927, "speed": 81.4367657708996, "cost": 57.275269058769744 }, "jevbench_score": 67.59602327344567 } }, { "threshold": 0.61, "public": { "n": 231, "escalation": 0.20346320346320346, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6576576576576577 }, "accuracy": 0.7792207792207793, "ece_hard": 0.16913924837006666, "calibration": 67.16949799851167, "p50_s": 0.5404984593391419, "p95_s": 1.4830813348293304, "cost_usd_per_1000": 0.027724732152990577, "axes": { "intelligence": 78.71413079746414, "calibration": 67.16949799851167, "speed": 80.96040570474639, "cost": 56.713979225560735 }, "jevbench_score": 70.19350044373662 }, "heldout": { "n": 303, "escalation": 0.15841584158415842, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9657534246575342, "hard": 0.5596330275229358 }, "accuracy": 0.8085808580858086, "ece_hard": 0.26813436915672983, "calibration": 55.95315371843927, "p50_s": 0.5717611402273178, "p95_s": 1.2563446298241583, "cost_usd_per_1000": 0.026555696592131122, "axes": { "intelligence": 79.9967533827657, "calibration": 55.95315371843927, "speed": 81.4367657708996, "cost": 57.275269058769744 }, "jevbench_score": 67.59602327344567 } }, { "threshold": 0.62, "public": { "n": 231, "escalation": 0.2077922077922078, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6576576576576577 }, "accuracy": 0.7792207792207793, "ece_hard": 0.16309811913200697, "calibration": 67.77361092231766, "p50_s": 0.5404984593391419, "p95_s": 1.4830813348293304, "cost_usd_per_1000": 0.027837075246850525, "axes": { "intelligence": 78.71413079746414, "calibration": 67.77361092231766, "speed": 80.96040570474639, "cost": 56.66129189822311 }, "jevbench_score": 70.33445382699068 }, "heldout": { "n": 303, "escalation": 0.1617161716171617, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9657534246575342, "hard": 0.5596330275229358 }, "accuracy": 0.8085808580858086, "ece_hard": 0.26813436915672983, "calibration": 55.95315371843927, "p50_s": 0.5717611402273178, "p95_s": 1.2563446298241583, "cost_usd_per_1000": 0.026641344297351082, "axes": { "intelligence": 79.9967533827657, "calibration": 55.95315371843927, "speed": 81.4367657708996, "cost": 57.23331594564687 }, "jevbench_score": 67.58364165155196 } }, { "threshold": 0.63, "public": { "n": 231, "escalation": 0.21645021645021645, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6576576576576577 }, "accuracy": 0.7792207792207793, "ece_hard": 0.1568173788366799, "calibration": 68.40168495185036, "p50_s": 0.5404984593391419, "p95_s": 1.4830813348293304, "cost_usd_per_1000": 0.028061761434570422, "axes": { "intelligence": 78.71413079746414, "calibration": 68.40168495185036, "speed": 80.96040570474639, "cost": 56.55655215464164 }, "jevbench_score": 70.46424068930426 }, "heldout": { "n": 303, "escalation": 0.16831683168316833, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9726027397260274, "hard": 0.5596330275229358 }, "accuracy": 0.8118811881188119, "ece_hard": 0.26813436915672983, "calibration": 55.95315371843927, "p50_s": 0.5727051436901093, "p95_s": 1.2563446298241583, "cost_usd_per_1000": 0.026812639707791002, "axes": { "intelligence": 80.1885342046835, "calibration": 55.95315371843927, "speed": 81.4296012856976, "cost": 57.149812826958794 }, "jevbench_score": 67.59794467864833 } }, { "threshold": 0.64, "public": { "n": 231, "escalation": 0.21645021645021645, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6576576576576577 }, "accuracy": 0.7792207792207793, "ece_hard": 0.1568173788366799, "calibration": 68.40168495185036, "p50_s": 0.5404984593391419, "p95_s": 1.4830813348293304, "cost_usd_per_1000": 0.028061761434570422, "axes": { "intelligence": 78.71413079746414, "calibration": 68.40168495185036, "speed": 80.96040570474639, "cost": 56.55655215464164 }, "jevbench_score": 70.46424068930426 }, "heldout": { "n": 303, "escalation": 0.17491749174917492, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9726027397260274, "hard": 0.5688073394495413 }, "accuracy": 0.8151815181518152, "ece_hard": 0.2644068311813767, "calibration": 57.44102787114536, "p50_s": 0.5727051436901093, "p95_s": 1.2563446298241583, "cost_usd_per_1000": 0.026983935118230922, "axes": { "intelligence": 80.46376356248169, "calibration": 57.44102787114536, "speed": 81.4296012856976, "cost": 57.06684148182791 }, "jevbench_score": 68.07649261993889 } }, { "threshold": 0.65, "public": { "n": 231, "escalation": 0.22077922077922077, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6576576576576577 }, "accuracy": 0.7792207792207793, "ece_hard": 0.1568173788366799, "calibration": 68.40168495185036, "p50_s": 0.5407783031463623, "p95_s": 1.4830813348293304, "cost_usd_per_1000": 0.028174104528430367, "axes": { "intelligence": 78.71413079746414, "calibration": 68.40168495185036, "speed": 80.95815772109964, "cost": 56.50449635379722 }, "jevbench_score": 70.44753188171057 }, "heldout": { "n": 303, "escalation": 0.17491749174917492, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9726027397260274, "hard": 0.5688073394495413 }, "accuracy": 0.8151815181518152, "ece_hard": 0.2644068311813767, "calibration": 57.44102787114536, "p50_s": 0.5727051436901093, "p95_s": 1.2563446298241583, "cost_usd_per_1000": 0.026983935118230922, "axes": { "intelligence": 80.46376356248169, "calibration": 57.44102787114536, "speed": 81.4296012856976, "cost": 57.06684148182791 }, "jevbench_score": 68.07649261993889 } }, { "threshold": 0.66, "public": { "n": 231, "escalation": 0.24242424242424243, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6396396396396397 }, "accuracy": 0.7705627705627706, "ece_hard": 0.17817063170600736, "calibration": 66.26635966491762, "p50_s": 0.5409701555967331, "p95_s": 1.4830813348293304, "cost_usd_per_1000": 0.02873581999773011, "axes": { "intelligence": 77.96338004671338, "calibration": 66.26635966491762, "speed": 80.95661724353786, "cost": 56.247292163242875 }, "jevbench_score": 69.64411698080356 }, "heldout": { "n": 303, "escalation": 0.1848184818481848, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9726027397260274, "hard": 0.5688073394495413 }, "accuracy": 0.8151815181518152, "ece_hard": 0.2723254240229221, "calibration": 56.64916858699082, "p50_s": 0.5745811700820923, "p95_s": 1.2837200626730914, "cost_usd_per_1000": 0.027240878233890802, "axes": { "intelligence": 80.46376356248169, "calibration": 56.64916858699082, "speed": 81.32178281969736, "cost": 56.94336685213413 }, "jevbench_score": 67.78146930488172 } }, { "threshold": 0.67, "public": { "n": 231, "escalation": 0.24242424242424243, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6396396396396397 }, "accuracy": 0.7705627705627706, "ece_hard": 0.17817063170600736, "calibration": 66.26635966491762, "p50_s": 0.5409701555967331, "p95_s": 1.4830813348293304, "cost_usd_per_1000": 0.02873581999773011, "axes": { "intelligence": 77.96338004671338, "calibration": 66.26635966491762, "speed": 80.95661724353786, "cost": 56.247292163242875 }, "jevbench_score": 69.64411698080356 }, "heldout": { "n": 303, "escalation": 0.1848184818481848, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9726027397260274, "hard": 0.5688073394495413 }, "accuracy": 0.8151815181518152, "ece_hard": 0.2723254240229221, "calibration": 56.64916858699082, "p50_s": 0.5745811700820923, "p95_s": 1.2837200626730914, "cost_usd_per_1000": 0.027240878233890802, "axes": { "intelligence": 80.46376356248169, "calibration": 56.64916858699082, "speed": 81.32178281969736, "cost": 56.94336685213413 }, "jevbench_score": 67.78146930488172 } }, { "threshold": 0.68, "public": { "n": 231, "escalation": 0.26406926406926406, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "hard": 0.6396396396396397 }, "accuracy": 0.7748917748917749, "ece_hard": 0.18714228246059084, "calibration": 65.36919458945927, "p50_s": 0.5424583941698075, "p95_s": 1.5242826543748378, "cost_usd_per_1000": 0.029297535467029845, "axes": { "intelligence": 78.50350350350352, "calibration": 65.36919458945927, "speed": 80.82568058415923, "cost": 55.995067339636236 }, "jevbench_score": 69.42091421715394 }, "heldout": { "n": 303, "escalation": 0.19141914191419143, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9726027397260274, "hard": 0.5871559633027523 }, "accuracy": 0.8217821782178217, "ece_hard": 0.2597140726888643, "calibration": 57.9103037203966, "p50_s": 0.5745811700820923, "p95_s": 1.3258842036128033, "cost_usd_per_1000": 0.027412173644330722, "axes": { "intelligence": 81.01422227807801, "calibration": 57.9103037203966, "speed": 81.18143014094701, "cost": 56.861695773314985 }, "jevbench_score": 68.21790906353766 } }, { "threshold": 0.69, "public": { "n": 231, "escalation": 0.2683982683982684, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "hard": 0.6396396396396397 }, "accuracy": 0.7748917748917749, "ece_hard": 0.1878901690939302, "calibration": 65.29440592612534, "p50_s": 0.5424583941698075, "p95_s": 1.5242826543748378, "cost_usd_per_1000": 0.029409878560889793, "axes": { "intelligence": 78.50350350350352, "calibration": 65.29440592612534, "speed": 80.82568058415923, "cost": 55.94520306326905 }, "jevbench_score": 69.38559385892817 }, "heldout": { "n": 303, "escalation": 0.19141914191419143, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9726027397260274, "hard": 0.5871559633027523 }, "accuracy": 0.8217821782178217, "ece_hard": 0.2597140726888643, "calibration": 57.9103037203966, "p50_s": 0.5745811700820923, "p95_s": 1.3258842036128033, "cost_usd_per_1000": 0.027412173644330722, "axes": { "intelligence": 81.01422227807801, "calibration": 57.9103037203966, "speed": 81.18143014094701, "cost": 56.861695773314985 }, "jevbench_score": 68.21790906353766 } }, { "threshold": 0.7, "public": { "n": 231, "escalation": 0.2727272727272727, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "hard": 0.6396396396396397 }, "accuracy": 0.7748917748917749, "ece_hard": 0.1878901690939302, "calibration": 65.29440592612534, "p50_s": 0.543133819103241, "p95_s": 1.5242826543748378, "cost_usd_per_1000": 0.029522221654749742, "axes": { "intelligence": 78.50350350350352, "calibration": 65.29440592612534, "speed": 80.82027646729485, "cost": 55.89552890131006 }, "jevbench_score": 69.36902715596159 }, "heldout": { "n": 303, "escalation": 0.19141914191419143, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9726027397260274, "hard": 0.5871559633027523 }, "accuracy": 0.8217821782178217, "ece_hard": 0.2597140726888643, "calibration": 57.9103037203966, "p50_s": 0.5745811700820923, "p95_s": 1.3258842036128033, "cost_usd_per_1000": 0.027412173644330722, "axes": { "intelligence": 81.01422227807801, "calibration": 57.9103037203966, "speed": 81.18143014094701, "cost": 56.861695773314985 }, "jevbench_score": 68.21790906353766 } }, { "threshold": 0.71, "public": { "n": 231, "escalation": 0.27705627705627706, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "hard": 0.6396396396396397 }, "accuracy": 0.7748917748917749, "ece_hard": 0.19054794286680823, "calibration": 64.0057105365427, "p50_s": 0.543133819103241, "p95_s": 1.5242826543748378, "cost_usd_per_1000": 0.02963456474860969, "axes": { "intelligence": 78.50350350350352, "calibration": 64.0057105365427, "speed": 80.82027646729485, "cost": 55.84604340958898 }, "jevbench_score": 69.00890334773142 }, "heldout": { "n": 303, "escalation": 0.2079207920792079, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9726027397260274, "hard": 0.6055045871559633 }, "accuracy": 0.8283828382838284, "ece_hard": 0.2516413827741623, "calibration": 58.717572711866794, "p50_s": 0.5753569543361664, "p95_s": 1.3288795724511142, "cost_usd_per_1000": 0.027840412170430526, "axes": { "intelligence": 81.56468099367433, "calibration": 58.717572711866794, "speed": 81.1657700772831, "cost": 56.6597301817381 }, "jevbench_score": 68.50612662095692 } }, { "threshold": 0.72, "public": { "n": 231, "escalation": 0.2813852813852814, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "hard": 0.6486486486486487 }, "accuracy": 0.7792207792207793, "ece_hard": 0.18395261526302037, "calibration": 64.66524329692149, "p50_s": 0.543133819103241, "p95_s": 1.5242826543748378, "cost_usd_per_1000": 0.02974690784246964, "axes": { "intelligence": 78.8788788788789, "calibration": 64.66524329692149, "speed": 80.82027646729485, "cost": 55.79674516032886 }, "jevbench_score": 69.25325818279254 }, "heldout": { "n": 303, "escalation": 0.2079207920792079, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9726027397260274, "hard": 0.6055045871559633 }, "accuracy": 0.8283828382838284, "ece_hard": 0.2516413827741623, "calibration": 58.717572711866794, "p50_s": 0.5753569543361664, "p95_s": 1.3288795724511142, "cost_usd_per_1000": 0.027840412170430526, "axes": { "intelligence": 81.56468099367433, "calibration": 58.717572711866794, "speed": 81.1657700772831, "cost": 56.6597301817381 }, "jevbench_score": 68.50612662095692 } }, { "threshold": 0.73, "public": { "n": 231, "escalation": 0.2813852813852814, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "hard": 0.6486486486486487 }, "accuracy": 0.7792207792207793, "ece_hard": 0.18395261526302037, "calibration": 64.66524329692149, "p50_s": 0.543133819103241, "p95_s": 1.5242826543748378, "cost_usd_per_1000": 0.02974690784246964, "axes": { "intelligence": 78.8788788788789, "calibration": 64.66524329692149, "speed": 80.82027646729485, "cost": 55.79674516032886 }, "jevbench_score": 69.25325818279254 }, "heldout": { "n": 303, "escalation": 0.2079207920792079, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9726027397260274, "hard": 0.6055045871559633 }, "accuracy": 0.8283828382838284, "ece_hard": 0.2516413827741623, "calibration": 58.717572711866794, "p50_s": 0.5753569543361664, "p95_s": 1.3288795724511142, "cost_usd_per_1000": 0.027840412170430526, "axes": { "intelligence": 81.56468099367433, "calibration": 58.717572711866794, "speed": 81.1657700772831, "cost": 56.6597301817381 }, "jevbench_score": 68.50612662095692 } }, { "threshold": 0.74, "public": { "n": 231, "escalation": 0.2987012987012987, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6396396396396397 }, "accuracy": 0.7705627705627706, "ece_hard": 0.19970104463122387, "calibration": 63.09040036010113, "p50_s": 0.545120233297348, "p95_s": 1.5242826543748378, "cost_usd_per_1000": 0.03019628021790943, "axes": { "intelligence": 77.96338004671338, "calibration": 63.09040036010113, "speed": 80.80442190319786, "cost": 55.60139659241263 }, "jevbench_score": 68.56361264140065 }, "heldout": { "n": 303, "escalation": 0.21122112211221122, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9726027397260274, "hard": 0.6055045871559633 }, "accuracy": 0.8283828382838284, "ece_hard": 0.2516413827741623, "calibration": 58.717572711866794, "p50_s": 0.5755759567022324, "p95_s": 1.3288795724511142, "cost_usd_per_1000": 0.027926059875650486, "axes": { "intelligence": 81.56468099367433, "calibration": 58.717572711866794, "speed": 81.16411730466113, "cost": 56.619710054612426 }, "jevbench_score": 68.49367784784121 } }, { "threshold": 0.75, "public": { "n": 231, "escalation": 0.2987012987012987, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6396396396396397 }, "accuracy": 0.7705627705627706, "ece_hard": 0.19970104463122387, "calibration": 63.09040036010113, "p50_s": 0.545120233297348, "p95_s": 1.5242826543748378, "cost_usd_per_1000": 0.03019628021790943, "axes": { "intelligence": 77.96338004671338, "calibration": 63.09040036010113, "speed": 80.80442190319786, "cost": 55.60139659241263 }, "jevbench_score": 68.56361264140065 }, "heldout": { "n": 303, "escalation": 0.21122112211221122, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9726027397260274, "hard": 0.6055045871559633 }, "accuracy": 0.8283828382838284, "ece_hard": 0.2516413827741623, "calibration": 58.717572711866794, "p50_s": 0.5755759567022324, "p95_s": 1.3288795724511142, "cost_usd_per_1000": 0.027926059875650486, "axes": { "intelligence": 81.56468099367433, "calibration": 58.717572711866794, "speed": 81.16411730466113, "cost": 56.619710054612426 }, "jevbench_score": 68.49367784784121 } }, { "threshold": 0.76, "public": { "n": 231, "escalation": 0.30303030303030304, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6396396396396397 }, "accuracy": 0.7705627705627706, "ece_hard": 0.2009908227920189, "calibration": 62.24559566478037, "p50_s": 0.545120233297348, "p95_s": 1.5242826543748378, "cost_usd_per_1000": 0.030308623311769377, "axes": { "intelligence": 77.96338004671338, "calibration": 62.24559566478037, "speed": 80.80442190319786, "cost": 55.55301369552401 }, "jevbench_score": 68.31805751112206 }, "heldout": { "n": 303, "escalation": 0.22112211221122113, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9657534246575342, "hard": 0.6055045871559633 }, "accuracy": 0.8250825082508251, "ece_hard": 0.2538768479616441, "calibration": 58.494026193118614, "p50_s": 0.5755759567022324, "p95_s": 1.3288795724511142, "cost_usd_per_1000": 0.028183002991310366, "axes": { "intelligence": 81.37290017175654, "calibration": 58.494026193118614, "speed": 81.16411730466113, "cost": 56.500381997832136 }, "jevbench_score": 68.352073076319 } }, { "threshold": 0.77, "public": { "n": 231, "escalation": 0.30303030303030304, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6396396396396397 }, "accuracy": 0.7705627705627706, "ece_hard": 0.2009908227920189, "calibration": 62.24559566478037, "p50_s": 0.545120233297348, "p95_s": 1.5242826543748378, "cost_usd_per_1000": 0.030308623311769377, "axes": { "intelligence": 77.96338004671338, "calibration": 62.24559566478037, "speed": 80.80442190319786, "cost": 55.55301369552401 }, "jevbench_score": 68.31805751112206 }, "heldout": { "n": 303, "escalation": 0.22442244224422442, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9657534246575342, "hard": 0.6055045871559633 }, "accuracy": 0.8250825082508251, "ece_hard": 0.25142083761588935, "calibration": 58.73962722769409, "p50_s": 0.5755759567022324, "p95_s": 1.3288795724511142, "cost_usd_per_1000": 0.028268650696530326, "axes": { "intelligence": 81.37290017175654, "calibration": 58.73962722769409, "speed": 81.16411730466113, "cost": 56.460847615533304 }, "jevbench_score": 68.4117359966245 } }, { "threshold": 0.78, "public": { "n": 231, "escalation": 0.30735930735930733, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6396396396396397 }, "accuracy": 0.7705627705627706, "ece_hard": 0.19685587538985405, "calibration": 62.65909040499686, "p50_s": 0.545120233297348, "p95_s": 1.5242826543748378, "cost_usd_per_1000": 0.030420966405629322, "axes": { "intelligence": 77.96338004671338, "calibration": 62.65909040499686, "speed": 80.80442190319786, "cost": 55.50480980517159 }, "jevbench_score": 68.41638497832196 }, "heldout": { "n": 303, "escalation": 0.23432343234323433, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.958904109589041, "hard": 0.6055045871559633 }, "accuracy": 0.8217821782178217, "ece_hard": 0.25142083761588935, "calibration": 58.73962722769409, "p50_s": 0.5812294721603394, "p95_s": 1.3288795724511142, "cost_usd_per_1000": 0.028525593812190206, "axes": { "intelligence": 81.18111934983871, "calibration": 58.73962722769409, "speed": 81.1216674673135, "cost": 56.342959185550185 }, "jevbench_score": 68.32673773405585 } }, { "threshold": 0.79, "public": { "n": 231, "escalation": 0.30735930735930733, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6396396396396397 }, "accuracy": 0.7705627705627706, "ece_hard": 0.19685587538985405, "calibration": 62.65909040499686, "p50_s": 0.545120233297348, "p95_s": 1.5242826543748378, "cost_usd_per_1000": 0.030420966405629322, "axes": { "intelligence": 77.96338004671338, "calibration": 62.65909040499686, "speed": 80.80442190319786, "cost": 55.50480980517159 }, "jevbench_score": 68.41638497832196 }, "heldout": { "n": 303, "escalation": 0.24092409240924093, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.958904109589041, "hard": 0.6146788990825688 }, "accuracy": 0.8250825082508251, "ece_hard": 0.24491031114997508, "calibration": 59.390679874285524, "p50_s": 0.5812294721603394, "p95_s": 1.3288795724511142, "cost_usd_per_1000": 0.028696889222630126, "axes": { "intelligence": 81.45634870763688, "calibration": 59.390679874285524, "speed": 81.1216674673135, "cost": 56.264955362790175 }, "jevbench_score": 68.54953622127334 } }, { "threshold": 0.8, "public": { "n": 231, "escalation": 0.3203463203463203, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6306306306306306 }, "accuracy": 0.7662337662337663, "ece_hard": 0.20644200154912623, "calibration": 61.70047778906964, "p50_s": 0.5452094167470932, "p95_s": 1.5242826543748378, "cost_usd_per_1000": 0.030757995687209168, "axes": { "intelligence": 77.58800467133801, "calibration": 61.70047778906964, "speed": 80.80371144136302, "cost": 55.36125904890021 }, "jevbench_score": 68.02680820252058 }, "heldout": { "n": 303, "escalation": 0.26732673267326734, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9452054794520548, "hard": 0.6146788990825688 }, "accuracy": 0.8184818481848185, "ece_hard": 0.2502314561642916, "calibration": 58.85856537285387, "p50_s": 0.5844195425510407, "p95_s": 1.3489860884845255, "cost_usd_per_1000": 0.02938207086438981, "axes": { "intelligence": 81.07278706380127, "calibration": 58.85856537285387, "speed": 81.03267799011175, "cost": 55.9575279446479 }, "jevbench_score": 68.20259071433898 } }, { "threshold": 0.81, "public": { "n": 231, "escalation": 0.3246753246753247, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6306306306306306 }, "accuracy": 0.7662337662337663, "ece_hard": 0.20801334467869964, "calibration": 61.5433434761123, "p50_s": 0.5452094167470932, "p95_s": 1.5242826543748378, "cost_usd_per_1000": 0.030870338781069116, "axes": { "intelligence": 77.58800467133801, "calibration": 61.5433434761123, "speed": 80.80371144136302, "cost": 55.313758132114835 }, "jevbench_score": 67.96886789989549 }, "heldout": { "n": 303, "escalation": 0.26732673267326734, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9452054794520548, "hard": 0.6146788990825688 }, "accuracy": 0.8184818481848185, "ece_hard": 0.2502314561642916, "calibration": 58.85856537285387, "p50_s": 0.5844195425510407, "p95_s": 1.3489860884845255, "cost_usd_per_1000": 0.02938207086438981, "axes": { "intelligence": 81.07278706380127, "calibration": 58.85856537285387, "speed": 81.03267799011175, "cost": 55.9575279446479 }, "jevbench_score": 68.20259071433898 } }, { "threshold": 0.82, "public": { "n": 231, "escalation": 0.35064935064935066, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6306306306306306 }, "accuracy": 0.7662337662337663, "ece_hard": 0.21730373588920177, "calibration": 59.71280635591178, "p50_s": 0.5457840502262116, "p95_s": 1.5451002843677997, "cost_usd_per_1000": 0.0315443973442288, "axes": { "intelligence": 77.58800467133801, "calibration": 59.71280635591178, "speed": 80.74022487672036, "cost": 55.032332961269255 }, "jevbench_score": 67.3585122173941 }, "heldout": { "n": 303, "escalation": 0.29042904290429045, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9383561643835616, "hard": 0.6146788990825688 }, "accuracy": 0.8184818481848185, "ece_hard": 0.25271734990017763, "calibration": 58.16651051488086, "p50_s": 0.5914658546447754, "p95_s": 1.3489860884845255, "cost_usd_per_1000": 0.02998160480092953, "axes": { "intelligence": 82.04767290855013, "calibration": 58.16651051488086, "speed": 80.98062851153796, "cost": 55.694353742161994 }, "jevbench_score": 68.11345304775448 } }, { "threshold": 0.83, "public": { "n": 231, "escalation": 0.354978354978355, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6306306306306306 }, "accuracy": 0.7662337662337663, "ece_hard": 0.21855529972703935, "calibration": 59.58764997212802, "p50_s": 0.5457840502262116, "p95_s": 1.5451002843677997, "cost_usd_per_1000": 0.03165674043808875, "axes": { "intelligence": 77.58800467133801, "calibration": 59.58764997212802, "speed": 80.74022487672036, "cost": 54.9860141396382 }, "jevbench_score": 67.30901867247445 }, "heldout": { "n": 303, "escalation": 0.29372937293729373, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9383561643835616, "hard": 0.6146788990825688 }, "accuracy": 0.8184818481848185, "ece_hard": 0.2543304626906764, "calibration": 58.005199235830986, "p50_s": 0.5914658546447754, "p95_s": 1.3746332585811607, "cost_usd_per_1000": 0.03006725250614949, "axes": { "intelligence": 82.04767290855013, "calibration": 58.005199235830986, "speed": 80.89883474912601, "cost": 55.6571876550895 }, "jevbench_score": 68.03763023610458 } }, { "threshold": 0.84, "public": { "n": 231, "escalation": 0.36796536796536794, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6306306306306306 }, "accuracy": 0.7662337662337663, "ece_hard": 0.2229600570944021, "calibration": 59.147174235391745, "p50_s": 0.5469208598136902, "p95_s": 1.5451002843677997, "cost_usd_per_1000": 0.031993769719668594, "axes": { "intelligence": 77.58800467133801, "calibration": 59.147174235391745, "speed": 80.73118839661666, "cost": 54.8480375627201 }, "jevbench_score": 67.1402193507087 }, "heldout": { "n": 303, "escalation": 0.30363036303630364, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9383561643835616, "hard": 0.6238532110091743 }, "accuracy": 0.8217821782178217, "ece_hard": 0.24612898253641055, "calibration": 58.82534725125757, "p50_s": 0.5918420493602753, "p95_s": 1.4031180121004574, "cost_usd_per_1000": 0.03032419562180937, "axes": { "intelligence": 82.3229022663483, "calibration": 58.82534725125757, "speed": 80.80699982415595, "cost": 55.54632131150971 }, "jevbench_score": 68.28060559450705 } }, { "threshold": 0.85, "public": { "n": 231, "escalation": 0.36796536796536794, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6306306306306306 }, "accuracy": 0.7662337662337663, "ece_hard": 0.2229600570944021, "calibration": 59.147174235391745, "p50_s": 0.5469208598136902, "p95_s": 1.5451002843677997, "cost_usd_per_1000": 0.031993769719668594, "axes": { "intelligence": 77.58800467133801, "calibration": 59.147174235391745, "speed": 80.73118839661666, "cost": 54.8480375627201 }, "jevbench_score": 67.1402193507087 }, "heldout": { "n": 303, "escalation": 0.3069306930693069, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9383561643835616, "hard": 0.6238532110091743 }, "accuracy": 0.8217821782178217, "ece_hard": 0.24612898253641055, "calibration": 58.82534725125757, "p50_s": 0.5918420493602753, "p95_s": 1.4031180121004574, "cost_usd_per_1000": 0.03040984332702933, "axes": { "intelligence": 82.3229022663483, "calibration": 58.82534725125757, "speed": 80.80699982415595, "cost": 55.50957452068285 }, "jevbench_score": 68.26930999574644 } }, { "threshold": 0.86, "public": { "n": 231, "escalation": 0.38095238095238093, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6396396396396397 }, "accuracy": 0.7705627705627706, "ece_hard": 0.21553581821729184, "calibration": 59.88959812310277, "p50_s": 0.5478601396083832, "p95_s": 1.5451002843677997, "cost_usd_per_1000": 0.03233079900124844, "axes": { "intelligence": 77.96338004671338, "calibration": 59.88959812310277, "speed": 80.72373623710521, "cost": 54.71150687197307 }, "jevbench_score": 67.38767882498594 }, "heldout": { "n": 303, "escalation": 0.3333333333333333, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9383561643835616, "hard": 0.6513761467889908 }, "accuracy": 0.8316831683168316, "ece_hard": 0.2194968025462811, "calibration": 61.48856525027051, "p50_s": 0.6031645923852921, "p95_s": 1.4046614453196522, "cost_usd_per_1000": 0.031095024968789013, "axes": { "intelligence": 83.1485903397428, "calibration": 61.48856525027051, "speed": 80.71992498595725, "cost": 55.21927270357169 }, "jevbench_score": 69.09238280263969 } }, { "threshold": 0.87, "public": { "n": 231, "escalation": 0.3939393939393939, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6396396396396397 }, "accuracy": 0.7705627705627706, "ece_hard": 0.21670485776467963, "calibration": 59.772694168364, "p50_s": 0.550955080986023, "p95_s": 1.5451002843677997, "cost_usd_per_1000": 0.032667828282828285, "axes": { "intelligence": 77.96338004671338, "calibration": 59.772694168364, "speed": 80.69927134999068, "cost": 54.57639207788881 }, "jevbench_score": 67.30804580078511 }, "heldout": { "n": 303, "escalation": 0.34983498349834985, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9383561643835616, "hard": 0.6513761467889908 }, "accuracy": 0.8316831683168316, "ece_hard": 0.22193091103563575, "calibration": 61.24515440133504, "p50_s": 0.6054746598005295, "p95_s": 1.4171127848327156, "cost_usd_per_1000": 0.03152326349488881, "axes": { "intelligence": 83.1485903397428, "calibration": 61.24515440133504, "speed": 80.66499611994738, "cost": 55.0410648352958 }, "jevbench_score": 68.95640985243115 } }, { "threshold": 0.88, "public": { "n": 231, "escalation": 0.39826839826839827, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6396396396396397 }, "accuracy": 0.7705627705627706, "ece_hard": 0.21771071878205228, "calibration": 59.672108066626734, "p50_s": 0.550955080986023, "p95_s": 1.551507832109928, "cost_usd_per_1000": 0.03278017137668823, "axes": { "intelligence": 77.96338004671338, "calibration": 59.672108066626734, "speed": 80.68129835183348, "cost": 54.53166340726504 }, "jevbench_score": 67.26217644710216 }, "heldout": { "n": 303, "escalation": 0.34983498349834985, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9383561643835616, "hard": 0.6513761467889908 }, "accuracy": 0.8316831683168316, "ece_hard": 0.22193091103563575, "calibration": 61.24515440133504, "p50_s": 0.6054746598005295, "p95_s": 1.4171127848327156, "cost_usd_per_1000": 0.03152326349488881, "axes": { "intelligence": 83.1485903397428, "calibration": 61.24515440133504, "speed": 80.66499611994738, "cost": 55.0410648352958 }, "jevbench_score": 68.95640985243115 } }, { "threshold": 0.89, "public": { "n": 231, "escalation": 0.4025974025974026, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6396396396396397 }, "accuracy": 0.7705627705627706, "ece_hard": 0.21871768762509677, "calibration": 59.57141118232228, "p50_s": 0.550955080986023, "p95_s": 1.551507832109928, "cost_usd_per_1000": 0.03289251447054818, "axes": { "intelligence": 77.96338004671338, "calibration": 59.57141118232228, "speed": 80.68129835183348, "cost": 54.48708776729617 }, "jevbench_score": 67.2200382983184 }, "heldout": { "n": 303, "escalation": 0.36303630363036304, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9383561643835616, "hard": 0.6513761467889908 }, "accuracy": 0.8316831683168316, "ece_hard": 0.22273413826421867, "calibration": 61.16483167847676, "p50_s": 0.6071688771247864, "p95_s": 1.4196723394095896, "cost_usd_per_1000": 0.03186585431576865, "axes": { "intelligence": 83.1485903397428, "calibration": 61.16483167847676, "speed": 80.64502377330602, "cost": 54.9002330109725 }, "jevbench_score": 68.88538807857921 } }, { "threshold": 0.9, "public": { "n": 231, "escalation": 0.4155844155844156, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6396396396396397 }, "accuracy": 0.7705627705627706, "ece_hard": 0.2205828348555867, "calibration": 59.384896459273286, "p50_s": 0.5544787734746933, "p95_s": 1.551507832109928, "cost_usd_per_1000": 0.03322954375212802, "axes": { "intelligence": 77.96338004671338, "calibration": 59.384896459273286, "speed": 80.65361102561792, "cost": 54.35426864875284 }, "jevbench_score": 67.12063151798652 }, "heldout": { "n": 303, "escalation": 0.36633663366336633, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9383561643835616, "hard": 0.6513761467889908 }, "accuracy": 0.8316831683168316, "ece_hard": 0.22368682331478, "calibration": 61.06956317342063, "p50_s": 0.6071688771247864, "p95_s": 1.4203328676521778, "cost_usd_per_1000": 0.03195150202098861, "axes": { "intelligence": 83.1485903397428, "calibration": 61.06956317342063, "speed": 80.64300360957282, "cost": 54.86526163306016 }, "jevbench_score": 68.84714945859825 } }, { "threshold": 0.91, "public": { "n": 231, "escalation": 0.4329004329004329, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6396396396396397 }, "accuracy": 0.7705627705627706, "ece_hard": 0.22308218796424184, "calibration": 59.13496114840777, "p50_s": 0.5592252552509308, "p95_s": 1.551507832109928, "cost_usd_per_1000": 0.033678916127567814, "axes": { "intelligence": 77.96338004671338, "calibration": 59.13496114840777, "speed": 80.61659251313093, "cost": 54.17925680916581 }, "jevbench_score": 66.9881700383039 }, "heldout": { "n": 303, "escalation": 0.37293729372937295, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9383561643835616, "hard": 0.6513761467889908 }, "accuracy": 0.8316831683168316, "ece_hard": 0.22537525213478515, "calibration": 60.90072029142011, "p50_s": 0.6102951854467392, "p95_s": 1.4203328676521778, "cost_usd_per_1000": 0.03212279743142853, "axes": { "intelligence": 83.1485903397428, "calibration": 60.90072029142011, "speed": 80.62069919042764, "cost": 54.79559922920026 }, "jevbench_score": 68.7729082768948 } }, { "threshold": 0.92, "public": { "n": 231, "escalation": 0.45021645021645024, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6396396396396397 }, "accuracy": 0.7705627705627706, "ece_hard": 0.22387365768606912, "calibration": 59.05581417622504, "p50_s": 0.5608406305313111, "p95_s": 1.551507832109928, "cost_usd_per_1000": 0.0341282885030076, "axes": { "intelligence": 77.96338004671338, "calibration": 59.05581417622504, "speed": 80.60406558831728, "cost": 54.00656471730474 }, "jevbench_score": 66.90971891498785 }, "heldout": { "n": 303, "escalation": 0.3795379537953795, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9383561643835616, "hard": 0.6513761467889908 }, "accuracy": 0.8316831683168316, "ece_hard": 0.22613692850137992, "calibration": 60.82455265476064, "p50_s": 0.6141898572444916, "p95_s": 1.4203328676521778, "cost_usd_per_1000": 0.03229409284186845, "axes": { "intelligence": 83.1485903397428, "calibration": 60.82455265476064, "speed": 80.59307222700205, "cost": 54.72630731532876 }, "jevbench_score": 68.72376081904207 } }, { "threshold": 0.93, "public": { "n": 231, "escalation": 0.45454545454545453, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6396396396396397 }, "accuracy": 0.7705627705627706, "ece_hard": 0.22453835912483017, "calibration": 58.98934403234894, "p50_s": 0.5718684434890747, "p95_s": 1.551507832109928, "cost_usd_per_1000": 0.03424063159686755, "axes": { "intelligence": 77.96338004671338, "calibration": 58.98934403234894, "speed": 80.51949895505149, "cost": 53.963746989965244 }, "jevbench_score": 66.86007306258796 }, "heldout": { "n": 303, "escalation": 0.39933993399339934, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9383561643835616, "hard": 0.6697247706422018 }, "accuracy": 0.8382838283828383, "ece_hard": 0.2093846592732479, "calibration": 62.49977957757383, "p50_s": 0.6410244047641754, "p95_s": 1.4203328676521778, "cost_usd_per_1000": 0.03280797907318822, "axes": { "intelligence": 83.69904905533912, "calibration": 62.49977957757383, "speed": 80.40735298831945, "cost": 54.520615622152796 }, "jevbench_score": 69.20124078597432 } }, { "threshold": 0.94, "public": { "n": 231, "escalation": 0.46320346320346323, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6396396396396397 }, "accuracy": 0.7705627705627706, "ece_hard": 0.22477681345581194, "calibration": 58.965498599250765, "p50_s": 0.577006408572197, "p95_s": 1.551507832109928, "cost_usd_per_1000": 0.034465317784587446, "axes": { "intelligence": 77.96338004671338, "calibration": 58.965498599250765, "speed": 80.4806539101599, "cost": 53.87853138325879 }, "jevbench_score": 66.81884587731786 }, "heldout": { "n": 303, "escalation": 0.42244224422442245, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9383561643835616, "hard": 0.6697247706422018 }, "accuracy": 0.8382838283828383, "ece_hard": 0.20993313420083387, "calibration": 62.449570305288816, "p50_s": 0.6549608945846558, "p95_s": 1.4203328676521778, "cost_usd_per_1000": 0.03340751300972794, "axes": { "intelligence": 83.69904905533912, "calibration": 62.449570305288816, "speed": 80.31394492343068, "cost": 54.28467561384352 }, "jevbench_score": 69.09228343209567 } }, { "threshold": 0.95, "public": { "n": 231, "escalation": 0.48917748917748916, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6396396396396397 }, "accuracy": 0.7705627705627706, "ece_hard": 0.2255657839388291, "calibration": 58.88660155094905, "p50_s": 0.7446192368865013, "p95_s": 1.551507832109928, "cost_usd_per_1000": 0.03513937634774714, "axes": { "intelligence": 77.96338004671338, "calibration": 58.88660155094905, "speed": 79.37311775907952, "cost": 53.626178518405155 }, "jevbench_score": 66.48740150150113 }, "heldout": { "n": 303, "escalation": 0.44554455445544555, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9315068493150684, "hard": 0.6697247706422018 }, "accuracy": 0.834983498349835, "ece_hard": 0.20958345032033615, "calibration": 62.48453869333858, "p50_s": 0.6615447193384171, "p95_s": 1.4203328676521778, "cost_usd_per_1000": 0.03400704694626766, "axes": { "intelligence": 83.50726823342131, "calibration": 62.48453869333858, "speed": 80.2705065579951, "cost": 54.05293237158859 }, "jevbench_score": 68.9791799419127 } }, { "threshold": 0.96, "public": { "n": 231, "escalation": 0.49783549783549785, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6396396396396397 }, "accuracy": 0.7705627705627706, "ece_hard": 0.2259927712968861, "calibration": 58.84390281514335, "p50_s": 0.7483932241797447, "p95_s": 1.551507832109928, "cost_usd_per_1000": 0.03536406253546703, "axes": { "intelligence": 77.96338004671338, "calibration": 58.84390281514335, "speed": 79.35116180908065, "cost": 53.543135503724805 }, "jevbench_score": 66.44499983604358 }, "heldout": { "n": 303, "escalation": 0.49504950495049505, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9315068493150684, "hard": 0.6697247706422018 }, "accuracy": 0.834983498349835, "ece_hard": 0.21079494361822548, "calibration": 62.36338936354965, "p50_s": 0.7645105361938477, "p95_s": 1.4203328676521778, "cost_usd_per_1000": 0.03529176252456706, "axes": { "intelligence": 83.50726823342131, "calibration": 62.36338936354965, "speed": 79.64226387839096, "cost": 53.56979955304389 }, "jevbench_score": 68.65614293096789 } }, { "threshold": 0.97, "public": { "n": 231, "escalation": 0.5151515151515151, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6486486486486487 }, "accuracy": 0.7748917748917749, "ece_hard": 0.2166973700380424, "calibration": 59.77344294102772, "p50_s": 0.748915196955204, "p95_s": 1.5689974851906299, "cost_usd_per_1000": 0.03581343491090682, "axes": { "intelligence": 78.33875542208877, "calibration": 59.77344294102772, "speed": 79.2994510914429, "cost": 53.3786206980607 }, "jevbench_score": 66.72377619875526 }, "heldout": { "n": 303, "escalation": 0.5544554455445545, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6788990825688074 }, "accuracy": 0.834983498349835, "ece_hard": 0.19774100895750696, "calibration": 63.668782829621506, "p50_s": 0.7783614560961724, "p95_s": 1.4203328676521778, "cost_usd_per_1000": 0.03683342121852635, "axes": { "intelligence": 82.6158309245528, "calibration": 63.668782829621506, "speed": 79.56428541070335, "cost": 53.01273821388997 }, "jevbench_score": 68.63127333686315 } }, { "threshold": 0.98, "public": { "n": 231, "escalation": 0.5584415584415584, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6486486486486487 }, "accuracy": 0.7748917748917749, "ece_hard": 0.21392581370049468, "calibration": 60.050598574782484, "p50_s": 0.7521987199783325, "p95_s": 1.5794555447995662, "cost_usd_per_1000": 0.0369368658495063, "axes": { "intelligence": 78.33875542208877, "calibration": 60.050598574782484, "speed": 79.2516000315145, "cost": 52.976198736833396 }, "jevbench_score": 66.66466637342864 }, "heldout": { "n": 303, "escalation": 0.6204620462046204, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6788990825688074 }, "accuracy": 0.834983498349835, "ece_hard": 0.19792727608459854, "calibration": 63.65015611691234, "p50_s": 0.7825702965259552, "p95_s": 1.4203328676521778, "cost_usd_per_1000": 0.03854637532292555, "axes": { "intelligence": 82.6158309245528, "calibration": 63.65015611691234, "speed": 79.54086503245043, "cost": 52.4204936266499 }, "jevbench_score": 68.42873965932324 } }, { "threshold": 0.99, "public": { "n": 231, "escalation": 0.6103896103896104, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6486486486486487 }, "accuracy": 0.7748917748917749, "ece_hard": 0.21463304375919556, "calibration": 59.979875568912405, "p50_s": 0.7564209192991257, "p95_s": 1.5794555447995662, "cost_usd_per_1000": 0.03828498297582567, "axes": { "intelligence": 78.33875542208877, "calibration": 59.979875568912405, "speed": 79.2272906216882, "cost": 52.50914625062874 }, "jevbench_score": 66.49255135499726 }, "heldout": { "n": 303, "escalation": 0.7194719471947195, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6788990825688074 }, "accuracy": 0.834983498349835, "ece_hard": 0.19845912161133178, "calibration": 63.59697156423902, "p50_s": 0.7912625685334206, "p95_s": 1.4203328676521778, "cost_usd_per_1000": 0.041115806479524356, "axes": { "intelligence": 82.6158309245528, "calibration": 63.59697156423902, "speed": 79.49289241431339, "cost": 51.57973560127476 }, "jevbench_score": 68.12817893528835 } }, { "threshold": 1.0, "public": { "n": 231, "escalation": 1.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6486486486486487 }, "accuracy": 0.7748917748917749, "ece_hard": 0.21041641822205612, "calibration": 61.35164946854124, "p50_s": 0.778744050860405, "p95_s": 1.5930537745356559, "cost_usd_per_1000": 0.04839586142322097, "axes": { "intelligence": 78.33875542208877, "calibration": 61.35164946854124, "speed": 79.06374822137728, "cost": 49.45575326507256 }, "jevbench_score": 65.84143426980515 }, "heldout": { "n": 303, "escalation": 1.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6788990825688074 }, "accuracy": 0.834983498349835, "ece_hard": 0.1935286337386598, "calibration": 64.09002035150621, "p50_s": 0.8086212813854218, "p95_s": 1.4217803001403808, "cost_usd_per_1000": 0.04839586142322097, "axes": { "intelligence": 82.6158309245528, "calibration": 64.09002035150621, "speed": 79.39422340591312, "cost": 49.45575326507256 }, "jevbench_score": 67.52504039892447 } } ] }, { "cheap": "openjev-verdict-1.4", "backend": "open-alternative-jev", "chosen": { "threshold": 0.68, "public": { "n": 231, "escalation": 0.7402597402597403, "tiers": { "easy": 1.0, "standard": 0.6527777777777778, "hard": 0.5225225225225225 }, "accuracy": 0.6623376623376623, "ece_hard": 0.2534466232636447, "calibration": 55.915249867556724, "p50_s": 1.173205989599228, "p95_s": 2.807708340883255, "cost_usd_per_1000": 0.020285519480519484, "axes": { "intelligence": 66.60201868535202, "calibration": 55.915249867556724, "speed": 74.82273736310684, "cost": 60.78441598638495 }, "jevbench_score": 64.15209977518022 }, "heldout": { "n": 303, "escalation": 0.900990099009901, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5504587155963303 }, "accuracy": 0.5775577557755776, "ece_hard": 0.2617424941288704, "calibration": 61.03872240012119, "p50_s": 1.7833678156137465, "p95_s": 2.8249304682016367, "cost_usd_per_1000": 0.023849137278896433, "axes": { "intelligence": 64.53887562314105, "calibration": 61.03872240012119, "speed": 72.97751312600565, "cost": 58.67581979656682 }, "jevbench_score": 64.08682613379641 } }, "full": { "tiers": { "easy": 1.0, "standard": 0.6666666666666666, "judge": 0.5068493150684932, "hard": 0.5363636363636364 }, "accuracy": 0.6142322097378277, "ece_hard": 0.24883080877070302, "calibration": 59.353361126394404, "p50_s": 1.5713767513632773, "p95_s": 2.8257861554622647, "cost_usd_per_1000": 0.02230757227622775, "axes": { "intelligence": 62.94935657949357, "calibration": 59.353361126394404, "speed": 73.52580383529104, "cost": 59.546430736945915 }, "jevbench_score": 63.596473217251365 }, "full_escalation": 0.8314606741573034, "backend_heldout_accuracy": 0.5643564356435643, "useful": false, "curve": [ { "threshold": 0.0, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.01, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.02, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.03, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.04, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.05, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.06, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.07, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.08, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.09, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.1, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.11, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.12, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.13, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.14, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.15, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.16, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.17, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.18, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.19, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.2, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.21, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.22, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.23, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.24, "public": { "n": 231, "escalation": 0.004329004329004329, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12720247334713877, "calibration": 73.67128823420694, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003968901454350892, "axes": { "intelligence": 53.09371871871872, "calibration": 73.67128823420694, "speed": 79.65074585196557, "cost": 82.03989052759192 }, "jevbench_score": 71.1031894186663 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.25, "public": { "n": 231, "escalation": 0.008658008658008658, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.130421498111999, "calibration": 73.34938575772091, "p50_s": 0.6819422334432602, "p95_s": 1.6338174432516097, "cost_usd_per_1000": 0.004064881560387179, "axes": { "intelligence": 53.09371871871872, "calibration": 73.34938575772091, "speed": 79.53048883940582, "cost": 81.72856312143065 }, "jevbench_score": 70.93111476949332 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.26, "public": { "n": 231, "escalation": 0.008658008658008658, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.130421498111999, "calibration": 73.34938575772091, "p50_s": 0.6819422334432602, "p95_s": 1.6338174432516097, "cost_usd_per_1000": 0.004064881560387179, "axes": { "intelligence": 53.09371871871872, "calibration": 73.34938575772091, "speed": 79.53048883940582, "cost": 81.72856312143065 }, "jevbench_score": 70.93111476949332 }, "heldout": { "n": 303, "escalation": 0.0033003300330033004, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5684931506849316, "hard": 0.3577981651376147 }, "accuracy": 0.504950495049505, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9941574066877361, "cost_usd_per_1000": 0.00394609430044128, "axes": { "intelligence": 51.151753173306524, "calibration": 70.2289302105776, "speed": 76.12358067087503, "cost": 82.11497621667138 }, "jevbench_score": 68.83820526594195 } }, { "threshold": 0.27, "public": { "n": 231, "escalation": 0.008658008658008658, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.130421498111999, "calibration": 73.34938575772091, "p50_s": 0.6819422334432602, "p95_s": 1.6338174432516097, "cost_usd_per_1000": 0.004064881560387179, "axes": { "intelligence": 53.09371871871872, "calibration": 73.34938575772091, "speed": 79.53048883940582, "cost": 81.72856312143065 }, "jevbench_score": 70.93111476949332 }, "heldout": { "n": 303, "escalation": 0.006600660066006601, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5684931506849316, "hard": 0.3669724770642202 }, "accuracy": 0.5082508250825083, "ece_hard": 0.14532480903696277, "calibration": 70.47853653835716, "p50_s": 1.2242849111557006, "p95_s": 1.9994657054543494, "cost_usd_per_1000": 0.0040192672525679535, "axes": { "intelligence": 51.42698253110469, "calibration": 70.47853653835716, "speed": 76.11203543410345, "cost": 81.87559346099192 }, "jevbench_score": 68.93883332247182 } }, { "threshold": 0.28, "public": { "n": 231, "escalation": 0.008658008658008658, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.130421498111999, "calibration": 73.34938575772091, "p50_s": 0.6819422334432602, "p95_s": 1.6338174432516097, "cost_usd_per_1000": 0.004064881560387179, "axes": { "intelligence": 53.09371871871872, "calibration": 73.34938575772091, "speed": 79.53048883940582, "cost": 81.72856312143065 }, "jevbench_score": 70.93111476949332 }, "heldout": { "n": 303, "escalation": 0.013201320132013201, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5684931506849316, "hard": 0.3853211009174312 }, "accuracy": 0.5148514851485149, "ece_hard": 0.14016487796223248, "calibration": 70.9945296458302, "p50_s": 1.234055870771408, "p95_s": 2.002006083726883, "cost_usd_per_1000": 0.0041656131568213, "axes": { "intelligence": 51.97744124670103, "calibration": 70.9945296458302, "speed": 76.07199784678924, "cost": 81.40963190917007 }, "jevbench_score": 69.14091122271694 } }, { "threshold": 0.29, "public": { "n": 231, "escalation": 0.017316017316017316, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36936936936936937 }, "accuracy": 0.5281385281385281, "ece_hard": 0.13770553662822418, "calibration": 72.6209819060984, "p50_s": 0.6819422334432602, "p95_s": 1.6590973153710364, "cost_usd_per_1000": 0.00425684177245975, "axes": { "intelligence": 53.46909409409409, "calibration": 72.6209819060984, "speed": 79.46380551799933, "cost": 81.12737476970153 }, "jevbench_score": 70.73354610923833 }, "heldout": { "n": 303, "escalation": 0.019801980198019802, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5684931506849316, "hard": 0.4036697247706422 }, "accuracy": 0.5214521452145214, "ece_hard": 0.12668252967071672, "calibration": 74.23174557516751, "p50_s": 1.234055870771408, "p95_s": 2.06843184530735, "cost_usd_per_1000": 0.004311959061074647, "axes": { "intelligence": 52.527899962297354, "calibration": 74.23174557516751, "speed": 75.93023962072186, "cost": 80.9597611333903 }, "jevbench_score": 69.97065268350649 } }, { "threshold": 0.3, "public": { "n": 231, "escalation": 0.030303030303030304, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36936936936936937 }, "accuracy": 0.5281385281385281, "ece_hard": 0.14119620592829235, "calibration": 72.27191497609158, "p50_s": 0.6859634757041931, "p95_s": 1.669744887948036, "cost_usd_per_1000": 0.004544782090568607, "axes": { "intelligence": 53.46909409409409, "calibration": 72.27191497609158, "speed": 79.41048885282348, "cost": 80.27460805400204 }, "jevbench_score": 70.4501809503078 }, "heldout": { "n": 303, "escalation": 0.033003300330033, "tiers": { "easy": 0.875, "standard": 0.4583333333333333, "judge": 0.5821917808219178, "hard": 0.41284403669724773 }, "accuracy": 0.5346534653465347, "ece_hard": 0.12390774543586146, "calibration": 74.50922399865303, "p50_s": 1.2708158284425735, "p95_s": 2.0718681916594504, "cost_usd_per_1000": 0.00460465086958134, "axes": { "intelligence": 53.77002429726447, "calibration": 74.50922399865303, "speed": 75.79555261568528, "cost": 80.10409878917591 }, "jevbench_score": 70.22830455892748 } }, { "threshold": 0.31, "public": { "n": 231, "escalation": 0.030303030303030304, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36936936936936937 }, "accuracy": 0.5281385281385281, "ece_hard": 0.14119620592829235, "calibration": 72.27191497609158, "p50_s": 0.6859634757041931, "p95_s": 1.669744887948036, "cost_usd_per_1000": 0.004544782090568607, "axes": { "intelligence": 53.46909409409409, "calibration": 72.27191497609158, "speed": 79.41048885282348, "cost": 80.27460805400204 }, "jevbench_score": 70.4501809503078 }, "heldout": { "n": 303, "escalation": 0.0462046204620462, "tiers": { "easy": 0.875, "standard": 0.4583333333333333, "judge": 0.5958904109589042, "hard": 0.42201834862385323 }, "accuracy": 0.5445544554455446, "ece_hard": 0.11504014992459426, "calibration": 75.39598354977976, "p50_s": 1.2708158284425735, "p95_s": 2.129326386749744, "cost_usd_per_1000": 0.0048973426780880335, "axes": { "intelligence": 54.42881529889825, "calibration": 75.39598354977976, "speed": 75.67675148592534, "cost": 79.30118519068992 }, "jevbench_score": 70.44575255547522 } }, { "threshold": 0.32, "public": { "n": 231, "escalation": 0.05194805194805195, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36936936936936937 }, "accuracy": 0.5281385281385281, "ece_hard": 0.1573498739641968, "calibration": 70.65654817250112, "p50_s": 0.6935107052326203, "p95_s": 1.683034674823284, "cost_usd_per_1000": 0.005024682620750037, "axes": { "intelligence": 53.46909409409409, "calibration": 70.65654817250112, "speed": 79.32853766997002, "cost": 78.96674094504498 }, "jevbench_score": 69.74807510036959 }, "heldout": { "n": 303, "escalation": 0.0627062706270627, "tiers": { "easy": 0.875, "standard": 0.4583333333333333, "judge": 0.6095890410958904, "hard": 0.44036697247706424 }, "accuracy": 0.5577557755775577, "ece_hard": 0.10565512872306354, "calibration": 76.33448566993283, "p50_s": 1.2723965734243392, "p95_s": 2.1683346524834626, "cost_usd_per_1000": 0.0052632074387214, "axes": { "intelligence": 55.3628356583302, "calibration": 76.33448566993283, "speed": 75.59251198732296, "cost": 78.36248538409122 }, "jevbench_score": 70.73453817211652 } }, { "threshold": 0.33, "public": { "n": 231, "escalation": 0.05627705627705628, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36936936936936937 }, "accuracy": 0.5281385281385281, "ece_hard": 0.16143069222151973, "calibration": 70.24846634676884, "p50_s": 0.6935107052326203, "p95_s": 1.6896898761391639, "cost_usd_per_1000": 0.005120662726786323, "axes": { "intelligence": 53.46909409409409, "calibration": 70.24846634676884, "speed": 79.3113982865718, "cost": 78.72021484283172 }, "jevbench_score": 69.58896666943401 }, "heldout": { "n": 303, "escalation": 0.10561056105610561, "tiers": { "easy": 0.875, "standard": 0.4583333333333333, "judge": 0.6506849315068494, "hard": 0.44954128440366975 }, "accuracy": 0.5808580858085809, "ece_hard": 0.11186370431941225, "calibration": 75.71362811029795, "p50_s": 1.3009985327720641, "p95_s": 2.342191968262195, "cost_usd_per_1000": 0.006214455816368154, "axes": { "intelligence": 56.78874994763521, "calibration": 75.71362811029795, "speed": 75.16100705842331, "cost": 76.1979068616555 }, "jevbench_score": 70.44383637319724 } }, { "threshold": 0.34, "public": { "n": 231, "escalation": 0.06926406926406926, "tiers": { "easy": 0.875, "standard": 0.5555555555555556, "hard": 0.3783783783783784 }, "accuracy": 0.5367965367965368, "ece_hard": 0.1592350957596271, "calibration": 70.4680259929581, "p50_s": 0.6951739430427551, "p95_s": 1.7372671008110045, "cost_usd_per_1000": 0.00540860304489518, "axes": { "intelligence": 54.3845929262596, "calibration": 70.4680259929581, "speed": 79.18039919403824, "cost": 78.0074467505138 }, "jevbench_score": 69.7518024358571 }, "heldout": { "n": 303, "escalation": 0.1353135313531353, "tiers": { "easy": 0.875, "standard": 0.4583333333333333, "judge": 0.6712328767123288, "hard": 0.46788990825688076 }, "accuracy": 0.5973597359735974, "ece_hard": 0.10872652204037109, "calibration": 76.02734633820208, "p50_s": 1.340339982509613, "p95_s": 2.394378633797168, "cost_usd_per_1000": 0.006873012385508213, "axes": { "intelligence": 57.91455112898496, "calibration": 76.02734633820208, "speed": 74.9359219872866, "cost": 74.88558620410303 }, "jevbench_score": 70.50362609671934 } }, { "threshold": 0.35, "public": { "n": 231, "escalation": 0.08225108225108226, "tiers": { "easy": 0.875, "standard": 0.5555555555555556, "hard": 0.38738738738738737 }, "accuracy": 0.5411255411255411, "ece_hard": 0.15904712245225536, "calibration": 70.48682332369526, "p50_s": 0.6951739430427551, "p95_s": 1.897957244515419, "cost_usd_per_1000": 0.0056965433630040375, "axes": { "intelligence": 54.75996830163497, "calibration": 70.48682332369526, "speed": 78.79620089956148, "cost": 77.33165777056956 }, "jevbench_score": 69.63994691467784 }, "heldout": { "n": 303, "escalation": 0.16501650165016502, "tiers": { "easy": 0.875, "standard": 0.4583333333333333, "judge": 0.7191780821917808, "hard": 0.46788990825688076 }, "accuracy": 0.6204620462046204, "ece_hard": 0.11148409914845227, "calibration": 75.3763940563288, "p50_s": 1.340339982509613, "p95_s": 2.500080558657644, "cost_usd_per_1000": 0.007531568954648274, "axes": { "intelligence": 59.257016882409616, "calibration": 75.3763940563288, "speed": 74.74831024238847, "cost": 73.6934363021513 }, "jevbench_score": 70.42896923469269 } }, { "threshold": 0.36, "public": { "n": 231, "escalation": 0.1038961038961039, "tiers": { "easy": 0.875, "standard": 0.5694444444444444, "hard": 0.3963963963963964 }, "accuracy": 0.5497835497835498, "ece_hard": 0.16501942850177967, "calibration": 69.88959271874285, "p50_s": 0.7169283866882324, "p95_s": 2.0118878990411755, "cost_usd_per_1000": 0.006176443893185467, "axes": { "intelligence": 55.675467133800474, "calibration": 69.88959271874285, "speed": 78.40920444959538, "cost": 76.27784498071982 }, "jevbench_score": 69.45611064903125 }, "heldout": { "n": 303, "escalation": 0.1782178217821782, "tiers": { "easy": 0.875, "standard": 0.4583333333333333, "judge": 0.7328767123287672, "hard": 0.47706422018348627 }, "accuracy": 0.6303630363036303, "ece_hard": 0.11269726019529266, "calibration": 75.25507795164475, "p50_s": 1.3478259533643722, "p95_s": 2.500080558657644, "cost_usd_per_1000": 0.007824260763154968, "axes": { "intelligence": 59.9158078840434, "calibration": 75.25507795164475, "speed": 74.72412182310421, "cost": 73.1967005207236 }, "jevbench_score": 70.47050578339177 } }, { "threshold": 0.37, "public": { "n": 231, "escalation": 0.12554112554112554, "tiers": { "easy": 0.875, "standard": 0.5694444444444444, "hard": 0.40540540540540543 }, "accuracy": 0.5541125541125541, "ece_hard": 0.16676096922754569, "calibration": 69.54366617534258, "p50_s": 0.7518308401107788, "p95_s": 2.0313757389783857, "cost_usd_per_1000": 0.006656344423366895, "axes": { "intelligence": 56.050842509175844, "calibration": 69.54366617534258, "speed": 78.16089602187992, "cost": 75.30292642499799 }, "jevbench_score": 69.2086340184298 }, "heldout": { "n": 303, "escalation": 0.2079207920792079, "tiers": { "easy": 0.875, "standard": 0.4583333333333333, "judge": 0.7602739726027398, "hard": 0.48623853211009177 }, "accuracy": 0.6468646864686468, "ece_hard": 0.10386723095944511, "calibration": 76.13808087522952, "p50_s": 1.3478259533643722, "p95_s": 2.529576647281646, "cost_usd_per_1000": 0.008482817332295028, "axes": { "intelligence": 60.9581605295128, "calibration": 76.13808087522952, "speed": 74.67318341971983, "cost": 72.14379654787928 }, "jevbench_score": 70.71301580613317 } }, { "threshold": 0.38, "public": { "n": 231, "escalation": 0.12987012987012986, "tiers": { "easy": 0.875, "standard": 0.5694444444444444, "hard": 0.3963963963963964 }, "accuracy": 0.5497835497835498, "ece_hard": 0.17737027894762708, "calibration": 68.48273520333444, "p50_s": 0.7518308401107788, "p95_s": 2.0672476321458815, "cost_usd_per_1000": 0.0067523245294031815, "axes": { "intelligence": 55.675467133800474, "calibration": 68.48273520333444, "speed": 78.08487360336494, "cost": 75.11640078616097 }, "jevbench_score": 68.76804252335796 }, "heldout": { "n": 303, "escalation": 0.22772277227722773, "tiers": { "easy": 0.875, "standard": 0.4583333333333333, "judge": 0.773972602739726, "hard": 0.48623853211009177 }, "accuracy": 0.6534653465346535, "ece_hard": 0.10386723095944511, "calibration": 76.13808087522952, "p50_s": 1.3478259533643722, "p95_s": 2.529576647281646, "cost_usd_per_1000": 0.008921855045055067, "axes": { "intelligence": 61.34172217334842, "calibration": 76.13808087522952, "speed": 74.67318341971983, "cost": 71.48634511287278 }, "jevbench_score": 70.66207925753855 } }, { "threshold": 0.39, "public": { "n": 231, "escalation": 0.1645021645021645, "tiers": { "easy": 0.875, "standard": 0.5555555555555556, "hard": 0.3963963963963964 }, "accuracy": 0.5454545454545454, "ece_hard": 0.19162842164754143, "calibration": 65.41932608563516, "p50_s": 0.7781706541776657, "p95_s": 2.1605932667851446, "cost_usd_per_1000": 0.007520165377693468, "axes": { "intelligence": 55.135343677010354, "calibration": 65.41932608563516, "speed": 77.74352132696994, "cost": 73.7131782589929 }, "jevbench_score": 67.42739387382998 }, "heldout": { "n": 303, "escalation": 0.24422442244224424, "tiers": { "easy": 0.875, "standard": 0.4583333333333333, "judge": 0.8013698630136986, "hard": 0.48623853211009177 }, "accuracy": 0.6666666666666666, "ece_hard": 0.10386723095944511, "calibration": 76.13808087522952, "p50_s": 1.3673696160316466, "p95_s": 2.529576647281646, "cost_usd_per_1000": 0.009287719805688436, "axes": { "intelligence": 62.108845461019655, "calibration": 76.13808087522952, "speed": 74.61066231417485, "cost": 70.9627268493729 }, "jevbench_score": 70.73700098468126 } }, { "threshold": 0.4, "public": { "n": 231, "escalation": 0.18614718614718614, "tiers": { "easy": 0.875, "standard": 0.5555555555555556, "hard": 0.4144144144144144 }, "accuracy": 0.5541125541125541, "ece_hard": 0.18737036970351245, "calibration": 64.11365639649502, "p50_s": 0.8036727398633957, "p95_s": 2.227668511867523, "cost_usd_per_1000": 0.008000065907874897, "axes": { "intelligence": 55.8860944277611, "calibration": 64.11365639649502, "speed": 77.47070196317091, "cost": 72.90719305283491 }, "jevbench_score": 67.07188989645087 }, "heldout": { "n": 303, "escalation": 0.2739273927392739, "tiers": { "easy": 0.875, "standard": 0.4583333333333333, "judge": 0.815068493150685, "hard": 0.4954128440366973 }, "accuracy": 0.6765676567656765, "ece_hard": 0.10252259766233845, "calibration": 76.27254420494017, "p50_s": 1.3726330131292344, "p95_s": 2.54213126450777, "cost_usd_per_1000": 0.009946276374828495, "axes": { "intelligence": 62.76763646265344, "calibration": 76.27254420494017, "speed": 74.57247588265422, "cost": 70.07018431894328 }, "jevbench_score": 70.72190617493011 } }, { "threshold": 0.41, "public": { "n": 231, "escalation": 0.21645021645021645, "tiers": { "easy": 0.875, "standard": 0.5555555555555556, "hard": 0.40540540540540543 }, "accuracy": 0.5497835497835498, "ece_hard": 0.20970751485776573, "calibration": 62.519671903351465, "p50_s": 0.8354266524314881, "p95_s": 2.227668511867523, "cost_usd_per_1000": 0.008671926650128899, "axes": { "intelligence": 55.510719052385724, "calibration": 62.519671903351465, "speed": 77.30241107514414, "cost": 71.85653212566623 }, "jevbench_score": 66.26179693558066 }, "heldout": { "n": 303, "escalation": 0.29042904290429045, "tiers": { "easy": 0.875, "standard": 0.5, "judge": 0.821917808219178, "hard": 0.5045871559633027 }, "accuracy": 0.6864686468646864, "ece_hard": 0.1035290062558904, "calibration": 75.7850705238042, "p50_s": 1.3770495146512984, "p95_s": 2.5632856556773183, "cost_usd_per_1000": 0.01031214113546186, "axes": { "intelligence": 64.40131330903607, "calibration": 75.7850705238042, "speed": 74.5225343632224, "cost": 69.59953455137106 }, "jevbench_score": 70.9321434246586 } }, { "threshold": 0.42, "public": { "n": 231, "escalation": 0.22943722943722944, "tiers": { "easy": 0.875, "standard": 0.5694444444444444, "hard": 0.40540540540540543 }, "accuracy": 0.5541125541125541, "ece_hard": 0.21462980077310312, "calibration": 62.02744331181773, "p50_s": 0.8431772083044052, "p95_s": 2.275622035562992, "cost_usd_per_1000": 0.008959866968237754, "axes": { "intelligence": 56.050842509175844, "calibration": 62.02744331181773, "speed": 77.16981010743837, "cost": 71.43095315451961 }, "jevbench_score": 66.16449095822873 }, "heldout": { "n": 303, "escalation": 0.3102310231023102, "tiers": { "easy": 0.875, "standard": 0.5, "judge": 0.8287671232876712, "hard": 0.5045871559633027 }, "accuracy": 0.6897689768976898, "ece_hard": 0.1158338533274832, "calibration": 74.5545858166449, "p50_s": 1.3870477318763732, "p95_s": 2.5802142208814614, "cost_usd_per_1000": 0.0107511788482219, "axes": { "intelligence": 64.59309413095387, "calibration": 74.5545858166449, "speed": 74.46252828957421, "cost": 69.05631740489244 }, "jevbench_score": 70.54242918024339 } }, { "threshold": 0.43, "public": { "n": 231, "escalation": 0.24242424242424243, "tiers": { "easy": 0.875, "standard": 0.5833333333333334, "hard": 0.40540540540540543 }, "accuracy": 0.5584415584415584, "ece_hard": 0.21581867870102356, "calibration": 61.90855551902568, "p50_s": 0.8909249544143677, "p95_s": 2.3557583793997763, "cost_usd_per_1000": 0.009247807286346613, "axes": { "intelligence": 56.59096596596597, "calibration": 61.90855551902568, "speed": 76.78028131925373, "cost": 71.01883687078261 }, "jevbench_score": 66.1119943028962 }, "heldout": { "n": 303, "escalation": 0.3234323432343234, "tiers": { "easy": 0.875, "standard": 0.5, "judge": 0.8287671232876712, "hard": 0.5045871559633027 }, "accuracy": 0.6897689768976898, "ece_hard": 0.12048317407073117, "calibration": 74.554240515775, "p50_s": 1.3870477318763732, "p95_s": 2.5802142208814614, "cost_usd_per_1000": 0.011043870656728594, "axes": { "intelligence": 64.59309413095387, "calibration": 74.554240515775, "speed": 74.46252828957421, "cost": 68.70636065080029 }, "jevbench_score": 70.45280527792428 } }, { "threshold": 0.44, "public": { "n": 231, "escalation": 0.2597402597402597, "tiers": { "easy": 0.875, "standard": 0.5833333333333334, "hard": 0.3963963963963964 }, "accuracy": 0.5541125541125541, "ece_hard": 0.23535527222641792, "calibration": 58.52717077206927, "p50_s": 0.8988664388656616, "p95_s": 2.4928061693906782, "cost_usd_per_1000": 0.009631727710491755, "axes": { "intelligence": 56.2155905905906, "calibration": 58.52717077206927, "speed": 76.4961632377962, "cost": 70.48887410329687 }, "jevbench_score": 64.89984114660112 }, "heldout": { "n": 303, "escalation": 0.33003300330033003, "tiers": { "easy": 0.875, "standard": 0.5, "judge": 0.8287671232876712, "hard": 0.5229357798165137 }, "accuracy": 0.6963696369636964, "ece_hard": 0.10872694113565112, "calibration": 75.729863809283, "p50_s": 1.3870477318763732, "p95_s": 2.585548315942287, "cost_usd_per_1000": 0.01119021656098194, "axes": { "intelligence": 65.14355284655021, "calibration": 75.729863809283, "speed": 74.45355935669087, "cost": 68.53484525849434 }, "jevbench_score": 70.83271351849626 } }, { "threshold": 0.45, "public": { "n": 231, "escalation": 0.27705627705627706, "tiers": { "easy": 0.875, "standard": 0.5833333333333334, "hard": 0.40540540540540543 }, "accuracy": 0.5584415584415584, "ece_hard": 0.22972644492335256, "calibration": 59.09005350237581, "p50_s": 0.935780817270279, "p95_s": 2.4942908465862272, "cost_usd_per_1000": 0.010015648134636899, "axes": { "intelligence": 56.59096596596597, "calibration": 59.09005350237581, "speed": 76.31878768760012, "cost": 69.9796282392744 }, "jevbench_score": 65.00790137209947 }, "heldout": { "n": 303, "escalation": 0.35313531353135313, "tiers": { "easy": 0.875, "standard": 0.5, "judge": 0.8287671232876712, "hard": 0.5229357798165137 }, "accuracy": 0.6963696369636964, "ece_hard": 0.12079049061528724, "calibration": 75.75564335729561, "p50_s": 1.4169122666120528, "p95_s": 2.585548315942287, "cost_usd_per_1000": 0.011702427225868655, "axes": { "intelligence": 65.14355284655021, "calibration": 75.75564335729561, "speed": 74.36104382065184, "cost": 67.95172152845169 }, "jevbench_score": 70.665606905663 } }, { "threshold": 0.46, "public": { "n": 231, "escalation": 0.30735930735930733, "tiers": { "easy": 0.875, "standard": 0.5833333333333334, "hard": 0.42342342342342343 }, "accuracy": 0.5670995670995671, "ece_hard": 0.22691820866114776, "calibration": 59.37087712859629, "p50_s": 0.9514630824327469, "p95_s": 2.549408945441246, "cost_usd_per_1000": 0.010687508876890899, "axes": { "intelligence": 57.34171671671673, "calibration": 59.37087712859629, "speed": 76.15168552699843, "cost": 69.13370534634643 }, "jevbench_score": 65.06589045410155 }, "heldout": { "n": 303, "escalation": 0.3795379537953795, "tiers": { "easy": 0.875, "standard": 0.5, "judge": 0.8287671232876712, "hard": 0.5137614678899083 }, "accuracy": 0.693069306930693, "ece_hard": 0.13563677159020557, "calibration": 74.27101525980376, "p50_s": 1.4302849113941192, "p95_s": 2.6178028911352147, "cost_usd_per_1000": 0.012287810842882042, "axes": { "intelligence": 64.86832348875204, "calibration": 74.27101525980376, "speed": 74.26640499858416, "cost": 67.31576448218169 }, "jevbench_score": 70.05518827381181 } }, { "threshold": 0.47, "public": { "n": 231, "escalation": 0.3203463203463203, "tiers": { "easy": 0.875, "standard": 0.5833333333333334, "hard": 0.44144144144144143 }, "accuracy": 0.5757575757575758, "ece_hard": 0.22044723893732107, "calibration": 60.017974100978954, "p50_s": 0.9591678708791733, "p95_s": 2.5662340775132177, "cost_usd_per_1000": 0.010975449194999758, "axes": { "intelligence": 58.09246746746747, "calibration": 60.017974100978954, "speed": 76.08809109412823, "cost": 68.78733088724256 }, "jevbench_score": 65.3591770825509 }, "heldout": { "n": 303, "escalation": 0.38943894389438943, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "judge": 0.8287671232876712, "hard": 0.5229357798165137 }, "accuracy": 0.6996699669966997, "ece_hard": 0.12963543589120555, "calibration": 74.87114882970377, "p50_s": 1.4302849113941192, "p95_s": 2.63591176867485, "cost_usd_per_1000": 0.012507329699262063, "axes": { "intelligence": 66.31021951321688, "calibration": 74.87114882970377, "speed": 74.23646573752958, "cost": 67.08506205371464 }, "jevbench_score": 70.51548853208877 } }, { "threshold": 0.48, "public": { "n": 231, "escalation": 0.354978354978355, "tiers": { "easy": 0.875, "standard": 0.5833333333333334, "hard": 0.4774774774774775 }, "accuracy": 0.5930735930735931, "ece_hard": 0.20044702937587944, "calibration": 62.01799505712312, "p50_s": 1.0030515521764756, "p95_s": 2.5871212884783743, "cost_usd_per_1000": 0.011743290043290045, "axes": { "intelligence": 59.59396896896897, "calibration": 62.01799505712312, "speed": 75.85859956063405, "cost": 67.90630637509219 }, "jevbench_score": 66.05546292364518 }, "heldout": { "n": 303, "escalation": 0.41254125412541254, "tiers": { "easy": 0.875, "standard": 0.5833333333333334, "judge": 0.8287671232876712, "hard": 0.5412844036697247 }, "accuracy": 0.7095709570957096, "ece_hard": 0.1263953350461071, "calibration": 75.19515891421361, "p50_s": 1.4307569205760955, "p95_s": 2.63591176867485, "cost_usd_per_1000": 0.013019540364148773, "axes": { "intelligence": 68.02734489547989, "calibration": 75.19515891421361, "speed": 74.23503275614001, "cost": 66.56213042888142 }, "jevbench_score": 70.90508469990051 } }, { "threshold": 0.49, "public": { "n": 231, "escalation": 0.36363636363636365, "tiers": { "easy": 0.875, "standard": 0.5833333333333334, "hard": 0.4864864864864865 }, "accuracy": 0.5974025974025974, "ece_hard": 0.19184857979784575, "calibration": 62.87784001492649, "p50_s": 1.0043737918138504, "p95_s": 2.60593341588974, "cost_usd_per_1000": 0.011935250255362614, "axes": { "intelligence": 59.96934434434435, "calibration": 62.87784001492649, "speed": 75.8214131344011, "cost": 67.69505411148822 }, "jevbench_score": 66.32754642826083 }, "heldout": { "n": 303, "escalation": 0.42244224422442245, "tiers": { "easy": 0.875, "standard": 0.5833333333333334, "judge": 0.8287671232876712, "hard": 0.5412844036697247 }, "accuracy": 0.7095709570957096, "ece_hard": 0.12964007886734685, "calibration": 74.87068453208964, "p50_s": 1.4307569205760955, "p95_s": 2.6401963800191877, "cost_usd_per_1000": 0.013239059220528794, "axes": { "intelligence": 68.02734489547989, "calibration": 74.87068453208964, "speed": 74.22797913471851, "cost": 66.34428625470578 }, "jevbench_score": 70.76876585590865 } }, { "threshold": 0.5, "public": { "n": 231, "escalation": 0.36363636363636365, "tiers": { "easy": 0.875, "standard": 0.5833333333333334, "hard": 0.4864864864864865 }, "accuracy": 0.5974025974025974, "ece_hard": 0.19184857979784575, "calibration": 62.87784001492649, "p50_s": 1.0043737918138504, "p95_s": 2.60593341588974, "cost_usd_per_1000": 0.011935250255362614, "axes": { "intelligence": 59.96934434434435, "calibration": 62.87784001492649, "speed": 75.8214131344011, "cost": 67.69505411148822 }, "jevbench_score": 66.32754642826083 }, "heldout": { "n": 303, "escalation": 0.43234323432343236, "tiers": { "easy": 0.875, "standard": 0.5833333333333334, "judge": 0.8287671232876712, "hard": 0.5412844036697247 }, "accuracy": 0.7095709570957096, "ece_hard": 0.13494885656671743, "calibration": 74.33980676215259, "p50_s": 1.4320855915546418, "p95_s": 2.64613357335329, "cost_usd_per_1000": 0.013458578076908815, "axes": { "intelligence": 68.02734489547989, "calibration": 74.33980676215259, "speed": 74.2141926172449, "cost": 66.13002465055175 }, "jevbench_score": 70.582599670731 } }, { "threshold": 0.51, "public": { "n": 231, "escalation": 0.38095238095238093, "tiers": { "easy": 0.875, "standard": 0.5833333333333334, "hard": 0.4864864864864865 }, "accuracy": 0.5974025974025974, "ece_hard": 0.19951887936247442, "calibration": 62.11081005846363, "p50_s": 1.0217471450567246, "p95_s": 2.6404057845473288, "cost_usd_per_1000": 0.01231917067950776, "axes": { "intelligence": 59.96934434434435, "calibration": 62.11081005846363, "speed": 75.68985891586595, "cost": 67.28255583033396 }, "jevbench_score": 65.99471649923575 }, "heldout": { "n": 303, "escalation": 0.45544554455445546, "tiers": { "easy": 0.875, "standard": 0.5833333333333334, "judge": 0.8287671232876712, "hard": 0.5504587155963303 }, "accuracy": 0.7128712871287128, "ece_hard": 0.14336308919393417, "calibration": 73.76283147880162, "p50_s": 1.4508444547653199, "p95_s": 2.64613357335329, "cost_usd_per_1000": 0.013970788741795529, "axes": { "intelligence": 68.30257425327805, "calibration": 73.76283147880162, "speed": 74.15767382817194, "cost": 65.64337223345403 }, "jevbench_score": 70.3728935328373 } }, { "threshold": 0.52, "public": { "n": 231, "escalation": 0.4025974025974026, "tiers": { "easy": 0.8958333333333334, "standard": 0.5833333333333334, "hard": 0.4864864864864865 }, "accuracy": 0.6017316017316018, "ece_hard": 0.20814395439222855, "calibration": 59.56677430115697, "p50_s": 1.0325120270252228, "p95_s": 2.6704053461551664, "cost_usd_per_1000": 0.012799071209689188, "axes": { "intelligence": 60.37443693693695, "calibration": 59.56677430115697, "speed": 75.59527692049025, "cost": 66.78464633980325 }, "jevbench_score": 65.2765386069181 }, "heldout": { "n": 303, "escalation": 0.46864686468646866, "tiers": { "easy": 0.875, "standard": 0.5833333333333334, "judge": 0.8287671232876712, "hard": 0.5688073394495413 }, "accuracy": 0.7194719471947195, "ece_hard": 0.12969886863063324, "calibration": 75.66867624037617, "p50_s": 1.458793941140175, "p95_s": 2.6478858068585396, "cost_usd_per_1000": 0.014263480550302222, "axes": { "intelligence": 68.85303296887437, "calibration": 75.66867624037617, "speed": 74.13106797405845, "cost": 65.37323457250176 }, "jevbench_score": 70.88590047906696 } }, { "threshold": 0.53, "public": { "n": 231, "escalation": 0.42424242424242425, "tiers": { "easy": 0.8958333333333334, "standard": 0.5833333333333334, "hard": 0.5045045045045045 }, "accuracy": 0.6103896103896104, "ece_hard": 0.20264788849886622, "calibration": 60.116380890493204, "p50_s": 1.0397221624851227, "p95_s": 2.6704053461551664, "cost_usd_per_1000": 0.013278971739870617, "axes": { "intelligence": 61.1251876876877, "calibration": 60.116380890493204, "speed": 75.56505509811007, "cost": 66.30506660067348 }, "jevbench_score": 65.50435701686649 }, "heldout": { "n": 303, "escalation": 0.4884488448844885, "tiers": { "easy": 0.875, "standard": 0.5833333333333334, "judge": 0.8287671232876712, "hard": 0.5688073394495413 }, "accuracy": 0.7194719471947195, "ece_hard": 0.15358418761312218, "calibration": 73.28014434212727, "p50_s": 1.4600932538509368, "p95_s": 2.6478858068585396, "cost_usd_per_1000": 0.014702518263062264, "axes": { "intelligence": 68.85303296887437, "calibration": 73.28014434212727, "speed": 74.12720153907742, "cost": 64.97824817370925 }, "jevbench_score": 70.21238875448898 } }, { "threshold": 0.54, "public": { "n": 231, "escalation": 0.4458874458874459, "tiers": { "easy": 0.8958333333333334, "standard": 0.5833333333333334, "hard": 0.4954954954954955 }, "accuracy": 0.6060606060606061, "ece_hard": 0.22159182871986277, "calibration": 58.22198686839354, "p50_s": 1.0450961768627167, "p95_s": 2.6869689926505087, "cost_usd_per_1000": 0.013758872270052045, "axes": { "intelligence": 60.74981231231232, "calibration": 58.22198686839354, "speed": 75.51581086232105, "cost": 65.8425148325018 }, "jevbench_score": 64.758095170788 }, "heldout": { "n": 303, "escalation": 0.5082508250825083, "tiers": { "easy": 0.875, "standard": 0.5833333333333334, "judge": 0.8287671232876712, "hard": 0.5963302752293578 }, "accuracy": 0.7293729372937293, "ece_hard": 0.16623164586083694, "calibration": 72.0153985173558, "p50_s": 1.4737490206956863, "p95_s": 2.6478858068585396, "cost_usd_per_1000": 0.015141555975822302, "axes": { "intelligence": 69.67872104226886, "calibration": 72.0153985173558, "speed": 74.08677218765774, "cost": 64.59488480784265 }, "jevbench_score": 70.00290860215236 } }, { "threshold": 0.55, "public": { "n": 231, "escalation": 0.4675324675324675, "tiers": { "easy": 0.8958333333333334, "standard": 0.5694444444444444, "hard": 0.4954954954954955 }, "accuracy": 0.6017316017316018, "ece_hard": 0.2256330040932608, "calibration": 57.81786933105374, "p50_s": 1.049175101518631, "p95_s": 2.6869689926505087, "cost_usd_per_1000": 0.014238772800233473, "axes": { "intelligence": 60.209688855522195, "calibration": 57.81786933105374, "speed": 75.49889369663786, "cost": 65.39582319117198 }, "jevbench_score": 64.38797492532701 }, "heldout": { "n": 303, "escalation": 0.528052805280528, "tiers": { "easy": 0.875, "standard": 0.6666666666666666, "judge": 0.821917808219178, "hard": 0.5688073394495413 }, "accuracy": 0.7227722772277227, "ece_hard": 0.1744814465616235, "calibration": 70.37277530838168, "p50_s": 1.4903221815824508, "p95_s": 2.6831919223070133, "cost_usd_per_1000": 0.01558059368858234, "axes": { "intelligence": 70.9945854802899, "calibration": 70.37277530838168, "speed": 73.98068096739362, "cost": 64.22247993502003 }, "jevbench_score": 69.80054694602734 } }, { "threshold": 0.56, "public": { "n": 231, "escalation": 0.48484848484848486, "tiers": { "easy": 0.8958333333333334, "standard": 0.5694444444444444, "hard": 0.4864864864864865 }, "accuracy": 0.5974025974025974, "ece_hard": 0.2422810959992791, "calibration": 56.15306014045191, "p50_s": 1.0502359300851822, "p95_s": 2.726760859787464, "cost_usd_per_1000": 0.014622693224378619, "axes": { "intelligence": 59.834313480146825, "calibration": 56.15306014045191, "speed": 75.43066076492069, "cost": 65.04917893468487 }, "jevbench_score": 63.72038161931311 }, "heldout": { "n": 303, "escalation": 0.5445544554455446, "tiers": { "easy": 0.875, "standard": 0.6666666666666666, "judge": 0.821917808219178, "hard": 0.5871559633027523 }, "accuracy": 0.7293729372937293, "ece_hard": 0.19086463650249552, "calibration": 68.73445631429446, "p50_s": 1.4966442912817002, "p95_s": 2.6831919223070133, "cost_usd_per_1000": 0.01594645844921571, "axes": { "intelligence": 71.54504419588623, "calibration": 68.73445631429446, "speed": 73.9622966870117, "cost": 63.92007263227188 }, "jevbench_score": 69.43851532941218 } }, { "threshold": 0.57, "public": { "n": 231, "escalation": 0.5108225108225108, "tiers": { "easy": 0.8958333333333334, "standard": 0.5416666666666666, "hard": 0.4954954954954955 }, "accuracy": 0.5930735930735931, "ece_hard": 0.2412087101508127, "calibration": 56.260298725298554, "p50_s": 1.0590612471103669, "p95_s": 2.726760859787464, "cost_usd_per_1000": 0.015198573860596334, "axes": { "intelligence": 59.12944194194195, "calibration": 56.260298725298554, "speed": 75.39431871798439, "cost": 64.54591484888941 }, "jevbench_score": 63.43125425256365 }, "heldout": { "n": 303, "escalation": 0.5742574257425742, "tiers": { "easy": 0.875, "standard": 0.6666666666666666, "judge": 0.8082191780821918, "hard": 0.5779816513761468 }, "accuracy": 0.7194719471947195, "ece_hard": 0.19090115376545874, "calibration": 68.73080458799814, "p50_s": 1.5044434458017348, "p95_s": 2.747519150376318, "cost_usd_per_1000": 0.01660501501835577, "axes": { "intelligence": 70.88625319425245, "calibration": 68.73080458799814, "speed": 73.83683405207657, "cost": 63.392821818072505 }, "jevbench_score": 69.10455109547833 } }, { "threshold": 0.58, "public": { "n": 231, "escalation": 0.5238095238095238, "tiers": { "easy": 0.8958333333333334, "standard": 0.5416666666666666, "hard": 0.4954954954954955 }, "accuracy": 0.5930735930735931, "ece_hard": 0.2432864638426187, "calibration": 56.421713483876985, "p50_s": 1.0590612471103669, "p95_s": 2.7566434919834135, "cost_usd_per_1000": 0.015486514178705193, "axes": { "intelligence": 59.12944194194195, "calibration": 56.421713483876985, "speed": 75.34698319460404, "cost": 64.30138976544994 }, "jevbench_score": 63.40654216004295 }, "heldout": { "n": 303, "escalation": 0.5841584158415841, "tiers": { "easy": 0.875, "standard": 0.6666666666666666, "judge": 0.8082191780821918, "hard": 0.5779816513761468 }, "accuracy": 0.7194719471947195, "ece_hard": 0.20517003938139278, "calibration": 67.30391602640474, "p50_s": 1.5044434458017348, "p95_s": 2.753682453930378, "cost_usd_per_1000": 0.016824533874735786, "axes": { "intelligence": 70.88625319425245, "calibration": 67.30391602640474, "speed": 73.82710276049238, "cost": 63.22170877318081 }, "jevbench_score": 68.6943633278022 } }, { "threshold": 0.59, "public": { "n": 231, "escalation": 0.5584415584415584, "tiers": { "easy": 0.9166666666666666, "standard": 0.5277777777777778, "hard": 0.4864864864864865 }, "accuracy": 0.5887445887445888, "ece_hard": 0.2619120411828723, "calibration": 53.81603100098088, "p50_s": 1.066072928905487, "p95_s": 2.765017020702362, "cost_usd_per_1000": 0.016254355026995477, "axes": { "intelligence": 58.61903570236903, "calibration": 53.81603100098088, "speed": 75.30515275624393, "cost": 63.670907759088145 }, "jevbench_score": 62.36328986566223 }, "heldout": { "n": 303, "escalation": 0.6171617161716172, "tiers": { "easy": 0.875, "standard": 0.6666666666666666, "judge": 0.8013698630136986, "hard": 0.5688073394495413 }, "accuracy": 0.7128712871287128, "ece_hard": 0.22903841735440306, "calibration": 64.91707822910371, "p50_s": 1.5274034827947616, "p95_s": 2.810947927832602, "cost_usd_per_1000": 0.01755626339600252, "axes": { "intelligence": 70.41924301453648, "calibration": 64.91707822910371, "speed": 73.67193422975339, "cost": 62.667037362200276 }, "jevbench_score": 67.7794262526339 } }, { "threshold": 0.6, "public": { "n": 231, "escalation": 0.5670995670995671, "tiers": { "easy": 0.9166666666666666, "standard": 0.5416666666666666, "hard": 0.4954954954954955 }, "accuracy": 0.5974025974025974, "ece_hard": 0.2553828855984491, "calibration": 54.4689465594232, "p50_s": 1.066072928905487, "p95_s": 2.765017020702362, "cost_usd_per_1000": 0.01644631523906805, "axes": { "intelligence": 59.53453453453454, "calibration": 54.4689465594232, "speed": 75.30515275624393, "cost": 63.51794168630496 }, "jevbench_score": 62.756651385732205 }, "heldout": { "n": 303, "escalation": 0.6303630363036303, "tiers": { "easy": 0.875, "standard": 0.6666666666666666, "judge": 0.8013698630136986, "hard": 0.5504587155963303 }, "accuracy": 0.7062706270627063, "ece_hard": 0.2382177722257695, "calibration": 63.99841936793911, "p50_s": 1.5508419513702392, "p95_s": 2.810947927832602, "cost_usd_per_1000": 0.017848955204509215, "axes": { "intelligence": 69.86878429894014, "calibration": 63.99841936793911, "speed": 73.60579659057406, "cost": 62.45161601209362 }, "jevbench_score": 67.33285556256669 } }, { "threshold": 0.61, "public": { "n": 231, "escalation": 0.5887445887445888, "tiers": { "easy": 0.9166666666666666, "standard": 0.5416666666666666, "hard": 0.4954954954954955 }, "accuracy": 0.5974025974025974, "ece_hard": 0.2661743613601947, "calibration": 53.38979898324864, "p50_s": 1.066072928905487, "p95_s": 2.765017020702362, "cost_usd_per_1000": 0.016926215769249478, "axes": { "intelligence": 59.53453453453454, "calibration": 53.38979898324864, "speed": 75.30515275624393, "cost": 63.14320381592269 }, "jevbench_score": 62.35117454363574 }, "heldout": { "n": 303, "escalation": 0.66996699669967, "tiers": { "easy": 0.875, "standard": 0.6666666666666666, "judge": 0.7397260273972602, "hard": 0.5412844036697247 }, "accuracy": 0.6732673267326733, "ece_hard": 0.24752994989666366, "calibration": 63.067201600849685, "p50_s": 1.5818203419446943, "p95_s": 2.8249304682016367, "cost_usd_per_1000": 0.018727030630029295, "axes": { "intelligence": 67.8675275438817, "calibration": 63.067201600849685, "speed": 73.49835081082148, "cost": 61.82593237531415 }, "jevbench_score": 66.40923301370064 } }, { "threshold": 0.62, "public": { "n": 231, "escalation": 0.6017316017316018, "tiers": { "easy": 0.9375, "standard": 0.5416666666666666, "hard": 0.5045045045045045 }, "accuracy": 0.6060606060606061, "ece_hard": 0.2575898061684545, "calibration": 54.24825450242266, "p50_s": 1.066072928905487, "p95_s": 2.765017020702362, "cost_usd_per_1000": 0.017214156087358337, "axes": { "intelligence": 60.315002502502516, "calibration": 54.24825450242266, "speed": 75.30515275624393, "cost": 62.923427903133934 }, "jevbench_score": 62.74975675799463 }, "heldout": { "n": 303, "escalation": 0.7227722772277227, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6438356164383562, "hard": 0.5412844036697247 }, "accuracy": 0.6303630363036303, "ece_hard": 0.24752994989666366, "calibration": 63.067201600849685, "p50_s": 1.672505873441696, "p95_s": 2.8249304682016367, "cost_usd_per_1000": 0.019897797864056068, "axes": { "intelligence": 65.76592937036571, "calibration": 63.067201600849685, "speed": 73.25624585696285, "cost": 61.03584955958175 }, "jevbench_score": 65.62337220839939 } }, { "threshold": 0.63, "public": { "n": 231, "escalation": 0.6190476190476191, "tiers": { "easy": 0.9375, "standard": 0.5694444444444444, "hard": 0.5045045045045045 }, "accuracy": 0.6147186147186147, "ece_hard": 0.26019927502769774, "calibration": 55.23998469115142, "p50_s": 1.0779939621686936, "p95_s": 2.7812519550323485, "cost_usd_per_1000": 0.01759807651150348, "axes": { "intelligence": 61.39524941608276, "calibration": 55.23998469115142, "speed": 75.23143338015859, "cost": 62.63604395323109 }, "jevbench_score": 63.227060810943485 }, "heldout": { "n": 303, "escalation": 0.768976897689769, "tiers": { "easy": 0.9583333333333334, "standard": 0.6666666666666666, "judge": 0.5821917808219178, "hard": 0.5321100917431193 }, "accuracy": 0.6006600660066007, "ece_hard": 0.2634221139940554, "calibration": 61.47798519111051, "p50_s": 1.7178881496191023, "p95_s": 2.8249304682016367, "cost_usd_per_1000": 0.020922219193829496, "axes": { "intelligence": 64.3480059486406, "calibration": 61.47798519111051, "speed": 73.13997353442628, "cost": 60.38176756854212 }, "jevbench_score": 64.65153416680249 } }, { "threshold": 0.64, "public": { "n": 231, "escalation": 0.6536796536796536, "tiers": { "easy": 0.9375, "standard": 0.5972222222222222, "hard": 0.5045045045045045 }, "accuracy": 0.6233766233766234, "ece_hard": 0.26069719485847376, "calibration": 55.190192708073816, "p50_s": 1.1012055903673172, "p95_s": 2.7812519550323485, "cost_usd_per_1000": 0.018365917359793763, "axes": { "intelligence": 62.475496329663, "calibration": 55.190192708073816, "speed": 75.1389125893405, "cost": 62.079621225638114 }, "jevbench_score": 63.32809111859155 }, "heldout": { "n": 303, "escalation": 0.8085808580858086, "tiers": { "easy": 0.9583333333333334, "standard": 0.6666666666666666, "judge": 0.547945205479452, "hard": 0.5321100917431193 }, "accuracy": 0.5841584158415841, "ece_hard": 0.266189366655055, "calibration": 61.201259925010554, "p50_s": 1.739775541424751, "p95_s": 2.8249304682016367, "cost_usd_per_1000": 0.021800294619349576, "axes": { "intelligence": 63.38910183905158, "calibration": 61.201259925010554, "speed": 73.08499016241223, "cost": 59.846129112926285 }, "jevbench_score": 64.18149141848248 } }, { "threshold": 0.65, "public": { "n": 231, "escalation": 0.6926406926406926, "tiers": { "easy": 0.9583333333333334, "standard": 0.6527777777777778, "hard": 0.5135135135135135 }, "accuracy": 0.6493506493506493, "ece_hard": 0.2566698333661038, "calibration": 55.59292885731081, "p50_s": 1.1418727666139603, "p95_s": 2.7812519550323485, "cost_usd_per_1000": 0.019229738314120337, "axes": { "intelligence": 65.41645812479146, "calibration": 55.59292885731081, "speed": 74.98141951133215, "cost": 61.480798773168246 }, "jevbench_score": 63.98820612650035 }, "heldout": { "n": 303, "escalation": 0.8283828382838284, "tiers": { "easy": 0.9583333333333334, "standard": 0.6666666666666666, "judge": 0.5068493150684932, "hard": 0.5321100917431193 }, "accuracy": 0.5643564356435643, "ece_hard": 0.266189366655055, "calibration": 61.201259925010554, "p50_s": 1.739775541424751, "p95_s": 2.8249304682016367, "cost_usd_per_1000": 0.022239332332109618, "axes": { "intelligence": 62.238416907544725, "calibration": 61.201259925010554, "speed": 73.08499016241223, "cost": 59.58634765770814 }, "jevbench_score": 63.81877517184909 } }, { "threshold": 0.66, "public": { "n": 231, "escalation": 0.7056277056277056, "tiers": { "easy": 0.9583333333333334, "standard": 0.6388888888888888, "hard": 0.5225225225225225 }, "accuracy": 0.6493506493506493, "ece_hard": 0.2505521881710988, "calibration": 56.204693376811306, "p50_s": 1.1566056340932847, "p95_s": 2.7812519550323485, "cost_usd_per_1000": 0.019517678632229196, "axes": { "intelligence": 65.25171004337672, "calibration": 56.204693376811306, "speed": 74.92574362873691, "cost": 61.28715511416442 }, "jevbench_score": 64.06063675793497 }, "heldout": { "n": 303, "escalation": 0.8646864686468647, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5, "hard": 0.5229357798165137 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2780139052243787, "calibration": 59.41158129057036, "p50_s": 1.7833678156137465, "p95_s": 2.8249304682016367, "cost_usd_per_1000": 0.023044234805503026, "axes": { "intelligence": 63.521406727828754, "calibration": 59.41158129057036, "speed": 72.97751312600565, "cost": 59.12313124748776 }, "jevbench_score": 63.52350112153608 } }, { "threshold": 0.67, "public": { "n": 231, "escalation": 0.7272727272727273, "tiers": { "easy": 0.9791666666666666, "standard": 0.6388888888888888, "hard": 0.5225225225225225 }, "accuracy": 0.6536796536796536, "ece_hard": 0.2534466232636447, "calibration": 55.915249867556724, "p50_s": 1.1566056340932847, "p95_s": 2.807708340883255, "cost_usd_per_1000": 0.019997579162410625, "axes": { "intelligence": 65.6568026359693, "calibration": 55.915249867556724, "speed": 74.88462704318341, "cost": 60.970677260141855 }, "jevbench_score": 63.98540489389448 }, "heldout": { "n": 303, "escalation": 0.8811881188118812, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5321100917431193 }, "accuracy": 0.570957095709571, "ece_hard": 0.2739235376890208, "calibration": 59.820618044106155, "p50_s": 1.7833678156137465, "p95_s": 2.8249304682016367, "cost_usd_per_1000": 0.02341009956613639, "axes": { "intelligence": 63.98841690754473, "calibration": 59.820618044106155, "speed": 72.97751312600565, "cost": 58.91790217569034 }, "jevbench_score": 63.69379841241209 } }, { "threshold": 0.68, "public": { "n": 231, "escalation": 0.7402597402597403, "tiers": { "easy": 1.0, "standard": 0.6527777777777778, "hard": 0.5225225225225225 }, "accuracy": 0.6623376623376623, "ece_hard": 0.2534466232636447, "calibration": 55.915249867556724, "p50_s": 1.173205989599228, "p95_s": 2.807708340883255, "cost_usd_per_1000": 0.020285519480519484, "axes": { "intelligence": 66.60201868535202, "calibration": 55.915249867556724, "speed": 74.82273736310684, "cost": 60.78441598638495 }, "jevbench_score": 64.15209977518022 }, "heldout": { "n": 303, "escalation": 0.900990099009901, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5504587155963303 }, "accuracy": 0.5775577557755776, "ece_hard": 0.2617424941288704, "calibration": 61.03872240012119, "p50_s": 1.7833678156137465, "p95_s": 2.8249304682016367, "cost_usd_per_1000": 0.023849137278896433, "axes": { "intelligence": 64.53887562314105, "calibration": 61.03872240012119, "speed": 72.97751312600565, "cost": 58.67581979656682 }, "jevbench_score": 64.08682613379641 } }, { "threshold": 0.69, "public": { "n": 231, "escalation": 0.7619047619047619, "tiers": { "easy": 1.0, "standard": 0.6388888888888888, "hard": 0.5225225225225225 }, "accuracy": 0.658008658008658, "ece_hard": 0.25501587883833704, "calibration": 55.75832431008749, "p50_s": 1.1978708386421204, "p95_s": 2.807708340883255, "cost_usd_per_1000": 0.020765420010700912, "axes": { "intelligence": 66.0618952285619, "calibration": 55.75832431008749, "speed": 74.73238015403479, "cost": 60.47977840690723 }, "jevbench_score": 63.8770626300525 }, "heldout": { "n": 303, "escalation": 0.9075907590759076, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5412844036697247 }, "accuracy": 0.5742574257425742, "ece_hard": 0.2708434295292974, "calibration": 60.12862886007849, "p50_s": 1.7862060844898222, "p95_s": 2.8249304682016367, "cost_usd_per_1000": 0.023995483183149778, "axes": { "intelligence": 64.26364626534289, "calibration": 60.12862886007849, "speed": 72.97060672874197, "cost": 58.59611501521048 }, "jevbench_score": 63.75523648703337 } }, { "threshold": 0.7, "public": { "n": 231, "escalation": 0.7965367965367965, "tiers": { "easy": 1.0, "standard": 0.6527777777777778, "hard": 0.5315315315315315 }, "accuracy": 0.6666666666666666, "ece_hard": 0.24614350930148388, "calibration": 56.645561263772805, "p50_s": 1.2072503596544266, "p95_s": 2.807708340883255, "cost_usd_per_1000": 0.0215332608589912, "axes": { "intelligence": 66.9773940607274, "calibration": 56.645561263772805, "speed": 74.69850664385876, "cost": 60.00670595271855 }, "jevbench_score": 64.21721385755238 }, "heldout": { "n": 303, "escalation": 0.9141914191419142, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5412844036697247 }, "accuracy": 0.5742574257425742, "ece_hard": 0.2708434295292974, "calibration": 60.12862886007849, "p50_s": 1.7862060844898222, "p95_s": 2.8249304682016367, "cost_usd_per_1000": 0.024141829087403123, "axes": { "intelligence": 64.26364626534289, "calibration": 60.12862886007849, "speed": 72.97060672874197, "cost": 58.51689486998544 }, "jevbench_score": 63.7336767737334 } }, { "threshold": 0.71, "public": { "n": 231, "escalation": 0.8051948051948052, "tiers": { "easy": 1.0, "standard": 0.6527777777777778, "hard": 0.5315315315315315 }, "accuracy": 0.6666666666666666, "ece_hard": 0.24614350930148388, "calibration": 56.645561263772805, "p50_s": 1.2072503596544266, "p95_s": 2.807708340883255, "cost_usd_per_1000": 0.021725221071063772, "axes": { "intelligence": 66.9773940607274, "calibration": 56.645561263772805, "speed": 74.69850664385876, "cost": 59.891073867155015 }, "jevbench_score": 64.1862550582272 }, "heldout": { "n": 303, "escalation": 0.9174917491749175, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5321100917431193 }, "accuracy": 0.570957095709571, "ece_hard": 0.2814821934445667, "calibration": 59.06475246855156, "p50_s": 1.7862060844898222, "p95_s": 2.8249304682016367, "cost_usd_per_1000": 0.0242150020395298, "axes": { "intelligence": 63.98841690754473, "calibration": 59.06475246855156, "speed": 72.97060672874197, "cost": 58.47746470115819 }, "jevbench_score": 63.371146106399216 } }, { "threshold": 0.72, "public": { "n": 231, "escalation": 0.8225108225108225, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23902528298420433, "calibration": 57.35738389550076, "p50_s": 1.2131326526403428, "p95_s": 2.807708340883255, "cost_usd_per_1000": 0.022109141495208914, "axes": { "intelligence": 66.27252252252254, "calibration": 57.35738389550076, "speed": 74.67739715497721, "cost": 59.66284362648144 }, "jevbench_score": 64.15108252055501 }, "heldout": { "n": 303, "escalation": 0.9207920792079208, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5321100917431193 }, "accuracy": 0.570957095709571, "ece_hard": 0.27029556262357857, "calibration": 60.18341555065037, "p50_s": 1.7862060844898222, "p95_s": 2.8249304682016367, "cost_usd_per_1000": 0.02428817499165647, "axes": { "intelligence": 63.98841690754473, "calibration": 60.18341555065037, "speed": 72.97060672874197, "cost": 58.43815350293934 }, "jevbench_score": 63.65839135361288 } }, { "threshold": 0.73, "public": { "n": 231, "escalation": 0.8225108225108225, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23902528298420433, "calibration": 57.35738389550076, "p50_s": 1.2131326526403428, "p95_s": 2.807708340883255, "cost_usd_per_1000": 0.022109141495208914, "axes": { "intelligence": 66.27252252252254, "calibration": 57.35738389550076, "speed": 74.67739715497721, "cost": 59.66284362648144 }, "jevbench_score": 64.15108252055501 }, "heldout": { "n": 303, "escalation": 0.9273927392739274, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5229357798165137 }, "accuracy": 0.5676567656765676, "ece_hard": 0.2750920635431081, "calibration": 59.70376545869742, "p50_s": 1.7862060844898222, "p95_s": 2.8249304682016367, "cost_usd_per_1000": 0.024434520895909816, "axes": { "intelligence": 63.71318754974655, "calibration": 59.70376545869742, "speed": 72.97060672874197, "cost": 58.359885161692475 }, "jevbench_score": 63.44148796412869 } }, { "threshold": 0.74, "public": { "n": 231, "escalation": 0.8268398268398268, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23902528298420433, "calibration": 57.35738389550076, "p50_s": 1.2131326526403428, "p95_s": 2.807708340883255, "cost_usd_per_1000": 0.022205121601245197, "axes": { "intelligence": 66.27252252252254, "calibration": 57.35738389550076, "speed": 74.67739715497721, "cost": 59.60640532509622 }, "jevbench_score": 64.13590614378498 }, "heldout": { "n": 303, "escalation": 0.9273927392739274, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5229357798165137 }, "accuracy": 0.5676567656765676, "ece_hard": 0.2750920635431081, "calibration": 59.70376545869742, "p50_s": 1.7862060844898222, "p95_s": 2.8249304682016367, "cost_usd_per_1000": 0.024434520895909816, "axes": { "intelligence": 63.71318754974655, "calibration": 59.70376545869742, "speed": 72.97060672874197, "cost": 58.359885161692475 }, "jevbench_score": 63.44148796412869 } }, { "threshold": 0.75, "public": { "n": 231, "escalation": 0.8311688311688312, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 1.2131326526403428, "p95_s": 2.812245886027813, "cost_usd_per_1000": 0.022301101707281487, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 74.67038417527554, "cost": 59.550210449038076 }, "jevbench_score": 64.2123829038157 }, "heldout": { "n": 303, "escalation": 0.933993399339934, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5229357798165137 }, "accuracy": 0.5676567656765676, "ece_hard": 0.2727809090922032, "calibration": 59.934880903787914, "p50_s": 1.7862060844898222, "p95_s": 2.8249304682016367, "cost_usd_per_1000": 0.024580866800163165, "axes": { "intelligence": 63.71318754974655, "calibration": 59.934880903787914, "speed": 72.97060672874197, "cost": 58.282084196916145 }, "jevbench_score": 63.481620252001356 } }, { "threshold": 0.76, "public": { "n": 231, "escalation": 0.8311688311688312, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 1.2131326526403428, "p95_s": 2.812245886027813, "cost_usd_per_1000": 0.022301101707281487, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 74.67038417527554, "cost": 59.550210449038076 }, "jevbench_score": 64.2123829038157 }, "heldout": { "n": 303, "escalation": 0.933993399339934, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5229357798165137 }, "accuracy": 0.5676567656765676, "ece_hard": 0.2727809090922032, "calibration": 59.934880903787914, "p50_s": 1.7862060844898222, "p95_s": 2.8249304682016367, "cost_usd_per_1000": 0.024580866800163165, "axes": { "intelligence": 63.71318754974655, "calibration": 59.934880903787914, "speed": 72.97060672874197, "cost": 58.282084196916145 }, "jevbench_score": 63.481620252001356 } }, { "threshold": 0.77, "public": { "n": 231, "escalation": 0.8441558441558441, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 1.2135097563266755, "p95_s": 2.812245886027813, "cost_usd_per_1000": 0.02258904202539034, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 74.66903437564464, "cost": 59.383065598164144 }, "jevbench_score": 64.1669877686557 }, "heldout": { "n": 303, "escalation": 0.9372937293729373, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2834191807784519, "calibration": 58.871053735163045, "p50_s": 1.7934504777193068, "p95_s": 2.8284195110201833, "cost_usd_per_1000": 0.024654039752289837, "axes": { "intelligence": 63.4379581919484, "calibration": 58.871053735163045, "speed": 72.94766786496196, "cost": 58.24335724286356 }, "jevbench_score": 63.11421785312238 } }, { "threshold": 0.78, "public": { "n": 231, "escalation": 0.8441558441558441, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 1.2135097563266755, "p95_s": 2.812245886027813, "cost_usd_per_1000": 0.02258904202539034, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 74.66903437564464, "cost": 59.383065598164144 }, "jevbench_score": 64.1669877686557 }, "heldout": { "n": 303, "escalation": 0.9405940594059405, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2834191807784519, "calibration": 58.871053735163045, "p50_s": 1.7934504777193068, "p95_s": 2.8284195110201833, "cost_usd_per_1000": 0.02472721270441651, "axes": { "intelligence": 63.4379581919484, "calibration": 58.871053735163045, "speed": 72.94766786496196, "cost": 58.20474505988575 }, "jevbench_score": 63.10375492686779 } }, { "threshold": 0.79, "public": { "n": 231, "escalation": 0.8441558441558441, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 1.2135097563266755, "p95_s": 2.812245886027813, "cost_usd_per_1000": 0.02258904202539034, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 74.66903437564464, "cost": 59.383065598164144 }, "jevbench_score": 64.1669877686557 }, "heldout": { "n": 303, "escalation": 0.9438943894389439, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2834191807784519, "calibration": 58.871053735163045, "p50_s": 1.7934504777193068, "p95_s": 2.8284195110201833, "cost_usd_per_1000": 0.024800385656543186, "axes": { "intelligence": 63.4379581919484, "calibration": 58.871053735163045, "speed": 72.94766786496196, "cost": 58.16624696972196 }, "jevbench_score": 63.093317732935084 } }, { "threshold": 0.8, "public": { "n": 231, "escalation": 0.8484848484848485, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 1.2135097563266755, "p95_s": 2.812245886027813, "cost_usd_per_1000": 0.02268502213142663, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 74.66903437564464, "cost": 59.32782378104834 }, "jevbench_score": 64.1520595302689 }, "heldout": { "n": 303, "escalation": 0.9438943894389439, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2834191807784519, "calibration": 58.871053735163045, "p50_s": 1.7934504777193068, "p95_s": 2.8284195110201833, "cost_usd_per_1000": 0.024800385656543186, "axes": { "intelligence": 63.4379581919484, "calibration": 58.871053735163045, "speed": 72.94766786496196, "cost": 58.16624696972196 }, "jevbench_score": 63.093317732935084 } }, { "threshold": 0.81, "public": { "n": 231, "escalation": 0.8484848484848485, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 1.2135097563266755, "p95_s": 2.812245886027813, "cost_usd_per_1000": 0.02268502213142663, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 74.66903437564464, "cost": 59.32782378104834 }, "jevbench_score": 64.1520595302689 }, "heldout": { "n": 303, "escalation": 0.9471947194719472, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2834191807784519, "calibration": 58.871053735163045, "p50_s": 1.7934504777193068, "p95_s": 2.8284195110201833, "cost_usd_per_1000": 0.024873558608669858, "axes": { "intelligence": 63.4379581919484, "calibration": 58.871053735163045, "speed": 72.94766786496196, "cost": 58.12786230010615 }, "jevbench_score": 63.08290612898196 } }, { "threshold": 0.82, "public": { "n": 231, "escalation": 0.8484848484848485, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 1.2135097563266755, "p95_s": 2.812245886027813, "cost_usd_per_1000": 0.02268502213142663, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 74.66903437564464, "cost": 59.32782378104834 }, "jevbench_score": 64.1520595302689 }, "heldout": { "n": 303, "escalation": 0.9504950495049505, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.28384975837017223, "calibration": 58.82799597599101, "p50_s": 1.8024227499961851, "p95_s": 2.8284195110201833, "cost_usd_per_1000": 0.02494673156079653, "axes": { "intelligence": 63.4379581919484, "calibration": 58.82799597599101, "speed": 72.92599515193916, "cost": 58.089590384696635 }, "jevbench_score": 63.05629977291645 } }, { "threshold": 0.83, "public": { "n": 231, "escalation": 0.8528138528138528, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 1.2135097563266755, "p95_s": 2.812245886027813, "cost_usd_per_1000": 0.022781002237462912, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 74.66903437564464, "cost": 59.27281519880732 }, "jevbench_score": 64.13718395721946 }, "heldout": { "n": 303, "escalation": 0.9504950495049505, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.28384975837017223, "calibration": 58.82799597599101, "p50_s": 1.8024227499961851, "p95_s": 2.8284195110201833, "cost_usd_per_1000": 0.02494673156079653, "axes": { "intelligence": 63.4379581919484, "calibration": 58.82799597599101, "speed": 72.92599515193916, "cost": 58.089590384696635 }, "jevbench_score": 63.05629977291645 } }, { "threshold": 0.84, "public": { "n": 231, "escalation": 0.8571428571428571, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 1.2135097563266755, "p95_s": 2.812245886027813, "cost_usd_per_1000": 0.0228769823434992, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 74.66903437564464, "cost": 59.21803789025005 }, "jevbench_score": 64.12236063344488 }, "heldout": { "n": 303, "escalation": 0.9504950495049505, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.28384975837017223, "calibration": 58.82799597599101, "p50_s": 1.8024227499961851, "p95_s": 2.8284195110201833, "cost_usd_per_1000": 0.02494673156079653, "axes": { "intelligence": 63.4379581919484, "calibration": 58.82799597599101, "speed": 72.92599515193916, "cost": 58.089590384696635 }, "jevbench_score": 63.05629977291645 } }, { "threshold": 0.85, "public": { "n": 231, "escalation": 0.8571428571428571, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 1.2135097563266755, "p95_s": 2.812245886027813, "cost_usd_per_1000": 0.0228769823434992, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 74.66903437564464, "cost": 59.21803789025005 }, "jevbench_score": 64.12236063344488 }, "heldout": { "n": 303, "escalation": 0.9537953795379538, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.28384975837017223, "calibration": 58.82799597599101, "p50_s": 1.8024227499961851, "p95_s": 2.8284195110201833, "cost_usd_per_1000": 0.025019904512923203, "axes": { "intelligence": 63.4379581919484, "calibration": 58.82799597599101, "speed": 72.92599515193916, "cost": 58.0514305630066 }, "jevbench_score": 63.04594159118136 } }, { "threshold": 0.86, "public": { "n": 231, "escalation": 0.8658008658008658, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 1.2135097563266755, "p95_s": 2.812245886027813, "cost_usd_per_1000": 0.02306894255557177, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 74.66903437564464, "cost": 59.10916937217658 }, "jevbench_score": 64.09286909453668 }, "heldout": { "n": 303, "escalation": 0.9537953795379538, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.28384975837017223, "calibration": 58.82799597599101, "p50_s": 1.8024227499961851, "p95_s": 2.8284195110201833, "cost_usd_per_1000": 0.025019904512923203, "axes": { "intelligence": 63.4379581919484, "calibration": 58.82799597599101, "speed": 72.92599515193916, "cost": 58.0514305630066 }, "jevbench_score": 63.04594159118136 } }, { "threshold": 0.87, "public": { "n": 231, "escalation": 0.8701298701298701, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 1.2135097563266755, "p95_s": 2.812245886027813, "cost_usd_per_1000": 0.023164922661608058, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 74.66903437564464, "cost": 59.05507436180828 }, "jevbench_score": 64.07820007210128 }, "heldout": { "n": 303, "escalation": 0.9537953795379538, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.28384975837017223, "calibration": 58.82799597599101, "p50_s": 1.8024227499961851, "p95_s": 2.8284195110201833, "cost_usd_per_1000": 0.025019904512923203, "axes": { "intelligence": 63.4379581919484, "calibration": 58.82799597599101, "speed": 72.92599515193916, "cost": 58.0514305630066 }, "jevbench_score": 63.04594159118136 } }, { "threshold": 0.88, "public": { "n": 231, "escalation": 0.8831168831168831, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 1.2135097563266755, "p95_s": 2.812245886027813, "cost_usd_per_1000": 0.023452862979716913, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 74.66903437564464, "cost": 58.89412401226488 }, "jevbench_score": 64.03449524965284 }, "heldout": { "n": 303, "escalation": 0.9537953795379538, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.28384975837017223, "calibration": 58.82799597599101, "p50_s": 1.8024227499961851, "p95_s": 2.8284195110201833, "cost_usd_per_1000": 0.025019904512923203, "axes": { "intelligence": 63.4379581919484, "calibration": 58.82799597599101, "speed": 72.92599515193916, "cost": 58.0514305630066 }, "jevbench_score": 63.04594159118136 } }, { "threshold": 0.89, "public": { "n": 231, "escalation": 0.8961038961038961, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 1.2135097563266755, "p95_s": 2.812245886027813, "cost_usd_per_1000": 0.023740803297825772, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 74.66903437564464, "cost": 58.73513770814356 }, "jevbench_score": 63.99123571303705 }, "heldout": { "n": 303, "escalation": 0.9570957095709571, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.28384975837017223, "calibration": 58.82799597599101, "p50_s": 1.8024227499961851, "p95_s": 2.8284195110201833, "cost_usd_per_1000": 0.02509307746504988, "axes": { "intelligence": 63.4379581919484, "calibration": 58.82799597599101, "speed": 72.92599515193916, "cost": 58.01338218033576 }, "jevbench_score": 63.03560857311134 } }, { "threshold": 0.9, "public": { "n": 231, "escalation": 0.8961038961038961, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 1.2135097563266755, "p95_s": 2.812245886027813, "cost_usd_per_1000": 0.023740803297825772, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 74.66903437564464, "cost": 58.73513770814356 }, "jevbench_score": 63.99123571303705 }, "heldout": { "n": 303, "escalation": 0.9570957095709571, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.28384975837017223, "calibration": 58.82799597599101, "p50_s": 1.8024227499961851, "p95_s": 2.8284195110201833, "cost_usd_per_1000": 0.02509307746504988, "axes": { "intelligence": 63.4379581919484, "calibration": 58.82799597599101, "speed": 72.92599515193916, "cost": 58.01338218033576 }, "jevbench_score": 63.03560857311134 } }, { "threshold": 0.91, "public": { "n": 231, "escalation": 0.9090909090909091, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 1.2135097563266755, "p95_s": 2.812245886027813, "cost_usd_per_1000": 0.024028743615934628, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 74.66903437564464, "cost": 58.578068093025784 }, "jevbench_score": 63.94841137317201 }, "heldout": { "n": 303, "escalation": 0.9603960396039604, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.28384975837017223, "calibration": 58.82799597599101, "p50_s": 1.8024227499961851, "p95_s": 2.8284195110201833, "cost_usd_per_1000": 0.02516625041717655, "axes": { "intelligence": 63.4379581919484, "calibration": 58.82799597599101, "speed": 72.92599515193916, "cost": 57.97544458770295 }, "jevbench_score": 63.02530058114446 } }, { "threshold": 0.92, "public": { "n": 231, "escalation": 0.922077922077922, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 1.2135097563266755, "p95_s": 2.812245886027813, "cost_usd_per_1000": 0.024316683934043487, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 74.66903437564464, "cost": 58.42286950287379 }, "jevbench_score": 63.90601248807256 }, "heldout": { "n": 303, "escalation": 0.9636963696369637, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.28384975837017223, "calibration": 58.82799597599101, "p50_s": 1.8024227499961851, "p95_s": 2.8284195110201833, "cost_usd_per_1000": 0.025239423369303224, "axes": { "intelligence": 63.4379581919484, "calibration": 58.82799597599101, "speed": 72.92599515193916, "cost": 57.9376171417797 }, "jevbench_score": 63.01501747887128 } }, { "threshold": 0.93, "public": { "n": 231, "escalation": 0.922077922077922, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 1.2135097563266755, "p95_s": 2.812245886027813, "cost_usd_per_1000": 0.024316683934043487, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 74.66903437564464, "cost": 58.42286950287379 }, "jevbench_score": 63.90601248807256 }, "heldout": { "n": 303, "escalation": 0.9702970297029703, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.28384975837017223, "calibration": 58.82799597599101, "p50_s": 1.8024227499961851, "p95_s": 2.8284195110201833, "cost_usd_per_1000": 0.025385769273556572, "axes": { "intelligence": 63.4379581919484, "calibration": 58.82799597599101, "speed": 72.92599515193916, "cost": 57.86229014461965 }, "jevbench_score": 62.99452540345339 } }, { "threshold": 0.94, "public": { "n": 231, "escalation": 0.9264069264069265, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 1.2135097563266755, "p95_s": 2.812245886027813, "cost_usd_per_1000": 0.024412664040079773, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 74.66903437564464, "cost": 58.37154476401103 }, "jevbench_score": 63.89197243607092 }, "heldout": { "n": 303, "escalation": 0.9735973597359736, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.28384975837017223, "calibration": 58.82799597599101, "p50_s": 1.8024227499961851, "p95_s": 2.8284195110201833, "cost_usd_per_1000": 0.025458942225683245, "axes": { "intelligence": 63.4379581919484, "calibration": 58.82799597599101, "speed": 72.92599515193916, "cost": 57.824789334405 }, "jevbench_score": 62.98431616313664 } }, { "threshold": 0.95, "public": { "n": 231, "escalation": 0.935064935064935, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 1.2135097563266755, "p95_s": 2.812245886027813, "cost_usd_per_1000": 0.024604624252152343, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 74.66903437564464, "cost": 58.269497886338 }, "jevbench_score": 63.864029646940494 }, "heldout": { "n": 303, "escalation": 0.976897689768977, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.28384975837017223, "calibration": 58.82799597599101, "p50_s": 1.8024227499961851, "p95_s": 2.8284195110201833, "cost_usd_per_1000": 0.02553211517780992, "axes": { "intelligence": 63.4379581919484, "calibration": 58.82799597599101, "speed": 72.92599515193916, "cost": 57.7873961528179 }, "jevbench_score": 62.974131278144405 } }, { "threshold": 0.96, "public": { "n": 231, "escalation": 0.9523809523809523, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 1.2135097563266755, "p95_s": 2.812245886027813, "cost_usd_per_1000": 0.024988544676297485, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 74.66903437564464, "cost": 58.06777108866495 }, "jevbench_score": 63.80868402961088 }, "heldout": { "n": 303, "escalation": 0.976897689768977, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.28384975837017223, "calibration": 58.82799597599101, "p50_s": 1.8024227499961851, "p95_s": 2.8284195110201833, "cost_usd_per_1000": 0.02553211517780992, "axes": { "intelligence": 63.4379581919484, "calibration": 58.82799597599101, "speed": 72.92599515193916, "cost": 57.7873961528179 }, "jevbench_score": 62.974131278144405 } }, { "threshold": 0.97, "public": { "n": 231, "escalation": 0.9696969696969697, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 1.216033148765564, "p95_s": 2.8300808921456335, "cost_usd_per_1000": 0.02537246510044263, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 74.63255737008592, "cost": 57.86912008562125 }, "jevbench_score": 63.746253542604805 }, "heldout": { "n": 303, "escalation": 0.9933993399339934, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 1.8024227499961851, "p95_s": 2.8284195110201833, "cost_usd_per_1000": 0.025897979938443286, "axes": { "intelligence": 63.4379581919484, "calibration": 59.97952523584675, "speed": 72.92599515193916, "cost": 57.602023296637405 }, "jevbench_score": 63.22925732430593 } }, { "threshold": 0.98, "public": { "n": 231, "escalation": 0.9826839826839827, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 1.216033148765564, "p95_s": 2.8300808921456335, "cost_usd_per_1000": 0.02566040541855149, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 74.63255737008592, "cost": 57.72209458967192 }, "jevbench_score": 63.70572559251683 }, "heldout": { "n": 303, "escalation": 0.9966996699669967, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 1.8024227499961851, "p95_s": 2.8284195110201833, "cost_usd_per_1000": 0.025971152890569955, "axes": { "intelligence": 63.4379581919484, "calibration": 59.97952523584675, "speed": 72.92599515193916, "cost": 57.56526313272346 }, "jevbench_score": 63.21916707841273 } }, { "threshold": 0.99, "public": { "n": 231, "escalation": 1.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 1.216033148765564, "p95_s": 2.8300808921456335, "cost_usd_per_1000": 0.02604432584269663, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 74.63255737008592, "cost": 57.528606393918096 }, "jevbench_score": 63.65227192445073 }, "heldout": { "n": 303, "escalation": 1.0, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 1.8024227499961851, "p95_s": 2.8284195110201833, "cost_usd_per_1000": 0.02604432584269663, "axes": { "intelligence": 63.4379581919484, "calibration": 59.97952523584675, "speed": 72.92599515193916, "cost": 57.528606393918096 }, "jevbench_score": 63.20910040817032 } }, { "threshold": 1.0, "public": { "n": 231, "escalation": 1.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 1.216033148765564, "p95_s": 2.8300808921456335, "cost_usd_per_1000": 0.02604432584269663, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 74.63255737008592, "cost": 57.528606393918096 }, "jevbench_score": 63.65227192445073 }, "heldout": { "n": 303, "escalation": 1.0, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 1.8024227499961851, "p95_s": 2.8284195110201833, "cost_usd_per_1000": 0.02604432584269663, "axes": { "intelligence": 63.4379581919484, "calibration": 59.97952523584675, "speed": 72.92599515193916, "cost": 57.528606393918096 }, "jevbench_score": 63.20910040817032 } } ] }, { "cheap": "laya", "backend": "semif-qwen3.5-4b", "chosen": { "threshold": 0.96, "public": { "n": 231, "escalation": 0.8874458874458875, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6036036036036037 }, "accuracy": 0.7532467532467533, "ece_hard": 0.12914387221294593, "calibration": 72.24478620042561, "p50_s": 1.688392999768257, "p95_s": 5.500179085135461, "cost_usd_per_1000": 0.022801501288973198, "axes": { "intelligence": 76.46187854521189, "calibration": 72.24478620042561, "speed": 70.32149626840659, "cost": 59.26109672149705 }, "jevbench_score": 69.26714942420028 }, "heldout": { "n": 303, "escalation": 0.9537953795379538, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7986798679867987, "ece_hard": 0.17685813516393573, "calibration": 66.51802582183663, "p50_s": 3.3309142172336577, "p95_s": 5.625163489580155, "cost_usd_per_1000": 0.024290689546482735, "axes": { "intelligence": 79.33796238113192, "calibration": 66.51802582183663, "speed": 67.27301402488074, "cost": 58.43680469749049 }, "jevbench_score": 67.4896125784508 } }, "full": { "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.952054794520548, "hard": 0.5772727272727273 }, "accuracy": 0.7790262172284644, "ece_hard": 0.13898290825804124, "calibration": 70.78321555417108, "p50_s": 2.7824564978480337, "p95_s": 5.5497759498655785, "cost_usd_per_1000": 0.023646490131717372, "axes": { "intelligence": 80.72571606475717, "calibration": 70.78321555417108, "speed": 68.11296166327722, "cost": 58.78699938505723 }, "jevbench_score": 69.16132628228146 }, "full_escalation": 0.9250936329588015, "backend_heldout_accuracy": 0.7986798679867987, "useful": false, "curve": [ { "threshold": 0.0, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.01, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.02, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.03, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.04, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.05, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.06, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.07, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.08, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.09, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.1, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.11, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.12, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.13, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.14, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.15, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.16, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.17, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.18, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.19, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.2, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.21, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.22, "public": { "n": 231, "escalation": 0.004329004329004329, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2113480459308089, "calibration": 63.54769540691912, "p50_s": 1.1657145261764525, "p95_s": 4.4528342053294185, "cost_usd_per_1000": 0.0029802901405710396, "axes": { "intelligence": 53.58796296296297, "calibration": 63.54769540691912, "speed": 72.84771266552642, "cost": 85.77224361816316 }, "jevbench_score": 67.91761644837008 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.23, "public": { "n": 231, "escalation": 0.004329004329004329, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2113480459308089, "calibration": 63.54769540691912, "p50_s": 1.1657145261764525, "p95_s": 4.4528342053294185, "cost_usd_per_1000": 0.0029802901405710396, "axes": { "intelligence": 53.58796296296297, "calibration": 63.54769540691912, "speed": 72.84771266552642, "cost": 85.77224361816316 }, "jevbench_score": 67.91761644837008 }, "heldout": { "n": 303, "escalation": 0.0033003300330033004, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.30275229357798167 }, "accuracy": 0.5676567656765676, "ece_hard": 0.26903563195753477, "calibration": 54.44518680424652, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.002957201950532132, "axes": { "intelligence": 59.95243182103809, "calibration": 54.44518680424652, "speed": 68.8164811989393, "cost": 85.87357048058219 }, "jevbench_score": 66.27192254893268 } }, { "threshold": 0.24, "public": { "n": 231, "escalation": 0.004329004329004329, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2113480459308089, "calibration": 63.54769540691912, "p50_s": 1.1657145261764525, "p95_s": 4.4528342053294185, "cost_usd_per_1000": 0.0029802901405710396, "axes": { "intelligence": 53.58796296296297, "calibration": 63.54769540691912, "speed": 72.84771266552642, "cost": 85.77224361816316 }, "jevbench_score": 67.91761644837008 }, "heldout": { "n": 303, "escalation": 0.006600660066006601, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.3119266055045872 }, "accuracy": 0.570957095709571, "ece_hard": 0.27079911881794677, "calibration": 54.26883811820532, "p50_s": 2.7680240362882613, "p95_s": 4.776861734688282, "cost_usd_per_1000": 0.0030312765602402937, "axes": { "intelligence": 60.22766117883624, "calibration": 54.26883811820532, "speed": 68.78687472081019, "cost": 85.55123316162373 }, "jevbench_score": 66.22463807828377 } }, { "threshold": 0.25, "public": { "n": 231, "escalation": 0.008658008658008658, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.34234234234234234 }, "accuracy": 0.5324675324675324, "ece_hard": 0.21349238406622065, "calibration": 63.33326159337794, "p50_s": 1.1657145261764525, "p95_s": 4.4528342053294185, "cost_usd_per_1000": 0.003077452940318109, "axes": { "intelligence": 53.963338338338346, "calibration": 63.33326159337794, "speed": 72.84771266552642, "cost": 85.3542573842285 }, "jevbench_score": 67.89580514178415 }, "heldout": { "n": 303, "escalation": 0.009900990099009901, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.30275229357798167 }, "accuracy": 0.5676567656765676, "ece_hard": 0.2684903431880919, "calibration": 54.49971568119081, "p50_s": 2.7680240362882613, "p95_s": 4.808721804618835, "cost_usd_per_1000": 0.0031053511699484556, "axes": { "intelligence": 59.95243182103809, "calibration": 54.49971568119081, "speed": 68.75800490091856, "cost": 85.23667841006063 }, "jevbench_score": 66.15119684652457 } }, { "threshold": 0.26, "public": { "n": 231, "escalation": 0.008658008658008658, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.34234234234234234 }, "accuracy": 0.5324675324675324, "ece_hard": 0.21349238406622065, "calibration": 63.33326159337794, "p50_s": 1.1657145261764525, "p95_s": 4.4528342053294185, "cost_usd_per_1000": 0.003077452940318109, "axes": { "intelligence": 53.963338338338346, "calibration": 63.33326159337794, "speed": 72.84771266552642, "cost": 85.3542573842285 }, "jevbench_score": 67.89580514178415 }, "heldout": { "n": 303, "escalation": 0.013201320132013201, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.30275229357798167 }, "accuracy": 0.5676567656765676, "ece_hard": 0.2799787465788868, "calibration": 53.35087534211132, "p50_s": 2.7680240362882613, "p95_s": 4.8142633035779, "cost_usd_per_1000": 0.0031794257796566175, "axes": { "intelligence": 59.95243182103809, "calibration": 53.35087534211132, "speed": 68.75300303786179, "cost": 84.92953926126778 }, "jevbench_score": 65.73924294265706 } }, { "threshold": 0.27, "public": { "n": 231, "escalation": 0.021645021645021644, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.34234234234234234 }, "accuracy": 0.5324675324675324, "ece_hard": 0.2179634713218332, "calibration": 62.88615286781669, "p50_s": 1.1657145261764525, "p95_s": 4.455602088570595, "cost_usd_per_1000": 0.003368941339559317, "axes": { "intelligence": 53.963338338338346, "calibration": 62.88615286781669, "speed": 72.84501392824589, "cost": 84.17519652833566 }, "jevbench_score": 67.5397504047398 }, "heldout": { "n": 303, "escalation": 0.013201320132013201, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.30275229357798167 }, "accuracy": 0.5676567656765676, "ece_hard": 0.2799787465788868, "calibration": 53.35087534211132, "p50_s": 2.7680240362882613, "p95_s": 4.8142633035779, "cost_usd_per_1000": 0.0031794257796566175, "axes": { "intelligence": 59.95243182103809, "calibration": 53.35087534211132, "speed": 68.75300303786179, "cost": 84.92953926126778 }, "jevbench_score": 65.73924294265706 } }, { "threshold": 0.28, "public": { "n": 231, "escalation": 0.021645021645021644, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.34234234234234234 }, "accuracy": 0.5324675324675324, "ece_hard": 0.2179634713218332, "calibration": 62.88615286781669, "p50_s": 1.1657145261764525, "p95_s": 4.455602088570595, "cost_usd_per_1000": 0.003368941339559317, "axes": { "intelligence": 53.963338338338346, "calibration": 62.88615286781669, "speed": 72.84501392824589, "cost": 84.17519652833566 }, "jevbench_score": 67.5397504047398 }, "heldout": { "n": 303, "escalation": 0.019801980198019802, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.30275229357798167 }, "accuracy": 0.5676567656765676, "ece_hard": 0.2924038957947795, "calibration": 52.10836042052205, "p50_s": 2.7680240362882613, "p95_s": 4.822164891660214, "cost_usd_per_1000": 0.0033275749990729414, "axes": { "intelligence": 59.95243182103809, "calibration": 52.10836042052205, "speed": 68.74588086188128, "cost": 84.3361644198529 }, "jevbench_score": 65.23695545252534 } }, { "threshold": 0.29, "public": { "n": 231, "escalation": 0.03896103896103896, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2230055439322114, "calibration": 62.381945606778864, "p50_s": 1.1657145261764525, "p95_s": 4.538626746833325, "cost_usd_per_1000": 0.0037575925385475947, "axes": { "intelligence": 53.58796296296297, "calibration": 62.381945606778864, "speed": 72.76483323506824, "cost": 82.75270945607825 }, "jevbench_score": 66.98190287890112 }, "heldout": { "n": 303, "escalation": 0.0231023102310231, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2931477384554014, "calibration": 52.033976154459864, "p50_s": 2.7680240362882613, "p95_s": 4.840475116670132, "cost_usd_per_1000": 0.003401649608781103, "axes": { "intelligence": 59.67720246323992, "calibration": 52.033976154459864, "speed": 68.72942151082292, "cost": 84.0493127041549 }, "jevbench_score": 65.07933119965146 } }, { "threshold": 0.3, "public": { "n": 231, "escalation": 0.04329004329004329, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.34234234234234234 }, "accuracy": 0.5324675324675324, "ece_hard": 0.21916585331229338, "calibration": 62.76591466877066, "p50_s": 1.1657145261764525, "p95_s": 4.646009536087513, "cost_usd_per_1000": 0.003854755338294664, "axes": { "intelligence": 53.963338338338346, "calibration": 62.76591466877066, "speed": 72.66327694805801, "cost": 82.42009544356975 }, "jevbench_score": 67.11084249050388 }, "heldout": { "n": 303, "escalation": 0.033003300330033, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6986301369863014, "hard": 0.30275229357798167 }, "accuracy": 0.570957095709571, "ece_hard": 0.27898294821124164, "calibration": 53.450455178875835, "p50_s": 2.7861666113138197, "p95_s": 4.844832879900933, "cost_usd_per_1000": 0.003623873437905588, "axes": { "intelligence": 60.1442126429559, "calibration": 53.450455178875835, "speed": 68.69714115678303, "cost": 83.22480934789505 }, "jevbench_score": 65.4763077418565 } }, { "threshold": 0.31, "public": { "n": 231, "escalation": 0.05627705627705628, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2373984928224887, "calibration": 60.942650717751135, "p50_s": 1.1657145261764525, "p95_s": 4.646009536087513, "cost_usd_per_1000": 0.004146243737535873, "axes": { "intelligence": 53.58796296296297, "calibration": 60.942650717751135, "speed": 72.66327694805801, "cost": 81.4703551428401 }, "jevbench_score": 66.30950630001462 }, "heldout": { "n": 303, "escalation": 0.0429042904290429, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6986301369863014, "hard": 0.3211009174311927 }, "accuracy": 0.5775577557755776, "ece_hard": 0.2733769173089533, "calibration": 54.011058269104666, "p50_s": 2.7861666113138197, "p95_s": 4.969698165357111, "cost_usd_per_1000": 0.003846097267030074, "axes": { "intelligence": 60.69467135855222, "calibration": 54.011058269104666, "speed": 68.58662903894668, "cost": 82.44939210214399 }, "jevbench_score": 65.61679922866553 } }, { "threshold": 0.32, "public": { "n": 231, "escalation": 0.06060606060606061, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.32432432432432434 }, "accuracy": 0.5238095238095238, "ece_hard": 0.24637849168494952, "calibration": 60.04465083150505, "p50_s": 1.1657145261764525, "p95_s": 4.7284474208951, "cost_usd_per_1000": 0.004243406537282942, "axes": { "intelligence": 53.2125875875876, "calibration": 60.04465083150505, "speed": 72.58689227517124, "cost": 81.16856076672155 }, "jevbench_score": 65.86939696940792 }, "heldout": { "n": 303, "escalation": 0.0462046204620462, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6986301369863014, "hard": 0.3211009174311927 }, "accuracy": 0.5775577557755776, "ece_hard": 0.27829158404589394, "calibration": 53.5195915954106, "p50_s": 2.7861666113138197, "p95_s": 5.039579792916774, "cost_usd_per_1000": 0.003920171876738236, "axes": { "intelligence": 60.69467135855222, "calibration": 53.5195915954106, "speed": 68.52598590666544, "cost": 82.20084673824407 }, "jevbench_score": 65.4031603899457 } }, { "threshold": 0.33, "public": { "n": 231, "escalation": 0.06493506493506493, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.32432432432432434 }, "accuracy": 0.5238095238095238, "ece_hard": 0.25162408909346384, "calibration": 59.52009109065362, "p50_s": 1.1657145261764525, "p95_s": 4.7342256441712385, "cost_usd_per_1000": 0.004340569337030011, "axes": { "intelligence": 53.2125875875876, "calibration": 59.52009109065362, "speed": 72.58158838106294, "cost": 80.87359905669612 }, "jevbench_score": 65.66407049206533 }, "heldout": { "n": 303, "escalation": 0.0594059405940594, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6986301369863014, "hard": 0.3394495412844037 }, "accuracy": 0.5841584158415841, "ece_hard": 0.2674150550660732, "calibration": 55.79346848179763, "p50_s": 2.79284847676754, "p95_s": 5.0870937281846995, "cost_usd_per_1000": 0.004216470315570883, "axes": { "intelligence": 61.245130074148555, "calibration": 55.79346848179763, "speed": 68.47482886426148, "cost": 81.25152858425535 }, "jevbench_score": 66.03198523768147 } }, { "threshold": 0.34, "public": { "n": 231, "escalation": 0.0735930735930736, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.24972599025195558, "calibration": 59.70990097480445, "p50_s": 1.1657145261764525, "p95_s": 4.7731709912419324, "cost_usd_per_1000": 0.00453489493652415, "axes": { "intelligence": 53.58796296296297, "calibration": 59.70990097480445, "speed": 72.54600799122221, "cost": 80.3029831038698 }, "jevbench_score": 65.707463036431 }, "heldout": { "n": 303, "escalation": 0.06930693069306931, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7054794520547946, "hard": 0.3486238532110092 }, "accuracy": 0.5907590759075908, "ece_hard": 0.2688748615420892, "calibration": 55.64748783419603, "p50_s": 2.79284847676754, "p95_s": 5.103181174397469, "cost_usd_per_1000": 0.0044386941446953685, "axes": { "intelligence": 61.71214025386453, "calibration": 55.64748783419603, "speed": 68.4611163884171, "cost": 80.58234339094275 }, "jevbench_score": 65.97433332765077 } }, { "threshold": 0.35, "public": { "n": 231, "escalation": 0.0735930735930736, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.24972599025195558, "calibration": 59.70990097480445, "p50_s": 1.1657145261764525, "p95_s": 4.7731709912419324, "cost_usd_per_1000": 0.00453489493652415, "axes": { "intelligence": 53.58796296296297, "calibration": 59.70990097480445, "speed": 72.54600799122221, "cost": 80.3029831038698 }, "jevbench_score": 65.707463036431 }, "heldout": { "n": 303, "escalation": 0.07920792079207921, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7054794520547946, "hard": 0.3669724770642202 }, "accuracy": 0.5973597359735974, "ece_hard": 0.26123499544196843, "calibration": 56.41147444420811, "p50_s": 2.79284847676754, "p95_s": 5.116016503274441, "cost_usd_per_1000": 0.004660917973819853, "axes": { "intelligence": 62.26259896946086, "calibration": 56.41147444420811, "speed": 68.45020689119092, "cost": 79.94585620078581 }, "jevbench_score": 66.2127087049311 } }, { "threshold": 0.36, "public": { "n": 231, "escalation": 0.08658008658008658, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.26132522598583663, "calibration": 58.54997740141634, "p50_s": 1.1657145261764525, "p95_s": 4.87819176465273, "cost_usd_per_1000": 0.004826383335765358, "axes": { "intelligence": 53.58796296296297, "calibration": 58.54997740141634, "speed": 72.45148922984038, "cost": 79.49134561581697 }, "jevbench_score": 65.19890267696223 }, "heldout": { "n": 303, "escalation": 0.08250825082508251, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7054794520547946, "hard": 0.3669724770642202 }, "accuracy": 0.5973597359735974, "ece_hard": 0.2610865065096464, "calibration": 56.57301013652551, "p50_s": 2.79284847676754, "p95_s": 5.116016503274441, "cost_usd_per_1000": 0.004734992583528016, "axes": { "intelligence": 62.26259896946086, "calibration": 56.57301013652551, "speed": 68.45020689119092, "cost": 79.74042090701477 }, "jevbench_score": 66.2174504618403 } }, { "threshold": 0.37, "public": { "n": 231, "escalation": 0.11255411255411256, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5411255411255411, "ece_hard": 0.24557888942245945, "calibration": 60.124611057754066, "p50_s": 1.169974410533905, "p95_s": 5.172874148190022, "cost_usd_per_1000": 0.0054093601342477746, "axes": { "intelligence": 54.7140890890891, "calibration": 60.124611057754066, "speed": 72.18091724452356, "cost": 78.00562311852376 }, "jevbench_score": 65.60318732532508 }, "heldout": { "n": 303, "escalation": 0.0891089108910891, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7054794520547946, "hard": 0.3669724770642202 }, "accuracy": 0.5973597359735974, "ece_hard": 0.2672236793666213, "calibration": 55.95929285082802, "p50_s": 2.79284847676754, "p95_s": 5.127418529987335, "cost_usd_per_1000": 0.004883141802944339, "axes": { "intelligence": 62.26259896946086, "calibration": 55.95929285082802, "speed": 68.44053857326249, "cost": 79.33901991795149 }, "jevbench_score": 65.95153804886063 } }, { "threshold": 0.38, "public": { "n": 231, "escalation": 0.12554112554112554, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5411255411255411, "ece_hard": 0.24655102732618286, "calibration": 60.02739726738172, "p50_s": 1.2405554592609405, "p95_s": 5.271691578626633, "cost_usd_per_1000": 0.005700848533488983, "axes": { "intelligence": 54.7140890890891, "calibration": 60.02739726738172, "speed": 71.8443382189935, "cost": 77.32181492990301 }, "jevbench_score": 65.35605335236309 }, "heldout": { "n": 303, "escalation": 0.09900990099009901, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7123287671232876, "hard": 0.3761467889908257 }, "accuracy": 0.6039603960396039, "ece_hard": 0.26920539184748765, "calibration": 55.76112160274138, "p50_s": 2.8226395934820174, "p95_s": 5.149891981482506, "cost_usd_per_1000": 0.005105365632068825, "axes": { "intelligence": 62.72960914917684, "calibration": 55.76112160274138, "speed": 68.37546451013421, "cost": 78.75919448505762 }, "jevbench_score": 65.87966914794764 } }, { "threshold": 0.39, "public": { "n": 231, "escalation": 0.14285714285714285, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.36936936936936937 }, "accuracy": 0.5454545454545454, "ece_hard": 0.24945802487547789, "calibration": 59.73669751245222, "p50_s": 1.2405554592609405, "p95_s": 5.35441863834858, "cost_usd_per_1000": 0.00608949973247726, "axes": { "intelligence": 55.08946446446447, "calibration": 59.73669751245222, "speed": 71.77671491996624, "cost": 76.4625515281503 }, "jevbench_score": 65.19068296707567 }, "heldout": { "n": 303, "escalation": 0.11221122112211221, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7191780821917808, "hard": 0.3761467889908257 }, "accuracy": 0.6072607260726073, "ece_hard": 0.27994288152958313, "calibration": 54.68737263453184, "p50_s": 2.8282473921775817, "p95_s": 5.149891981482506, "cost_usd_per_1000": 0.005401664070901473, "axes": { "intelligence": 62.921389971094634, "calibration": 54.68737263453184, "speed": 68.36684484800057, "cost": 78.02417284153711 }, "jevbench_score": 65.45457682202584 } }, { "threshold": 0.4, "public": { "n": 231, "escalation": 0.1471861471861472, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3783783783783784 }, "accuracy": 0.5497835497835498, "ece_hard": 0.24256611613348253, "calibration": 60.42588838665175, "p50_s": 1.2405554592609405, "p95_s": 5.35441863834858, "cost_usd_per_1000": 0.00618666253222433, "axes": { "intelligence": 55.46483983983985, "calibration": 60.42588838665175, "speed": 71.77671491996624, "cost": 76.25630719205222 }, "jevbench_score": 65.44478402312033 }, "heldout": { "n": 303, "escalation": 0.132013201320132, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7191780821917808, "hard": 0.3853211009174312 }, "accuracy": 0.6105610561056105, "ece_hard": 0.2766254474239468, "calibration": 54.97062470104822, "p50_s": 2.8282473921775817, "p95_s": 5.185585963129998, "cost_usd_per_1000": 0.0058461117291504425, "axes": { "intelligence": 63.1966193288928, "calibration": 54.97062470104822, "speed": 68.33684766552014, "cost": 76.99398666376483 }, "jevbench_score": 65.3858941262437 } }, { "threshold": 0.41, "public": { "n": 231, "escalation": 0.1645021645021645, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.40540540540540543 }, "accuracy": 0.5627705627705628, "ece_hard": 0.22758299368559348, "calibration": 61.924200631440655, "p50_s": 1.2405554592609405, "p95_s": 5.4050926029682165, "cost_usd_per_1000": 0.006575313731212607, "axes": { "intelligence": 56.59096596596597, "calibration": 61.924200631440655, "speed": 71.7358067682186, "cost": 75.46250561906008 }, "jevbench_score": 65.99616217313717 }, "heldout": { "n": 303, "escalation": 0.13861386138613863, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7191780821917808, "hard": 0.3761467889908257 }, "accuracy": 0.6072607260726073, "ece_hard": 0.28792285669887485, "calibration": 52.408795718719645, "p50_s": 2.8427457749843597, "p95_s": 5.25048265069723, "cost_usd_per_1000": 0.005994260948566766, "axes": { "intelligence": 62.921389971094634, "calibration": 52.408795718719645, "speed": 68.26062749495006, "cost": 76.66793064423473 }, "jevbench_score": 64.45351792035089 } }, { "threshold": 0.42, "public": { "n": 231, "escalation": 0.16883116883116883, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.4144144144144144 }, "accuracy": 0.5670995670995671, "ece_hard": 0.22048911404995875, "calibration": 62.63358859500413, "p50_s": 1.2405554592609405, "p95_s": 5.4050926029682165, "cost_usd_per_1000": 0.006672476530959677, "axes": { "intelligence": 56.96634134134135, "calibration": 62.63358859500413, "speed": 71.7358067682186, "cost": 75.27138835233264 }, "jevbench_score": 66.25183081488382 }, "heldout": { "n": 303, "escalation": 0.15841584158415842, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.726027397260274, "hard": 0.3944954128440367 }, "accuracy": 0.6171617161716172, "ece_hard": 0.274314001970879, "calibration": 53.769681191519226, "p50_s": 2.8427457749843597, "p95_s": 5.25048265069723, "cost_usd_per_1000": 0.006438708606815738, "axes": { "intelligence": 63.66362950860878, "calibration": 53.769681191519226, "speed": 68.26062749495006, "cost": 75.73603687287174 }, "jevbench_score": 64.85977338283037 } }, { "threshold": 0.43, "public": { "n": 231, "escalation": 0.18614718614718614, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.43243243243243246 }, "accuracy": 0.5757575757575758, "ece_hard": 0.2133659887631854, "calibration": 63.56657896001409, "p50_s": 1.2405554592609405, "p95_s": 5.4050926029682165, "cost_usd_per_1000": 0.007061127729947955, "axes": { "intelligence": 57.7170920920921, "calibration": 63.56657896001409, "speed": 71.7358067682186, "cost": 74.5337779722087 }, "jevbench_score": 66.55115558628785 }, "heldout": { "n": 303, "escalation": 0.19141914191419143, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7397260273972602, "hard": 0.3944954128440367 }, "accuracy": 0.6237623762376238, "ece_hard": 0.283079185313694, "calibration": 52.893162857237726, "p50_s": 2.8707323759794234, "p95_s": 5.310694602131845, "cost_usd_per_1000": 0.007179454703897355, "axes": { "intelligence": 64.0471911524444, "calibration": 52.893162857237726, "speed": 68.16855965207009, "cost": 74.3172562051055 }, "jevbench_score": 64.36404837787786 } }, { "threshold": 0.44, "public": { "n": 231, "escalation": 0.19913419913419914, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.44144144144144143 }, "accuracy": 0.5800865800865801, "ece_hard": 0.2049388361404157, "calibration": 66.16482044313587, "p50_s": 1.2405554592609405, "p95_s": 5.4050926029682165, "cost_usd_per_1000": 0.007352616129189162, "axes": { "intelligence": 58.09246746746747, "calibration": 66.16482044313587, "speed": 71.7358067682186, "cost": 74.0067432220994 }, "jevbench_score": 67.21072193879486 }, "heldout": { "n": 303, "escalation": 0.20132013201320131, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7397260273972602, "hard": 0.3944954128440367 }, "accuracy": 0.6237623762376238, "ece_hard": 0.283079185313694, "calibration": 52.893162857237726, "p50_s": 2.8707323759794234, "p95_s": 5.310694602131845, "cost_usd_per_1000": 0.00740167853302184, "axes": { "intelligence": 64.0471911524444, "calibration": 52.893162857237726, "speed": 68.16855965207009, "cost": 73.92009342848279 }, "jevbench_score": 64.27788267594055 } }, { "threshold": 0.45, "public": { "n": 231, "escalation": 0.21212121212121213, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.4594594594594595 }, "accuracy": 0.5887445887445888, "ece_hard": 0.19691741943324684, "calibration": 66.96696211385276, "p50_s": 1.2405554592609405, "p95_s": 5.4050926029682165, "cost_usd_per_1000": 0.007644104528430371, "axes": { "intelligence": 58.84321821821823, "calibration": 66.96696211385276, "speed": 71.7358067682186, "cost": 73.50020148599062 }, "jevbench_score": 67.51423994562484 }, "heldout": { "n": 303, "escalation": 0.2145214521452145, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7465753424657534, "hard": 0.3944954128440367 }, "accuracy": 0.6270627062706271, "ece_hard": 0.28334489596890167, "calibration": 52.86659179171696, "p50_s": 2.9026939451694487, "p95_s": 5.310694602131845, "cost_usd_per_1000": 0.007697976971854488, "axes": { "intelligence": 64.23897197436219, "calibration": 52.86659179171696, "speed": 68.12047425472045, "cost": 73.40870177234605 }, "jevbench_score": 64.19501068607799 } }, { "threshold": 0.46, "public": { "n": 231, "escalation": 0.22510822510822512, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.4774774774774775 }, "accuracy": 0.5974025974025974, "ece_hard": 0.18217999028817924, "calibration": 68.44070502835952, "p50_s": 1.3233075231313705, "p95_s": 5.457395336031914, "cost_usd_per_1000": 0.007935592927671578, "axes": { "intelligence": 59.59396896896897, "calibration": 68.44070502835952, "speed": 71.41353803089638, "cost": 73.01261854895407 }, "jevbench_score": 67.90844952586104 }, "heldout": { "n": 303, "escalation": 0.21782178217821782, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7465753424657534, "hard": 0.3944954128440367 }, "accuracy": 0.6270627062706271, "ece_hard": 0.28341771011537686, "calibration": 52.7774257250435, "p50_s": 2.9026939451694487, "p95_s": 5.310694602131845, "cost_usd_per_1000": 0.007772051581562651, "axes": { "intelligence": 64.23897197436219, "calibration": 52.7774257250435, "speed": 68.12047425472045, "cost": 73.28392977209423 }, "jevbench_score": 64.1406415525424 } }, { "threshold": 0.47, "public": { "n": 231, "escalation": 0.24242424242424243, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.46846846846846846 }, "accuracy": 0.5930735930735931, "ece_hard": 0.19477099718951846, "calibration": 66.86193488068562, "p50_s": 1.3233075231313705, "p95_s": 5.478691536188126, "cost_usd_per_1000": 0.008324244126659856, "axes": { "intelligence": 59.2185935935936, "calibration": 66.86193488068562, "speed": 71.39662369213465, "cost": 72.38965574915144 }, "jevbench_score": 67.25859700389262 }, "heldout": { "n": 303, "escalation": 0.23432343234323433, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7671232876712328, "hard": 0.4036697247706422 }, "accuracy": 0.6402640264026402, "ece_hard": 0.2791242405837679, "calibration": 53.2067726782044, "p50_s": 2.984680587053299, "p95_s": 5.310694602131845, "cost_usd_per_1000": 0.00814242463010346, "axes": { "intelligence": 65.08954379791379, "calibration": 53.2067726782044, "speed": 67.99950811630167, "cost": 72.67738758300933 }, "jevbench_score": 64.31996597255379 } }, { "threshold": 0.48, "public": { "n": 231, "escalation": 0.2510822510822511, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.4594594594594595 }, "accuracy": 0.5887445887445888, "ece_hard": 0.20480971488012037, "calibration": 64.8805665576068, "p50_s": 1.3951993823051452, "p95_s": 5.478691536188126, "cost_usd_per_1000": 0.008518569726153995, "axes": { "intelligence": 58.84321821821823, "calibration": 64.8805665576068, "speed": 71.16686875410838, "cost": 72.08899952383919 }, "jevbench_score": 66.5257036243004 }, "heldout": { "n": 303, "escalation": 0.264026402640264, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7808219178082192, "hard": 0.42201834862385323 }, "accuracy": 0.6534653465346535, "ece_hard": 0.26963365312156146, "calibration": 54.155831424425045, "p50_s": 3.0486501187086104, "p95_s": 5.33113528341055, "cost_usd_per_1000": 0.008809096117476916, "axes": { "intelligence": 66.02356415734573, "calibration": 54.155831424425045, "speed": 67.89072711638636, "cost": 71.65205953991142 }, "jevbench_score": 64.57967168102512 } }, { "threshold": 0.49, "public": { "n": 231, "escalation": 0.2554112554112554, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.4594594594594595 }, "accuracy": 0.5887445887445888, "ece_hard": 0.20499699113329045, "calibration": 64.86183893228979, "p50_s": 1.3951993823051452, "p95_s": 5.478691536188126, "cost_usd_per_1000": 0.008615732525901065, "axes": { "intelligence": 58.84321821821823, "calibration": 64.86183893228979, "speed": 71.16686875410838, "cost": 71.94123376319753 }, "jevbench_score": 66.48678813567166 }, "heldout": { "n": 303, "escalation": 0.26732673267326734, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7808219178082192, "hard": 0.42201834862385323 }, "accuracy": 0.6534653465346535, "ece_hard": 0.27884455495962407, "calibration": 53.23474124061878, "p50_s": 3.0486501187086104, "p95_s": 5.33113528341055, "cost_usd_per_1000": 0.008883170727185079, "axes": { "intelligence": 66.02356415734573, "calibration": 53.23474124061878, "speed": 67.89072711638636, "cost": 71.54295973091597 }, "jevbench_score": 64.2788160489502 } }, { "threshold": 0.5, "public": { "n": 231, "escalation": 0.26406926406926406, "tiers": { "easy": 0.9583333333333334, "standard": 0.5555555555555556, "hard": 0.4594594594594595 }, "accuracy": 0.5930735930735931, "ece_hard": 0.2049991404636437, "calibration": 64.86162399925446, "p50_s": 1.4165491938591004, "p95_s": 5.478691536188126, "cost_usd_per_1000": 0.008810058125395204, "axes": { "intelligence": 59.38334167500835, "calibration": 64.86162399925446, "speed": 71.10091489446404, "cost": 71.6506367880865 }, "jevbench_score": 66.55595565538599 }, "heldout": { "n": 303, "escalation": 0.27722772277227725, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7808219178082192, "hard": 0.41284403669724773 }, "accuracy": 0.6501650165016502, "ece_hard": 0.2813784142434357, "calibration": 52.98135531223762, "p50_s": 3.0486501187086104, "p95_s": 5.347061344981194, "cost_usd_per_1000": 0.009105394556309564, "axes": { "intelligence": 65.74833479954758, "calibration": 52.98135531223762, "speed": 67.87777248363737, "cost": 71.22103691644934 }, "jevbench_score": 64.05985096908633 } }, { "threshold": 0.51, "public": { "n": 231, "escalation": 0.2813852813852814, "tiers": { "easy": 0.9583333333333334, "standard": 0.5972222222222222, "hard": 0.4594594594594595 }, "accuracy": 0.6060606060606061, "ece_hard": 0.20945145035029755, "calibration": 64.41639301058908, "p50_s": 1.4668009370565414, "p95_s": 5.478691536188126, "cost_usd_per_1000": 0.009198709324383481, "axes": { "intelligence": 61.00371204537873, "calibration": 64.41639301058908, "speed": 70.94951971989778, "cost": 71.08819313383128 }, "jevbench_score": 66.72289789967354 }, "heldout": { "n": 303, "escalation": 0.29042904290429045, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7808219178082192, "hard": 0.4036697247706422 }, "accuracy": 0.6468646864686468, "ece_hard": 0.2929336941834413, "calibration": 51.78445121878859, "p50_s": 3.0606545895338058, "p95_s": 5.347061344981194, "cost_usd_per_1000": 0.009401692995142211, "axes": { "intelligence": 65.47310544174941, "calibration": 51.78445121878859, "speed": 67.86070513343526, "cost": 70.80381803378376 }, "jevbench_score": 63.53080246090112 } }, { "threshold": 0.52, "public": { "n": 231, "escalation": 0.2943722943722944, "tiers": { "easy": 0.9583333333333334, "standard": 0.5972222222222222, "hard": 0.4594594594594595 }, "accuracy": 0.6060606060606061, "ece_hard": 0.21143305916678407, "calibration": 64.21823212894043, "p50_s": 1.4749168187379837, "p95_s": 5.478691536188126, "cost_usd_per_1000": 0.00949019772362469, "axes": { "intelligence": 61.00371204537873, "calibration": 64.21823212894043, "speed": 70.92555622737156, "cost": 70.68174217496434 }, "jevbench_score": 66.57039762468452 }, "heldout": { "n": 303, "escalation": 0.32673267326732675, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7945205479452054, "hard": 0.4036697247706422 }, "accuracy": 0.6534653465346535, "ece_hard": 0.2962278568165826, "calibration": 51.45503495547446, "p50_s": 3.0606545895338058, "p95_s": 5.352626000344754, "cost_usd_per_1000": 0.01021651370193199, "axes": { "intelligence": 65.85666708558502, "calibration": 51.45503495547446, "speed": 67.85618780641815, "cost": 69.72091834614858 }, "jevbench_score": 63.27687827434011 } }, { "threshold": 0.53, "public": { "n": 231, "escalation": 0.30735930735930733, "tiers": { "easy": 0.9583333333333334, "standard": 0.6111111111111112, "hard": 0.46846846846846846 }, "accuracy": 0.6147186147186147, "ece_hard": 0.2071850458798299, "calibration": 64.64303345763584, "p50_s": 1.4749168187379837, "p95_s": 5.478691536188126, "cost_usd_per_1000": 0.009781686122865898, "axes": { "intelligence": 61.91921087754421, "calibration": 64.64303345763584, "speed": 70.92555622737156, "cost": 70.2875883111578 }, "jevbench_score": 66.83548953583913 }, "heldout": { "n": 303, "escalation": 0.3465346534653465, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.8082191780821918, "hard": 0.41284403669724773 }, "accuracy": 0.6633663366336634, "ece_hard": 0.2890332299362947, "calibration": 52.17449764350325, "p50_s": 3.0606545895338058, "p95_s": 5.388026666045189, "cost_usd_per_1000": 0.010660961360180961, "axes": { "intelligence": 66.5154580872188, "calibration": 52.17449764350325, "speed": 67.82755943546307, "cost": 69.16610892139497 }, "jevbench_score": 63.52140556765892 } }, { "threshold": 0.54, "public": { "n": 231, "escalation": 0.3246753246753247, "tiers": { "easy": 0.9583333333333334, "standard": 0.6111111111111112, "hard": 0.4774774774774775 }, "accuracy": 0.6190476190476191, "ece_hard": 0.20429097506149393, "calibration": 64.93244053946944, "p50_s": 1.4749168187379837, "p95_s": 5.478691536188126, "cost_usd_per_1000": 0.010170337321854175, "axes": { "intelligence": 62.2945862529196, "calibration": 64.93244053946944, "speed": 70.92555622737156, "cost": 69.7799392748828 }, "jevbench_score": 66.89002256168823 }, "heldout": { "n": 303, "escalation": 0.36303630363036304, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.8082191780821918, "hard": 0.43119266055045874 }, "accuracy": 0.66996699669967, "ece_hard": 0.27188478357518725, "calibration": 53.889342279614, "p50_s": 3.0606545895338058, "p95_s": 5.405680187940597, "cost_usd_per_1000": 0.01103133440872177, "axes": { "intelligence": 67.06591680281512, "calibration": 53.889342279614, "speed": 67.81335331686957, "cost": 68.72115849454107 }, "jevbench_score": 64.06231231546542 } }, { "threshold": 0.55, "public": { "n": 231, "escalation": 0.3463203463203463, "tiers": { "easy": 0.9791666666666666, "standard": 0.6111111111111112, "hard": 0.4774774774774775 }, "accuracy": 0.6233766233766234, "ece_hard": 0.20483583747014453, "calibration": 64.84495429860436, "p50_s": 1.4749168187379837, "p95_s": 5.478691536188126, "cost_usd_per_1000": 0.010656151320589521, "axes": { "intelligence": 62.699678845512175, "calibration": 64.84495429860436, "speed": 70.92555622737156, "cost": 69.17198863048795 }, "jevbench_score": 66.82956421461556 }, "heldout": { "n": 303, "escalation": 0.37293729372937295, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.8082191780821918, "hard": 0.44036697247706424 }, "accuracy": 0.6732673267326733, "ece_hard": 0.2658661580005569, "calibration": 54.491204837077035, "p50_s": 3.0606545895338058, "p95_s": 5.452450767159462, "cost_usd_per_1000": 0.011253558237846257, "axes": { "intelligence": 67.34114616061329, "calibration": 54.491204837077035, "speed": 67.77593927170997, "cost": 68.46130411663182 }, "jevbench_score": 64.23650545918048 } }, { "threshold": 0.56, "public": { "n": 231, "escalation": 0.354978354978355, "tiers": { "easy": 0.9791666666666666, "standard": 0.625, "hard": 0.4774774774774775 }, "accuracy": 0.6277056277056277, "ece_hard": 0.20483583747014453, "calibration": 64.84495429860436, "p50_s": 1.4921533554792403, "p95_s": 5.478691536188126, "cost_usd_per_1000": 0.010850476920083662, "axes": { "intelligence": 63.23980230230231, "calibration": 64.84495429860436, "speed": 70.8750969084825, "cost": 68.93653517462401 }, "jevbench_score": 66.90405677647952 }, "heldout": { "n": 303, "escalation": 0.39933993399339934, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.815068493150685, "hard": 0.44036697247706424 }, "accuracy": 0.6765676567656765, "ece_hard": 0.2710408531441605, "calibration": 53.97373532271667, "p50_s": 3.0677541375160216, "p95_s": 5.469749297201634, "cost_usd_per_1000": 0.01184615511551155, "axes": { "intelligence": 67.53292698253111, "calibration": 53.97373532271667, "speed": 67.75212028328474, "cost": 67.79267754802513 }, "jevbench_score": 63.96625595954925 } }, { "threshold": 0.57, "public": { "n": 231, "escalation": 0.36363636363636365, "tiers": { "easy": 0.9791666666666666, "standard": 0.6388888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.6277056277056277, "ece_hard": 0.21307628657381836, "calibration": 63.4113465618905, "p50_s": 1.5156939208507538, "p95_s": 5.478691536188126, "cost_usd_per_1000": 0.0110448025195778, "axes": { "intelligence": 63.40455038371705, "calibration": 63.4113465618905, "speed": 70.80711641979104, "cost": 68.70526134649086 }, "jevbench_score": 66.50259263350071 }, "heldout": { "n": 303, "escalation": 0.40924092409240925, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.821917808219178, "hard": 0.44036697247706424 }, "accuracy": 0.6798679867986799, "ece_hard": 0.2698603915348144, "calibration": 54.09178148365128, "p50_s": 3.0780361533164977, "p95_s": 5.469749297201634, "cost_usd_per_1000": 0.012068378944636037, "axes": { "intelligence": 67.72470780444891, "calibration": 54.09178148365128, "speed": 67.73758862281503, "cost": 67.55053184574535 }, "jevbench_score": 63.98589256188605 } }, { "threshold": 0.58, "public": { "n": 231, "escalation": 0.36796536796536794, "tiers": { "easy": 0.9791666666666666, "standard": 0.6388888888888888, "hard": 0.4774774774774775 }, "accuracy": 0.6320346320346321, "ece_hard": 0.20448063932262944, "calibration": 64.27091128700938, "p50_s": 1.5156939208507538, "p95_s": 5.481852474808694, "cost_usd_per_1000": 0.011141965319324869, "axes": { "intelligence": 63.77992575909242, "calibration": 64.27091128700938, "speed": 70.80461147434814, "cost": 68.59114593001101 }, "jevbench_score": 66.79700917115322 }, "heldout": { "n": 303, "escalation": 0.42244224422442245, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.8287671232876712, "hard": 0.44954128440366975 }, "accuracy": 0.6864686468646864, "ece_hard": 0.2608848557545765, "calibration": 54.98933506167507, "p50_s": 3.0780361533164977, "p95_s": 5.469749297201634, "cost_usd_per_1000": 0.012364677383468683, "axes": { "intelligence": 68.1917179841649, "calibration": 54.98933506167507, "speed": 67.73758862281503, "cost": 67.23451632022207 }, "jevbench_score": 64.28476036538238 } }, { "threshold": 0.59, "public": { "n": 231, "escalation": 0.37662337662337664, "tiers": { "easy": 0.9791666666666666, "standard": 0.6388888888888888, "hard": 0.4774774774774775 }, "accuracy": 0.6320346320346321, "ece_hard": 0.20674175226950725, "calibration": 64.0447999923216, "p50_s": 1.5156939208507538, "p95_s": 5.481852474808694, "cost_usd_per_1000": 0.01133629091881901, "axes": { "intelligence": 63.77992575909242, "calibration": 64.0447999923216, "speed": 70.80461147434814, "cost": 68.36587052462292 }, "jevbench_score": 66.68331687822464 }, "heldout": { "n": 303, "escalation": 0.43564356435643564, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.8287671232876712, "hard": 0.44036697247706424 }, "accuracy": 0.6831683168316832, "ece_hard": 0.2737136284260401, "calibration": 53.70645779452872, "p50_s": 3.0828032076358793, "p95_s": 5.489500077068806, "cost_usd_per_1000": 0.01266097582230133, "axes": { "intelligence": 67.91648862636673, "calibration": 53.70645779452872, "speed": 67.71521402290799, "cost": 66.92598461650596 }, "jevbench_score": 63.76328593261288 } }, { "threshold": 0.6, "public": { "n": 231, "escalation": 0.3939393939393939, "tiers": { "easy": 0.9791666666666666, "standard": 0.6666666666666666, "hard": 0.4774774774774775 }, "accuracy": 0.6406926406926406, "ece_hard": 0.21045285712426504, "calibration": 63.673689506845825, "p50_s": 1.5156939208507538, "p95_s": 5.481852474808694, "cost_usd_per_1000": 0.011724942117807285, "axes": { "intelligence": 64.86017267267268, "calibration": 63.673689506845825, "speed": 70.80461147434814, "cost": 67.92667877730905 }, "jevbench_score": 66.75902881910628 }, "heldout": { "n": 303, "escalation": 0.44224422442244227, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.8287671232876712, "hard": 0.44954128440366975 }, "accuracy": 0.6864686468646864, "ece_hard": 0.26810039122702894, "calibration": 54.26778151442983, "p50_s": 3.0828032076358793, "p95_s": 5.489500077068806, "cost_usd_per_1000": 0.012809125041717655, "axes": { "intelligence": 68.1917179841649, "calibration": 54.26778151442983, "speed": 67.71521402290799, "cost": 66.77441604333077 }, "jevbench_score": 63.95765227233058 } }, { "threshold": 0.61, "public": { "n": 231, "escalation": 0.4199134199134199, "tiers": { "easy": 0.9791666666666666, "standard": 0.6666666666666666, "hard": 0.4954954954954955 }, "accuracy": 0.6493506493506493, "ece_hard": 0.19626142021982126, "calibration": 67.63133136454401, "p50_s": 1.5305662184953688, "p95_s": 5.481852474808694, "cost_usd_per_1000": 0.012307918916289701, "axes": { "intelligence": 65.61092342342343, "calibration": 67.63133136454401, "speed": 70.76220532501881, "cost": 67.29446120550618 }, "jevbench_score": 67.79945325573988 }, "heldout": { "n": 303, "escalation": 0.46534653465346537, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.8356164383561644, "hard": 0.44036697247706424 }, "accuracy": 0.6864686468646864, "ece_hard": 0.27950638023591245, "calibration": 53.12718261354148, "p50_s": 3.0828032076358793, "p95_s": 5.489500077068806, "cost_usd_per_1000": 0.013327647309674787, "axes": { "intelligence": 68.10826944828453, "calibration": 53.12718261354148, "speed": 67.71521402290799, "cost": 66.2573952563796 }, "jevbench_score": 63.47596489338959 } }, { "threshold": 0.62, "public": { "n": 231, "escalation": 0.44155844155844154, "tiers": { "easy": 0.9791666666666666, "standard": 0.6666666666666666, "hard": 0.5135135135135135 }, "accuracy": 0.658008658008658, "ece_hard": 0.18692542894170683, "calibration": 68.56493049235546, "p50_s": 1.5305662184953688, "p95_s": 5.485127615928651, "cost_usd_per_1000": 0.01279373291502505, "axes": { "intelligence": 66.36167417417417, "calibration": 68.56493049235546, "speed": 70.75961140080929, "cost": 66.79008159872463 }, "jevbench_score": 68.09719085959455 }, "heldout": { "n": 303, "escalation": 0.4752475247524752, "tiers": { "easy": 0.9166666666666666, "standard": 0.7083333333333334, "judge": 0.8356164383561644, "hard": 0.43119266055045874 }, "accuracy": 0.6864686468646864, "ece_hard": 0.28929609307238396, "calibration": 52.14821132989432, "p50_s": 3.0828032076358793, "p95_s": 5.489500077068806, "cost_usd_per_1000": 0.013549871138799272, "axes": { "intelligence": 68.99970675715304, "calibration": 52.14821132989432, "speed": 67.71521402290799, "cost": 66.0419450491659 }, "jevbench_score": 63.33564434505317 } }, { "threshold": 0.63, "public": { "n": 231, "escalation": 0.44155844155844154, "tiers": { "easy": 0.9791666666666666, "standard": 0.6666666666666666, "hard": 0.5135135135135135 }, "accuracy": 0.658008658008658, "ece_hard": 0.18692542894170683, "calibration": 68.56493049235546, "p50_s": 1.5305662184953688, "p95_s": 5.485127615928651, "cost_usd_per_1000": 0.01279373291502505, "axes": { "intelligence": 66.36167417417417, "calibration": 68.56493049235546, "speed": 70.75961140080929, "cost": 66.79008159872463 }, "jevbench_score": 68.09719085959455 }, "heldout": { "n": 303, "escalation": 0.49504950495049505, "tiers": { "easy": 0.9166666666666666, "standard": 0.75, "judge": 0.8356164383561644, "hard": 0.43119266055045874 }, "accuracy": 0.6897689768976898, "ece_hard": 0.2884066900324542, "calibration": 52.237151633887294, "p50_s": 3.099529358744621, "p95_s": 5.489500077068806, "cost_usd_per_1000": 0.013994318797048244, "axes": { "intelligence": 70.1663734238197, "calibration": 52.237151633887294, "speed": 67.69171450140712, "cost": 65.62144710646676 }, "jevbench_score": 63.52175068869053 } }, { "threshold": 0.64, "public": { "n": 231, "escalation": 0.4588744588744589, "tiers": { "easy": 0.9791666666666666, "standard": 0.6666666666666666, "hard": 0.5135135135135135 }, "accuracy": 0.658008658008658, "ece_hard": 0.18642571151424603, "calibration": 68.61490223510154, "p50_s": 1.5305662184953688, "p95_s": 5.485127615928651, "cost_usd_per_1000": 0.013182384114013327, "axes": { "intelligence": 66.36167417417417, "calibration": 68.61490223510154, "speed": 70.75961140080929, "cost": 66.40018113552725 }, "jevbench_score": 68.009975987614 }, "heldout": { "n": 303, "escalation": 0.5148514851485149, "tiers": { "easy": 0.9166666666666666, "standard": 0.75, "judge": 0.8424657534246576, "hard": 0.44036697247706424 }, "accuracy": 0.6963696369636964, "ece_hard": 0.2831831337591945, "calibration": 52.75950726121327, "p50_s": 3.1100841641426085, "p95_s": 5.489500077068806, "cost_usd_per_1000": 0.014438766455297215, "axes": { "intelligence": 70.63338360353568, "calibration": 52.75950726121327, "speed": 67.67695062555703, "cost": 65.21409724566672 }, "jevbench_score": 63.68296167003443 } }, { "threshold": 0.65, "public": { "n": 231, "escalation": 0.48917748917748916, "tiers": { "easy": 0.9791666666666666, "standard": 0.6805555555555556, "hard": 0.5225225225225225 }, "accuracy": 0.6666666666666666, "ece_hard": 0.1823497348656534, "calibration": 69.0224998999608, "p50_s": 1.5306500375270844, "p95_s": 5.485127615928651, "cost_usd_per_1000": 0.013862523712242811, "axes": { "intelligence": 67.27717300633968, "calibration": 69.0224998999608, "speed": 70.7593735728356, "cost": 65.74473093907092 }, "jevbench_score": 68.17510848212405 }, "heldout": { "n": 303, "escalation": 0.528052805280528, "tiers": { "easy": 0.9583333333333334, "standard": 0.75, "judge": 0.8561643835616438, "hard": 0.44954128440366975 }, "accuracy": 0.7095709570957096, "ece_hard": 0.27220310366646006, "calibration": 53.85751027048671, "p50_s": 3.1100841641426085, "p95_s": 5.489500077068806, "cost_usd_per_1000": 0.014735064894129861, "axes": { "intelligence": 71.8755079385028, "calibration": 53.85751027048671, "speed": 67.67695062555703, "cost": 64.94943841450899 }, "jevbench_score": 64.22599564955651 } }, { "threshold": 0.66, "public": { "n": 231, "escalation": 0.5151515151515151, "tiers": { "easy": 0.9791666666666666, "standard": 0.7083333333333334, "hard": 0.5045045045045045 }, "accuracy": 0.6666666666666666, "ece_hard": 0.2057161854848164, "calibration": 65.01427990370635, "p50_s": 1.5375520616769789, "p95_s": 5.485127615928651, "cost_usd_per_1000": 0.01444550051072523, "axes": { "intelligence": 67.60666916916918, "calibration": 65.01427990370635, "speed": 70.73983433805535, "cost": 65.20802218098952 }, "jevbench_score": 67.10283064126122 }, "heldout": { "n": 303, "escalation": 0.5478547854785478, "tiers": { "easy": 0.9583333333333334, "standard": 0.75, "judge": 0.8698630136986302, "hard": 0.45871559633027525 }, "accuracy": 0.7194719471947195, "ece_hard": 0.2622838588125359, "calibration": 54.84943475587913, "p50_s": 3.1124935925006865, "p95_s": 5.489500077068806, "cost_usd_per_1000": 0.015179512552378831, "axes": { "intelligence": 72.5342989401366, "calibration": 54.84943475587913, "speed": 67.67358738436126, "cost": 64.56226523110362 }, "jevbench_score": 64.56964270275284 } }, { "threshold": 0.67, "public": { "n": 231, "escalation": 0.5194805194805194, "tiers": { "easy": 0.9791666666666666, "standard": 0.7083333333333334, "hard": 0.5045045045045045 }, "accuracy": 0.6666666666666666, "ece_hard": 0.21423051638609364, "calibration": 64.16284681357862, "p50_s": 1.5375520616769789, "p95_s": 5.485127615928651, "cost_usd_per_1000": 0.014542663310472298, "axes": { "intelligence": 67.60666916916918, "calibration": 64.16284681357862, "speed": 70.73983433805535, "cost": 65.12068151451348 }, "jevbench_score": 66.85964017319392 }, "heldout": { "n": 303, "escalation": 0.570957095709571, "tiers": { "easy": 0.9583333333333334, "standard": 0.75, "judge": 0.8698630136986302, "hard": 0.47706422018348627 }, "accuracy": 0.7260726072607261, "ece_hard": 0.2465578152706393, "calibration": 56.422039110068795, "p50_s": 3.1124935925006865, "p95_s": 5.489500077068806, "cost_usd_per_1000": 0.015698034820335965, "axes": { "intelligence": 73.08475765573291, "calibration": 56.422039110068795, "speed": 67.67358738436126, "cost": 64.12464135785612 }, "jevbench_score": 65.03991039718211 } }, { "threshold": 0.68, "public": { "n": 231, "escalation": 0.5238095238095238, "tiers": { "easy": 0.9791666666666666, "standard": 0.7083333333333334, "hard": 0.5045045045045045 }, "accuracy": 0.6666666666666666, "ece_hard": 0.21423051638609364, "calibration": 64.16284681357862, "p50_s": 1.5375520616769789, "p95_s": 5.485127615928651, "cost_usd_per_1000": 0.014639826110219368, "axes": { "intelligence": 67.60666916916918, "calibration": 64.16284681357862, "speed": 70.73983433805535, "cost": 65.03392245205973 }, "jevbench_score": 66.83736007773099 }, "heldout": { "n": 303, "escalation": 0.5907590759075908, "tiers": { "easy": 0.9583333333333334, "standard": 0.75, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7260726072607261, "ece_hard": 0.25444332232836075, "calibration": 56.74056373106332, "p50_s": 3.1520356118679045, "p95_s": 5.6008212968707065, "cost_usd_per_1000": 0.016142482478584935, "axes": { "intelligence": 73.00130911985254, "calibration": 56.74056373106332, "speed": 67.5315716844011, "cost": 63.76089028944719 }, "jevbench_score": 64.98623565888123 } }, { "threshold": 0.69, "public": { "n": 231, "escalation": 0.5238095238095238, "tiers": { "easy": 0.9791666666666666, "standard": 0.7083333333333334, "hard": 0.5045045045045045 }, "accuracy": 0.6666666666666666, "ece_hard": 0.21423051638609364, "calibration": 64.16284681357862, "p50_s": 1.5375520616769789, "p95_s": 5.485127615928651, "cost_usd_per_1000": 0.014639826110219368, "axes": { "intelligence": 67.60666916916918, "calibration": 64.16284681357862, "speed": 70.73983433805535, "cost": 65.03392245205973 }, "jevbench_score": 66.83736007773099 }, "heldout": { "n": 303, "escalation": 0.594059405940594, "tiers": { "easy": 0.9583333333333334, "standard": 0.75, "judge": 0.8835616438356164, "hard": 0.46788990825688076 }, "accuracy": 0.7293729372937293, "ece_hard": 0.25444332232836075, "calibration": 56.74056373106332, "p50_s": 3.1520356118679045, "p95_s": 5.6008212968707065, "cost_usd_per_1000": 0.016216557088293098, "axes": { "intelligence": 73.19308994177037, "calibration": 56.74056373106332, "speed": 67.5315716844011, "cost": 63.70124034148421 }, "jevbench_score": 65.01366042275629 } }, { "threshold": 0.7, "public": { "n": 231, "escalation": 0.5367965367965368, "tiers": { "easy": 0.9791666666666666, "standard": 0.7222222222222222, "hard": 0.5135135135135135 }, "accuracy": 0.6753246753246753, "ece_hard": 0.22019166746040578, "calibration": 63.56673170614742, "p50_s": 1.5375520616769789, "p95_s": 5.485127615928651, "cost_usd_per_1000": 0.014931314509460577, "axes": { "intelligence": 68.52216800133468, "calibration": 63.56673170614742, "speed": 70.73983433805535, "cost": 64.77705869710073 }, "jevbench_score": 66.84001830955357 }, "heldout": { "n": 303, "escalation": 0.6105610561056105, "tiers": { "easy": 0.9583333333333334, "standard": 0.75, "judge": 0.8972602739726028, "hard": 0.47706422018348627 }, "accuracy": 0.7392739273927392, "ece_hard": 0.24344342821333326, "calibration": 57.840553142566066, "p50_s": 3.1520356118679045, "p95_s": 5.6008212968707065, "cost_usd_per_1000": 0.016586930136833905, "axes": { "intelligence": 73.85188094340414, "calibration": 57.840553142566066, "speed": 67.5315716844011, "cost": 63.407019536810566 }, "jevbench_score": 65.39725941247197 } }, { "threshold": 0.71, "public": { "n": 231, "escalation": 0.5454545454545454, "tiers": { "easy": 0.9791666666666666, "standard": 0.7222222222222222, "hard": 0.5135135135135135 }, "accuracy": 0.6753246753246753, "ece_hard": 0.22019166746040578, "calibration": 63.56673170614742, "p50_s": 1.5436184436082838, "p95_s": 5.485127615928651, "cost_usd_per_1000": 0.015125640108954714, "axes": { "intelligence": 68.52216800133468, "calibration": 63.56673170614742, "speed": 70.72273304750607, "cost": 64.60858711533035 }, "jevbench_score": 66.79247926713303 }, "heldout": { "n": 303, "escalation": 0.6303630363036303, "tiers": { "easy": 0.9583333333333334, "standard": 0.75, "judge": 0.910958904109589, "hard": 0.4954128440366973 }, "accuracy": 0.7524752475247525, "ece_hard": 0.2299158572888604, "calibration": 59.19331023501335, "p50_s": 3.1520356118679045, "p95_s": 5.6008212968707065, "cost_usd_per_1000": 0.017031377795082875, "axes": { "intelligence": 74.7859013028361, "calibration": 59.19331023501335, "speed": 67.5315716844011, "cost": 63.06250651896499 }, "jevbench_score": 65.89350533656707 } }, { "threshold": 0.72, "public": { "n": 231, "escalation": 0.5497835497835498, "tiers": { "easy": 0.9791666666666666, "standard": 0.7222222222222222, "hard": 0.5135135135135135 }, "accuracy": 0.6753246753246753, "ece_hard": 0.2215289265577226, "calibration": 63.43300579641574, "p50_s": 1.5436184436082838, "p95_s": 5.485127615928651, "cost_usd_per_1000": 0.015222802908701785, "axes": { "intelligence": 68.52216800133468, "calibration": 63.43300579641574, "speed": 70.72273304750607, "cost": 64.52516126328015 }, "jevbench_score": 66.73576298996257 }, "heldout": { "n": 303, "escalation": 0.6501650165016502, "tiers": { "easy": 0.9583333333333334, "standard": 0.75, "judge": 0.910958904109589, "hard": 0.5045871559633027 }, "accuracy": 0.7557755775577558, "ece_hard": 0.2227415423546794, "calibration": 59.910741728431454, "p50_s": 3.1520356118679045, "p95_s": 5.6008212968707065, "cost_usd_per_1000": 0.017475825453331848, "axes": { "intelligence": 75.06113066063426, "calibration": 59.910741728431454, "speed": 67.5315716844011, "cost": 62.726869048823794 }, "jevbench_score": 66.064791181878 } }, { "threshold": 0.73, "public": { "n": 231, "escalation": 0.5584415584415584, "tiers": { "easy": 1.0, "standard": 0.7222222222222222, "hard": 0.5135135135135135 }, "accuracy": 0.6796536796536796, "ece_hard": 0.2215289265577226, "calibration": 63.43300579641574, "p50_s": 1.550479757785797, "p95_s": 5.485127615928651, "cost_usd_per_1000": 0.015417128508195922, "axes": { "intelligence": 68.92726059392727, "calibration": 63.43300579641574, "speed": 70.70347163059931, "cost": 64.35989522663337 }, "jevbench_score": 66.78679367306376 }, "heldout": { "n": 303, "escalation": 0.6666666666666666, "tiers": { "easy": 1.0, "standard": 0.75, "judge": 0.9246575342465754, "hard": 0.5045871559633027 }, "accuracy": 0.7656765676567657, "ece_hard": 0.2227415423546794, "calibration": 59.910741728431454, "p50_s": 3.1520356118679045, "p95_s": 5.6008212968707065, "cost_usd_per_1000": 0.017846198501872655, "axes": { "intelligence": 76.0280256378032, "calibration": 59.910741728431454, "speed": 67.5315716844011, "cost": 62.45362842098621 }, "jevbench_score": 66.20422947020249 } }, { "threshold": 0.74, "public": { "n": 231, "escalation": 0.5627705627705628, "tiers": { "easy": 1.0, "standard": 0.7361111111111112, "hard": 0.5135135135135135 }, "accuracy": 0.683982683982684, "ece_hard": 0.2215289265577226, "calibration": 63.43300579641574, "p50_s": 1.550479757785797, "p95_s": 5.485127615928651, "cost_usd_per_1000": 0.015514291307942994, "axes": { "intelligence": 69.46738405071738, "calibration": 63.43300579641574, "speed": 70.70347163059931, "cost": 64.27804174531292 }, "jevbench_score": 66.89596192289648 }, "heldout": { "n": 303, "escalation": 0.6897689768976898, "tiers": { "easy": 1.0, "standard": 0.75, "judge": 0.9315068493150684, "hard": 0.5045871559633027 }, "accuracy": 0.768976897689769, "ece_hard": 0.2222064033779242, "calibration": 59.964255626106976, "p50_s": 3.1520356118679045, "p95_s": 5.613359278440476, "cost_usd_per_1000": 0.01836472076982979, "axes": { "intelligence": 76.21980645972101, "calibration": 59.964255626106976, "speed": 67.52186044702219, "cost": 62.080470117645056 }, "jevbench_score": 66.15915069335716 } }, { "threshold": 0.75, "public": { "n": 231, "escalation": 0.5844155844155844, "tiers": { "easy": 1.0, "standard": 0.75, "hard": 0.5225225225225225 }, "accuracy": 0.6926406926406926, "ece_hard": 0.21320076868559326, "calibration": 64.26582158362868, "p50_s": 1.563112351298332, "p95_s": 5.485127615928651, "cost_usd_per_1000": 0.01600010530667834, "axes": { "intelligence": 70.38288288288288, "calibration": 64.26582158362868, "speed": 70.66823068980668, "cost": 63.876314769149495 }, "jevbench_score": 67.22066375503421 }, "heldout": { "n": 303, "escalation": 0.7029702970297029, "tiers": { "easy": 1.0, "standard": 0.75, "judge": 0.9315068493150684, "hard": 0.5045871559633027 }, "accuracy": 0.768976897689769, "ece_hard": 0.2243813359160649, "calibration": 59.746762372292906, "p50_s": 3.1520356118679045, "p95_s": 5.613359278440476, "cost_usd_per_1000": 0.018661019208662435, "axes": { "intelligence": 76.21980645972101, "calibration": 59.746762372292906, "speed": 67.52186044702219, "cost": 61.87193920255688 }, "jevbench_score": 66.04350086100837 } }, { "threshold": 0.76, "public": { "n": 231, "escalation": 0.5974025974025974, "tiers": { "easy": 1.0, "standard": 0.7361111111111112, "hard": 0.5135135135135135 }, "accuracy": 0.683982683982684, "ece_hard": 0.21912187146931536, "calibration": 63.67371130525646, "p50_s": 1.563112351298332, "p95_s": 5.493131297826768, "cost_usd_per_1000": 0.016291593705919545, "axes": { "intelligence": 69.46738405071738, "calibration": 63.67371130525646, "speed": 70.66189825489799, "cost": 63.641092878052724 }, "jevbench_score": 66.78301308964089 }, "heldout": { "n": 303, "escalation": 0.7227722772277227, "tiers": { "easy": 1.0, "standard": 0.75, "judge": 0.9315068493150684, "hard": 0.5137614678899083 }, "accuracy": 0.7722772277227723, "ece_hard": 0.21576302775932843, "calibration": 60.60859318796655, "p50_s": 3.1520356118679045, "p95_s": 5.613359278440476, "cost_usd_per_1000": 0.019105466866911405, "axes": { "intelligence": 76.49503581751918, "calibration": 60.60859318796655, "speed": 67.52186044702219, "cost": 61.56527035887555 }, "jevbench_score": 66.25778475593002 } }, { "threshold": 0.77, "public": { "n": 231, "escalation": 0.6190476190476191, "tiers": { "easy": 1.0, "standard": 0.7361111111111112, "hard": 0.5315315315315315 }, "accuracy": 0.6926406926406926, "ece_hard": 0.20501924879154065, "calibration": 65.08397357303393, "p50_s": 1.563112351298332, "p95_s": 5.493131297826768, "cost_usd_per_1000": 0.016777407704654895, "axes": { "intelligence": 70.21813480146815, "calibration": 65.08397357303393, "speed": 70.66189825489799, "cost": 63.25825424896537 }, "jevbench_score": 67.22897082985568 }, "heldout": { "n": 303, "escalation": 0.735973597359736, "tiers": { "easy": 1.0, "standard": 0.75, "judge": 0.9383561643835616, "hard": 0.5137614678899083 }, "accuracy": 0.7755775577557755, "ece_hard": 0.21576302775932843, "calibration": 60.60859318796655, "p50_s": 3.1716890662908552, "p95_s": 5.613359278440476, "cost_usd_per_1000": 0.019401765305744056, "axes": { "intelligence": 76.68681663943697, "calibration": 60.60859318796655, "speed": 67.49486555169747, "cost": 61.36476259539577 }, "jevbench_score": 66.23860493979586 } }, { "threshold": 0.78, "public": { "n": 231, "escalation": 0.6406926406926406, "tiers": { "easy": 1.0, "standard": 0.7361111111111112, "hard": 0.5225225225225225 }, "accuracy": 0.6883116883116883, "ece_hard": 0.21953243363879607, "calibration": 63.6326550883084, "p50_s": 1.5920215278863905, "p95_s": 5.500179085135461, "cost_usd_per_1000": 0.01726322170339024, "axes": { "intelligence": 69.84275942609275, "calibration": 63.6326550883084, "speed": 70.57674233574949, "cost": 62.886344559716 }, "jevbench_score": 66.64304747412197 }, "heldout": { "n": 303, "escalation": 0.7557755775577558, "tiers": { "easy": 1.0, "standard": 0.75, "judge": 0.9383561643835616, "hard": 0.5045871559633027 }, "accuracy": 0.7722772277227723, "ece_hard": 0.22584183906161406, "calibration": 59.600712057737994, "p50_s": 3.2513657987117766, "p95_s": 5.613359278440476, "cost_usd_per_1000": 0.019846212963993026, "axes": { "intelligence": 76.4115872816388, "calibration": 59.600712057737994, "speed": 67.38711327176361, "cost": 61.0696705799977 }, "jevbench_score": 65.79657311973963 } }, { "threshold": 0.79, "public": { "n": 231, "escalation": 0.645021645021645, "tiers": { "easy": 1.0, "standard": 0.75, "hard": 0.5225225225225225 }, "accuracy": 0.6926406926406926, "ece_hard": 0.21953243363879607, "calibration": 63.6326550883084, "p50_s": 1.5920215278863905, "p95_s": 5.500179085135461, "cost_usd_per_1000": 0.01736038450313731, "axes": { "intelligence": 70.38288288288288, "calibration": 63.6326550883084, "speed": 70.57674233574949, "cost": 62.81321980495594 }, "jevbench_score": 66.75210109439337 }, "heldout": { "n": 303, "escalation": 0.7755775577557755, "tiers": { "easy": 1.0, "standard": 0.75, "judge": 0.9452054794520548, "hard": 0.5045871559633027 }, "accuracy": 0.7755775577557755, "ece_hard": 0.223632422364504, "calibration": 59.821653727449, "p50_s": 3.2513657987117766, "p95_s": 5.613359278440476, "cost_usd_per_1000": 0.020290660622241996, "axes": { "intelligence": 76.60336810355662, "calibration": 59.821653727449, "speed": 67.38711327176361, "cost": 60.78111438999948 }, "jevbench_score": 65.82076834837063 } }, { "threshold": 0.8, "public": { "n": 231, "escalation": 0.658008658008658, "tiers": { "easy": 1.0, "standard": 0.75, "hard": 0.5225225225225225 }, "accuracy": 0.6926406926406926, "ece_hard": 0.21953243363879607, "calibration": 63.6326550883084, "p50_s": 1.6035674840211867, "p95_s": 5.500179085135461, "cost_usd_per_1000": 0.017651872902378518, "axes": { "intelligence": 70.38288288288288, "calibration": 63.6326550883084, "speed": 70.54535928342996, "cost": 62.59627624706892 }, "jevbench_score": 66.68697397663303 }, "heldout": { "n": 303, "escalation": 0.7821782178217822, "tiers": { "easy": 1.0, "standard": 0.75, "judge": 0.9383561643835616, "hard": 0.5045871559633027 }, "accuracy": 0.7722772277227723, "ece_hard": 0.22416593430916745, "calibration": 59.76830253298265, "p50_s": 3.2513657987117766, "p95_s": 5.613359278440476, "cost_usd_per_1000": 0.02043880984165832, "axes": { "intelligence": 76.4115872816388, "calibration": 59.76830253298265, "speed": 67.38711327176361, "cost": 60.68633190717029 }, "jevbench_score": 65.7392084993624 } }, { "threshold": 0.81, "public": { "n": 231, "escalation": 0.6623376623376623, "tiers": { "easy": 1.0, "standard": 0.75, "hard": 0.5315315315315315 }, "accuracy": 0.696969696969697, "ece_hard": 0.21516776067325202, "calibration": 64.0691223848628, "p50_s": 1.6035674840211867, "p95_s": 5.500179085135461, "cost_usd_per_1000": 0.017749035702125587, "axes": { "intelligence": 70.75825825825827, "calibration": 64.0691223848628, "speed": 70.54535928342996, "cost": 62.52475711030361 }, "jevbench_score": 66.87081141961156 }, "heldout": { "n": 303, "escalation": 0.7887788778877888, "tiers": { "easy": 1.0, "standard": 0.75, "judge": 0.9383561643835616, "hard": 0.5045871559633027 }, "accuracy": 0.7722772277227723, "ece_hard": 0.2262766121525333, "calibration": 58.8996679995161, "p50_s": 3.275020423531532, "p95_s": 5.613359278440476, "cost_usd_per_1000": 0.020586959061074644, "axes": { "intelligence": 76.4115872816388, "calibration": 58.8996679995161, "speed": 67.35563147566381, "cost": 60.59223397328389 }, "jevbench_score": 65.46598946108234 } }, { "threshold": 0.82, "public": { "n": 231, "escalation": 0.670995670995671, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.5315315315315315 }, "accuracy": 0.7012987012987013, "ece_hard": 0.21516776067325202, "calibration": 64.0691223848628, "p50_s": 1.6035674840211867, "p95_s": 5.500179085135461, "cost_usd_per_1000": 0.017943361301619727, "axes": { "intelligence": 71.2983817150484, "calibration": 64.0691223848628, "speed": 70.54535928342996, "cost": 62.382885939081554 }, "jevbench_score": 66.96002233749599 }, "heldout": { "n": 303, "escalation": 0.801980198019802, "tiers": { "easy": 1.0, "standard": 0.75, "judge": 0.9452054794520548, "hard": 0.5045871559633027 }, "accuracy": 0.7755775577557755, "ece_hard": 0.2262766121525333, "calibration": 58.8996679995161, "p50_s": 3.305311831831932, "p95_s": 5.613359278440476, "cost_usd_per_1000": 0.02088325749990729, "axes": { "intelligence": 76.60336810355662, "calibration": 58.8996679995161, "speed": 67.31564721989025, "cost": 60.40605269323859 }, "jevbench_score": 65.44693291453554 } }, { "threshold": 0.83, "public": { "n": 231, "escalation": 0.696969696969697, "tiers": { "easy": 1.0, "standard": 0.7777777777777778, "hard": 0.5495495495495496 }, "accuracy": 0.7142857142857143, "ece_hard": 0.19737782717395846, "calibration": 65.84811573479215, "p50_s": 1.6066411167383192, "p95_s": 5.500179085135461, "cost_usd_per_1000": 0.01852633810010214, "axes": { "intelligence": 72.58925592258926, "calibration": 65.84811573479215, "speed": 70.53704292581986, "cost": 61.966312434056604 }, "jevbench_score": 67.60785379916256 }, "heldout": { "n": 303, "escalation": 0.8250825082508251, "tiers": { "easy": 1.0, "standard": 0.75, "judge": 0.9452054794520548, "hard": 0.5137614678899083 }, "accuracy": 0.7788778877887789, "ece_hard": 0.21805497747343935, "calibration": 59.72183146742549, "p50_s": 3.305311831831932, "p95_s": 5.613359278440476, "cost_usd_per_1000": 0.021401779767864424, "axes": { "intelligence": 76.87859746135479, "calibration": 59.72183146742549, "speed": 67.31564721989025, "cost": 60.0865032791185 }, "jevbench_score": 65.64594151924136 } }, { "threshold": 0.84, "public": { "n": 231, "escalation": 0.7056277056277056, "tiers": { "easy": 1.0, "standard": 0.7777777777777778, "hard": 0.5495495495495496 }, "accuracy": 0.7142857142857143, "ece_hard": 0.19821333906565902, "calibration": 65.7645645456221, "p50_s": 1.6066411167383192, "p95_s": 5.500179085135461, "cost_usd_per_1000": 0.01872066369959628, "axes": { "intelligence": 72.58925592258926, "calibration": 65.7645645456221, "speed": 70.53704292581986, "cost": 61.8303627513304 }, "jevbench_score": 67.54929710543843 }, "heldout": { "n": 303, "escalation": 0.834983498349835, "tiers": { "easy": 1.0, "standard": 0.75, "judge": 0.9452054794520548, "hard": 0.5137614678899083 }, "accuracy": 0.7788778877887789, "ece_hard": 0.21831035141657856, "calibration": 59.696294073111574, "p50_s": 3.305311831831932, "p95_s": 5.613359278440476, "cost_usd_per_1000": 0.02162400359698891, "axes": { "intelligence": 76.87859746135479, "calibration": 59.696294073111574, "speed": 67.31564721989025, "cost": 59.95191685203201 }, "jevbench_score": 65.60213604362771 } }, { "threshold": 0.85, "public": { "n": 231, "escalation": 0.70995670995671, "tiers": { "easy": 1.0, "standard": 0.7777777777777778, "hard": 0.5495495495495496 }, "accuracy": 0.7142857142857143, "ece_hard": 0.19570952375832387, "calibration": 66.01494607635561, "p50_s": 1.6066411167383192, "p95_s": 5.500179085135461, "cost_usd_per_1000": 0.01881782649934335, "axes": { "intelligence": 72.58925592258926, "calibration": 66.01494607635561, "speed": 70.53704292581986, "cost": 61.762916199610956 }, "jevbench_score": 67.59505333107806 }, "heldout": { "n": 303, "escalation": 0.8481848184818482, "tiers": { "easy": 1.0, "standard": 0.75, "judge": 0.9452054794520548, "hard": 0.5321100917431193 }, "accuracy": 0.7854785478547854, "ece_hard": 0.1993918727669265, "calibration": 61.588141938076774, "p50_s": 3.305311831831932, "p95_s": 5.613359278440476, "cost_usd_per_1000": 0.021920302035821557, "axes": { "intelligence": 77.42905617695112, "calibration": 61.588141938076774, "speed": 67.31564721989025, "cost": 59.77460398247275 }, "jevbench_score": 66.18482885671148 } }, { "threshold": 0.86, "public": { "n": 231, "escalation": 0.7229437229437229, "tiers": { "easy": 1.0, "standard": 0.7777777777777778, "hard": 0.5585585585585585 }, "accuracy": 0.7186147186147186, "ece_hard": 0.18425240581665406, "calibration": 67.1606578705226, "p50_s": 1.6066411167383192, "p95_s": 5.500179085135461, "cost_usd_per_1000": 0.019109314898584556, "axes": { "intelligence": 72.96463129796463, "calibration": 67.1606578705226, "speed": 70.53704292581986, "cost": 61.56264648583437 }, "jevbench_score": 67.91887223996014 }, "heldout": { "n": 303, "escalation": 0.8547854785478548, "tiers": { "easy": 1.0, "standard": 0.75, "judge": 0.9452054794520548, "hard": 0.5321100917431193 }, "accuracy": 0.7854785478547854, "ece_hard": 0.19906686796136358, "calibration": 61.797770037664854, "p50_s": 3.305311831831932, "p95_s": 5.613359278440476, "cost_usd_per_1000": 0.02206845125523788, "axes": { "intelligence": 77.42905617695112, "calibration": 61.797770037664854, "speed": 67.31564721989025, "cost": 59.68684432536578 }, "jevbench_score": 66.21674879355515 } }, { "threshold": 0.87, "public": { "n": 231, "escalation": 0.7532467532467533, "tiers": { "easy": 1.0, "standard": 0.7777777777777778, "hard": 0.5765765765765766 }, "accuracy": 0.7272727272727273, "ece_hard": 0.1678008554738095, "calibration": 68.80581290480706, "p50_s": 1.6066411167383192, "p95_s": 5.500179085135461, "cost_usd_per_1000": 0.019789454496814046, "axes": { "intelligence": 73.71538204871538, "calibration": 68.80581290480706, "speed": 70.53704292581986, "cost": 61.10698531320487 }, "jevbench_score": 68.37901486403412 }, "heldout": { "n": 303, "escalation": 0.8646864686468647, "tiers": { "easy": 1.0, "standard": 0.75, "judge": 0.9452054794520548, "hard": 0.5321100917431193 }, "accuracy": 0.7854785478547854, "ece_hard": 0.19906686796136358, "calibration": 61.797770037664854, "p50_s": 3.305311831831932, "p95_s": 5.613359278440476, "cost_usd_per_1000": 0.022290675084362367, "axes": { "intelligence": 77.42905617695112, "calibration": 61.797770037664854, "speed": 67.31564721989025, "cost": 59.556303354780326 }, "jevbench_score": 66.18051343344605 } }, { "threshold": 0.88, "public": { "n": 231, "escalation": 0.7792207792207793, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "hard": 0.5855855855855856 }, "accuracy": 0.7359307359307359, "ece_hard": 0.15172593231801446, "calibration": 70.41330522038655, "p50_s": 1.6066411167383192, "p95_s": 5.500179085135461, "cost_usd_per_1000": 0.02037243129529646, "axes": { "intelligence": 74.63088088088088, "calibration": 70.41330522038655, "speed": 70.53704292581986, "cost": 60.72871414454308 }, "jevbench_score": 68.88047895163933 }, "heldout": { "n": 303, "escalation": 0.8712871287128713, "tiers": { "easy": 1.0, "standard": 0.75, "judge": 0.9452054794520548, "hard": 0.5321100917431193 }, "accuracy": 0.7854785478547854, "ece_hard": 0.19906686796136358, "calibration": 61.797770037664854, "p50_s": 3.305311831831932, "p95_s": 5.613359278440476, "cost_usd_per_1000": 0.02243882430377869, "axes": { "intelligence": 77.42905617695112, "calibration": 61.797770037664854, "speed": 67.31564721989025, "cost": 59.46999705851591 }, "jevbench_score": 66.15652394248593 } }, { "threshold": 0.89, "public": { "n": 231, "escalation": 0.7878787878787878, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "hard": 0.5855855855855856 }, "accuracy": 0.7359307359307359, "ece_hard": 0.15172593231801446, "calibration": 70.41330522038655, "p50_s": 1.6066411167383192, "p95_s": 5.500179085135461, "cost_usd_per_1000": 0.020566756894790598, "axes": { "intelligence": 74.63088088088088, "calibration": 70.41330522038655, "speed": 70.53704292581986, "cost": 60.60502556185463 }, "jevbench_score": 68.84537923046649 }, "heldout": { "n": 303, "escalation": 0.8877887788778878, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5321100917431193 }, "accuracy": 0.7920792079207921, "ece_hard": 0.19906686796136358, "calibration": 61.797770037664854, "p50_s": 3.305311831831932, "p95_s": 5.613359278440476, "cost_usd_per_1000": 0.022809197352319496, "axes": { "intelligence": 78.7875036655356, "calibration": 61.797770037664854, "speed": 67.31564721989025, "cost": 59.256699913763384 }, "jevbench_score": 66.38514494793662 } }, { "threshold": 0.9, "public": { "n": 231, "escalation": 0.8008658008658008, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "hard": 0.5945945945945946 }, "accuracy": 0.7402597402597403, "ece_hard": 0.14200047714858782, "calibration": 71.3858507373292, "p50_s": 1.677485856413841, "p95_s": 5.500179085135461, "cost_usd_per_1000": 0.020858245294031807, "axes": { "intelligence": 75.00625625625626, "calibration": 71.3858507373292, "speed": 70.34964302564424, "cost": 60.42166688568254 }, "jevbench_score": 69.07025466742049 }, "heldout": { "n": 303, "escalation": 0.9042904290429042, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5412844036697247 }, "accuracy": 0.7953795379537953, "ece_hard": 0.18895079374548693, "calibration": 62.80937745925252, "p50_s": 3.3098835378885267, "p95_s": 5.625163489580155, "cost_usd_per_1000": 0.023179570400860307, "axes": { "intelligence": 79.06273302333375, "calibration": 62.80937745925252, "speed": 67.30052137824748, "cost": 59.0468385191598 }, "jevbench_score": 66.65041340680247 } }, { "threshold": 0.91, "public": { "n": 231, "escalation": 0.8138528138528138, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.5945945945945946 }, "accuracy": 0.7445887445887446, "ece_hard": 0.14200047714858782, "calibration": 71.3858507373292, "p50_s": 1.677485856413841, "p95_s": 5.500179085135461, "cost_usd_per_1000": 0.021149733693273016, "axes": { "intelligence": 75.54637971304639, "calibration": 71.3858507373292, "speed": 70.34964302564424, "cost": 60.24085290011519 }, "jevbench_score": 69.14244003372396 }, "heldout": { "n": 303, "escalation": 0.9108910891089109, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5412844036697247 }, "accuracy": 0.7953795379537953, "ece_hard": 0.18895079374548693, "calibration": 62.80937745925252, "p50_s": 3.3098835378885267, "p95_s": 5.625163489580155, "cost_usd_per_1000": 0.023327719620276632, "axes": { "intelligence": 79.06273302333375, "calibration": 62.80937745925252, "speed": 67.30052137824748, "cost": 58.96383139564042 }, "jevbench_score": 66.62697702065199 } }, { "threshold": 0.92, "public": { "n": 231, "escalation": 0.8225108225108225, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.5945945945945946 }, "accuracy": 0.7445887445887446, "ece_hard": 0.1427693510773586, "calibration": 70.88223831398435, "p50_s": 1.677485856413841, "p95_s": 5.500179085135461, "cost_usd_per_1000": 0.021344059292767156, "axes": { "intelligence": 75.54637971304639, "calibration": 70.88223831398435, "speed": 70.34964302564424, "cost": 60.12168943938242 }, "jevbench_score": 68.98601183560481 }, "heldout": { "n": 303, "escalation": 0.9207920792079208, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5412844036697247 }, "accuracy": 0.7953795379537953, "ece_hard": 0.18897832544267024, "calibration": 62.806624289534184, "p50_s": 3.3098835378885267, "p95_s": 5.625163489580155, "cost_usd_per_1000": 0.023549943449401117, "axes": { "intelligence": 79.06273302333375, "calibration": 62.806624289534184, "speed": 67.30052137824748, "cost": 58.84030394222138 }, "jevbench_score": 66.5913244379163 } }, { "threshold": 0.93, "public": { "n": 231, "escalation": 0.8398268398268398, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.5945945945945946 }, "accuracy": 0.7445887445887446, "ece_hard": 0.1427693510773586, "calibration": 70.88223831398435, "p50_s": 1.677485856413841, "p95_s": 5.500179085135461, "cost_usd_per_1000": 0.02173271049175543, "axes": { "intelligence": 75.54637971304639, "calibration": 70.88223831398435, "speed": 70.34964302564424, "cost": 59.886583159943314 }, "jevbench_score": 68.91847014064555 }, "heldout": { "n": 303, "escalation": 0.9240924092409241, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5412844036697247 }, "accuracy": 0.7953795379537953, "ece_hard": 0.18897832544267024, "calibration": 62.806624289534184, "p50_s": 3.3098835378885267, "p95_s": 5.625163489580155, "cost_usd_per_1000": 0.023624018059109277, "axes": { "intelligence": 79.06273302333375, "calibration": 62.806624289534184, "speed": 67.30052137824748, "cost": 58.79938702154267 }, "jevbench_score": 66.57974469337238 } }, { "threshold": 0.94, "public": { "n": 231, "escalation": 0.8528138528138528, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.5945945945945946 }, "accuracy": 0.7445887445887446, "ece_hard": 0.1427693510773586, "calibration": 70.88223831398435, "p50_s": 1.688392999768257, "p95_s": 5.500179085135461, "cost_usd_per_1000": 0.02202419889099664, "axes": { "intelligence": 75.54637971304639, "calibration": 70.88223831398435, "speed": 70.32149626840659, "cost": 59.712996390152206 }, "jevbench_score": 68.86158456239934 }, "heldout": { "n": 303, "escalation": 0.9405940594059405, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7986798679867987, "ece_hard": 0.17685813516393573, "calibration": 66.51802582183663, "p50_s": 3.3309142172336577, "p95_s": 5.625163489580155, "cost_usd_per_1000": 0.023994391107650087, "axes": { "intelligence": 79.33796238113192, "calibration": 66.51802582183663, "speed": 67.27301402488074, "cost": 58.59670799325482 }, "jevbench_score": 67.53573400561656 } }, { "threshold": 0.95, "public": { "n": 231, "escalation": 0.8614718614718615, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.5945945945945946 }, "accuracy": 0.7445887445887446, "ece_hard": 0.1427693510773586, "calibration": 70.88223831398435, "p50_s": 1.688392999768257, "p95_s": 5.500179085135461, "cost_usd_per_1000": 0.02221852449049078, "axes": { "intelligence": 75.54637971304639, "calibration": 70.88223831398435, "speed": 70.32149626840659, "cost": 59.59854356471008 }, "jevbench_score": 68.82856380209903 }, "heldout": { "n": 303, "escalation": 0.9471947194719472, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7986798679867987, "ece_hard": 0.17685813516393573, "calibration": 66.51802582183663, "p50_s": 3.3309142172336577, "p95_s": 5.625163489580155, "cost_usd_per_1000": 0.024142540327066413, "axes": { "intelligence": 79.33796238113192, "calibration": 66.51802582183663, "speed": 67.27301402488074, "cost": 58.51651103463892 }, "jevbench_score": 67.51261435165836 } }, { "threshold": 0.96, "public": { "n": 231, "escalation": 0.8874458874458875, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6036036036036037 }, "accuracy": 0.7532467532467533, "ece_hard": 0.12914387221294593, "calibration": 72.24478620042561, "p50_s": 1.688392999768257, "p95_s": 5.500179085135461, "cost_usd_per_1000": 0.022801501288973198, "axes": { "intelligence": 76.46187854521189, "calibration": 72.24478620042561, "speed": 70.32149626840659, "cost": 59.26109672149705 }, "jevbench_score": 69.26714942420028 }, "heldout": { "n": 303, "escalation": 0.9537953795379538, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7986798679867987, "ece_hard": 0.17685813516393573, "calibration": 66.51802582183663, "p50_s": 3.3309142172336577, "p95_s": 5.625163489580155, "cost_usd_per_1000": 0.024290689546482735, "axes": { "intelligence": 79.33796238113192, "calibration": 66.51802582183663, "speed": 67.27301402488074, "cost": 58.43680469749049 }, "jevbench_score": 67.4896125784508 } }, { "threshold": 0.97, "public": { "n": 231, "escalation": 0.8961038961038961, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6036036036036037 }, "accuracy": 0.7532467532467533, "ece_hard": 0.12914387221294593, "calibration": 72.24478620042561, "p50_s": 1.688392999768257, "p95_s": 5.500179085135461, "cost_usd_per_1000": 0.022995826888467335, "axes": { "intelligence": 76.46187854521189, "calibration": 72.24478620042561, "speed": 70.32149626840659, "cost": 59.15052908088238 }, "jevbench_score": 69.23481763290278 }, "heldout": { "n": 303, "escalation": 0.9603960396039604, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7986798679867987, "ece_hard": 0.17685813516393573, "calibration": 66.51802582183663, "p50_s": 3.3309142172336577, "p95_s": 5.625163489580155, "cost_usd_per_1000": 0.024438838765899057, "axes": { "intelligence": 79.33796238113192, "calibration": 66.51802582183663, "speed": 67.27301402488074, "cost": 58.35758301533574 }, "jevbench_score": 67.46672733967279 } }, { "threshold": 0.98, "public": { "n": 231, "escalation": 0.9090909090909091, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6036036036036037 }, "accuracy": 0.7532467532467533, "ece_hard": 0.12914387221294593, "calibration": 72.24478620042561, "p50_s": 1.688392999768257, "p95_s": 5.500179085135461, "cost_usd_per_1000": 0.02328731528770854, "axes": { "intelligence": 76.46187854521189, "calibration": 72.24478620042561, "speed": 70.32149626840659, "cost": 58.98641730566918 }, "jevbench_score": 69.18674498763033 }, "heldout": { "n": 303, "escalation": 0.9702970297029703, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7986798679867987, "ece_hard": 0.17685813516393573, "calibration": 66.51802582183663, "p50_s": 3.3309142172336577, "p95_s": 5.625163489580155, "cost_usd_per_1000": 0.024661062595023542, "axes": { "intelligence": 79.33796238113192, "calibration": 66.51802582183663, "speed": 67.27301402488074, "cost": 58.239646434139296 }, "jevbench_score": 67.43261509615236 } }, { "threshold": 0.99, "public": { "n": 231, "escalation": 0.9393939393939394, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6036036036036037 }, "accuracy": 0.7532467532467533, "ece_hard": 0.12914387221294593, "calibration": 72.24478620042561, "p50_s": 1.7035709500312803, "p95_s": 5.500179085135461, "cost_usd_per_1000": 0.02396745488593803, "axes": { "intelligence": 76.46187854521189, "calibration": 72.24478620042561, "speed": 70.28262943999344, "cost": 58.61134244291845 }, "jevbench_score": 69.0669512643166 }, "heldout": { "n": 303, "escalation": 0.9834983498349835, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7986798679867987, "ece_hard": 0.17685813516393573, "calibration": 66.51802582183663, "p50_s": 3.3309142172336577, "p95_s": 5.625163489580155, "cost_usd_per_1000": 0.024957361033856193, "axes": { "intelligence": 79.33796238113192, "calibration": 66.51802582183663, "speed": 67.27301402488074, "cost": 58.08404015265135 }, "jevbench_score": 67.38752781974331 } }, { "threshold": 1.0, "public": { "n": 231, "escalation": 1.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6036036036036037 }, "accuracy": 0.7532467532467533, "ece_hard": 0.12914387221294593, "calibration": 72.24478620042561, "p50_s": 1.7035709500312803, "p95_s": 5.500179085135461, "cost_usd_per_1000": 0.025327734082397, "axes": { "intelligence": 76.46187854521189, "calibration": 72.24478620042561, "speed": 70.28262943999344, "cost": 57.892109864352726 }, "jevbench_score": 68.85408528854965 }, "heldout": { "n": 303, "escalation": 1.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7986798679867987, "ece_hard": 0.17685813516393573, "calibration": 66.51802582183663, "p50_s": 3.3309142172336577, "p95_s": 5.625163489580155, "cost_usd_per_1000": 0.025327734082397, "axes": { "intelligence": 79.33796238113192, "calibration": 66.51802582183663, "speed": 67.27301402488074, "cost": 57.892109864352726 }, "jevbench_score": 67.33179062775422 } } ] }, { "cheap": "openjev-verdict-1.4", "backend": "semif-qwen3.5-4b", "chosen": { "threshold": 0.75, "public": { "n": 231, "escalation": 0.8311688311688312, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.6036036036036037 }, "accuracy": 0.7489177489177489, "ece_hard": 0.12914387221294593, "calibration": 72.24478620042561, "p50_s": 1.2058723896741868, "p95_s": 2.822939346730709, "cost_usd_per_1000": 0.022528178899751932, "axes": { "intelligence": 75.92175508842178, "calibration": 72.24478620042561, "speed": 74.67997101516198, "cost": 59.41821741173971 }, "jevbench_score": 70.23832191160102 }, "heldout": { "n": 303, "escalation": 0.933993399339934, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7986798679867987, "ece_hard": 0.16834264824186954, "calibration": 67.36957451404325, "p50_s": 1.7672429561614988, "p95_s": 2.845012076199054, "cost_usd_per_1000": 0.0248360358957244, "axes": { "intelligence": 79.33796238113192, "calibration": 67.36957451404325, "speed": 72.98619626483534, "cost": 58.147531634087755 }, "jevbench_score": 69.012731232157 } }, "full": { "tiers": { "easy": 1.0, "standard": 0.8020833333333334, "judge": 0.952054794520548, "hard": 0.5772727272727273 }, "accuracy": 0.7771535580524345, "ece_hard": 0.1347638715557448, "calibration": 71.20511922440073, "p50_s": 1.571361045539379, "p95_s": 2.830469681322574, "cost_usd_per_1000": 0.023837693262635188, "axes": { "intelligence": 80.43404939809051, "calibration": 71.20511922440073, "speed": 73.51865510049427, "cost": 58.68207318747417 }, "jevbench_score": 70.50396450693064 }, "full_escalation": 0.8895131086142322, "backend_heldout_accuracy": 0.7986798679867987, "useful": false, "curve": [ { "threshold": 0.0, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.01, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.02, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.03, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.04, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.05, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.06, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.07, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.08, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.09, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.1, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.11, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.12, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.13, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.14, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.15, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.16, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.17, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.18, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.19, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.2, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.21, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.22, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.23, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.24, "public": { "n": 231, "escalation": 0.004329004329004329, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12703664082271265, "calibration": 73.68787148664954, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003970084148061676, "axes": { "intelligence": 53.09371871871872, "calibration": 73.68787148664954, "speed": 79.65074585196557, "cost": 82.03600864104949 }, "jevbench_score": 71.10634921490106 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.25, "public": { "n": 231, "escalation": 0.008658008658008658, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.13016021024529512, "calibration": 73.3755145443913, "p50_s": 0.6819422334432602, "p95_s": 1.6338174432516097, "cost_usd_per_1000": 0.004067246947808746, "axes": { "intelligence": 53.09371871871872, "calibration": 73.3755145443913, "speed": 79.53048883940582, "cost": 81.72098374243275 }, "jevbench_score": 70.93578604740493 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.26, "public": { "n": 231, "escalation": 0.008658008658008658, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.13016021024529512, "calibration": 73.3755145443913, "p50_s": 0.6819422334432602, "p95_s": 1.6338174432516097, "cost_usd_per_1000": 0.004067246947808746, "axes": { "intelligence": 53.09371871871872, "calibration": 73.3755145443913, "speed": 79.53048883940582, "cost": 81.72098374243275 }, "jevbench_score": 70.93578604740493 }, "heldout": { "n": 303, "escalation": 0.0033003300330033004, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5684931506849316, "hard": 0.3577981651376147 }, "accuracy": 0.504950495049505, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9941574066877361, "cost_usd_per_1000": 0.003946995958022769, "axes": { "intelligence": 51.151753173306524, "calibration": 70.2289302105776, "speed": 76.12358067087503, "cost": 82.11199955049055 }, "jevbench_score": 68.83758141158394 } }, { "threshold": 0.27, "public": { "n": 231, "escalation": 0.008658008658008658, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.13016021024529512, "calibration": 73.3755145443913, "p50_s": 0.6819422334432602, "p95_s": 1.6338174432516097, "cost_usd_per_1000": 0.004067246947808746, "axes": { "intelligence": 53.09371871871872, "calibration": 73.3755145443913, "speed": 79.53048883940582, "cost": 81.72098374243275 }, "jevbench_score": 70.93578604740493 }, "heldout": { "n": 303, "escalation": 0.006600660066006601, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5684931506849316, "hard": 0.3669724770642202 }, "accuracy": 0.5082508250825083, "ece_hard": 0.14492037042241382, "calibration": 70.51898039981205, "p50_s": 1.2242849111557006, "p95_s": 1.9994657054543494, "cost_usd_per_1000": 0.004021070567730931, "axes": { "intelligence": 51.42698253110469, "calibration": 70.51898039981205, "speed": 76.11203543410345, "cost": 81.86974915552828 }, "jevbench_score": 68.94749084369555 } }, { "threshold": 0.28, "public": { "n": 231, "escalation": 0.008658008658008658, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.13016021024529512, "calibration": 73.3755145443913, "p50_s": 0.6819422334432602, "p95_s": 1.6338174432516097, "cost_usd_per_1000": 0.004067246947808746, "axes": { "intelligence": 53.09371871871872, "calibration": 73.3755145443913, "speed": 79.53048883940582, "cost": 81.72098374243275 }, "jevbench_score": 70.93578604740493 }, "heldout": { "n": 303, "escalation": 0.013201320132013201, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5684931506849316, "hard": 0.3853211009174312 }, "accuracy": 0.5148514851485149, "ece_hard": 0.1388683393552082, "calibration": 71.12418350653262, "p50_s": 1.234055870771408, "p95_s": 2.002006083726883, "cost_usd_per_1000": 0.004169219787147254, "axes": { "intelligence": 51.97744124670103, "calibration": 71.12418350653262, "speed": 76.07199784678924, "cost": 81.39835629207887 }, "jevbench_score": 69.17006149687398 } }, { "threshold": 0.29, "public": { "n": 231, "escalation": 0.017316017316017316, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36936936936936937 }, "accuracy": 0.5281385281385281, "ece_hard": 0.1358904048687984, "calibration": 72.80249508204096, "p50_s": 0.6819422334432602, "p95_s": 1.6590973153710364, "cost_usd_per_1000": 0.004261572547302884, "axes": { "intelligence": 53.46909409409409, "calibration": 72.80249508204096, "speed": 79.46380551799933, "cost": 81.1129034170981 }, "jevbench_score": 70.7745471334041 }, "heldout": { "n": 303, "escalation": 0.019801980198019802, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5684931506849316, "hard": 0.3944954128440367 }, "accuracy": 0.5181518151815182, "ece_hard": 0.13233371391351925, "calibration": 73.02662094373292, "p50_s": 1.234055870771408, "p95_s": 2.06843184530735, "cost_usd_per_1000": 0.0043173690065635775, "axes": { "intelligence": 52.25267060449919, "calibration": 73.02662094373292, "speed": 75.93023962072186, "cost": 80.94342491511719 }, "jevbench_score": 69.58994786187007 } }, { "threshold": 0.3, "public": { "n": 231, "escalation": 0.030303030303030304, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.3783783783783784 }, "accuracy": 0.5324675324675324, "ece_hard": 0.12547115470818282, "calibration": 73.84442009810252, "p50_s": 0.6859634757041931, "p95_s": 1.669744887948036, "cost_usd_per_1000": 0.0045530609465440925, "axes": { "intelligence": 53.84446946946947, "calibration": 73.84442009810252, "speed": 79.41048885282348, "cost": 80.25089608769719 }, "jevbench_score": 70.94905848862001 }, "heldout": { "n": 303, "escalation": 0.033003300330033, "tiers": { "easy": 0.875, "standard": 0.4583333333333333, "judge": 0.5821917808219178, "hard": 0.4036697247706422 }, "accuracy": 0.5313531353135313, "ece_hard": 0.12948130155034454, "calibration": 73.31186218005038, "p50_s": 1.2708158284425735, "p95_s": 2.0718681916594504, "cost_usd_per_1000": 0.004613667445396225, "axes": { "intelligence": 53.4947949394663, "calibration": 73.31186218005038, "speed": 75.79555261568528, "cost": 80.07861138309785 }, "jevbench_score": 69.84921165398127 } }, { "threshold": 0.31, "public": { "n": 231, "escalation": 0.030303030303030304, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.3783783783783784 }, "accuracy": 0.5324675324675324, "ece_hard": 0.12547115470818282, "calibration": 73.84442009810252, "p50_s": 0.6859634757041931, "p95_s": 1.669744887948036, "cost_usd_per_1000": 0.0045530609465440925, "axes": { "intelligence": 53.84446946946947, "calibration": 73.84442009810252, "speed": 79.41048885282348, "cost": 80.25089608769719 }, "jevbench_score": 70.94905848862001 }, "heldout": { "n": 303, "escalation": 0.0462046204620462, "tiers": { "easy": 0.875, "standard": 0.4583333333333333, "judge": 0.5958904109589042, "hard": 0.41284403669724773 }, "accuracy": 0.5412541254125413, "ece_hard": 0.12029571396882864, "calibration": 74.23042093820197, "p50_s": 1.2708158284425735, "p95_s": 2.117987248897552, "cost_usd_per_1000": 0.004909965884228872, "axes": { "intelligence": 54.15358594110009, "calibration": 74.23042093820197, "speed": 75.69994043367416, "cost": 79.2676457638626 }, "jevbench_score": 70.08097902395942 } }, { "threshold": 0.32, "public": { "n": 231, "escalation": 0.05194805194805195, "tiers": { "easy": 0.875, "standard": 0.5555555555555556, "hard": 0.3783783783783784 }, "accuracy": 0.5367965367965368, "ece_hard": 0.14389768329079908, "calibration": 72.00176723984089, "p50_s": 0.6935107052326203, "p95_s": 1.683034674823284, "cost_usd_per_1000": 0.005038874945279439, "axes": { "intelligence": 54.3845929262596, "calibration": 72.00176723984089, "speed": 79.32853766997002, "cost": 78.92999259472688 }, "jevbench_score": 70.36758351488145 }, "heldout": { "n": 303, "escalation": 0.0627062706270627, "tiers": { "easy": 0.875, "standard": 0.4583333333333333, "judge": 0.6164383561643836, "hard": 0.43119266055045874 }, "accuracy": 0.5577557755775577, "ece_hard": 0.10613773862920607, "calibration": 75.64621847216424, "p50_s": 1.2723965734243392, "p95_s": 2.14650867074728, "cost_usd_per_1000": 0.005280338932769682, "axes": { "intelligence": 55.27938712244984, "calibration": 75.64621847216424, "speed": 75.63644861985641, "cost": 78.32014600632994 }, "jevbench_score": 70.54865930351288 } }, { "threshold": 0.33, "public": { "n": 231, "escalation": 0.05627705627705628, "tiers": { "easy": 0.875, "standard": 0.5555555555555556, "hard": 0.3783783783783784 }, "accuracy": 0.5367965367965368, "ece_hard": 0.14375935841055637, "calibration": 72.01559972786518, "p50_s": 0.6935107052326203, "p95_s": 1.6896898761391639, "cost_usd_per_1000": 0.005136037745026509, "axes": { "intelligence": 54.3845929262596, "calibration": 72.01559972786518, "speed": 79.3113982865718, "cost": 78.68115379819656 }, "jevbench_score": 70.31163510463725 }, "heldout": { "n": 303, "escalation": 0.10561056105610561, "tiers": { "easy": 0.875, "standard": 0.4583333333333333, "judge": 0.6712328767123288, "hard": 0.44036697247706424 }, "accuracy": 0.5874587458745875, "ece_hard": 0.10641824330612357, "calibration": 75.61816800447248, "p50_s": 1.3009985327720641, "p95_s": 2.3409621635079376, "cost_usd_per_1000": 0.006243308858975785, "axes": { "intelligence": 57.088863055590465, "calibration": 75.61816800447248, "speed": 75.16328798882125, "cost": 76.13755539448832 }, "jevbench_score": 70.501046073086 } }, { "threshold": 0.34, "public": { "n": 231, "escalation": 0.06926406926406926, "tiers": { "easy": 0.875, "standard": 0.5694444444444444, "hard": 0.38738738738738737 }, "accuracy": 0.5454545454545454, "ece_hard": 0.14169239188186278, "calibration": 72.22229638073453, "p50_s": 0.6951739430427551, "p95_s": 1.7372671008110045, "cost_usd_per_1000": 0.005427526144267717, "axes": { "intelligence": 55.30009175842509, "calibration": 72.22229638073453, "speed": 79.18039919403824, "cost": 77.96194227646133 }, "jevbench_score": 70.46514834577351 }, "heldout": { "n": 303, "escalation": 0.1353135313531353, "tiers": { "easy": 0.875, "standard": 0.4583333333333333, "judge": 0.7123287671232876, "hard": 0.45871559633027525 }, "accuracy": 0.6138613861386139, "ece_hard": 0.10389325849512336, "calibration": 75.8706664855725, "p50_s": 1.340339982509613, "p95_s": 2.352632534503937, "cost_usd_per_1000": 0.006909980346349241, "axes": { "intelligence": 58.79000670269365, "calibration": 75.8706664855725, "speed": 75.01230928583325, "cost": 74.81569563586903 }, "jevbench_score": 70.73358429238117 } }, { "threshold": 0.35, "public": { "n": 231, "escalation": 0.08225108225108226, "tiers": { "easy": 0.875, "standard": 0.5694444444444444, "hard": 0.3963963963963964 }, "accuracy": 0.5497835497835498, "ece_hard": 0.1376842017762312, "calibration": 72.62311539129769, "p50_s": 0.6951739430427551, "p95_s": 1.897957244515419, "cost_usd_per_1000": 0.005719014543508925, "axes": { "intelligence": 55.675467133800474, "calibration": 72.62311539129769, "speed": 78.79620089956148, "cost": 77.28036397110651 }, "jevbench_score": 70.44145138484458 }, "heldout": { "n": 303, "escalation": 0.16501650165016502, "tiers": { "easy": 0.875, "standard": 0.4583333333333333, "judge": 0.7602739726027398, "hard": 0.45871559633027525 }, "accuracy": 0.636963696369637, "ece_hard": 0.10390526416974895, "calibration": 76.44628598770277, "p50_s": 1.340339982509613, "p95_s": 2.5098082795739143, "cost_usd_per_1000": 0.007576651833722698, "axes": { "intelligence": 60.1324724561183, "calibration": 76.44628598770277, "speed": 74.73144479502548, "cost": 73.61568007659315 }, "jevbench_score": 70.91445829739662 } }, { "threshold": 0.36, "public": { "n": 231, "escalation": 0.1038961038961039, "tiers": { "easy": 0.875, "standard": 0.5833333333333334, "hard": 0.40540540540540543 }, "accuracy": 0.5584415584415584, "ece_hard": 0.13788997923543472, "calibration": 72.60253764537734, "p50_s": 0.7169283866882324, "p95_s": 1.991252316534519, "cost_usd_per_1000": 0.006204828542244273, "axes": { "intelligence": 56.59096596596597, "calibration": 72.60253764537734, "speed": 78.45397929390384, "cost": 76.21810644519607 }, "jevbench_score": 70.40329896177106 }, "heldout": { "n": 303, "escalation": 0.1782178217821782, "tiers": { "easy": 0.875, "standard": 0.4583333333333333, "judge": 0.773972602739726, "hard": 0.46788990825688076 }, "accuracy": 0.6468646864686468, "ece_hard": 0.10044049820147256, "calibration": 76.7927625845304, "p50_s": 1.3478259533643722, "p95_s": 2.5098082795739143, "cost_usd_per_1000": 0.007872950272555345, "axes": { "intelligence": 60.79126345775209, "calibration": 76.7927625845304, "speed": 74.70725637574124, "cost": 73.1158747497269 }, "jevbench_score": 71.06143644543656 } }, { "threshold": 0.37, "public": { "n": 231, "escalation": 0.12554112554112554, "tiers": { "easy": 0.875, "standard": 0.5833333333333334, "hard": 0.40540540540540543 }, "accuracy": 0.5584415584415584, "ece_hard": 0.145321519914975, "calibration": 73.21257878910296, "p50_s": 0.7518308401107788, "p95_s": 2.008351984620094, "cost_usd_per_1000": 0.006690642540979619, "axes": { "intelligence": 56.59096596596597, "calibration": 73.21257878910296, "speed": 78.2104003371287, "cost": 75.23596517278625 }, "jevbench_score": 70.26769447172211 }, "heldout": { "n": 303, "escalation": 0.2079207920792079, "tiers": { "easy": 0.875, "standard": 0.5, "judge": 0.815068493150685, "hard": 0.47706422018348627 }, "accuracy": 0.6732673267326733, "ece_hard": 0.09242487116769495, "calibration": 77.59432528790816, "p50_s": 1.3478259533643722, "p95_s": 2.523379400372505, "cost_usd_per_1000": 0.008539621759928801, "axes": { "intelligence": 63.383844413723764, "calibration": 77.59432528790816, "speed": 74.68383631857643, "cost": 72.05684094448301 }, "jevbench_score": 71.72616214948452 } }, { "threshold": 0.38, "public": { "n": 231, "escalation": 0.12987012987012986, "tiers": { "easy": 0.875, "standard": 0.5833333333333334, "hard": 0.3963963963963964 }, "accuracy": 0.5541125541125541, "ece_hard": 0.15440831043618058, "calibration": 72.3038997369824, "p50_s": 0.7518308401107788, "p95_s": 2.027668859064579, "cost_usd_per_1000": 0.006787805340726689, "axes": { "intelligence": 56.2155905905906, "calibration": 72.3038997369824, "speed": 78.1688283217121, "cost": 75.0481186234849 }, "jevbench_score": 69.87920859108527 }, "heldout": { "n": 303, "escalation": 0.22772277227722773, "tiers": { "easy": 0.875, "standard": 0.5, "judge": 0.8561643835616438, "hard": 0.47706422018348627 }, "accuracy": 0.693069306930693, "ece_hard": 0.09242487116769495, "calibration": 77.59432528790816, "p50_s": 1.3478259533643722, "p95_s": 2.523379400372505, "cost_usd_per_1000": 0.008984069418177773, "axes": { "intelligence": 64.53452934523062, "calibration": 77.59432528790816, "speed": 74.68383631857643, "cost": 71.39580703070915 }, "jevbench_score": 71.88368930645743 } }, { "threshold": 0.39, "public": { "n": 231, "escalation": 0.1645021645021645, "tiers": { "easy": 0.875, "standard": 0.5833333333333334, "hard": 0.38738738738738737 }, "accuracy": 0.5497835497835498, "ece_hard": 0.17614245422591293, "calibration": 69.1397534203007, "p50_s": 0.7781706541776657, "p95_s": 2.087223435938358, "cost_usd_per_1000": 0.007565107738703244, "axes": { "intelligence": 55.84021521521522, "calibration": 69.1397534203007, "speed": 77.89356208528866, "cost": 73.63554647737537 }, "jevbench_score": 68.59871762915041 }, "heldout": { "n": 303, "escalation": 0.24422442244224424, "tiers": { "easy": 0.875, "standard": 0.5, "judge": 0.8904109589041096, "hard": 0.47706422018348627 }, "accuracy": 0.7095709570957096, "ece_hard": 0.09242487116769495, "calibration": 77.59432528790816, "p50_s": 1.3673696160316466, "p95_s": 2.523379400372505, "cost_usd_per_1000": 0.009354442466718581, "axes": { "intelligence": 65.49343345481967, "calibration": 77.59432528790816, "speed": 74.62131521303141, "cost": 70.869462752166 }, "jevbench_score": 72.00081956070932 } }, { "threshold": 0.4, "public": { "n": 231, "escalation": 0.18614718614718614, "tiers": { "easy": 0.875, "standard": 0.5833333333333334, "hard": 0.4144144144144144 }, "accuracy": 0.5627705627705628, "ece_hard": 0.1649290136391093, "calibration": 70.75423683372514, "p50_s": 0.8036727398633957, "p95_s": 2.154814641177654, "cost_usd_per_1000": 0.008050921737438592, "axes": { "intelligence": 56.96634134134135, "calibration": 70.75423683372514, "speed": 77.61510844695718, "cost": 72.82463185268831 }, "jevbench_score": 69.08739798588554 }, "heldout": { "n": 303, "escalation": 0.2739273927392739, "tiers": { "easy": 0.875, "standard": 0.5, "judge": 0.9383561643835616, "hard": 0.48623853211009177 }, "accuracy": 0.735973597359736, "ece_hard": 0.09007073726949426, "calibration": 77.82973867772822, "p50_s": 1.3695352911949157, "p95_s": 2.5376301407814017, "cost_usd_per_1000": 0.010021113954092036, "axes": { "intelligence": 67.11112856604248, "calibration": 77.82973867772822, "speed": 74.58998448031039, "cost": 69.97251997913692 }, "jevbench_score": 72.25818297865914 } }, { "threshold": 0.41, "public": { "n": 231, "escalation": 0.21645021645021645, "tiers": { "easy": 0.875, "standard": 0.5833333333333334, "hard": 0.4144144144144144 }, "accuracy": 0.5627705627705628, "ece_hard": 0.17916820406563622, "calibration": 71.21124948952698, "p50_s": 0.8354266524314881, "p95_s": 2.154814641177654, "cost_usd_per_1000": 0.008731061335668077, "axes": { "intelligence": 56.96634134134135, "calibration": 71.21124948952698, "speed": 77.44681755893038, "cost": 71.7679888255758 }, "jevbench_score": 68.90890068500732 }, "heldout": { "n": 303, "escalation": 0.29042904290429045, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "judge": 0.9452054794520548, "hard": 0.4954128440366973 }, "accuracy": 0.7458745874587459, "ece_hard": 0.08597394618080242, "calibration": 77.28678660388707, "p50_s": 1.3770495146512984, "p95_s": 2.555801300704479, "cost_usd_per_1000": 0.010391487002632847, "axes": { "intelligence": 68.74480541242511, "calibration": 77.28678660388707, "speed": 74.53523356684612, "cost": 69.4996690376842 }, "jevbench_score": 72.43064835153353 } }, { "threshold": 0.42, "public": { "n": 231, "escalation": 0.22943722943722944, "tiers": { "easy": 0.875, "standard": 0.5972222222222222, "hard": 0.4144144144144144 }, "accuracy": 0.5670995670995671, "ece_hard": 0.18190313679368497, "calibration": 70.9377562167221, "p50_s": 0.8431772083044052, "p95_s": 2.1951754197478293, "cost_usd_per_1000": 0.009022549734909284, "axes": { "intelligence": 57.50646479813148, "calibration": 70.9377562167221, "speed": 77.32611910183965, "cost": 71.34012145914383 }, "jevbench_score": 68.87530642618873 }, "heldout": { "n": 303, "escalation": 0.3102310231023102, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "judge": 0.952054794520548, "hard": 0.4954128440366973 }, "accuracy": 0.7491749174917491, "ece_hard": 0.09187428651550812, "calibration": 76.6967525704165, "p50_s": 1.3846236795186997, "p95_s": 2.5595119938254354, "cost_usd_per_1000": 0.010835934660881816, "axes": { "intelligence": 68.93658623434294, "calibration": 76.6967525704165, "speed": 74.50511076468922, "cost": 68.9540086709159 }, "jevbench_score": 72.19266650786034 } }, { "threshold": 0.43, "public": { "n": 231, "escalation": 0.24242424242424243, "tiers": { "easy": 0.875, "standard": 0.6111111111111112, "hard": 0.4144144144144144 }, "accuracy": 0.5714285714285714, "ece_hard": 0.18227149279457908, "calibration": 70.90092061663267, "p50_s": 0.8909249544143677, "p95_s": 2.246106307208538, "cost_usd_per_1000": 0.009314038134150493, "axes": { "intelligence": 58.0465882549216, "calibration": 70.90092061663267, "speed": 76.98728569121974, "cost": 70.92585964847432 }, "jevbench_score": 68.8514431084197 }, "heldout": { "n": 303, "escalation": 0.3234323432343234, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "judge": 0.952054794520548, "hard": 0.4954128440366973 }, "accuracy": 0.7491749174917491, "ece_hard": 0.09898201631863625, "calibration": 75.4351394664535, "p50_s": 1.3846236795186997, "p95_s": 2.5595119938254354, "cost_usd_per_1000": 0.011132233099714464, "axes": { "intelligence": 68.93658623434294, "calibration": 75.4351394664535, "speed": 74.50511076468922, "cost": 68.60253125431203 }, "jevbench_score": 71.80214508477557 } }, { "threshold": 0.44, "public": { "n": 231, "escalation": 0.2597402597402597, "tiers": { "easy": 0.875, "standard": 0.6111111111111112, "hard": 0.42342342342342343 }, "accuracy": 0.5757575757575758, "ece_hard": 0.18128667080576769, "calibration": 69.17094069564074, "p50_s": 0.8988664388656616, "p95_s": 2.503174464404583, "cost_usd_per_1000": 0.00970268933313877, "axes": { "intelligence": 58.42196363029698, "calibration": 69.17094069564074, "speed": 76.47813714800125, "cost": 70.39323621738546 }, "jevbench_score": 68.29548855544245 }, "heldout": { "n": 303, "escalation": 0.33003300330033003, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "judge": 0.952054794520548, "hard": 0.5045871559633027 }, "accuracy": 0.7524752475247525, "ece_hard": 0.09324954995175026, "calibration": 76.0083861031421, "p50_s": 1.3846236795186997, "p95_s": 2.564928230047226, "cost_usd_per_1000": 0.01128038231913079, "axes": { "intelligence": 69.21181559214111, "calibration": 76.0083861031421, "speed": 74.49593027949683, "cost": 68.43028542474235 }, "jevbench_score": 71.96240400259745 } }, { "threshold": 0.45, "public": { "n": 231, "escalation": 0.27705627705627706, "tiers": { "easy": 0.875, "standard": 0.6111111111111112, "hard": 0.43243243243243246 }, "accuracy": 0.5800865800865801, "ece_hard": 0.17951557334230506, "calibration": 69.348050441987, "p50_s": 0.935780817270279, "p95_s": 2.5111605182290075, "cost_usd_per_1000": 0.010091340532127048, "axes": { "intelligence": 58.79733900567235, "calibration": 69.348050441987, "speed": 76.28951387186783, "cost": 69.88153414955725 }, "jevbench_score": 68.28177490575554 }, "heldout": { "n": 303, "escalation": 0.35313531353135313, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "judge": 0.952054794520548, "hard": 0.5045871559633027 }, "accuracy": 0.7524752475247525, "ece_hard": 0.11529096099403194, "calibration": 75.00342972713813, "p50_s": 1.4158916115760802, "p95_s": 2.572696069478988, "cost_usd_per_1000": 0.011798904587087922, "axes": { "intelligence": 69.21181559214111, "calibration": 75.00342972713813, "speed": 74.38581508199574, "cost": 67.84474932454324 }, "jevbench_score": 71.54296473963682 } }, { "threshold": 0.46, "public": { "n": 231, "escalation": 0.30735930735930733, "tiers": { "easy": 0.875, "standard": 0.6111111111111112, "hard": 0.4594594594594595 }, "accuracy": 0.5930735930735931, "ece_hard": 0.16253238967033348, "calibration": 71.04636880918414, "p50_s": 0.9484117716550827, "p95_s": 2.5690802589058874, "cost_usd_per_1000": 0.010771480130356535, "axes": { "intelligence": 59.92346513179848, "calibration": 71.04636880918414, "speed": 76.13225392232206, "cost": 69.03173846017127 }, "jevbench_score": 68.77634237947707 }, "heldout": { "n": 303, "escalation": 0.3795379537953795, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "judge": 0.952054794520548, "hard": 0.4954128440366973 }, "accuracy": 0.7491749174917491, "ece_hard": 0.1212672088187505, "calibration": 74.40580494466627, "p50_s": 1.420183914899826, "p95_s": 2.6179904133081426, "cost_usd_per_1000": 0.012391501464753214, "axes": { "intelligence": 68.93658623434294, "calibration": 74.40580494466627, "speed": 74.29687358612638, "cost": 67.20628202336769 }, "jevbench_score": 71.13924478787256 } }, { "threshold": 0.47, "public": { "n": 231, "escalation": 0.3203463203463203, "tiers": { "easy": 0.875, "standard": 0.6111111111111112, "hard": 0.4774774774774775 }, "accuracy": 0.6017316017316018, "ece_hard": 0.14737776474181222, "calibration": 72.56183130203628, "p50_s": 0.9526323765516281, "p95_s": 2.587468113005161, "cost_usd_per_1000": 0.011062968529597742, "axes": { "intelligence": 60.67421588254922, "calibration": 72.56183130203628, "speed": 76.08199656050346, "cost": 68.68384969083021 }, "jevbench_score": 69.25677020739312 }, "heldout": { "n": 303, "escalation": 0.38943894389438943, "tiers": { "easy": 0.875, "standard": 0.5833333333333334, "judge": 0.952054794520548, "hard": 0.5045871559633027 }, "accuracy": 0.7557755775577558, "ece_hard": 0.11600526496182384, "calibration": 74.93199933035893, "p50_s": 1.420183914899826, "p95_s": 2.6283281639218323, "cost_usd_per_1000": 0.012613725293877703, "axes": { "intelligence": 70.37848225880778, "calibration": 74.93199933035893, "speed": 74.27975821735308, "cost": 66.97469894370613 }, "jevbench_score": 71.56853333458064 } }, { "threshold": 0.48, "public": { "n": 231, "escalation": 0.354978354978355, "tiers": { "easy": 0.875, "standard": 0.6111111111111112, "hard": 0.5135135135135135 }, "accuracy": 0.6190476190476191, "ece_hard": 0.1254240166949213, "calibration": 74.75720610672536, "p50_s": 1.000770527124405, "p95_s": 2.6075733706355093, "cost_usd_per_1000": 0.011840270927574299, "axes": { "intelligence": 62.17571738405072, "calibration": 74.75720610672536, "speed": 75.83428955940357, "cost": 67.7991508008482 }, "jevbench_score": 69.91831218288648 }, "heldout": { "n": 303, "escalation": 0.41254125412541254, "tiers": { "easy": 0.875, "standard": 0.625, "judge": 0.952054794520548, "hard": 0.5229357798165137 }, "accuracy": 0.7656765676567657, "ece_hard": 0.10898617014489505, "calibration": 75.63390881205181, "p50_s": 1.4251453012228013, "p95_s": 2.6283281639218323, "cost_usd_per_1000": 0.013132247561834835, "axes": { "intelligence": 72.09560764107077, "calibration": 75.63390881205181, "speed": 74.26461265968456, "cost": 66.44982816361738 }, "jevbench_score": 72.02367603220388 } }, { "threshold": 0.49, "public": { "n": 231, "escalation": 0.36363636363636365, "tiers": { "easy": 0.875, "standard": 0.6111111111111112, "hard": 0.5135135135135135 }, "accuracy": 0.6190476190476191, "ece_hard": 0.12452053832131407, "calibration": 74.84755394408609, "p50_s": 1.0029189229011535, "p95_s": 2.6227237239480017, "cost_usd_per_1000": 0.012034596527068436, "axes": { "intelligence": 62.17571738405072, "calibration": 74.84755394408609, "speed": 75.79981629722401, "cost": 67.58705396018553 }, "jevbench_score": 69.87672169105389 }, "heldout": { "n": 303, "escalation": 0.42244224422442245, "tiers": { "easy": 0.875, "standard": 0.625, "judge": 0.952054794520548, "hard": 0.5229357798165137 }, "accuracy": 0.7656765676567657, "ece_hard": 0.11181060494067484, "calibration": 75.35146533247382, "p50_s": 1.4251453012228013, "p95_s": 2.65018840432167, "cost_usd_per_1000": 0.01335447139095932, "axes": { "intelligence": 72.09560764107077, "calibration": 75.35146533247382, "speed": 74.22864105314672, "cost": 66.23119893845 }, "jevbench_score": 71.88837369898054 } }, { "threshold": 0.5, "public": { "n": 231, "escalation": 0.36363636363636365, "tiers": { "easy": 0.875, "standard": 0.6111111111111112, "hard": 0.5135135135135135 }, "accuracy": 0.6190476190476191, "ece_hard": 0.12452053832131407, "calibration": 74.84755394408609, "p50_s": 1.0029189229011535, "p95_s": 2.6227237239480017, "cost_usd_per_1000": 0.012034596527068436, "axes": { "intelligence": 62.17571738405072, "calibration": 74.84755394408609, "speed": 75.79981629722401, "cost": 67.58705396018553 }, "jevbench_score": 69.87672169105389 }, "heldout": { "n": 303, "escalation": 0.43234323432343236, "tiers": { "easy": 0.875, "standard": 0.625, "judge": 0.952054794520548, "hard": 0.5229357798165137 }, "accuracy": 0.7656765676567657, "ece_hard": 0.1181754697113737, "calibration": 74.71497885540394, "p50_s": 1.4268825143575667, "p95_s": 2.656192147731781, "cost_usd_per_1000": 0.013576695220083805, "axes": { "intelligence": 72.09560764107077, "calibration": 74.71497885540394, "speed": 74.21352295547283, "cost": 66.01617793778084 }, "jevbench_score": 71.6741378100223 } }, { "threshold": 0.51, "public": { "n": 231, "escalation": 0.38095238095238093, "tiers": { "easy": 0.875, "standard": 0.6111111111111112, "hard": 0.5135135135135135 }, "accuracy": 0.6190476190476191, "ece_hard": 0.13030410263361925, "calibration": 74.26919751285558, "p50_s": 1.0093481719493866, "p95_s": 2.6447725400328634, "cost_usd_per_1000": 0.012423247726056713, "axes": { "intelligence": 62.17571738405072, "calibration": 74.26919751285558, "speed": 75.73570672113247, "cost": 67.17294563897562 }, "jevbench_score": 69.61954056350083 }, "heldout": { "n": 303, "escalation": 0.45544554455445546, "tiers": { "easy": 0.875, "standard": 0.625, "judge": 0.952054794520548, "hard": 0.5229357798165137 }, "accuracy": 0.7656765676567657, "ece_hard": 0.12307394915062195, "calibration": 73.40750864670284, "p50_s": 1.4403502821922303, "p95_s": 2.656192147731781, "cost_usd_per_1000": 0.014095217488040938, "axes": { "intelligence": 72.09560764107077, "calibration": 73.40750864670284, "speed": 74.1727238946002, "cost": 65.52784655838148 }, "jevbench_score": 71.21637338074932 } }, { "threshold": 0.52, "public": { "n": 231, "escalation": 0.4025974025974026, "tiers": { "easy": 0.8958333333333334, "standard": 0.6111111111111112, "hard": 0.5135135135135135 }, "accuracy": 0.6233766233766234, "ece_hard": 0.1340163403645818, "calibration": 73.21156034096936, "p50_s": 1.0194248735904694, "p95_s": 2.6739173635840414, "cost_usd_per_1000": 0.012909061724792063, "axes": { "intelligence": 62.58080997664332, "calibration": 73.21156034096936, "speed": 75.64496792568207, "cost": 66.67315967829809 }, "jevbench_score": 69.33267886847811 }, "heldout": { "n": 303, "escalation": 0.46864686468646866, "tiers": { "easy": 0.875, "standard": 0.625, "judge": 0.952054794520548, "hard": 0.5412844036697247 }, "accuracy": 0.7722772277227723, "ece_hard": 0.10881465421742245, "calibration": 75.22041034992591, "p50_s": 1.4444589495658875, "p95_s": 2.6681172206997865, "cost_usd_per_1000": 0.014391515926873585, "axes": { "intelligence": 72.64606635666709, "calibration": 75.22041034992591, "speed": 74.1408989006762, "cost": 65.25680372046837 }, "jevbench_score": 71.70639047060087 } }, { "threshold": 0.53, "public": { "n": 231, "escalation": 0.42424242424242425, "tiers": { "easy": 0.8958333333333334, "standard": 0.6111111111111112, "hard": 0.5405405405405406 }, "accuracy": 0.6363636363636364, "ece_hard": 0.11533256306317706, "calibration": 75.07993807110981, "p50_s": 1.019511866569519, "p95_s": 2.6739173635840414, "cost_usd_per_1000": 0.013394875723527409, "axes": { "intelligence": 63.706936102769454, "calibration": 75.07993807110981, "speed": 75.64459733477511, "cost": 66.1918393414865 }, "jevbench_score": 69.95572266425629 }, "heldout": { "n": 303, "escalation": 0.4884488448844885, "tiers": { "easy": 0.875, "standard": 0.625, "judge": 0.952054794520548, "hard": 0.5412844036697247 }, "accuracy": 0.7722772277227723, "ece_hard": 0.12506803226982283, "calibration": 73.59507254468586, "p50_s": 1.4600932538509368, "p95_s": 2.6681172206997865, "cost_usd_per_1000": 0.014835963585122555, "axes": { "intelligence": 72.64606635666709, "calibration": 73.59507254468586, "speed": 74.09414499728375, "cost": 64.860527239542 }, "jevbench_score": 71.19611593513888 } }, { "threshold": 0.54, "public": { "n": 231, "escalation": 0.4458874458874459, "tiers": { "easy": 0.8958333333333334, "standard": 0.6111111111111112, "hard": 0.5405405405405406 }, "accuracy": 0.6363636363636364, "ece_hard": 0.12286899311234808, "calibration": 74.32629506619271, "p50_s": 1.0325586676597596, "p95_s": 2.702420528233051, "cost_usd_per_1000": 0.013880689722262755, "axes": { "intelligence": 63.706936102769454, "calibration": 74.32629506619271, "speed": 75.54332332091151, "cost": 65.72766860621027 }, "jevbench_score": 69.63352482679244 }, "heldout": { "n": 303, "escalation": 0.5082508250825083, "tiers": { "easy": 0.875, "standard": 0.625, "judge": 0.952054794520548, "hard": 0.5596330275229358 }, "accuracy": 0.7788778877887789, "ece_hard": 0.12918275257255363, "calibration": 73.18360051441279, "p50_s": 1.4737490206956863, "p95_s": 2.6681172206997865, "cost_usd_per_1000": 0.015280411243371528, "axes": { "intelligence": 73.19652507226341, "calibration": 73.18360051441279, "speed": 74.05371564586407, "cost": 64.47594872165097 }, "jevbench_score": 71.1151626837017 } }, { "threshold": 0.55, "public": { "n": 231, "escalation": 0.4675324675324675, "tiers": { "easy": 0.8958333333333334, "standard": 0.6111111111111112, "hard": 0.5315315315315315 }, "accuracy": 0.6320346320346321, "ece_hard": 0.13042424430043517, "calibration": 73.57076994738401, "p50_s": 1.0435183227062226, "p95_s": 2.702420528233051, "cost_usd_per_1000": 0.0143665037209981, "axes": { "intelligence": 63.331560727394084, "calibration": 73.57076994738401, "speed": 75.49746989690286, "cost": 65.27946730968074 }, "jevbench_score": 69.22430797302005 }, "heldout": { "n": 303, "escalation": 0.528052805280528, "tiers": { "easy": 0.875, "standard": 0.7083333333333334, "judge": 0.9452054794520548, "hard": 0.5412844036697247 }, "accuracy": 0.7755775577557755, "ece_hard": 0.13968519404921778, "calibration": 70.84754090883929, "p50_s": 1.481574121117592, "p95_s": 2.674653121829033, "cost_usd_per_1000": 0.0157248589016205, "axes": { "intelligence": 74.7876188680826, "calibration": 70.84754090883929, "speed": 74.02009150517216, "cost": 64.10239728304211 }, "jevbench_score": 70.8099999311269 } }, { "threshold": 0.56, "public": { "n": 231, "escalation": 0.48484848484848486, "tiers": { "easy": 0.8958333333333334, "standard": 0.6111111111111112, "hard": 0.5315315315315315 }, "accuracy": 0.6320346320346321, "ece_hard": 0.13862527458179288, "calibration": 72.75066691924823, "p50_s": 1.0443254739046097, "p95_s": 2.7424878135323523, "cost_usd_per_1000": 0.014755154919986378, "axes": { "intelligence": 63.331560727394084, "calibration": 72.75066691924823, "speed": 75.43019421829214, "cost": 64.93168678951162 }, "jevbench_score": 68.92309483035781 }, "heldout": { "n": 303, "escalation": 0.5445544554455446, "tiers": { "easy": 0.875, "standard": 0.7083333333333334, "judge": 0.9452054794520548, "hard": 0.5596330275229358 }, "accuracy": 0.7821782178217822, "ece_hard": 0.13539045589221751, "calibration": 71.27701472453931, "p50_s": 1.4903221815824508, "p95_s": 2.674653121829033, "cost_usd_per_1000": 0.01609523195016131, "axes": { "intelligence": 75.33807758367894, "calibration": 71.27701472453931, "speed": 73.99452368601915, "cost": 63.79908280802898 }, "jevbench_score": 70.95688022574511 } }, { "threshold": 0.57, "public": { "n": 231, "escalation": 0.5108225108225108, "tiers": { "easy": 0.8958333333333334, "standard": 0.6111111111111112, "hard": 0.5315315315315315 }, "accuracy": 0.6320346320346321, "ece_hard": 0.14373703252857298, "calibration": 72.23949112457024, "p50_s": 1.0495290338993073, "p95_s": 2.7424878135323523, "cost_usd_per_1000": 0.015338131718468796, "axes": { "intelligence": 63.331560727394084, "calibration": 72.23949112457024, "speed": 75.40860836277173, "cost": 64.42682610947328 }, "jevbench_score": 68.66266141536647 }, "heldout": { "n": 303, "escalation": 0.5742574257425742, "tiers": { "easy": 0.875, "standard": 0.7083333333333334, "judge": 0.9452054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7788778877887789, "ece_hard": 0.14739322908258554, "calibration": 70.07673740550251, "p50_s": 1.4949380844831466, "p95_s": 2.7296174198389047, "cost_usd_per_1000": 0.016761903437534764, "axes": { "intelligence": 75.06284822588077, "calibration": 70.07673740550251, "speed": 73.89275012973206, "cost": 63.270299967091155 }, "jevbench_score": 70.42067106857121 } }, { "threshold": 0.58, "public": { "n": 231, "escalation": 0.5238095238095238, "tiers": { "easy": 0.8958333333333334, "standard": 0.6111111111111112, "hard": 0.5225225225225225 }, "accuracy": 0.6277056277056277, "ece_hard": 0.15638168954753112, "calibration": 70.12410822733281, "p50_s": 1.0495290338993073, "p95_s": 2.786542299389839, "cost_usd_per_1000": 0.015629620117710005, "axes": { "intelligence": 62.95618535201869, "calibration": 70.12410822733281, "speed": 75.33939902723094, "cost": 64.18154732504291 }, "jevbench_score": 67.97270025067252 }, "heldout": { "n": 303, "escalation": 0.5841584158415841, "tiers": { "easy": 0.875, "standard": 0.7083333333333334, "judge": 0.9452054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7788778877887789, "ece_hard": 0.15122672033858384, "calibration": 69.69338827990268, "p50_s": 1.4949380844831466, "p95_s": 2.743682792782783, "cost_usd_per_1000": 0.01698412726665925, "axes": { "intelligence": 75.06284822588077, "calibration": 69.69338827990268, "speed": 73.87042894483457, "cost": 63.0987029363155 }, "jevbench_score": 70.27112685292668 } }, { "threshold": 0.59, "public": { "n": 231, "escalation": 0.5584415584415584, "tiers": { "easy": 0.9166666666666666, "standard": 0.6111111111111112, "hard": 0.5135135135135135 }, "accuracy": 0.6277056277056277, "ece_hard": 0.1702485605817337, "calibration": 67.03060394513267, "p50_s": 1.050069946050644, "p95_s": 2.8044095143675802, "cost_usd_per_1000": 0.016406922515686557, "axes": { "intelligence": 62.9859025692359, "calibration": 67.03060394513267, "speed": 75.30940339392758, "cost": 63.54918618777843 }, "jevbench_score": 67.04539016715088 }, "heldout": { "n": 303, "escalation": 0.6171617161716172, "tiers": { "easy": 0.875, "standard": 0.7083333333333334, "judge": 0.958904109589041, "hard": 0.5412844036697247 }, "accuracy": 0.7821782178217822, "ece_hard": 0.17133980129800483, "calibration": 67.68208018396058, "p50_s": 1.5264378130435943, "p95_s": 2.8136056095361703, "cost_usd_per_1000": 0.017724873363740866, "axes": { "intelligence": 75.17118051191822, "calibration": 67.68208018396058, "speed": 73.67057662957465, "cost": 62.5425057691847 }, "jevbench_score": 69.58227176997103 } }, { "threshold": 0.6, "public": { "n": 231, "escalation": 0.5670995670995671, "tiers": { "easy": 0.9166666666666666, "standard": 0.625, "hard": 0.5225225225225225 }, "accuracy": 0.6363636363636364, "ece_hard": 0.16097292518725306, "calibration": 67.95816748458074, "p50_s": 1.050069946050644, "p95_s": 2.8044095143675802, "cost_usd_per_1000": 0.016601248115180697, "axes": { "intelligence": 63.9014014014014, "calibration": 67.95816748458074, "speed": 75.30940339392758, "cost": 63.395777788028965 }, "jevbench_score": 67.47849626408487 }, "heldout": { "n": 303, "escalation": 0.6303630363036303, "tiers": { "easy": 0.875, "standard": 0.7083333333333334, "judge": 0.958904109589041, "hard": 0.5229357798165137 }, "accuracy": 0.7755775577557755, "ece_hard": 0.1922077185760751, "calibration": 66.40335970138702, "p50_s": 1.5387233823537827, "p95_s": 2.8136056095361703, "cost_usd_per_1000": 0.018021171802573514, "axes": { "intelligence": 74.62072179632189, "calibration": 66.40335970138702, "speed": 73.63576225479495, "cost": 62.32650919064371 }, "jevbench_score": 69.05621523359095 } }, { "threshold": 0.61, "public": { "n": 231, "escalation": 0.5887445887445888, "tiers": { "easy": 0.9166666666666666, "standard": 0.625, "hard": 0.5315315315315315 }, "accuracy": 0.6406926406926406, "ece_hard": 0.1567238524944534, "calibration": 68.38307475386071, "p50_s": 1.050069946050644, "p95_s": 2.8044095143675802, "cost_usd_per_1000": 0.017087062113916047, "axes": { "intelligence": 64.27677677677679, "calibration": 68.38307475386071, "speed": 75.30940339392758, "cost": 63.019978055292796 }, "jevbench_score": 67.5822337934907 }, "heldout": { "n": 303, "escalation": 0.66996699669967, "tiers": { "easy": 0.875, "standard": 0.7083333333333334, "judge": 0.958904109589041, "hard": 0.5229357798165137 }, "accuracy": 0.7755775577557755, "ece_hard": 0.18953466976129374, "calibration": 66.67066458286516, "p50_s": 1.5736957371234892, "p95_s": 2.845012076199054, "cost_usd_per_1000": 0.018910067119071457, "axes": { "intelligence": 74.62072179632189, "calibration": 66.67066458286516, "speed": 73.48995117428329, "cost": 61.69920789024273 }, "jevbench_score": 68.91685404253379 } }, { "threshold": 0.62, "public": { "n": 231, "escalation": 0.6017316017316018, "tiers": { "easy": 0.9375, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6493506493506493, "ece_hard": 0.14878020087038468, "calibration": 69.17743991626759, "p50_s": 1.0534505426883698, "p95_s": 2.8044095143675802, "cost_usd_per_1000": 0.017378550513157256, "axes": { "intelligence": 65.05724474474475, "calibration": 69.17743991626759, "speed": 75.2954441695019, "cost": 62.799593482861795 }, "jevbench_score": 67.91980354530952 }, "heldout": { "n": 303, "escalation": 0.7227722772277227, "tiers": { "easy": 0.9166666666666666, "standard": 0.7083333333333334, "judge": 0.9452054794520548, "hard": 0.5229357798165137 }, "accuracy": 0.7722772277227723, "ece_hard": 0.18953466976129374, "calibration": 66.67066458286516, "p50_s": 1.6521354943513868, "p95_s": 2.845012076199054, "cost_usd_per_1000": 0.020095260874402044, "axes": { "intelligence": 74.82049348581961, "calibration": 66.67066458286516, "speed": 73.27870224168542, "cost": 60.90719054420635 }, "jevbench_score": 68.69109228555114 } }, { "threshold": 0.63, "public": { "n": 231, "escalation": 0.6190476190476191, "tiers": { "easy": 0.9375, "standard": 0.6527777777777778, "hard": 0.5495495495495496 }, "accuracy": 0.6623376623376623, "ece_hard": 0.14037276907560514, "calibration": 71.12189651415969, "p50_s": 1.0553882002830506, "p95_s": 2.8135541334748266, "cost_usd_per_1000": 0.01776720171214553, "axes": { "intelligence": 66.51286703370037, "calibration": 71.12189651415969, "speed": 75.27332490237046, "cost": 62.51142901165537 }, "jevbench_score": 68.6874480045509 }, "heldout": { "n": 303, "escalation": 0.768976897689769, "tiers": { "easy": 0.9583333333333334, "standard": 0.7083333333333334, "judge": 0.9452054794520548, "hard": 0.5137614678899083 }, "accuracy": 0.7722772277227723, "ece_hard": 0.2041298532446619, "calibration": 65.21114623452834, "p50_s": 1.7052855372428892, "p95_s": 2.845012076199054, "cost_usd_per_1000": 0.021132305410316313, "axes": { "intelligence": 75.12859746135479, "calibration": 65.21114623452834, "speed": 73.14118777055506, "cost": 60.25159364163043 }, "jevbench_score": 68.16546718942796 } }, { "threshold": 0.64, "public": { "n": 231, "escalation": 0.6536796536796536, "tiers": { "easy": 0.9375, "standard": 0.6944444444444444, "hard": 0.5585585585585585 }, "accuracy": 0.6796536796536796, "ece_hard": 0.12999366652019767, "calibration": 72.15980676970042, "p50_s": 1.0909338027238846, "p95_s": 2.8135541334748266, "cost_usd_per_1000": 0.018544504110122085, "axes": { "intelligence": 68.50861277944611, "calibration": 72.15980676970042, "speed": 75.12946325864377, "cost": 61.95354326334833 }, "jevbench_score": 69.25948258786866 }, "heldout": { "n": 303, "escalation": 0.8085808580858086, "tiers": { "easy": 0.9583333333333334, "standard": 0.7083333333333334, "judge": 0.9452054794520548, "hard": 0.5137614678899083 }, "accuracy": 0.7722772277227723, "ece_hard": 0.20713094346487576, "calibration": 64.91103721250695, "p50_s": 1.7254607558250425, "p95_s": 2.845012076199054, "cost_usd_per_1000": 0.022021200726814252, "axes": { "intelligence": 75.12859746135479, "calibration": 64.91103721250695, "speed": 73.09010799747075, "cost": 59.714770132027944 }, "jevbench_score": 67.92287334808184 } }, { "threshold": 0.65, "public": { "n": 231, "escalation": 0.6926406926406926, "tiers": { "easy": 0.9583333333333334, "standard": 0.75, "hard": 0.5675675675675675 }, "accuracy": 0.7056277056277056, "ece_hard": 0.13223539618884675, "calibration": 71.93563380283553, "p50_s": 1.133615779876709, "p95_s": 2.8135541334748266, "cost_usd_per_1000": 0.01941896930784571, "axes": { "intelligence": 71.44957457457458, "calibration": 71.93563380283553, "speed": 74.96278841857136, "cost": 61.35321474024636 }, "jevbench_score": 69.72791746198942 }, "heldout": { "n": 303, "escalation": 0.8283828382838284, "tiers": { "easy": 0.9583333333333334, "standard": 0.7083333333333334, "judge": 0.9452054794520548, "hard": 0.5137614678899083 }, "accuracy": 0.7722772277227723, "ece_hard": 0.20713094346487576, "calibration": 64.91103721250695, "p50_s": 1.7254607558250425, "p95_s": 2.845012076199054, "cost_usd_per_1000": 0.022465648385063226, "axes": { "intelligence": 75.12859746135479, "calibration": 64.91103721250695, "speed": 73.09010799747075, "cost": 59.45443127919982 }, "jevbench_score": 67.8487210641147 } }, { "threshold": 0.66, "public": { "n": 231, "escalation": 0.7056277056277056, "tiers": { "easy": 0.9583333333333334, "standard": 0.75, "hard": 0.5765765765765766 }, "accuracy": 0.70995670995671, "ece_hard": 0.12613888742069376, "calibration": 72.54528467965082, "p50_s": 1.1354727476835251, "p95_s": 2.8135541334748266, "cost_usd_per_1000": 0.019710457707086917, "axes": { "intelligence": 71.82494994994995, "calibration": 72.54528467965082, "speed": 74.9556800925469, "cost": 61.15909871856621 }, "jevbench_score": 69.90971582243851 }, "heldout": { "n": 303, "escalation": 0.8646864686468647, "tiers": { "easy": 1.0, "standard": 0.75, "judge": 0.952054794520548, "hard": 0.5137614678899083 }, "accuracy": 0.7821782178217822, "ece_hard": 0.21109516818279833, "calibration": 63.09432251995037, "p50_s": 1.7585037529468535, "p95_s": 2.845012076199054, "cost_usd_per_1000": 0.023280469091853006, "axes": { "intelligence": 77.0703782832726, "calibration": 63.09432251995037, "speed": 73.00772586306402, "cost": 58.99024819237739 }, "jevbench_score": 67.6482714608102 } }, { "threshold": 0.67, "public": { "n": 231, "escalation": 0.7272727272727273, "tiers": { "easy": 0.9791666666666666, "standard": 0.75, "hard": 0.5765765765765766 }, "accuracy": 0.7142857142857143, "ece_hard": 0.12738486996150838, "calibration": 72.42068642556936, "p50_s": 1.13800046145916, "p95_s": 2.822939346730709, "cost_usd_per_1000": 0.020196271705822267, "axes": { "intelligence": 72.23004254254253, "calibration": 72.42068642556936, "speed": 74.931560147762, "cost": 60.84186385773747 }, "jevbench_score": 69.88145649888784 }, "heldout": { "n": 303, "escalation": 0.8811881188118812, "tiers": { "easy": 1.0, "standard": 0.75, "judge": 0.952054794520548, "hard": 0.5321100917431193 }, "accuracy": 0.7887788778877888, "ece_hard": 0.19260504523128988, "calibration": 64.94333481510122, "p50_s": 1.7585037529468535, "p95_s": 2.845012076199054, "cost_usd_per_1000": 0.023650842140393813, "axes": { "intelligence": 77.62083699886892, "calibration": 64.94333481510122, "speed": 73.00772586306402, "cost": 58.78460171914089 }, "jevbench_score": 68.20030658957857 } }, { "threshold": 0.68, "public": { "n": 231, "escalation": 0.7402597402597403, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.5765765765765766 }, "accuracy": 0.7229437229437229, "ece_hard": 0.12738486996150838, "calibration": 72.42068642556936, "p50_s": 1.1484469056129456, "p95_s": 2.822939346730709, "cost_usd_per_1000": 0.020487760105063476, "axes": { "intelligence": 73.17525859192527, "calibration": 72.42068642556936, "speed": 74.89187531018243, "cost": 60.65516559230525 }, "jevbench_score": 70.04584039884124 }, "heldout": { "n": 303, "escalation": 0.900990099009901, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7986798679867987, "ece_hard": 0.18104412892996566, "calibration": 66.09942644523363, "p50_s": 1.7629361689090728, "p95_s": 2.845012076199054, "cost_usd_per_1000": 0.024095289798642786, "axes": { "intelligence": 79.33796238113192, "calibration": 66.09942644523363, "speed": 72.99679297848728, "cost": 58.542035379665634 }, "jevbench_score": 68.80382505456382 } }, { "threshold": 0.69, "public": { "n": 231, "escalation": 0.7619047619047619, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.5855855855855856 }, "accuracy": 0.7272727272727273, "ece_hard": 0.1338310720757613, "calibration": 71.77606621414407, "p50_s": 1.1846259266138077, "p95_s": 2.822939346730709, "cost_usd_per_1000": 0.020973574103798822, "axes": { "intelligence": 73.55063396730064, "calibration": 71.77606621414407, "speed": 74.75717219603763, "cost": 60.34982665500418 }, "jevbench_score": 69.85922144740114 }, "heldout": { "n": 303, "escalation": 0.9075907590759076, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7986798679867987, "ece_hard": 0.17289245962103897, "calibration": 66.9145933761263, "p50_s": 1.7672429561614988, "p95_s": 2.845012076199054, "cost_usd_per_1000": 0.02424343901805911, "axes": { "intelligence": 79.33796238113192, "calibration": 66.9145933761263, "speed": 72.98619626483534, "cost": 58.462173217562665 }, "jevbench_score": 68.98892740964862 } }, { "threshold": 0.7, "public": { "n": 231, "escalation": 0.7965367965367965, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "hard": 0.5855855855855856 }, "accuracy": 0.7359307359307359, "ece_hard": 0.1489936018519762, "calibration": 70.25981323652258, "p50_s": 1.1955331593751908, "p95_s": 2.822939346730709, "cost_usd_per_1000": 0.021750876501775377, "axes": { "intelligence": 74.63088088088088, "calibration": 70.25981323652258, "speed": 74.71736826706581, "cost": 59.875697123743024 }, "jevbench_score": 69.5944211634915 }, "heldout": { "n": 303, "escalation": 0.9141914191419142, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7986798679867987, "ece_hard": 0.17289245962103897, "calibration": 66.9145933761263, "p50_s": 1.7672429561614988, "p95_s": 2.845012076199054, "cost_usd_per_1000": 0.02439158823747543, "axes": { "intelligence": 79.33796238113192, "calibration": 66.9145933761263, "speed": 72.98619626483534, "cost": 58.3827976015175 }, "jevbench_score": 68.96549846135484 } }, { "threshold": 0.71, "public": { "n": 231, "escalation": 0.8051948051948052, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.5855855855855856 }, "accuracy": 0.7402597402597403, "ece_hard": 0.1489936018519762, "calibration": 70.25981323652258, "p50_s": 1.1955331593751908, "p95_s": 2.822939346730709, "cost_usd_per_1000": 0.021945202101269518, "axes": { "intelligence": 75.17100433767102, "calibration": 70.25981323652258, "speed": 74.71736826706581, "cost": 59.759812456789 }, "jevbench_score": 69.6862402810031 }, "heldout": { "n": 303, "escalation": 0.9174917491749175, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7986798679867987, "ece_hard": 0.17183976335421564, "calibration": 67.01986300280865, "p50_s": 1.7672429561614988, "p95_s": 2.845012076199054, "cost_usd_per_1000": 0.024465662847183593, "axes": { "intelligence": 79.33796238113192, "calibration": 67.01986300280865, "speed": 72.98619626483534, "cost": 58.34329040267063 }, "jevbench_score": 68.98093184165116 } }, { "threshold": 0.72, "public": { "n": 231, "escalation": 0.8225108225108225, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.5945945945945946 }, "accuracy": 0.7445887445887446, "ece_hard": 0.14091530937447033, "calibration": 71.06764248427316, "p50_s": 1.2058723896741868, "p95_s": 2.822939346730709, "cost_usd_per_1000": 0.022333853300257795, "axes": { "intelligence": 75.54637971304639, "calibration": 71.06764248427316, "speed": 74.67997101516198, "cost": 59.53109022545522 }, "jevbench_score": 69.89697579832769 }, "heldout": { "n": 303, "escalation": 0.9207920792079208, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7986798679867987, "ece_hard": 0.16987858950045584, "calibration": 67.21598038818462, "p50_s": 1.7672429561614988, "p95_s": 2.845012076199054, "cost_usd_per_1000": 0.024539737456891756, "axes": { "intelligence": 79.33796238113192, "calibration": 67.21598038818462, "speed": 72.98619626483534, "cost": 58.30390263901996 }, "jevbench_score": 69.01968677715462 } }, { "threshold": 0.73, "public": { "n": 231, "escalation": 0.8225108225108225, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.5945945945945946 }, "accuracy": 0.7445887445887446, "ece_hard": 0.14091530937447033, "calibration": 71.06764248427316, "p50_s": 1.2058723896741868, "p95_s": 2.822939346730709, "cost_usd_per_1000": 0.022333853300257795, "axes": { "intelligence": 75.54637971304639, "calibration": 71.06764248427316, "speed": 74.67997101516198, "cost": 59.53109022545522 }, "jevbench_score": 69.89697579832769 }, "heldout": { "n": 303, "escalation": 0.9273927392739274, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7986798679867987, "ece_hard": 0.17155720063770452, "calibration": 67.04811927445975, "p50_s": 1.7672429561614988, "p95_s": 2.845012076199054, "cost_usd_per_1000": 0.024687886676308078, "axes": { "intelligence": 79.33796238113192, "calibration": 67.04811927445975, "speed": 72.98619626483534, "cost": 58.2254825439613 }, "jevbench_score": 68.95334949545214 } }, { "threshold": 0.74, "public": { "n": 231, "escalation": 0.8268398268398268, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.5945945945945946 }, "accuracy": 0.7445887445887446, "ece_hard": 0.14091530937447033, "calibration": 71.06764248427316, "p50_s": 1.2058723896741868, "p95_s": 2.822939346730709, "cost_usd_per_1000": 0.022431016100004864, "axes": { "intelligence": 75.54637971304639, "calibration": 71.06764248427316, "speed": 74.67997101516198, "cost": 59.47453158747736 }, "jevbench_score": 69.88036814435482 }, "heldout": { "n": 303, "escalation": 0.9273927392739274, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7986798679867987, "ece_hard": 0.17155720063770452, "calibration": 67.04811927445975, "p50_s": 1.7672429561614988, "p95_s": 2.845012076199054, "cost_usd_per_1000": 0.024687886676308078, "axes": { "intelligence": 79.33796238113192, "calibration": 67.04811927445975, "speed": 72.98619626483534, "cost": 58.2254825439613 }, "jevbench_score": 68.95334949545214 } }, { "threshold": 0.75, "public": { "n": 231, "escalation": 0.8311688311688312, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.6036036036036037 }, "accuracy": 0.7489177489177489, "ece_hard": 0.12914387221294593, "calibration": 72.24478620042561, "p50_s": 1.2058723896741868, "p95_s": 2.822939346730709, "cost_usd_per_1000": 0.022528178899751932, "axes": { "intelligence": 75.92175508842178, "calibration": 72.24478620042561, "speed": 74.67997101516198, "cost": 59.41821741173971 }, "jevbench_score": 70.23832191160102 }, "heldout": { "n": 303, "escalation": 0.933993399339934, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7986798679867987, "ece_hard": 0.16834264824186954, "calibration": 67.36957451404325, "p50_s": 1.7672429561614988, "p95_s": 2.845012076199054, "cost_usd_per_1000": 0.0248360358957244, "axes": { "intelligence": 79.33796238113192, "calibration": 67.36957451404325, "speed": 72.98619626483534, "cost": 58.147531634087755 }, "jevbench_score": 69.012731232157 } }, { "threshold": 0.76, "public": { "n": 231, "escalation": 0.8311688311688312, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.6036036036036037 }, "accuracy": 0.7489177489177489, "ece_hard": 0.12914387221294593, "calibration": 72.24478620042561, "p50_s": 1.2058723896741868, "p95_s": 2.822939346730709, "cost_usd_per_1000": 0.022528178899751932, "axes": { "intelligence": 75.92175508842178, "calibration": 72.24478620042561, "speed": 74.67997101516198, "cost": 59.41821741173971 }, "jevbench_score": 70.23832191160102 }, "heldout": { "n": 303, "escalation": 0.933993399339934, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7986798679867987, "ece_hard": 0.16834264824186954, "calibration": 67.36957451404325, "p50_s": 1.7672429561614988, "p95_s": 2.845012076199054, "cost_usd_per_1000": 0.0248360358957244, "axes": { "intelligence": 79.33796238113192, "calibration": 67.36957451404325, "speed": 72.98619626483534, "cost": 58.147531634087755 }, "jevbench_score": 69.012731232157 } }, { "threshold": 0.77, "public": { "n": 231, "escalation": 0.8441558441558441, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.6036036036036037 }, "accuracy": 0.7489177489177489, "ece_hard": 0.12914387221294593, "calibration": 72.24478620042561, "p50_s": 1.2058723896741868, "p95_s": 2.822939346730709, "cost_usd_per_1000": 0.022819667298993138, "axes": { "intelligence": 75.92175508842178, "calibration": 72.24478620042561, "speed": 74.67997101516198, "cost": 59.25072075099342 }, "jevbench_score": 70.18877001541033 }, "heldout": { "n": 303, "escalation": 0.9372937293729373, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5412844036697247 }, "accuracy": 0.7953795379537953, "ece_hard": 0.17751077643252838, "calibration": 66.45276169497737, "p50_s": 1.771337929368019, "p95_s": 2.855087894201278, "cost_usd_per_1000": 0.024910110505432563, "axes": { "intelligence": 79.06273302333375, "calibration": 66.45276169497737, "speed": 72.96079095201704, "cost": 58.108730375679436 }, "jevbench_score": 68.69955822511686 } }, { "threshold": 0.78, "public": { "n": 231, "escalation": 0.8441558441558441, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.6036036036036037 }, "accuracy": 0.7489177489177489, "ece_hard": 0.12914387221294593, "calibration": 72.24478620042561, "p50_s": 1.2058723896741868, "p95_s": 2.822939346730709, "cost_usd_per_1000": 0.022819667298993138, "axes": { "intelligence": 75.92175508842178, "calibration": 72.24478620042561, "speed": 74.67997101516198, "cost": 59.25072075099342 }, "jevbench_score": 70.18877001541033 }, "heldout": { "n": 303, "escalation": 0.9405940594059405, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5412844036697247 }, "accuracy": 0.7953795379537953, "ece_hard": 0.17751077643252838, "calibration": 66.45276169497737, "p50_s": 1.771337929368019, "p95_s": 2.855087894201278, "cost_usd_per_1000": 0.024984185115140726, "axes": { "intelligence": 79.06273302333375, "calibration": 66.45276169497737, "speed": 72.96079095201704, "cost": 58.0700443285299 }, "jevbench_score": 68.68812113938549 } }, { "threshold": 0.79, "public": { "n": 231, "escalation": 0.8441558441558441, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.6036036036036037 }, "accuracy": 0.7489177489177489, "ece_hard": 0.12914387221294593, "calibration": 72.24478620042561, "p50_s": 1.2058723896741868, "p95_s": 2.822939346730709, "cost_usd_per_1000": 0.022819667298993138, "axes": { "intelligence": 75.92175508842178, "calibration": 72.24478620042561, "speed": 74.67997101516198, "cost": 59.25072075099342 }, "jevbench_score": 70.18877001541033 }, "heldout": { "n": 303, "escalation": 0.9438943894389439, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5412844036697247 }, "accuracy": 0.7953795379537953, "ece_hard": 0.17751077643252838, "calibration": 66.45276169497737, "p50_s": 1.771337929368019, "p95_s": 2.855087894201278, "cost_usd_per_1000": 0.02505825972484889, "axes": { "intelligence": 79.06273302333375, "calibration": 66.45276169497737, "speed": 72.96079095201704, "cost": 58.03147281047841 }, "jevbench_score": 68.67671222229512 } }, { "threshold": 0.8, "public": { "n": 231, "escalation": 0.8484848484848485, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.6036036036036037 }, "accuracy": 0.7489177489177489, "ece_hard": 0.12914387221294593, "calibration": 72.24478620042561, "p50_s": 1.2058723896741868, "p95_s": 2.822939346730709, "cost_usd_per_1000": 0.02291683009874021, "axes": { "intelligence": 75.92175508842178, "calibration": 72.24478620042561, "speed": 74.67997101516198, "cost": 59.19536365065882 }, "jevbench_score": 70.17237017769011 }, "heldout": { "n": 303, "escalation": 0.9438943894389439, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5412844036697247 }, "accuracy": 0.7953795379537953, "ece_hard": 0.17751077643252838, "calibration": 66.45276169497737, "p50_s": 1.771337929368019, "p95_s": 2.855087894201278, "cost_usd_per_1000": 0.02505825972484889, "axes": { "intelligence": 79.06273302333375, "calibration": 66.45276169497737, "speed": 72.96079095201704, "cost": 58.03147281047841 }, "jevbench_score": 68.67671222229512 } }, { "threshold": 0.81, "public": { "n": 231, "escalation": 0.8484848484848485, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.6036036036036037 }, "accuracy": 0.7489177489177489, "ece_hard": 0.12914387221294593, "calibration": 72.24478620042561, "p50_s": 1.2058723896741868, "p95_s": 2.822939346730709, "cost_usd_per_1000": 0.02291683009874021, "axes": { "intelligence": 75.92175508842178, "calibration": 72.24478620042561, "speed": 74.67997101516198, "cost": 59.19536365065882 }, "jevbench_score": 70.17237017769011 }, "heldout": { "n": 303, "escalation": 0.9471947194719472, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5412844036697247 }, "accuracy": 0.7953795379537953, "ece_hard": 0.17751077643252838, "calibration": 66.45276169497737, "p50_s": 1.771337929368019, "p95_s": 2.855087894201278, "cost_usd_per_1000": 0.02513233433455705, "axes": { "intelligence": 79.06273302333375, "calibration": 66.45276169497737, "speed": 72.96079095201704, "cost": 57.99301514540493 }, "jevbench_score": 68.66533131774352 } }, { "threshold": 0.82, "public": { "n": 231, "escalation": 0.8484848484848485, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.6036036036036037 }, "accuracy": 0.7489177489177489, "ece_hard": 0.12914387221294593, "calibration": 72.24478620042561, "p50_s": 1.2058723896741868, "p95_s": 2.822939346730709, "cost_usd_per_1000": 0.02291683009874021, "axes": { "intelligence": 75.92175508842178, "calibration": 72.24478620042561, "speed": 74.67997101516198, "cost": 59.19536365065882 }, "jevbench_score": 70.17237017769011 }, "heldout": { "n": 303, "escalation": 0.9504950495049505, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5412844036697247 }, "accuracy": 0.7953795379537953, "ece_hard": 0.1787210719898652, "calibration": 66.33173213924368, "p50_s": 1.7966659694910048, "p95_s": 2.855087894201278, "cost_usd_per_1000": 0.02520640894426521, "axes": { "intelligence": 79.06273302333375, "calibration": 66.33173213924368, "speed": 72.89913175463856, "cost": 57.95467066315897 }, "jevbench_score": 68.60819438262894 } }, { "threshold": 0.83, "public": { "n": 231, "escalation": 0.8528138528138528, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.6036036036036037 }, "accuracy": 0.7489177489177489, "ece_hard": 0.12914387221294593, "calibration": 72.24478620042561, "p50_s": 1.2058723896741868, "p95_s": 2.822939346730709, "cost_usd_per_1000": 0.023013992898487278, "axes": { "intelligence": 75.92175508842178, "calibration": 72.24478620042561, "speed": 74.67997101516198, "cost": 59.140240757620504 }, "jevbench_score": 70.1560282916993 }, "heldout": { "n": 303, "escalation": 0.9504950495049505, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5412844036697247 }, "accuracy": 0.7953795379537953, "ece_hard": 0.1787210719898652, "calibration": 66.33173213924368, "p50_s": 1.7966659694910048, "p95_s": 2.855087894201278, "cost_usd_per_1000": 0.02520640894426521, "axes": { "intelligence": 79.06273302333375, "calibration": 66.33173213924368, "speed": 72.89913175463856, "cost": 57.95467066315897 }, "jevbench_score": 68.60819438262894 } }, { "threshold": 0.84, "public": { "n": 231, "escalation": 0.8571428571428571, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.6036036036036037 }, "accuracy": 0.7489177489177489, "ece_hard": 0.12914387221294593, "calibration": 72.24478620042561, "p50_s": 1.2058723896741868, "p95_s": 2.822939346730709, "cost_usd_per_1000": 0.023111155698234347, "axes": { "intelligence": 75.92175508842178, "calibration": 72.24478620042561, "speed": 74.67997101516198, "cost": 59.08535009843544 }, "jevbench_score": 70.13974389870233 }, "heldout": { "n": 303, "escalation": 0.9504950495049505, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5412844036697247 }, "accuracy": 0.7953795379537953, "ece_hard": 0.1787210719898652, "calibration": 66.33173213924368, "p50_s": 1.7966659694910048, "p95_s": 2.855087894201278, "cost_usd_per_1000": 0.02520640894426521, "axes": { "intelligence": 79.06273302333375, "calibration": 66.33173213924368, "speed": 72.89913175463856, "cost": 57.95467066315897 }, "jevbench_score": 68.60819438262894 } }, { "threshold": 0.85, "public": { "n": 231, "escalation": 0.8571428571428571, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.6036036036036037 }, "accuracy": 0.7489177489177489, "ece_hard": 0.12914387221294593, "calibration": 72.24478620042561, "p50_s": 1.2058723896741868, "p95_s": 2.822939346730709, "cost_usd_per_1000": 0.023111155698234347, "axes": { "intelligence": 75.92175508842178, "calibration": 72.24478620042561, "speed": 74.67997101516198, "cost": 59.08535009843544 }, "jevbench_score": 70.13974389870233 }, "heldout": { "n": 303, "escalation": 0.9537953795379538, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5412844036697247 }, "accuracy": 0.7953795379537953, "ece_hard": 0.1787210719898652, "calibration": 66.33173213924368, "p50_s": 1.7966659694910048, "p95_s": 2.855087894201278, "cost_usd_per_1000": 0.025280483553973374, "axes": { "intelligence": 79.06273302333375, "calibration": 66.33173213924368, "speed": 72.89913175463856, "cost": 57.916438699489554 }, "jevbench_score": 68.59687659276717 } }, { "threshold": 0.86, "public": { "n": 231, "escalation": 0.8658008658008658, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.6036036036036037 }, "accuracy": 0.7489177489177489, "ece_hard": 0.12914387221294593, "calibration": 72.24478620042561, "p50_s": 1.2058723896741868, "p95_s": 2.822939346730709, "cost_usd_per_1000": 0.023305481297728487, "axes": { "intelligence": 75.92175508842178, "calibration": 72.24478620042561, "speed": 74.67997101516198, "cost": 58.97625771162744 }, "jevbench_score": 70.10734578444121 }, "heldout": { "n": 303, "escalation": 0.9537953795379538, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5412844036697247 }, "accuracy": 0.7953795379537953, "ece_hard": 0.1787210719898652, "calibration": 66.33173213924368, "p50_s": 1.7966659694910048, "p95_s": 2.855087894201278, "cost_usd_per_1000": 0.025280483553973374, "axes": { "intelligence": 79.06273302333375, "calibration": 66.33173213924368, "speed": 72.89913175463856, "cost": 57.916438699489554 }, "jevbench_score": 68.59687659276717 } }, { "threshold": 0.87, "public": { "n": 231, "escalation": 0.8701298701298701, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.6036036036036037 }, "accuracy": 0.7489177489177489, "ece_hard": 0.12914387221294593, "calibration": 72.24478620042561, "p50_s": 1.2058723896741868, "p95_s": 2.822939346730709, "cost_usd_per_1000": 0.023402644097475556, "axes": { "intelligence": 75.92175508842178, "calibration": 72.24478620042561, "speed": 74.67997101516198, "cost": 58.9220521596563 }, "jevbench_score": 70.0912311731294 }, "heldout": { "n": 303, "escalation": 0.9537953795379538, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5412844036697247 }, "accuracy": 0.7953795379537953, "ece_hard": 0.1787210719898652, "calibration": 66.33173213924368, "p50_s": 1.7966659694910048, "p95_s": 2.855087894201278, "cost_usd_per_1000": 0.025280483553973374, "axes": { "intelligence": 79.06273302333375, "calibration": 66.33173213924368, "speed": 72.89913175463856, "cost": 57.916438699489554 }, "jevbench_score": 68.59687659276717 } }, { "threshold": 0.88, "public": { "n": 231, "escalation": 0.8831168831168831, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.6036036036036037 }, "accuracy": 0.7489177489177489, "ece_hard": 0.12914387221294593, "calibration": 72.24478620042561, "p50_s": 1.2058723896741868, "p95_s": 2.822939346730709, "cost_usd_per_1000": 0.023694132496716765, "axes": { "intelligence": 75.92175508842178, "calibration": 72.24478620042561, "speed": 74.67997101516198, "cost": 58.760775619427946 }, "jevbench_score": 70.0432198933073 }, "heldout": { "n": 303, "escalation": 0.9537953795379538, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5412844036697247 }, "accuracy": 0.7953795379537953, "ece_hard": 0.1787210719898652, "calibration": 66.33173213924368, "p50_s": 1.7966659694910048, "p95_s": 2.855087894201278, "cost_usd_per_1000": 0.025280483553973374, "axes": { "intelligence": 79.06273302333375, "calibration": 66.33173213924368, "speed": 72.89913175463856, "cost": 57.916438699489554 }, "jevbench_score": 68.59687659276717 } }, { "threshold": 0.89, "public": { "n": 231, "escalation": 0.8961038961038961, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6036036036036037 }, "accuracy": 0.7532467532467533, "ece_hard": 0.12914387221294593, "calibration": 72.24478620042561, "p50_s": 1.2058723896741868, "p95_s": 2.822939346730709, "cost_usd_per_1000": 0.023985620895957974, "axes": { "intelligence": 76.46187854521189, "calibration": 72.24478620042561, "speed": 74.67997101516198, "cost": 58.601471044900336 }, "jevbench_score": 70.119858845011 }, "heldout": { "n": 303, "escalation": 0.9570957095709571, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5412844036697247 }, "accuracy": 0.7953795379537953, "ece_hard": 0.1787210719898652, "calibration": 66.33173213924368, "p50_s": 1.7966659694910048, "p95_s": 2.855087894201278, "cost_usd_per_1000": 0.025354558163681536, "axes": { "intelligence": 79.06273302333375, "calibration": 66.33173213924368, "speed": 72.89913175463856, "cost": 57.878318595976175 }, "jevbench_score": 68.58558633665734 } }, { "threshold": 0.9, "public": { "n": 231, "escalation": 0.8961038961038961, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6036036036036037 }, "accuracy": 0.7532467532467533, "ece_hard": 0.12914387221294593, "calibration": 72.24478620042561, "p50_s": 1.2058723896741868, "p95_s": 2.822939346730709, "cost_usd_per_1000": 0.023985620895957974, "axes": { "intelligence": 76.46187854521189, "calibration": 72.24478620042561, "speed": 74.67997101516198, "cost": 58.601471044900336 }, "jevbench_score": 70.119858845011 }, "heldout": { "n": 303, "escalation": 0.9570957095709571, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5412844036697247 }, "accuracy": 0.7953795379537953, "ece_hard": 0.1787210719898652, "calibration": 66.33173213924368, "p50_s": 1.7966659694910048, "p95_s": 2.855087894201278, "cost_usd_per_1000": 0.025354558163681536, "axes": { "intelligence": 79.06273302333375, "calibration": 66.33173213924368, "speed": 72.89913175463856, "cost": 57.878318595976175 }, "jevbench_score": 68.58558633665734 } }, { "threshold": 0.91, "public": { "n": 231, "escalation": 0.9090909090909091, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6036036036036037 }, "accuracy": 0.7532467532467533, "ece_hard": 0.12914387221294593, "calibration": 72.24478620042561, "p50_s": 1.2058723896741868, "p95_s": 2.822939346730709, "cost_usd_per_1000": 0.02427710929519918, "axes": { "intelligence": 76.46187854521189, "calibration": 72.24478620042561, "speed": 74.67997101516198, "cost": 58.44409079464499 }, "jevbench_score": 70.07273284278868 }, "heldout": { "n": 303, "escalation": 0.9603960396039604, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5412844036697247 }, "accuracy": 0.7953795379537953, "ece_hard": 0.1787210719898652, "calibration": 66.33173213924368, "p50_s": 1.7966659694910048, "p95_s": 2.855087894201278, "cost_usd_per_1000": 0.025428632773389696, "axes": { "intelligence": 79.06273302333375, "calibration": 66.33173213924368, "speed": 72.89913175463856, "cost": 57.84030969996074 }, "jevbench_score": 68.57432346351061 } }, { "threshold": 0.92, "public": { "n": 231, "escalation": 0.922077922077922, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6036036036036037 }, "accuracy": 0.7532467532467533, "ece_hard": 0.12914387221294593, "calibration": 72.24478620042561, "p50_s": 1.2058723896741868, "p95_s": 2.822939346730709, "cost_usd_per_1000": 0.024568597694440388, "axes": { "intelligence": 76.46187854521189, "calibration": 72.24478620042561, "speed": 74.67997101516198, "cost": 58.28858893312692 }, "jevbench_score": 70.02607573343855 }, "heldout": { "n": 303, "escalation": 0.9636963696369637, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5412844036697247 }, "accuracy": 0.7953795379537953, "ece_hard": 0.1787210719898652, "calibration": 66.33173213924368, "p50_s": 1.7966659694910048, "p95_s": 2.855087894201278, "cost_usd_per_1000": 0.02550270738309786, "axes": { "intelligence": 79.06273302333375, "calibration": 66.33173213924368, "speed": 72.89913175463856, "cost": 57.80241136448054 }, "jevbench_score": 68.56308782380398 } }, { "threshold": 0.93, "public": { "n": 231, "escalation": 0.922077922077922, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6036036036036037 }, "accuracy": 0.7532467532467533, "ece_hard": 0.12914387221294593, "calibration": 72.24478620042561, "p50_s": 1.2058723896741868, "p95_s": 2.822939346730709, "cost_usd_per_1000": 0.024568597694440388, "axes": { "intelligence": 76.46187854521189, "calibration": 72.24478620042561, "speed": 74.67997101516198, "cost": 58.28858893312692 }, "jevbench_score": 70.02607573343855 }, "heldout": { "n": 303, "escalation": 0.9702970297029703, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5412844036697247 }, "accuracy": 0.7953795379537953, "ece_hard": 0.1787210719898652, "calibration": 66.33173213924368, "p50_s": 1.7966659694910048, "p95_s": 2.855087894201278, "cost_usd_per_1000": 0.02565085660251418, "axes": { "intelligence": 79.06273302333375, "calibration": 66.33173213924368, "speed": 72.89913175463856, "cost": 57.72694381535639 }, "jevbench_score": 68.54069765286172 } }, { "threshold": 0.94, "public": { "n": 231, "escalation": 0.9264069264069265, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6036036036036037 }, "accuracy": 0.7532467532467533, "ece_hard": 0.12914387221294593, "calibration": 72.24478620042561, "p50_s": 1.2058723896741868, "p95_s": 2.822939346730709, "cost_usd_per_1000": 0.02466576049418746, "axes": { "intelligence": 76.46187854521189, "calibration": 72.24478620042561, "speed": 74.67997101516198, "cost": 58.2371646951237 }, "jevbench_score": 70.01062575590005 }, "heldout": { "n": 303, "escalation": 0.9735973597359736, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5412844036697247 }, "accuracy": 0.7953795379537953, "ece_hard": 0.1787210719898652, "calibration": 66.33173213924368, "p50_s": 1.7966659694910048, "p95_s": 2.855087894201278, "cost_usd_per_1000": 0.025724931212222343, "axes": { "intelligence": 79.06273302333375, "calibration": 66.33173213924368, "speed": 72.89913175463856, "cost": 57.689373335673984 }, "jevbench_score": 68.52954282878066 } }, { "threshold": 0.95, "public": { "n": 231, "escalation": 0.935064935064935, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6036036036036037 }, "accuracy": 0.7532467532467533, "ece_hard": 0.12914387221294593, "calibration": 72.24478620042561, "p50_s": 1.2058723896741868, "p95_s": 2.822939346730709, "cost_usd_per_1000": 0.024860086093681597, "axes": { "intelligence": 76.46187854521189, "calibration": 72.24478620042561, "speed": 74.67997101516198, "cost": 58.13492115022048 }, "jevbench_score": 69.97987712615739 }, "heldout": { "n": 303, "escalation": 0.976897689768977, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5412844036697247 }, "accuracy": 0.7953795379537953, "ece_hard": 0.1787210719898652, "calibration": 66.33173213924368, "p50_s": 1.7966659694910048, "p95_s": 2.855087894201278, "cost_usd_per_1000": 0.025799005821930506, "axes": { "intelligence": 79.06273302333375, "calibration": 66.33173213924368, "speed": 72.89913175463856, "cost": 57.651910884322476 }, "jevbench_score": 68.51841465242143 } }, { "threshold": 0.96, "public": { "n": 231, "escalation": 0.9523809523809523, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6036036036036037 }, "accuracy": 0.7532467532467533, "ece_hard": 0.12914387221294593, "calibration": 72.24478620042561, "p50_s": 1.2058723896741868, "p95_s": 2.822939346730709, "cost_usd_per_1000": 0.025248737292669875, "axes": { "intelligence": 76.46187854521189, "calibration": 72.24478620042561, "speed": 74.67997101516198, "cost": 57.93281009135537 }, "jevbench_score": 69.91897490206937 }, "heldout": { "n": 303, "escalation": 0.976897689768977, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5412844036697247 }, "accuracy": 0.7953795379537953, "ece_hard": 0.1787210719898652, "calibration": 66.33173213924368, "p50_s": 1.7966659694910048, "p95_s": 2.855087894201278, "cost_usd_per_1000": 0.025799005821930506, "axes": { "intelligence": 79.06273302333375, "calibration": 66.33173213924368, "speed": 72.89913175463856, "cost": 57.651910884322476 }, "jevbench_score": 68.51841465242143 } }, { "threshold": 0.97, "public": { "n": 231, "escalation": 0.9696969696969697, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6036036036036037 }, "accuracy": 0.7532467532467533, "ece_hard": 0.12914387221294593, "calibration": 72.24478620042561, "p50_s": 1.2091113656759263, "p95_s": 2.822939346730709, "cost_usd_per_1000": 0.025637388491658152, "axes": { "intelligence": 76.46187854521189, "calibration": 72.24478620042561, "speed": 74.66832149375296, "cost": 57.73378646674773 }, "jevbench_score": 69.8561225716608 }, "heldout": { "n": 303, "escalation": 0.9933993399339934, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7986798679867987, "ece_hard": 0.17685813516393573, "calibration": 66.51802582183663, "p50_s": 1.7966659694910048, "p95_s": 2.855087894201278, "cost_usd_per_1000": 0.026169378870471317, "axes": { "intelligence": 79.33796238113192, "calibration": 66.51802582183663, "speed": 72.89913175463856, "cost": 57.46619755618162 }, "jevbench_score": 68.57073470727173 } }, { "threshold": 0.98, "public": { "n": 231, "escalation": 0.9826839826839827, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6036036036036037 }, "accuracy": 0.7532467532467533, "ece_hard": 0.12914387221294593, "calibration": 72.24478620042561, "p50_s": 1.2091113656759263, "p95_s": 2.822939346730709, "cost_usd_per_1000": 0.02592887689089936, "axes": { "intelligence": 76.46187854521189, "calibration": 72.24478620042561, "speed": 74.66832149375296, "cost": 57.5864888285606 }, "jevbench_score": 69.81152346532748 }, "heldout": { "n": 303, "escalation": 0.9966996699669967, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7986798679867987, "ece_hard": 0.17685813516393573, "calibration": 66.51802582183663, "p50_s": 1.7966659694910048, "p95_s": 2.855087894201278, "cost_usd_per_1000": 0.026243453480179476, "axes": { "intelligence": 79.33796238113192, "calibration": 66.51802582183663, "speed": 72.89913175463856, "cost": 57.42937045004582 }, "jevbench_score": 68.55974620914046 } }, { "threshold": 0.99, "public": { "n": 231, "escalation": 1.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6036036036036037 }, "accuracy": 0.7532467532467533, "ece_hard": 0.12914387221294593, "calibration": 72.24478620042561, "p50_s": 1.2091113656759263, "p95_s": 2.822939346730709, "cost_usd_per_1000": 0.02631752808988764, "axes": { "intelligence": 76.46187854521189, "calibration": 72.24478620042561, "speed": 74.66832149375296, "cost": 57.3926471455153 }, "jevbench_score": 69.75270107990477 }, "heldout": { "n": 303, "escalation": 1.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7986798679867987, "ece_hard": 0.17685813516393573, "calibration": 66.51802582183663, "p50_s": 1.7966659694910048, "p95_s": 2.855087894201278, "cost_usd_per_1000": 0.02631752808988764, "axes": { "intelligence": 79.33796238113192, "calibration": 66.51802582183663, "speed": 72.89913175463856, "cost": 57.3926471455153 }, "jevbench_score": 68.54878341961265 } }, { "threshold": 1.0, "public": { "n": 231, "escalation": 1.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6036036036036037 }, "accuracy": 0.7532467532467533, "ece_hard": 0.12914387221294593, "calibration": 72.24478620042561, "p50_s": 1.2091113656759263, "p95_s": 2.822939346730709, "cost_usd_per_1000": 0.02631752808988764, "axes": { "intelligence": 76.46187854521189, "calibration": 72.24478620042561, "speed": 74.66832149375296, "cost": 57.3926471455153 }, "jevbench_score": 69.75270107990477 }, "heldout": { "n": 303, "escalation": 1.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7986798679867987, "ece_hard": 0.17685813516393573, "calibration": 66.51802582183663, "p50_s": 1.7966659694910048, "p95_s": 2.855087894201278, "cost_usd_per_1000": 0.02631752808988764, "axes": { "intelligence": 79.33796238113192, "calibration": 66.51802582183663, "speed": 72.89913175463856, "cost": 57.3926471455153 }, "jevbench_score": 68.54878341961265 } } ] }, { "cheap": "laya", "backend": "djev", "chosen": { "threshold": 0.91, "public": { "n": 231, "escalation": 0.8138528138528138, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.6486486486486487 }, "accuracy": 0.7705627705627706, "ece_hard": 0.21403948716400195, "calibration": 60.46595625889957, "p50_s": 1.4088232532143592, "p95_s": 4.751377658545971, "cost_usd_per_1000": 0.024003628986494156, "axes": { "intelligence": 77.79863196529865, "calibration": 60.46595625889957, "speed": 71.74323937512592, "cost": 58.591692836569436 }, "jevbench_score": 66.68451217536285 }, "heldout": { "n": 303, "escalation": 0.9108910891089109, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6788990825688074 }, "accuracy": 0.834983498349835, "ece_hard": 0.1936376942232804, "calibration": 63.32580633081179, "p50_s": 3.024282158911228, "p95_s": 4.98705321252346, "cost_usd_per_1000": 0.026521893981532988, "axes": { "intelligence": 82.6158309245528, "calibration": 63.32580633081179, "speed": 68.21533690332035, "cost": 57.291863959649056 }, "jevbench_score": 67.24418681143882 } }, "full": { "tiers": { "easy": 1.0, "standard": 0.8020833333333334, "judge": 0.9315068493150684, "hard": 0.6636363636363637 }, "accuracy": 0.8071161048689138, "ece_hard": 0.20393132611609935, "calibration": 61.88660775260986, "p50_s": 2.458402571827173, "p95_s": 4.905256568640471, "cost_usd_per_1000": 0.025432532157836415, "axes": { "intelligence": 82.44961602324616, "calibration": 61.88660775260986, "speed": 69.18685270407968, "cost": 57.83831193075474 }, "jevbench_score": 67.22117893279099 }, "full_escalation": 0.8689138576779026, "backend_heldout_accuracy": 0.834983498349835, "useful": false, "curve": [ { "threshold": 0.0, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.01, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.02, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.03, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.04, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.05, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.06, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.07, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.08, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.09, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.1, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.11, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.12, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.13, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.14, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.15, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.16, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.17, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.18, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.19, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.2, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.21, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.22, "public": { "n": 231, "escalation": 0.004329004329004329, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.34234234234234234 }, "accuracy": 0.5324675324675324, "ece_hard": 0.20499788359709112, "calibration": 64.18271164029089, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0029954704346839183, "axes": { "intelligence": 53.963338338338346, "calibration": 64.18271164029089, "speed": 72.85668019555155, "cost": 85.70604887626953 }, "jevbench_score": 68.1945127423754 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.23, "public": { "n": 231, "escalation": 0.004329004329004329, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.34234234234234234 }, "accuracy": 0.5324675324675324, "ece_hard": 0.20499788359709112, "calibration": 64.18271164029089, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0029954704346839183, "axes": { "intelligence": 53.963338338338346, "calibration": 64.18271164029089, "speed": 72.85668019555155, "cost": 85.70604887626953 }, "jevbench_score": 68.1945127423754 }, "heldout": { "n": 303, "escalation": 0.0033003300330033004, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.26638491846591456, "calibration": 54.710258153408546, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0029687750460439303, "axes": { "intelligence": 59.67720246323992, "calibration": 54.710258153408546, "speed": 68.8164811989393, "cost": 85.82268127282356 }, "jevbench_score": 66.26633325051776 } }, { "threshold": 0.24, "public": { "n": 231, "escalation": 0.004329004329004329, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.34234234234234234 }, "accuracy": 0.5324675324675324, "ece_hard": 0.20499788359709112, "calibration": 64.18271164029089, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0029954704346839183, "axes": { "intelligence": 53.963338338338346, "calibration": 64.18271164029089, "speed": 72.85668019555155, "cost": 85.70604887626953 }, "jevbench_score": 68.1945127423754 }, "heldout": { "n": 303, "escalation": 0.006600660066006601, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.30275229357798167 }, "accuracy": 0.5676567656765676, "ece_hard": 0.2685785710123605, "calibration": 54.490892898763946, "p50_s": 2.7680240362882613, "p95_s": 4.776861734688282, "cost_usd_per_1000": 0.0030544227512638907, "axes": { "intelligence": 59.95243182103809, "calibration": 54.490892898763946, "speed": 68.78687472081019, "cost": 85.45212562118809 }, "jevbench_score": 66.1972265290414 } }, { "threshold": 0.25, "public": { "n": 231, "escalation": 0.008658008658008658, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.35135135135135137 }, "accuracy": 0.5367965367965368, "ece_hard": 0.20721506441962992, "calibration": 63.96099355803702, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.003107813528543866, "axes": { "intelligence": 54.33871371371373, "calibration": 63.96099355803702, "speed": 72.85668019555155, "cost": 85.22635141383566 }, "jevbench_score": 68.15801871560345 }, "heldout": { "n": 303, "escalation": 0.009900990099009901, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2653040571677337, "calibration": 54.81834428322663, "p50_s": 2.7680240362882613, "p95_s": 4.776861734688282, "cost_usd_per_1000": 0.0031400704564838507, "axes": { "intelligence": 59.67720246323992, "calibration": 54.81834428322663, "speed": 68.78687472081019, "cost": 85.09181821526666 }, "jevbench_score": 66.15031825961717 } }, { "threshold": 0.26, "public": { "n": 231, "escalation": 0.008658008658008658, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.35135135135135137 }, "accuracy": 0.5367965367965368, "ece_hard": 0.20721506441962992, "calibration": 63.96099355803702, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.003107813528543866, "axes": { "intelligence": 54.33871371371373, "calibration": 63.96099355803702, "speed": 72.85668019555155, "cost": 85.22635141383566 }, "jevbench_score": 68.15801871560345 }, "heldout": { "n": 303, "escalation": 0.013201320132013201, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2719563772639535, "calibration": 54.15311227360465, "p50_s": 2.7680240362882613, "p95_s": 4.805826105177402, "cost_usd_per_1000": 0.003225718161703811, "axes": { "intelligence": 59.67720246323992, "calibration": 54.15311227360465, "speed": 68.76062090811617, "cost": 84.74120741743732 }, "jevbench_score": 65.87438576322377 } }, { "threshold": 0.27, "public": { "n": 231, "escalation": 0.021645021645021644, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.35135135135135137 }, "accuracy": 0.5367965367965368, "ece_hard": 0.2167309292550949, "calibration": 63.00940707449051, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0034448428101237095, "axes": { "intelligence": 54.33871371371373, "calibration": 63.00940707449051, "speed": 72.85668019555155, "cost": 83.88491771126874 }, "jevbench_score": 67.63430037029215 }, "heldout": { "n": 303, "escalation": 0.013201320132013201, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2719563772639535, "calibration": 54.15311227360465, "p50_s": 2.7680240362882613, "p95_s": 4.805826105177402, "cost_usd_per_1000": 0.003225718161703811, "axes": { "intelligence": 59.67720246323992, "calibration": 54.15311227360465, "speed": 68.76062090811617, "cost": 84.74120741743732 }, "jevbench_score": 65.87438576322377 } }, { "threshold": 0.28, "public": { "n": 231, "escalation": 0.021645021645021644, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.35135135135135137 }, "accuracy": 0.5367965367965368, "ece_hard": 0.2167309292550949, "calibration": 63.00940707449051, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0034448428101237095, "axes": { "intelligence": 54.33871371371373, "calibration": 63.00940707449051, "speed": 72.85668019555155, "cost": 83.88491771126874 }, "jevbench_score": 67.63430037029215 }, "heldout": { "n": 303, "escalation": 0.019801980198019802, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.30275229357798167 }, "accuracy": 0.5676567656765676, "ece_hard": 0.27631125485581776, "calibration": 53.71762451441822, "p50_s": 2.7680240362882613, "p95_s": 4.805826105177402, "cost_usd_per_1000": 0.0033970135721437316, "axes": { "intelligence": 59.95243182103809, "calibration": 53.71762451441822, "speed": 68.76062090811617, "cost": 84.06708153948036 }, "jevbench_score": 65.68592814352321 } }, { "threshold": 0.29, "public": { "n": 231, "escalation": 0.03896103896103896, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.34234234234234234 }, "accuracy": 0.5324675324675324, "ece_hard": 0.23428726529650074, "calibration": 61.25377347034993, "p50_s": 1.1657145261764525, "p95_s": 4.458208107948304, "cost_usd_per_1000": 0.0038942151855635007, "axes": { "intelligence": 53.963338338338346, "calibration": 61.25377347034993, "speed": 72.8424745430834, "cost": 82.28740162052938 }, "jevbench_score": 66.71714049488729 }, "heldout": { "n": 303, "escalation": 0.0231023102310231, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.27713259953055647, "calibration": 53.635490046944355, "p50_s": 2.7680240362882613, "p95_s": 4.805826105177402, "cost_usd_per_1000": 0.0034826612773636917, "axes": { "intelligence": 59.67720246323992, "calibration": 53.635490046944355, "speed": 68.76062090811617, "cost": 83.7426628853341 }, "jevbench_score": 65.52195035941706 } }, { "threshold": 0.3, "public": { "n": 231, "escalation": 0.04329004329004329, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.35135135135135137 }, "accuracy": 0.5367965367965368, "ece_hard": 0.23145343569395543, "calibration": 61.53715643060446, "p50_s": 1.1657145261764525, "p95_s": 4.458208107948304, "cost_usd_per_1000": 0.004006558279423448, "axes": { "intelligence": 54.33871371371373, "calibration": 61.53715643060446, "speed": 72.8424745430834, "cost": 81.91685606879712 }, "jevbench_score": 66.83457429595336 }, "heldout": { "n": 303, "escalation": 0.033003300330033, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6986301369863014, "hard": 0.29357798165137616 }, "accuracy": 0.5676567656765676, "ece_hard": 0.2766891866923046, "calibration": 53.679831330769545, "p50_s": 2.7680240362882613, "p95_s": 4.805826105177402, "cost_usd_per_1000": 0.003739604393023572, "axes": { "intelligence": 59.868983285157725, "calibration": 53.679831330769545, "speed": 68.76062090811617, "cost": 82.81523016137302 }, "jevbench_score": 65.40572395395589 } }, { "threshold": 0.31, "public": { "n": 231, "escalation": 0.05627705627705628, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.34234234234234234 }, "accuracy": 0.5324675324675324, "ece_hard": 0.25693116531842525, "calibration": 58.989383468157484, "p50_s": 1.1657145261764525, "p95_s": 4.458208107948304, "cost_usd_per_1000": 0.004343587561003292, "axes": { "intelligence": 53.963338338338346, "calibration": 58.989383468157484, "speed": 72.8424745430834, "cost": 80.86454257774314 }, "jevbench_score": 65.80423925282119 }, "heldout": { "n": 303, "escalation": 0.0429042904290429, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6986301369863014, "hard": 0.3119266055045872 }, "accuracy": 0.5742574257425742, "ece_hard": 0.2771799487596843, "calibration": 53.630755124031566, "p50_s": 2.7680240362882613, "p95_s": 4.8142633035779, "cost_usd_per_1000": 0.003996547508683453, "axes": { "intelligence": 60.41944200075407, "calibration": 53.630755124031566, "speed": 68.75300303786179, "cost": 81.94945060052964 }, "jevbench_score": 65.36677898006971 } }, { "threshold": 0.32, "public": { "n": 231, "escalation": 0.06060606060606061, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.34234234234234234 }, "accuracy": 0.5324675324675324, "ece_hard": 0.25806341794836873, "calibration": 58.87615820516313, "p50_s": 1.1657145261764525, "p95_s": 4.458208107948304, "cost_usd_per_1000": 0.0044559306548632395, "axes": { "intelligence": 53.963338338338346, "calibration": 58.87615820516313, "speed": 72.8424745430834, "cost": 80.5318472938578 }, "jevbench_score": 65.70488448589563 }, "heldout": { "n": 303, "escalation": 0.0462046204620462, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6986301369863014, "hard": 0.3119266055045872 }, "accuracy": 0.5742574257425742, "ece_hard": 0.28339131260983597, "calibration": 53.0096187390164, "p50_s": 2.7680240362882613, "p95_s": 4.8142633035779, "cost_usd_per_1000": 0.004082195213903413, "axes": { "intelligence": 60.41944200075407, "calibration": 53.0096187390164, "speed": 68.75300303786179, "cost": 81.67318692421424 }, "jevbench_score": 65.12168681295502 } }, { "threshold": 0.33, "public": { "n": 231, "escalation": 0.06493506493506493, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.34234234234234234 }, "accuracy": 0.5324675324675324, "ece_hard": 0.26412593020871994, "calibration": 58.26990697912801, "p50_s": 1.1657145261764525, "p95_s": 4.46199137121439, "cost_usd_per_1000": 0.004568273748723187, "axes": { "intelligence": 53.963338338338346, "calibration": 58.26990697912801, "speed": 72.83879065582862, "cost": 80.20743638105976 }, "jevbench_score": 65.46815830245839 }, "heldout": { "n": 303, "escalation": 0.0594059405940594, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6986301369863014, "hard": 0.3302752293577982 }, "accuracy": 0.5808580858085809, "ece_hard": 0.28356399879623506, "calibration": 53.61726160771801, "p50_s": 2.7680240362882613, "p95_s": 4.8142633035779, "cost_usd_per_1000": 0.004424786034783254, "axes": { "intelligence": 60.96990071635039, "calibration": 53.61726160771801, "speed": 68.75300303786179, "cost": 80.6232317568466 }, "jevbench_score": 65.2443634130602 } }, { "threshold": 0.34, "public": { "n": 231, "escalation": 0.0735930735930736, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.35135135135135137 }, "accuracy": 0.5367965367965368, "ece_hard": 0.26227544325831725, "calibration": 58.454955674168275, "p50_s": 1.1657145261764525, "p95_s": 4.481120200455189, "cost_usd_per_1000": 0.004792959936443083, "axes": { "intelligence": 54.33871371371373, "calibration": 58.454955674168275, "speed": 72.82021198198058, "cost": 79.58188603565684 }, "jevbench_score": 65.50119357835752 }, "heldout": { "n": 303, "escalation": 0.06930693069306931, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7054794520547946, "hard": 0.3394495412844037 }, "accuracy": 0.5874587458745875, "ece_hard": 0.28653347574668026, "calibration": 53.320313912673484, "p50_s": 2.7680240362882613, "p95_s": 4.8142633035779, "cost_usd_per_1000": 0.0046817291504431344, "axes": { "intelligence": 61.436910896066365, "calibration": 53.320313912673484, "speed": 68.75300303786179, "cost": 79.88781144752734 }, "jevbench_score": 65.1288734443149 } }, { "threshold": 0.35, "public": { "n": 231, "escalation": 0.0735930735930736, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.35135135135135137 }, "accuracy": 0.5367965367965368, "ece_hard": 0.26227544325831725, "calibration": 58.454955674168275, "p50_s": 1.1657145261764525, "p95_s": 4.481120200455189, "cost_usd_per_1000": 0.004792959936443083, "axes": { "intelligence": 54.33871371371373, "calibration": 58.454955674168275, "speed": 72.82021198198058, "cost": 79.58188603565684 }, "jevbench_score": 65.50119357835752 }, "heldout": { "n": 303, "escalation": 0.07920792079207921, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7054794520547946, "hard": 0.3577981651376147 }, "accuracy": 0.594059405940594, "ece_hard": 0.2801392597236976, "calibration": 53.959735514971754, "p50_s": 2.7680240362882613, "p95_s": 4.8142633035779, "cost_usd_per_1000": 0.004938672266103015, "axes": { "intelligence": 61.98736961166269, "calibration": 53.959735514971754, "speed": 68.75300303786179, "cost": 79.19169378943306 }, "jevbench_score": 65.32600218494143 } }, { "threshold": 0.36, "public": { "n": 231, "escalation": 0.08658008658008658, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5411255411255411, "ece_hard": 0.26771218867386204, "calibration": 57.9112811326138, "p50_s": 1.1657145261764525, "p95_s": 4.4887279063463215, "cost_usd_per_1000": 0.005129989218022926, "axes": { "intelligence": 54.7140890890891, "calibration": 57.9112811326138, "speed": 72.81284511145756, "cost": 78.69650643002603 }, "jevbench_score": 65.27643833183383 }, "heldout": { "n": 303, "escalation": 0.08250825082508251, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7054794520547946, "hard": 0.3577981651376147 }, "accuracy": 0.594059405940594, "ece_hard": 0.2844688296173663, "calibration": 54.844722003057356, "p50_s": 2.7680240362882613, "p95_s": 4.8142633035779, "cost_usd_per_1000": 0.005024319971322975, "axes": { "intelligence": 61.98736961166269, "calibration": 54.844722003057356, "speed": 68.75300303786179, "cost": 78.96768131685002 }, "jevbench_score": 65.5457861101556 } }, { "threshold": 0.37, "public": { "n": 231, "escalation": 0.11255411255411256, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3963963963963964 }, "accuracy": 0.5584415584415584, "ece_hard": 0.2575246670282414, "calibration": 58.93003329717587, "p50_s": 1.169974410533905, "p95_s": 4.579712373763323, "cost_usd_per_1000": 0.005804047781182613, "axes": { "intelligence": 56.2155905905906, "calibration": 58.93003329717587, "speed": 72.70985433717482, "cost": 77.08807062814623 }, "jevbench_score": 65.64376449658542 }, "heldout": { "n": 303, "escalation": 0.0891089108910891, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7054794520547946, "hard": 0.3669724770642202 }, "accuracy": 0.5973597359735974, "ece_hard": 0.2858808164297213, "calibration": 54.70352332182186, "p50_s": 2.7680240362882613, "p95_s": 4.8142633035779, "cost_usd_per_1000": 0.005195615381762896, "axes": { "intelligence": 62.26259896946086, "calibration": 54.70352332182186, "speed": 68.75300303786179, "cost": 78.53089018386258 }, "jevbench_score": 65.48527930995388 } }, { "threshold": 0.38, "public": { "n": 231, "escalation": 0.12554112554112554, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3963963963963964 }, "accuracy": 0.5584415584415584, "ece_hard": 0.2552971594021614, "calibration": 59.15278405978386, "p50_s": 1.1792236283421516, "p95_s": 4.579712373763323, "cost_usd_per_1000": 0.006141077062762457, "axes": { "intelligence": 56.2155905905906, "calibration": 59.15278405978386, "speed": 72.67565624109668, "cost": 76.35266358208892 }, "jevbench_score": 65.5407309198028 }, "heldout": { "n": 303, "escalation": 0.09900990099009901, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7123287671232876, "hard": 0.3761467889908257 }, "accuracy": 0.6039603960396039, "ece_hard": 0.2856826765914915, "calibration": 54.72333730564484, "p50_s": 2.7680240362882613, "p95_s": 4.8142633035779, "cost_usd_per_1000": 0.005452558497422776, "axes": { "intelligence": 62.72960914917684, "calibration": 54.72333730564484, "speed": 68.75300303786179, "cost": 77.90198999271644 }, "jevbench_score": 65.48191099280263 } }, { "threshold": 0.39, "public": { "n": 231, "escalation": 0.14285714285714285, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.40540540540540543 }, "accuracy": 0.5627705627705628, "ece_hard": 0.26113385854532956, "calibration": 58.56911414546705, "p50_s": 1.1792236283421516, "p95_s": 4.6298252247273926, "cost_usd_per_1000": 0.006590449438202247, "axes": { "intelligence": 56.59096596596597, "calibration": 58.56911414546705, "speed": 72.62839230581778, "cost": 75.43254902556865 }, "jevbench_score": 65.2785116939457 }, "heldout": { "n": 303, "escalation": 0.11221122112211221, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7191780821917808, "hard": 0.3853211009174312 }, "accuracy": 0.6105610561056105, "ece_hard": 0.28947732910729296, "calibration": 54.34387205406469, "p50_s": 2.7680240362882613, "p95_s": 4.8142633035779, "cost_usd_per_1000": 0.005795149318302617, "axes": { "intelligence": 63.1966193288928, "calibration": 54.34387205406469, "speed": 68.75300303786179, "cost": 77.10806108464698 }, "jevbench_score": 65.32192432404878 } }, { "threshold": 0.4, "public": { "n": 231, "escalation": 0.1471861471861472, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.4144144144144144 }, "accuracy": 0.5670995670995671, "ece_hard": 0.25758740976776034, "calibration": 58.923759023223965, "p50_s": 1.1792236283421516, "p95_s": 4.6298252247273926, "cost_usd_per_1000": 0.0067027925320621955, "axes": { "intelligence": 56.96634134134135, "calibration": 58.923759023223965, "speed": 72.62839230581778, "cost": 75.21232668646356 }, "jevbench_score": 65.43740322838681 }, "heldout": { "n": 303, "escalation": 0.132013201320132, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7191780821917808, "hard": 0.3853211009174312 }, "accuracy": 0.6105610561056105, "ece_hard": 0.3056268436556113, "calibration": 52.95780987592866, "p50_s": 2.7680240362882613, "p95_s": 4.818394577503205, "cost_usd_per_1000": 0.006309035549622378, "axes": { "intelligence": 63.1966193288928, "calibration": 52.95780987592866, "speed": 68.7492778155999, "cost": 76.00111076383412 }, "jevbench_score": 64.66629597351857 } }, { "threshold": 0.41, "public": { "n": 231, "escalation": 0.1645021645021645, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.43243243243243246 }, "accuracy": 0.5757575757575758, "ece_hard": 0.2577054114004288, "calibration": 58.911958859957124, "p50_s": 1.1792236283421516, "p95_s": 4.665233356505633, "cost_usd_per_1000": 0.0071521649075019875, "axes": { "intelligence": 57.7170920920921, "calibration": 58.911958859957124, "speed": 72.59530455172985, "cost": 74.36687441699974 }, "jevbench_score": 65.45592907122897 }, "heldout": { "n": 303, "escalation": 0.13861386138613863, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7123287671232876, "hard": 0.3853211009174312 }, "accuracy": 0.6072607260726073, "ece_hard": 0.3098322889760976, "calibration": 52.22029764421502, "p50_s": 2.79284847676754, "p95_s": 4.818394577503205, "cost_usd_per_1000": 0.0064803309600623, "axes": { "intelligence": 63.004838506975005, "calibration": 52.22029764421502, "speed": 68.71050264458512, "cost": 75.65208440509346 }, "jevbench_score": 64.3078984673794 } }, { "threshold": 0.42, "public": { "n": 231, "escalation": 0.16883116883116883, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.44144144144144143 }, "accuracy": 0.5800865800865801, "ece_hard": 0.25394636388435243, "calibration": 59.28786361156476, "p50_s": 1.1792236283421516, "p95_s": 4.665233356505633, "cost_usd_per_1000": 0.007264508001361934, "axes": { "intelligence": 58.09246746746747, "calibration": 59.28786361156476, "speed": 72.59530455172985, "cost": 74.16381380654964 }, "jevbench_score": 65.62156034108544 }, "heldout": { "n": 303, "escalation": 0.15841584158415842, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7191780821917808, "hard": 0.4036697247706422 }, "accuracy": 0.6171617161716172, "ece_hard": 0.3015970094357338, "calibration": 53.0438255982514, "p50_s": 2.79284847676754, "p95_s": 4.818394577503205, "cost_usd_per_1000": 0.00699421719138206, "axes": { "intelligence": 63.74707804448912, "calibration": 53.0438255982514, "speed": 68.71050264458512, "cost": 74.6578265702076 }, "jevbench_score": 64.53545813041144 } }, { "threshold": 0.43, "public": { "n": 231, "escalation": 0.18614718614718614, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.45045045045045046 }, "accuracy": 0.5844155844155844, "ece_hard": 0.2623346660308655, "calibration": 58.569332683094174, "p50_s": 1.1792236283421516, "p95_s": 4.665233356505633, "cost_usd_per_1000": 0.007713880376801726, "axes": { "intelligence": 58.46784284284285, "calibration": 58.569332683094174, "speed": 72.59530455172985, "cost": 73.38181300751805 }, "jevbench_score": 65.35383524656888 }, "heldout": { "n": 303, "escalation": 0.19141914191419143, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7328767123287672, "hard": 0.42201834862385323 }, "accuracy": 0.6303630363036303, "ece_hard": 0.29869441776037686, "calibration": 53.3340847657871, "p50_s": 2.8226395934820174, "p95_s": 4.842004146426917, "cost_usd_per_1000": 0.007850694243581662, "axes": { "intelligence": 64.68109840392108, "calibration": 53.3340847657871, "speed": 68.64319413850893, "cost": 73.15275809576346 }, "jevbench_score": 64.5137973397298 } }, { "threshold": 0.44, "public": { "n": 231, "escalation": 0.19913419913419914, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.4594594594594595 }, "accuracy": 0.5887445887445888, "ece_hard": 0.2628508345763212, "calibration": 59.54324341768803, "p50_s": 1.1792236283421516, "p95_s": 4.665233356505633, "cost_usd_per_1000": 0.00805090965838157, "axes": { "intelligence": 58.84321821821823, "calibration": 59.54324341768803, "speed": 72.59530455172985, "cost": 72.82465140028268 }, "jevbench_score": 65.6037949664464 }, "heldout": { "n": 303, "escalation": 0.20132013201320131, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7328767123287672, "hard": 0.42201834862385323 }, "accuracy": 0.6303630363036303, "ece_hard": 0.29869441776037686, "calibration": 53.3340847657871, "p50_s": 2.8226395934820174, "p95_s": 4.842004146426917, "cost_usd_per_1000": 0.008107637359241544, "axes": { "intelligence": 64.68109840392108, "calibration": 53.3340847657871, "speed": 68.64319413850893, "cost": 72.73317054382987 }, "jevbench_score": 64.42108858176823 } }, { "threshold": 0.45, "public": { "n": 231, "escalation": 0.21212121212121213, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.4864864864864865 }, "accuracy": 0.6017316017316018, "ece_hard": 0.2502979541508469, "calibration": 60.798531460235466, "p50_s": 1.1792236283421516, "p95_s": 4.665233356505633, "cost_usd_per_1000": 0.008387938939961414, "axes": { "intelligence": 59.96934434434435, "calibration": 60.798531460235466, "speed": 72.59530455172985, "cost": 72.29034218931362 }, "jevbench_score": 66.13826539573573 }, "heldout": { "n": 303, "escalation": 0.2145214521452145, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7397260273972602, "hard": 0.42201834862385323 }, "accuracy": 0.6336633663366337, "ece_hard": 0.3077091616410999, "calibration": 52.43261037771479, "p50_s": 2.8226395934820174, "p95_s": 4.842004146426917, "cost_usd_per_1000": 0.008450228180121384, "axes": { "intelligence": 64.87287922583889, "calibration": 52.43261037771479, "speed": 68.64319413850893, "cost": 72.19394691128569 }, "jevbench_score": 64.07531217959364 } }, { "threshold": 0.46, "public": { "n": 231, "escalation": 0.22510822510822512, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.4864864864864865 }, "accuracy": 0.6017316017316018, "ece_hard": 0.25348760136148857, "calibration": 60.47956673917129, "p50_s": 1.1792236283421516, "p95_s": 4.716590677946806, "cost_usd_per_1000": 0.008724968221541257, "axes": { "intelligence": 59.96934434434435, "calibration": 60.47956673917129, "speed": 72.54775639047833, "cost": 71.77708438516459 }, "jevbench_score": 65.92299695231577 }, "heldout": { "n": 303, "escalation": 0.21782178217821782, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7397260273972602, "hard": 0.42201834862385323 }, "accuracy": 0.6336633663366337, "ece_hard": 0.30947688514311517, "calibration": 51.25559535141044, "p50_s": 2.8226395934820174, "p95_s": 4.842004146426917, "cost_usd_per_1000": 0.008535875885341344, "axes": { "intelligence": 64.87287922583889, "calibration": 51.25559535141044, "speed": 68.64319413850893, "cost": 72.06255724999795 }, "jevbench_score": 63.683643484129234 } }, { "threshold": 0.47, "public": { "n": 231, "escalation": 0.24242424242424243, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.4864864864864865 }, "accuracy": 0.6017316017316018, "ece_hard": 0.2604639835569462, "calibration": 58.43934104979073, "p50_s": 1.1792236283421516, "p95_s": 4.716590677946806, "cost_usd_per_1000": 0.009174340596981048, "axes": { "intelligence": 59.96934434434435, "calibration": 58.43934104979073, "speed": 72.54775639047833, "cost": 71.12275422225046 }, "jevbench_score": 65.21038898360604 }, "heldout": { "n": 303, "escalation": 0.23432343234323433, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7534246575342466, "hard": 0.43119266055045874 }, "accuracy": 0.6435643564356436, "ece_hard": 0.3052319469918191, "calibration": 51.68008916654004, "p50_s": 2.8282473921775817, "p95_s": 4.842004146426917, "cost_usd_per_1000": 0.008964114411441145, "axes": { "intelligence": 65.53167022747267, "calibration": 51.68008916654004, "speed": 68.63457447637529, "cost": 71.42477827247455 }, "jevbench_score": 63.83246039262831 } }, { "threshold": 0.48, "public": { "n": 231, "escalation": 0.2510822510822511, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.4864864864864865 }, "accuracy": 0.6017316017316018, "ece_hard": 0.2647534656776358, "calibration": 57.54873026073903, "p50_s": 1.1792236283421516, "p95_s": 4.716590677946806, "cost_usd_per_1000": 0.009399026784700943, "axes": { "intelligence": 59.96934434434435, "calibration": 57.54873026073903, "speed": 72.54775639047833, "cost": 70.80751338322854 }, "jevbench_score": 64.88840423432964 }, "heldout": { "n": 303, "escalation": 0.264026402640264, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7534246575342466, "hard": 0.44036697247706424 }, "accuracy": 0.6468646864686468, "ece_hard": 0.3016570984098631, "calibration": 52.03757402473565, "p50_s": 2.8427457749843597, "p95_s": 4.842004146426917, "cost_usd_per_1000": 0.009734943758420786, "axes": { "intelligence": 65.80689958527084, "calibration": 52.03757402473565, "speed": 68.61236820250983, "cost": 70.34999659543732 }, "jevbench_score": 63.76226567240712 } }, { "threshold": 0.49, "public": { "n": 231, "escalation": 0.2554112554112554, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.4864864864864865 }, "accuracy": 0.6017316017316018, "ece_hard": 0.2663566212557762, "calibration": 57.38841470292499, "p50_s": 1.1792236283421516, "p95_s": 4.716590677946806, "cost_usd_per_1000": 0.009511369878560891, "axes": { "intelligence": 59.96934434434435, "calibration": 57.38841470292499, "speed": 72.54775639047833, "cost": 70.65270787403765 }, "jevbench_score": 64.80769593302819 }, "heldout": { "n": 303, "escalation": 0.26732673267326734, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7534246575342466, "hard": 0.44036697247706424 }, "accuracy": 0.6468646864686468, "ece_hard": 0.30538716250850134, "calibration": 51.66456761487183, "p50_s": 2.8427457749843597, "p95_s": 4.842004146426917, "cost_usd_per_1000": 0.009820591463640747, "axes": { "intelligence": 65.80689958527084, "calibration": 51.66456761487183, "speed": 68.61236820250983, "cost": 70.23587065663577 }, "jevbench_score": 63.62186594797809 } }, { "threshold": 0.5, "public": { "n": 231, "escalation": 0.26406926406926406, "tiers": { "easy": 0.9583333333333334, "standard": 0.5555555555555556, "hard": 0.4954954954954955 }, "accuracy": 0.6103896103896104, "ece_hard": 0.2585383006496236, "calibration": 58.17024676354025, "p50_s": 1.2183565288782119, "p95_s": 4.716590677946806, "cost_usd_per_1000": 0.009736056066280786, "axes": { "intelligence": 60.884843176509854, "calibration": 58.17024676354025, "speed": 72.40597416776316, "cost": 70.34850801497207 }, "jevbench_score": 65.17181972316237 }, "heldout": { "n": 303, "escalation": 0.27722772277227725, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7534246575342466, "hard": 0.43119266055045874 }, "accuracy": 0.6435643564356436, "ece_hard": 0.31100861699186555, "calibration": 51.102422166535405, "p50_s": 2.8427457749843597, "p95_s": 4.906408296525477, "cost_usd_per_1000": 0.01007753457930063, "axes": { "intelligence": 65.53167022747267, "calibration": 51.102422166535405, "speed": 68.5549829120949, "cost": 69.8993710889997 }, "jevbench_score": 63.29235486880842 } }, { "threshold": 0.51, "public": { "n": 231, "escalation": 0.2813852813852814, "tiers": { "easy": 0.9583333333333334, "standard": 0.5972222222222222, "hard": 0.4954954954954955 }, "accuracy": 0.6233766233766234, "ece_hard": 0.26302255873942704, "calibration": 57.7218209545599, "p50_s": 1.2405554592609405, "p95_s": 4.716590677946806, "cost_usd_per_1000": 0.010185428441720578, "axes": { "intelligence": 62.50521354688022, "calibration": 57.7218209545599, "speed": 72.32755627439042, "cost": 69.76062094111754 }, "jevbench_score": 65.31946199822896 }, "heldout": { "n": 303, "escalation": 0.29042904290429045, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7534246575342466, "hard": 0.44036697247706424 }, "accuracy": 0.6468646864686468, "ece_hard": 0.3079891242556981, "calibration": 53.065905136686, "p50_s": 2.8427457749843597, "p95_s": 4.906408296525477, "cost_usd_per_1000": 0.01042012540018047, "axes": { "intelligence": 65.80689958527084, "calibration": 53.065905136686, "speed": 68.5549829120949, "cost": 69.4638116356667 }, "jevbench_score": 63.85885994040476 } }, { "threshold": 0.52, "public": { "n": 231, "escalation": 0.2943722943722944, "tiers": { "easy": 0.9583333333333334, "standard": 0.5972222222222222, "hard": 0.4864864864864865 }, "accuracy": 0.6190476190476191, "ece_hard": 0.27635529072194515, "calibration": 56.38854775630809, "p50_s": 1.3017749428749084, "p95_s": 4.716590677946806, "cost_usd_per_1000": 0.010522457723300422, "axes": { "intelligence": 62.12983817150485, "calibration": 56.38854775630809, "speed": 72.11835903897321, "cost": 69.33648431401039 }, "jevbench_score": 64.69559276991971 }, "heldout": { "n": 303, "escalation": 0.32673267326732675, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7602739726027398, "hard": 0.44954128440366975 }, "accuracy": 0.6534653465346535, "ece_hard": 0.30428918970066104, "calibration": 53.4358985921897, "p50_s": 2.8427457749843597, "p95_s": 4.913613957166672, "cost_usd_per_1000": 0.011362250157600031, "axes": { "intelligence": 66.27390976498681, "calibration": 53.4358985921897, "speed": 68.54860944544535, "cost": 68.33606959856093 }, "jevbench_score": 63.81989549434319 } }, { "threshold": 0.53, "public": { "n": 231, "escalation": 0.30735930735930733, "tiers": { "easy": 0.9583333333333334, "standard": 0.6111111111111112, "hard": 0.4954954954954955 }, "accuracy": 0.6277056277056277, "ece_hard": 0.27326013158360507, "calibration": 56.698063670142105, "p50_s": 1.3017749428749084, "p95_s": 4.716590677946806, "cost_usd_per_1000": 0.010859487004880264, "axes": { "intelligence": 63.04533700367034, "calibration": 56.698063670142105, "speed": 72.11835903897321, "cost": 68.92572070156409 }, "jevbench_score": 64.92501931710717 }, "heldout": { "n": 303, "escalation": 0.3465346534653465, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.773972602739726, "hard": 0.45871559633027525 }, "accuracy": 0.6633663366336634, "ece_hard": 0.3037474463625721, "calibration": 53.4900729259986, "p50_s": 2.8427457749843597, "p95_s": 4.9281502470374114, "cost_usd_per_1000": 0.011876136388919791, "axes": { "intelligence": 66.93270076662058, "calibration": 53.4900729259986, "speed": 68.53578037346617, "cost": 67.75974470458915 }, "jevbench_score": 63.8557728628571 } }, { "threshold": 0.54, "public": { "n": 231, "escalation": 0.3246753246753247, "tiers": { "easy": 0.9583333333333334, "standard": 0.6111111111111112, "hard": 0.5045045045045045 }, "accuracy": 0.6320346320346321, "ece_hard": 0.27387231229484105, "calibration": 56.6368455990185, "p50_s": 1.3017749428749084, "p95_s": 4.716590677946806, "cost_usd_per_1000": 0.011308859380320056, "axes": { "intelligence": 63.42071237904572, "calibration": 56.6368455990185, "speed": 72.11835903897321, "cost": 68.39743588357197 }, "jevbench_score": 64.87897165984026 }, "heldout": { "n": 303, "escalation": 0.36303630363036304, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7808219178082192, "hard": 0.48623853211009177 }, "accuracy": 0.6765676567656765, "ece_hard": 0.285744852770426, "calibration": 55.29033228521321, "p50_s": 2.8427457749843597, "p95_s": 4.9281502470374114, "cost_usd_per_1000": 0.012304374915019593, "axes": { "intelligence": 67.9501696619329, "calibration": 55.29033228521321, "speed": 68.53578037346617, "cost": 67.29821333088707 }, "jevbench_score": 64.5193763075803 } }, { "threshold": 0.55, "public": { "n": 231, "escalation": 0.3463203463203463, "tiers": { "easy": 0.9791666666666666, "standard": 0.6111111111111112, "hard": 0.5135135135135135 }, "accuracy": 0.6406926406926406, "ece_hard": 0.27322812076997405, "calibration": 56.58314801864883, "p50_s": 1.3017749428749084, "p95_s": 4.716590677946806, "cost_usd_per_1000": 0.011870574849619795, "axes": { "intelligence": 64.20118034701368, "calibration": 56.58314801864883, "speed": 72.11835903897321, "cost": 67.76584747608145 }, "jevbench_score": 64.91150928650768 }, "heldout": { "n": 303, "escalation": 0.37293729372937295, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7808219178082192, "hard": 0.4954128440366973 }, "accuracy": 0.6798679867986799, "ece_hard": 0.28052945442437854, "calibration": 55.811872119817956, "p50_s": 2.8427457749843597, "p95_s": 4.9281502470374114, "cost_usd_per_1000": 0.012561318030679475, "axes": { "intelligence": 68.22539901973106, "calibration": 55.811872119817956, "speed": 68.53578037346617, "cost": 67.02894366011675 }, "jevbench_score": 64.6715251506132 } }, { "threshold": 0.56, "public": { "n": 231, "escalation": 0.354978354978355, "tiers": { "easy": 0.9791666666666666, "standard": 0.625, "hard": 0.5135135135135135 }, "accuracy": 0.645021645021645, "ece_hard": 0.27322812076997405, "calibration": 56.58314801864883, "p50_s": 1.3210672974586486, "p95_s": 4.716590677946806, "cost_usd_per_1000": 0.012095261037339692, "axes": { "intelligence": 64.7413038038038, "calibration": 56.58314801864883, "speed": 72.05446869804332, "cost": 67.52154263050697 }, "jevbench_score": 64.97450144379573 }, "heldout": { "n": 303, "escalation": 0.39933993399339934, "tiers": { "easy": 0.9166666666666666, "standard": 0.7083333333333334, "judge": 0.7876712328767124, "hard": 0.5137614678899083 }, "accuracy": 0.693069306930693, "ece_hard": 0.2652299023600466, "calibration": 57.34182732625115, "p50_s": 2.8707323759794234, "p95_s": 4.9281502470374114, "cost_usd_per_1000": 0.013246499672439157, "axes": { "intelligence": 70.13430522391187, "calibration": 57.34182732625115, "speed": 68.49323352802472, "cost": 66.33696600753501 }, "jevbench_score": 65.38096487122962 } }, { "threshold": 0.57, "public": { "n": 231, "escalation": 0.36363636363636365, "tiers": { "easy": 0.9791666666666666, "standard": 0.6388888888888888, "hard": 0.5045045045045045 }, "accuracy": 0.645021645021645, "ece_hard": 0.2833734423463175, "calibration": 54.28296238614389, "p50_s": 1.3210672974586486, "p95_s": 4.716590677946806, "cost_usd_per_1000": 0.012319947225059587, "axes": { "intelligence": 64.90605188521856, "calibration": 54.28296238614389, "speed": 72.05446869804332, "cost": 67.28173457662791 }, "jevbench_score": 64.28752528641753 }, "heldout": { "n": 303, "escalation": 0.40924092409240925, "tiers": { "easy": 0.9166666666666666, "standard": 0.7083333333333334, "judge": 0.7945205479452054, "hard": 0.5229357798165137 }, "accuracy": 0.6996699669966997, "ece_hard": 0.25975248623740144, "calibration": 57.889568938515666, "p50_s": 2.8755976274609565, "p95_s": 4.9281502470374114, "cost_usd_per_1000": 0.013503442788099037, "axes": { "intelligence": 70.60131540362784, "calibration": 57.889568938515666, "speed": 68.485879434629, "cost": 66.08666473791808 }, "jevbench_score": 65.58159803053003 } }, { "threshold": 0.58, "public": { "n": 231, "escalation": 0.36796536796536794, "tiers": { "easy": 0.9791666666666666, "standard": 0.6388888888888888, "hard": 0.5135135135135135 }, "accuracy": 0.6493506493506493, "ece_hard": 0.2755928252243116, "calibration": 55.06102409834448, "p50_s": 1.3210672974586486, "p95_s": 4.716590677946806, "cost_usd_per_1000": 0.012432290318919533, "axes": { "intelligence": 65.28142726059393, "calibration": 55.06102409834448, "speed": 72.05446869804332, "cost": 67.16346570332047 }, "jevbench_score": 64.5813301881859 }, "heldout": { "n": 303, "escalation": 0.42244224422442245, "tiers": { "easy": 0.9166666666666666, "standard": 0.7083333333333334, "judge": 0.8013698630136986, "hard": 0.5321100917431193 }, "accuracy": 0.7062706270627063, "ece_hard": 0.25438284516448584, "calibration": 58.426533045807226, "p50_s": 2.8755976274609565, "p95_s": 4.9281502470374114, "cost_usd_per_1000": 0.013846033608978877, "axes": { "intelligence": 71.06832558334382, "calibration": 58.426533045807226, "speed": 68.485879434629, "cost": 65.76023855633733 }, "jevbench_score": 65.76012865788752 } }, { "threshold": 0.59, "public": { "n": 231, "escalation": 0.37662337662337664, "tiers": { "easy": 0.9791666666666666, "standard": 0.6388888888888888, "hard": 0.5135135135135135 }, "accuracy": 0.6493506493506493, "ece_hard": 0.27931815810667654, "calibration": 54.68849081010799, "p50_s": 1.3210672974586486, "p95_s": 4.716590677946806, "cost_usd_per_1000": 0.01265697650663943, "axes": { "intelligence": 65.28142726059393, "calibration": 54.68849081010799, "speed": 72.05446869804332, "cost": 66.9301007805193 }, "jevbench_score": 64.41573917141933 }, "heldout": { "n": 303, "escalation": 0.43564356435643564, "tiers": { "easy": 0.9166666666666666, "standard": 0.7083333333333334, "judge": 0.8013698630136986, "hard": 0.5504587155963303 }, "accuracy": 0.7128712871287128, "ece_hard": 0.24551445432536392, "calibration": 59.31337212971942, "p50_s": 2.8755976274609565, "p95_s": 4.928930576890708, "cost_usd_per_1000": 0.014188624429858718, "axes": { "intelligence": 71.61878429894014, "calibration": 59.31337212971942, "speed": 68.48519182141752, "cost": 65.44179120494306 }, "jevbench_score": 66.05532804256426 } }, { "threshold": 0.6, "public": { "n": 231, "escalation": 0.3939393939393939, "tiers": { "easy": 0.9791666666666666, "standard": 0.6666666666666666, "hard": 0.5135135135135135 }, "accuracy": 0.658008658008658, "ece_hard": 0.280145879397107, "calibration": 54.60571868106494, "p50_s": 1.3210672974586486, "p95_s": 4.716590677946806, "cost_usd_per_1000": 0.01310634888207922, "axes": { "intelligence": 66.36167417417417, "calibration": 54.60571868106494, "speed": 72.05446869804332, "cost": 66.47554826808225 }, "jevbench_score": 64.5460362198616 }, "heldout": { "n": 303, "escalation": 0.44224422442244227, "tiers": { "easy": 0.9166666666666666, "standard": 0.7083333333333334, "judge": 0.8013698630136986, "hard": 0.5596330275229358 }, "accuracy": 0.7161716171617162, "ece_hard": 0.23953479209109135, "calibration": 59.911338353146675, "p50_s": 2.8755976274609565, "p95_s": 4.928930576890708, "cost_usd_per_1000": 0.014359919840298638, "axes": { "intelligence": 71.89401365673831, "calibration": 59.911338353146675, "speed": 68.48519182141752, "cost": 65.28543953194765 }, "jevbench_score": 66.24508932124053 } }, { "threshold": 0.61, "public": { "n": 231, "escalation": 0.4199134199134199, "tiers": { "easy": 0.9791666666666666, "standard": 0.6666666666666666, "hard": 0.5405405405405406 }, "accuracy": 0.670995670995671, "ece_hard": 0.2636968955138308, "calibration": 57.88761723749607, "p50_s": 1.3210672974586486, "p95_s": 4.716590677946806, "cost_usd_per_1000": 0.013780407445238908, "axes": { "intelligence": 67.48780030030031, "calibration": 57.88761723749607, "speed": 72.05446869804332, "cost": 65.82213824367264 }, "jevbench_score": 65.60862910671761 }, "heldout": { "n": 303, "escalation": 0.46534653465346537, "tiers": { "easy": 0.9166666666666666, "standard": 0.7083333333333334, "judge": 0.8082191780821918, "hard": 0.5596330275229358 }, "accuracy": 0.7194719471947195, "ece_hard": 0.2391165183802068, "calibration": 59.95316572423513, "p50_s": 2.8755976274609565, "p95_s": 4.928930576890708, "cost_usd_per_1000": 0.01495945377683836, "axes": { "intelligence": 72.08579447865611, "calibration": 59.95316572423513, "speed": 68.48519182141752, "cost": 64.75252791480683 }, "jevbench_score": 66.16507413716639 } }, { "threshold": 0.62, "public": { "n": 231, "escalation": 0.44155844155844154, "tiers": { "easy": 0.9791666666666666, "standard": 0.6666666666666666, "hard": 0.5495495495495496 }, "accuracy": 0.6753246753246753, "ece_hard": 0.2636312689246859, "calibration": 57.894179896410556, "p50_s": 1.3210672974586486, "p95_s": 4.716590677946806, "cost_usd_per_1000": 0.014342122914538647, "axes": { "intelligence": 67.86317567567568, "calibration": 57.894179896410556, "speed": 72.05446869804332, "cost": 65.30159679512705 }, "jevbench_score": 65.5712483028132 }, "heldout": { "n": 303, "escalation": 0.4752475247524752, "tiers": { "easy": 0.9166666666666666, "standard": 0.75, "judge": 0.8082191780821918, "hard": 0.5596330275229358 }, "accuracy": 0.7227722772277227, "ece_hard": 0.24256179259661234, "calibration": 59.60863830259457, "p50_s": 2.8755976274609565, "p95_s": 4.928930576890708, "cost_usd_per_1000": 0.01521639689249824, "axes": { "intelligence": 73.25246114532278, "calibration": 59.60863830259457, "speed": 68.48519182141752, "cost": 64.53064517392319 }, "jevbench_score": 66.27863053575264 } }, { "threshold": 0.63, "public": { "n": 231, "escalation": 0.44155844155844154, "tiers": { "easy": 0.9791666666666666, "standard": 0.6666666666666666, "hard": 0.5495495495495496 }, "accuracy": 0.6753246753246753, "ece_hard": 0.2636312689246859, "calibration": 57.894179896410556, "p50_s": 1.3210672974586486, "p95_s": 4.716590677946806, "cost_usd_per_1000": 0.014342122914538647, "axes": { "intelligence": 67.86317567567568, "calibration": 57.894179896410556, "speed": 72.05446869804332, "cost": 65.30159679512705 }, "jevbench_score": 65.5712483028132 }, "heldout": { "n": 303, "escalation": 0.49504950495049505, "tiers": { "easy": 0.9166666666666666, "standard": 0.75, "judge": 0.8082191780821918, "hard": 0.5596330275229358 }, "accuracy": 0.7227722772277227, "ece_hard": 0.24573858034357715, "calibration": 59.290959527898096, "p50_s": 2.9314894318580627, "p95_s": 4.928930576890708, "cost_usd_per_1000": 0.015730283123818002, "axes": { "intelligence": 73.25246114532278, "calibration": 59.290959527898096, "speed": 68.40158965888747, "cost": 64.09790381779479 }, "jevbench_score": 66.05872397406061 } }, { "threshold": 0.64, "public": { "n": 231, "escalation": 0.4588744588744589, "tiers": { "easy": 0.9791666666666666, "standard": 0.6666666666666666, "hard": 0.5405405405405406 }, "accuracy": 0.670995670995671, "ece_hard": 0.2767653257746059, "calibration": 56.58077421141856, "p50_s": 1.3210672974586486, "p95_s": 4.724898611754179, "cost_usd_per_1000": 0.014791495289978439, "axes": { "intelligence": 67.48780030030031, "calibration": 56.58077421141856, "speed": 72.0468256437982, "cost": 64.89963761303186 }, "jevbench_score": 65.00366153104409 }, "heldout": { "n": 303, "escalation": 0.5148514851485149, "tiers": { "easy": 0.9166666666666666, "standard": 0.75, "judge": 0.815068493150685, "hard": 0.5688073394495413 }, "accuracy": 0.7293729372937293, "ece_hard": 0.24648141564431553, "calibration": 59.216675997824254, "p50_s": 2.9615079179406165, "p95_s": 4.928930576890708, "cost_usd_per_1000": 0.016244169355137762, "axes": { "intelligence": 73.71947132503875, "calibration": 59.216675997824254, "speed": 68.35734400646604, "cost": 63.679074741395 }, "jevbench_score": 66.0240316860986 } }, { "threshold": 0.65, "public": { "n": 231, "escalation": 0.48917748917748916, "tiers": { "easy": 0.9791666666666666, "standard": 0.6666666666666666, "hard": 0.5495495495495496 }, "accuracy": 0.6753246753246753, "ece_hard": 0.27310161700061863, "calibration": 56.94714508881729, "p50_s": 1.3210672974586486, "p95_s": 4.724898611754179, "cost_usd_per_1000": 0.015577896946998073, "axes": { "intelligence": 67.86317567567568, "calibration": 56.94714508881729, "speed": 72.0468256437982, "cost": 64.2247352047103 }, "jevbench_score": 65.02881305961634 }, "heldout": { "n": 303, "escalation": 0.528052805280528, "tiers": { "easy": 0.9583333333333334, "standard": 0.75, "judge": 0.8287671232876712, "hard": 0.5779816513761468 }, "accuracy": 0.7425742574257426, "ece_hard": 0.24045546172838317, "calibration": 59.819271389417494, "p50_s": 2.9615079179406165, "p95_s": 4.928930576890708, "cost_usd_per_1000": 0.016586760176017602, "axes": { "intelligence": 74.96159566000587, "calibration": 59.819271389417494, "speed": 68.35734400646604, "cost": 63.407153039674974 }, "jevbench_score": 66.39736488822571 } }, { "threshold": 0.66, "public": { "n": 231, "escalation": 0.5151515151515151, "tiers": { "easy": 0.9791666666666666, "standard": 0.6944444444444444, "hard": 0.5225225225225225 }, "accuracy": 0.670995670995671, "ece_hard": 0.3099889288537704, "calibration": 50.871012089922715, "p50_s": 1.3210672974586486, "p95_s": 4.724898611754179, "cost_usd_per_1000": 0.01625195551015776, "axes": { "intelligence": 67.8172964631298, "calibration": 50.871012089922715, "speed": 72.0468256437982, "cost": 63.67283125684571 }, "jevbench_score": 63.07321457914135 }, "heldout": { "n": 303, "escalation": 0.5478547854785478, "tiers": { "easy": 0.9583333333333334, "standard": 0.75, "judge": 0.8424657534246576, "hard": 0.5871559633027523 }, "accuracy": 0.7524752475247525, "ece_hard": 0.2332357806120849, "calibration": 60.541239501047315, "p50_s": 2.9615079179406165, "p95_s": 4.928930576890708, "cost_usd_per_1000": 0.017100646407337362, "axes": { "intelligence": 75.62038666163966, "calibration": 60.541239501047315, "speed": 68.35734400646604, "cost": 63.00962418677263 }, "jevbench_score": 66.6377872847396 } }, { "threshold": 0.67, "public": { "n": 231, "escalation": 0.5194805194805194, "tiers": { "easy": 0.9791666666666666, "standard": 0.6944444444444444, "hard": 0.5315315315315315 }, "accuracy": 0.6753246753246753, "ece_hard": 0.3040217127511758, "calibration": 51.46773370018218, "p50_s": 1.3210672974586486, "p95_s": 4.724898611754179, "cost_usd_per_1000": 0.016364298604017707, "axes": { "intelligence": 68.19267183850518, "calibration": 51.46773370018218, "speed": 72.0468256437982, "cost": 63.58307813218746 }, "jevbench_score": 63.32238848859681 }, "heldout": { "n": 303, "escalation": 0.570957095709571, "tiers": { "easy": 0.9583333333333334, "standard": 0.75, "judge": 0.8424657534246576, "hard": 0.6055045871559633 }, "accuracy": 0.759075907590759, "ece_hard": 0.21882777880021376, "calibration": 61.98203968223443, "p50_s": 2.9615079179406165, "p95_s": 4.928930576890708, "cost_usd_per_1000": 0.017700180343877086, "axes": { "intelligence": 76.17084537723599, "calibration": 61.98203968223443, "speed": 68.35734400646604, "cost": 62.5606692600752 }, "jevbench_score": 67.03248217464537 } }, { "threshold": 0.68, "public": { "n": 231, "escalation": 0.5238095238095238, "tiers": { "easy": 0.9791666666666666, "standard": 0.6944444444444444, "hard": 0.5315315315315315 }, "accuracy": 0.6753246753246753, "ece_hard": 0.3040217127511758, "calibration": 51.46773370018218, "p50_s": 1.3210672974586486, "p95_s": 4.724898611754179, "cost_usd_per_1000": 0.016476641697877655, "axes": { "intelligence": 68.19267183850518, "calibration": 51.46773370018218, "speed": 72.0468256437982, "cost": 63.493939071715495 }, "jevbench_score": 63.300183413521516 }, "heldout": { "n": 303, "escalation": 0.5907590759075908, "tiers": { "easy": 0.9583333333333334, "standard": 0.75, "judge": 0.8493150684931506, "hard": 0.6055045871559633 }, "accuracy": 0.7623762376237624, "ece_hard": 0.22224792840174656, "calibration": 61.38404930697417, "p50_s": 2.974942201375961, "p95_s": 4.9330528065562245, "cost_usd_per_1000": 0.01821406657519685, "axes": { "intelligence": 76.3626261991538, "calibration": 61.38404930697417, "speed": 68.33405703084625, "cost": 62.1877924103372 }, "jevbench_score": 66.80664834462996 } }, { "threshold": 0.69, "public": { "n": 231, "escalation": 0.5238095238095238, "tiers": { "easy": 0.9791666666666666, "standard": 0.6944444444444444, "hard": 0.5315315315315315 }, "accuracy": 0.6753246753246753, "ece_hard": 0.3040217127511758, "calibration": 51.46773370018218, "p50_s": 1.3210672974586486, "p95_s": 4.724898611754179, "cost_usd_per_1000": 0.016476641697877655, "axes": { "intelligence": 68.19267183850518, "calibration": 51.46773370018218, "speed": 72.0468256437982, "cost": 63.493939071715495 }, "jevbench_score": 63.300183413521516 }, "heldout": { "n": 303, "escalation": 0.594059405940594, "tiers": { "easy": 0.9583333333333334, "standard": 0.75, "judge": 0.8561643835616438, "hard": 0.6055045871559633 }, "accuracy": 0.7656765676567657, "ece_hard": 0.22224792840174656, "calibration": 61.38404930697417, "p50_s": 2.974942201375961, "p95_s": 4.9330528065562245, "cost_usd_per_1000": 0.018299714280416806, "axes": { "intelligence": 76.55440702107161, "calibration": 61.38404930697417, "speed": 68.33405703084625, "cost": 62.12667073006589 }, "jevbench_score": 66.83212257420496 } }, { "threshold": 0.7, "public": { "n": 231, "escalation": 0.5367965367965368, "tiers": { "easy": 0.9791666666666666, "standard": 0.7083333333333334, "hard": 0.5405405405405406 }, "accuracy": 0.683982683982684, "ece_hard": 0.29437456070136797, "calibration": 52.43244890516296, "p50_s": 1.3210672974586486, "p95_s": 4.724898611754179, "cost_usd_per_1000": 0.016813670979457497, "axes": { "intelligence": 69.10817067067069, "calibration": 52.43244890516296, "speed": 72.0468256437982, "cost": 63.23012366153998 }, "jevbench_score": 63.74074061783342 }, "heldout": { "n": 303, "escalation": 0.6105610561056105, "tiers": { "easy": 0.9583333333333334, "standard": 0.75, "judge": 0.8698630136986302, "hard": 0.6055045871559633 }, "accuracy": 0.7722772277227723, "ece_hard": 0.2261491714244706, "calibration": 60.993925004701765, "p50_s": 2.974942201375961, "p95_s": 4.9330528065562245, "cost_usd_per_1000": 0.018727952806516606, "axes": { "intelligence": 76.93796866490723, "calibration": 60.993925004701765, "speed": 68.33405703084625, "cost": 61.82529081129818 }, "jevbench_score": 66.72793232258005 } }, { "threshold": 0.71, "public": { "n": 231, "escalation": 0.5454545454545454, "tiers": { "easy": 0.9791666666666666, "standard": 0.7083333333333334, "hard": 0.5405405405405406 }, "accuracy": 0.683982683982684, "ece_hard": 0.29437456070136797, "calibration": 52.43244890516296, "p50_s": 1.3210672974586486, "p95_s": 4.724898611754179, "cost_usd_per_1000": 0.01703835716717739, "axes": { "intelligence": 69.10817067067069, "calibration": 52.43244890516296, "speed": 72.0468256437982, "cost": 63.05716846247546 }, "jevbench_score": 63.697107859936395 }, "heldout": { "n": 303, "escalation": 0.6303630363036303, "tiers": { "easy": 0.9583333333333334, "standard": 0.75, "judge": 0.8835616438356164, "hard": 0.6238532110091743 }, "accuracy": 0.7854785478547854, "ece_hard": 0.2131808385287789, "calibration": 62.29075829427094, "p50_s": 2.974942201375961, "p95_s": 4.9330528065562245, "cost_usd_per_1000": 0.019241839037836366, "axes": { "intelligence": 77.87198902433917, "calibration": 62.29075829427094, "speed": 68.33405703084625, "cost": 61.47260267921223 }, "jevbench_score": 67.1863306694381 } }, { "threshold": 0.72, "public": { "n": 231, "escalation": 0.5497835497835498, "tiers": { "easy": 0.9791666666666666, "standard": 0.7083333333333334, "hard": 0.5405405405405406 }, "accuracy": 0.683982683982684, "ece_hard": 0.2968808593460725, "calibration": 52.1818190406925, "p50_s": 1.3210672974586486, "p95_s": 4.724898611754179, "cost_usd_per_1000": 0.017150700261037342, "axes": { "intelligence": 69.10817067067069, "calibration": 52.1818190406925, "speed": 72.0468256437982, "cost": 62.97154429202569 }, "jevbench_score": 63.59924374910914 }, "heldout": { "n": 303, "escalation": 0.6501650165016502, "tiers": { "easy": 0.9583333333333334, "standard": 0.75, "judge": 0.8835616438356164, "hard": 0.6330275229357798 }, "accuracy": 0.7887788778877888, "ece_hard": 0.2092161573518775, "calibration": 62.68722641196108, "p50_s": 2.974942201375961, "p95_s": 4.9330528065562245, "cost_usd_per_1000": 0.01975572526915613, "axes": { "intelligence": 78.14721838213734, "calibration": 62.68722641196108, "speed": 68.33405703084625, "cost": 61.12921065812362 }, "jevbench_score": 67.25810752558476 } }, { "threshold": 0.73, "public": { "n": 231, "escalation": 0.5584415584415584, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "hard": 0.5405405405405406 }, "accuracy": 0.6883116883116883, "ece_hard": 0.2968808593460725, "calibration": 52.1818190406925, "p50_s": 1.3210672974586486, "p95_s": 4.724898611754179, "cost_usd_per_1000": 0.017375386448757236, "axes": { "intelligence": 69.51326326326327, "calibration": 52.1818190406925, "speed": 72.0468256437982, "cost": 62.801965822515776 }, "jevbench_score": 63.649316598887246 }, "heldout": { "n": 303, "escalation": 0.6666666666666666, "tiers": { "easy": 1.0, "standard": 0.75, "judge": 0.8972602739726028, "hard": 0.6330275229357798 }, "accuracy": 0.7986798679867987, "ece_hard": 0.2092161573518775, "calibration": 62.68722641196108, "p50_s": 2.974942201375961, "p95_s": 4.9330528065562245, "cost_usd_per_1000": 0.02018396379525593, "axes": { "intelligence": 79.11411335930627, "calibration": 62.68722641196108, "speed": 68.33405703084625, "cost": 60.849806245009745 }, "jevbench_score": 67.38796729695093 } }, { "threshold": 0.74, "public": { "n": 231, "escalation": 0.5627705627705628, "tiers": { "easy": 1.0, "standard": 0.7222222222222222, "hard": 0.5405405405405406 }, "accuracy": 0.6926406926406926, "ece_hard": 0.2968808593460725, "calibration": 52.1818190406925, "p50_s": 1.3210672974586486, "p95_s": 4.724898611754179, "cost_usd_per_1000": 0.017487729542617184, "axes": { "intelligence": 70.0533867200534, "calibration": 52.1818190406925, "speed": 72.0468256437982, "cost": 62.71799715838404 }, "jevbench_score": 63.75127069173333 }, "heldout": { "n": 303, "escalation": 0.6897689768976898, "tiers": { "easy": 1.0, "standard": 0.75, "judge": 0.9041095890410958, "hard": 0.6330275229357798 }, "accuracy": 0.801980198019802, "ece_hard": 0.21393475314649518, "calibration": 62.21536683249931, "p50_s": 2.974942201375961, "p95_s": 4.9330528065562245, "cost_usd_per_1000": 0.020783497731795653, "axes": { "intelligence": 79.30589418122409, "calibration": 62.21536683249931, "speed": 68.33405703084625, "cost": 60.46844084804096 }, "jevbench_score": 67.1958229607099 } }, { "threshold": 0.75, "public": { "n": 231, "escalation": 0.5844155844155844, "tiers": { "easy": 1.0, "standard": 0.7361111111111112, "hard": 0.5495495495495496 }, "accuracy": 0.7012987012987013, "ece_hard": 0.2900366322257912, "calibration": 52.86624175272064, "p50_s": 1.3233075231313705, "p95_s": 4.745200221985579, "cost_usd_per_1000": 0.018049445011916923, "axes": { "intelligence": 70.9688855522189, "calibration": 52.86624175272064, "speed": 72.02084676442195, "cost": 62.30608441989799 }, "jevbench_score": 64.0558467310482 }, "heldout": { "n": 303, "escalation": 0.7029702970297029, "tiers": { "easy": 1.0, "standard": 0.75, "judge": 0.9041095890410958, "hard": 0.6330275229357798 }, "accuracy": 0.801980198019802, "ece_hard": 0.21629595281172764, "calibration": 61.979246865976066, "p50_s": 2.974942201375961, "p95_s": 4.9330528065562245, "cost_usd_per_1000": 0.021126088552675493, "axes": { "intelligence": 79.30589418122409, "calibration": 61.979246865976066, "speed": 68.33405703084625, "cost": 60.25542712426116 }, "jevbench_score": 67.07277642484387 } }, { "threshold": 0.76, "public": { "n": 231, "escalation": 0.5974025974025974, "tiers": { "easy": 1.0, "standard": 0.7361111111111112, "hard": 0.5495495495495496 }, "accuracy": 0.7012987012987013, "ece_hard": 0.29215115698184185, "calibration": 52.65478927711557, "p50_s": 1.3951993823051452, "p95_s": 4.745200221985579, "cost_usd_per_1000": 0.018386474293496765, "axes": { "intelligence": 70.9688855522189, "calibration": 52.65478927711557, "speed": 71.79109182639567, "cost": 62.065046233261036 }, "jevbench_score": 63.87867165772347 }, "heldout": { "n": 303, "escalation": 0.7227722772277227, "tiers": { "easy": 1.0, "standard": 0.75, "judge": 0.9041095890410958, "hard": 0.6422018348623854 }, "accuracy": 0.8052805280528053, "ece_hard": 0.20943487558294865, "calibration": 62.66535458885396, "p50_s": 2.974942201375961, "p95_s": 4.9330528065562245, "cost_usd_per_1000": 0.021639974783995253, "axes": { "intelligence": 79.58112353902224, "calibration": 62.66535458885396, "speed": 68.33405703084625, "cost": 59.94229748881908 }, "jevbench_score": 67.2282860928141 } }, { "threshold": 0.77, "public": { "n": 231, "escalation": 0.6190476190476191, "tiers": { "easy": 1.0, "standard": 0.7361111111111112, "hard": 0.5675675675675675 }, "accuracy": 0.70995670995671, "ece_hard": 0.2783628277269573, "calibration": 54.03362220260402, "p50_s": 1.3951993823051452, "p95_s": 4.745200221985579, "cost_usd_per_1000": 0.018948189762796507, "axes": { "intelligence": 71.71963630296963, "calibration": 54.03362220260402, "speed": 71.79109182639567, "cost": 61.67296823617753 }, "jevbench_score": 64.36012700233707 }, "heldout": { "n": 303, "escalation": 0.735973597359736, "tiers": { "easy": 1.0, "standard": 0.75, "judge": 0.910958904109589, "hard": 0.6422018348623854 }, "accuracy": 0.8085808580858086, "ece_hard": 0.20943487558294865, "calibration": 62.66535458885396, "p50_s": 2.984680587053299, "p95_s": 4.9330528065562245, "cost_usd_per_1000": 0.021982565604875097, "axes": { "intelligence": 79.77290436094006, "calibration": 62.66535458885396, "speed": 68.3198637468859, "cost": 59.73764866172135 }, "jevbench_score": 67.20777307852282 } }, { "threshold": 0.78, "public": { "n": 231, "escalation": 0.6406926406926406, "tiers": { "easy": 1.0, "standard": 0.7361111111111112, "hard": 0.5675675675675675 }, "accuracy": 0.70995670995671, "ece_hard": 0.2838276729980621, "calibration": 53.48713767549354, "p50_s": 1.3951993823051452, "p95_s": 4.751377658545971, "cost_usd_per_1000": 0.019509905232096243, "axes": { "intelligence": 71.71963630296963, "calibration": 53.48713767549354, "speed": 71.78544173428324, "cost": 61.292345204598526 }, "jevbench_score": 64.09623398935545 }, "heldout": { "n": 303, "escalation": 0.7557755775577558, "tiers": { "easy": 1.0, "standard": 0.75, "judge": 0.9178082191780822, "hard": 0.6422018348623854 }, "accuracy": 0.8118811881188119, "ece_hard": 0.2215704792740007, "calibration": 61.45179421974876, "p50_s": 3.0078381687402724, "p95_s": 4.9330528065562245, "cost_usd_per_1000": 0.022496451836194857, "axes": { "intelligence": 79.96468518285786, "calibration": 61.45179421974876, "speed": 68.2862976942293, "cost": 59.43657921595916 }, "jevbench_score": 66.82747362523314 } }, { "threshold": 0.79, "public": { "n": 231, "escalation": 0.645021645021645, "tiers": { "easy": 1.0, "standard": 0.75, "hard": 0.5675675675675675 }, "accuracy": 0.7142857142857143, "ece_hard": 0.2838276729980621, "calibration": 53.48713767549354, "p50_s": 1.3951993823051452, "p95_s": 4.751377658545971, "cost_usd_per_1000": 0.01962224832595619, "axes": { "intelligence": 72.25975975975977, "calibration": 53.48713767549354, "speed": 71.78544173428324, "cost": 61.21753697349259 }, "jevbench_score": 64.19696919662994 }, "heldout": { "n": 303, "escalation": 0.7755775577557755, "tiers": { "easy": 1.0, "standard": 0.75, "judge": 0.9246575342465754, "hard": 0.6422018348623854 }, "accuracy": 0.8151815181518152, "ece_hard": 0.2196187877726698, "calibration": 61.646963369881846, "p50_s": 3.0078381687402724, "p95_s": 4.9330528065562245, "cost_usd_per_1000": 0.023010338067514617, "axes": { "intelligence": 80.15646600477568, "calibration": 61.646963369881846, "speed": 68.2862976942293, "cost": 59.14231001910946 }, "jevbench_score": 66.83755049264848 } }, { "threshold": 0.8, "public": { "n": 231, "escalation": 0.658008658008658, "tiers": { "easy": 1.0, "standard": 0.75, "hard": 0.5675675675675675 }, "accuracy": 0.7142857142857143, "ece_hard": 0.2838276729980621, "calibration": 53.48713767549354, "p50_s": 1.3951993823051452, "p95_s": 4.751377658545971, "cost_usd_per_1000": 0.019959277607536036, "axes": { "intelligence": 72.25975975975977, "calibration": 53.48713767549354, "speed": 71.78544173428324, "cost": 60.99565543966327 }, "jevbench_score": 64.13871986138656 }, "heldout": { "n": 303, "escalation": 0.7821782178217822, "tiers": { "easy": 1.0, "standard": 0.75, "judge": 0.9246575342465754, "hard": 0.6422018348623854 }, "accuracy": 0.8151815181518152, "ece_hard": 0.21880044608051838, "calibration": 61.72879753909699, "p50_s": 3.0078381687402724, "p95_s": 4.9330528065562245, "cost_usd_per_1000": 0.02318163347795454, "axes": { "intelligence": 80.15646600477568, "calibration": 61.72879753909699, "speed": 68.2862976942293, "cost": 59.045678950913 }, "jevbench_score": 66.83239374745094 } }, { "threshold": 0.81, "public": { "n": 231, "escalation": 0.6623376623376623, "tiers": { "easy": 1.0, "standard": 0.75, "hard": 0.5765765765765766 }, "accuracy": 0.7186147186147186, "ece_hard": 0.27657251822098716, "calibration": 54.212653153201046, "p50_s": 1.3951993823051452, "p95_s": 4.751377658545971, "cost_usd_per_1000": 0.020071620701395985, "axes": { "intelligence": 72.63513513513514, "calibration": 54.212653153201046, "speed": 71.78544173428324, "cost": 60.92252675772917 }, "jevbench_score": 64.4192143440257 }, "heldout": { "n": 303, "escalation": 0.7887788778877888, "tiers": { "easy": 1.0, "standard": 0.75, "judge": 0.9246575342465754, "hard": 0.6422018348623854 }, "accuracy": 0.8151815181518152, "ece_hard": 0.2217630466917029, "calibration": 60.77960894932215, "p50_s": 3.0078381687402724, "p95_s": 4.9330528065562245, "cost_usd_per_1000": 0.023352928888394457, "axes": { "intelligence": 80.15646600477568, "calibration": 60.77960894932215, "speed": 68.2862976942293, "cost": 58.94975929409452 }, "jevbench_score": 66.54692869891292 } }, { "threshold": 0.82, "public": { "n": 231, "escalation": 0.670995670995671, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.5765765765765766 }, "accuracy": 0.7229437229437229, "ece_hard": 0.27657251822098716, "calibration": 54.212653153201046, "p50_s": 1.3951993823051452, "p95_s": 4.751377658545971, "cost_usd_per_1000": 0.020296306889115878, "axes": { "intelligence": 73.17525859192527, "calibration": 54.212653153201046, "speed": 71.78544173428324, "cost": 60.77748937037443 }, "jevbench_score": 64.50019316796478 }, "heldout": { "n": 303, "escalation": 0.801980198019802, "tiers": { "easy": 1.0, "standard": 0.75, "judge": 0.9315068493150684, "hard": 0.6422018348623854 }, "accuracy": 0.8184818481848185, "ece_hard": 0.2217630466917029, "calibration": 60.77960894932215, "p50_s": 3.024282158911228, "p95_s": 4.9330528065562245, "cost_usd_per_1000": 0.0236955197092743, "axes": { "intelligence": 80.34824682669348, "calibration": 60.77960894932215, "speed": 68.26261928101945, "cost": 58.760012846884464 }, "jevbench_score": 66.52728257315981 } }, { "threshold": 0.83, "public": { "n": 231, "escalation": 0.696969696969697, "tiers": { "easy": 1.0, "standard": 0.7777777777777778, "hard": 0.5945945945945946 }, "accuracy": 0.7359307359307359, "ece_hard": 0.2632608985682129, "calibration": 55.54381511847847, "p50_s": 1.4024939998984336, "p95_s": 4.751377658545971, "cost_usd_per_1000": 0.020970365452275565, "axes": { "intelligence": 74.46613279946614, "calibration": 55.54381511847847, "speed": 71.76279436793746, "cost": 60.35182002949647 }, "jevbench_score": 65.05730295925486 }, "heldout": { "n": 303, "escalation": 0.8250825082508251, "tiers": { "easy": 1.0, "standard": 0.75, "judge": 0.9315068493150684, "hard": 0.6513761467889908 }, "accuracy": 0.8217821782178217, "ece_hard": 0.21423718589539312, "calibration": 61.532195028953126, "p50_s": 3.024282158911228, "p95_s": 4.98705321252346, "cost_usd_per_1000": 0.02429505364581402, "axes": { "intelligence": 80.62347618449165, "calibration": 61.532195028953126, "speed": 68.21533690332035, "cost": 58.4344641290785 }, "jevbench_score": 66.68509210303827 } }, { "threshold": 0.84, "public": { "n": 231, "escalation": 0.7056277056277056, "tiers": { "easy": 1.0, "standard": 0.7777777777777778, "hard": 0.5945945945945946 }, "accuracy": 0.7359307359307359, "ece_hard": 0.2647611118986746, "calibration": 55.3937937854323, "p50_s": 1.4024939998984336, "p95_s": 4.751377658545971, "cost_usd_per_1000": 0.02119505163999546, "axes": { "intelligence": 74.46613279946614, "calibration": 55.3937937854323, "speed": 71.76279436793746, "cost": 60.21296562915341 }, "jevbench_score": 64.97590203873678 }, "heldout": { "n": 303, "escalation": 0.834983498349835, "tiers": { "easy": 1.0, "standard": 0.75, "judge": 0.9315068493150684, "hard": 0.6513761467889908 }, "accuracy": 0.8217821782178217, "ece_hard": 0.21569775265938476, "calibration": 61.386138352553964, "p50_s": 3.024282158911228, "p95_s": 4.98705321252346, "cost_usd_per_1000": 0.0245519967614739, "axes": { "intelligence": 80.62347618449165, "calibration": 61.386138352553964, "speed": 68.21533690332035, "cost": 58.29739545585264 }, "jevbench_score": 66.60636814687341 } }, { "threshold": 0.85, "public": { "n": 231, "escalation": 0.70995670995671, "tiers": { "easy": 1.0, "standard": 0.7777777777777778, "hard": 0.5945945945945946 }, "accuracy": 0.7359307359307359, "ece_hard": 0.2659896329582105, "calibration": 55.27094167947871, "p50_s": 1.4024939998984336, "p95_s": 4.751377658545971, "cost_usd_per_1000": 0.02130739473385541, "axes": { "intelligence": 74.46613279946614, "calibration": 55.27094167947871, "speed": 71.76279436793746, "cost": 60.14408945350025 }, "jevbench_score": 64.92126746586631 }, "heldout": { "n": 303, "escalation": 0.8481848184818482, "tiers": { "easy": 1.0, "standard": 0.75, "judge": 0.9315068493150684, "hard": 0.6697247706422018 }, "accuracy": 0.8283828382838284, "ece_hard": 0.20136910614378173, "calibration": 62.819003004114265, "p50_s": 3.024282158911228, "p95_s": 4.98705321252346, "cost_usd_per_1000": 0.024894587582353744, "axes": { "intelligence": 81.17393490008797, "calibration": 62.819003004114265, "speed": 68.21533690332035, "cost": 58.11685192281848 }, "jevbench_score": 67.05372932786317 } }, { "threshold": 0.86, "public": { "n": 231, "escalation": 0.7229437229437229, "tiers": { "easy": 1.0, "standard": 0.7777777777777778, "hard": 0.6126126126126126 }, "accuracy": 0.7445887445887446, "ece_hard": 0.25012862268188996, "calibration": 56.857042707110764, "p50_s": 1.4024939998984336, "p95_s": 4.751377658545971, "cost_usd_per_1000": 0.02164442401543525, "axes": { "intelligence": 75.21688355021689, "calibration": 56.857042707110764, "speed": 71.76279436793746, "cost": 59.93961900411304 }, "jevbench_score": 65.49048858942326 }, "heldout": { "n": 303, "escalation": 0.8547854785478548, "tiers": { "easy": 1.0, "standard": 0.75, "judge": 0.9315068493150684, "hard": 0.6697247706422018 }, "accuracy": 0.8283828382838284, "ece_hard": 0.20185779947470392, "calibration": 62.50379580566943, "p50_s": 3.024282158911228, "p95_s": 4.98705321252346, "cost_usd_per_1000": 0.025065882992793664, "axes": { "intelligence": 81.17393490008797, "calibration": 62.50379580566943, "speed": 68.21533690332035, "cost": 58.027509758286186 }, "jevbench_score": 66.94370400394784 } }, { "threshold": 0.87, "public": { "n": 231, "escalation": 0.7532467532467533, "tiers": { "easy": 1.0, "standard": 0.7777777777777778, "hard": 0.6306306306306306 }, "accuracy": 0.7532467532467533, "ece_hard": 0.23455796419027403, "calibration": 58.41410855627235, "p50_s": 1.4024939998984336, "p95_s": 4.751377658545971, "cost_usd_per_1000": 0.02243082567245489, "axes": { "intelligence": 75.96763430096765, "calibration": 58.41410855627235, "speed": 71.76279436793746, "cost": 59.47464219591102 }, "jevbench_score": 65.96968095820746 }, "heldout": { "n": 303, "escalation": 0.8646864686468647, "tiers": { "easy": 1.0, "standard": 0.75, "judge": 0.9315068493150684, "hard": 0.6697247706422018 }, "accuracy": 0.8283828382838284, "ece_hard": 0.20185779947470392, "calibration": 62.50379580566943, "p50_s": 3.024282158911228, "p95_s": 4.98705321252346, "cost_usd_per_1000": 0.025322826108453544, "axes": { "intelligence": 81.17393490008797, "calibration": 62.50379580566943, "speed": 68.21533690332035, "cost": 57.89463481886412 }, "jevbench_score": 66.90534810556026 } }, { "threshold": 0.88, "public": { "n": 231, "escalation": 0.7792207792207793, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "hard": 0.6396396396396397 }, "accuracy": 0.7619047619047619, "ece_hard": 0.22212255959849037, "calibration": 59.657649015450716, "p50_s": 1.4024939998984336, "p95_s": 4.751377658545971, "cost_usd_per_1000": 0.023104884235614576, "axes": { "intelligence": 76.88313313313314, "calibration": 59.657649015450716, "speed": 71.76279436793746, "cost": 59.088886093682774 }, "jevbench_score": 66.40879318740514 }, "heldout": { "n": 303, "escalation": 0.8712871287128713, "tiers": { "easy": 1.0, "standard": 0.75, "judge": 0.9315068493150684, "hard": 0.6697247706422018 }, "accuracy": 0.8283828382838284, "ece_hard": 0.20185779947470392, "calibration": 62.50379580566943, "p50_s": 3.024282158911228, "p95_s": 4.98705321252346, "cost_usd_per_1000": 0.025494121518893464, "axes": { "intelligence": 81.17393490008797, "calibration": 62.50379580566943, "speed": 68.21533690332035, "cost": 57.80679845312333 }, "jevbench_score": 66.8799568533492 } }, { "threshold": 0.89, "public": { "n": 231, "escalation": 0.7878787878787878, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "hard": 0.6396396396396397 }, "accuracy": 0.7619047619047619, "ece_hard": 0.22212255959849037, "calibration": 59.657649015450716, "p50_s": 1.4024939998984336, "p95_s": 4.751377658545971, "cost_usd_per_1000": 0.02332957042333447, "axes": { "intelligence": 76.88313313313314, "calibration": 59.657649015450716, "speed": 71.76279436793746, "cost": 58.96279773909313 }, "jevbench_score": 66.37333777110685 }, "heldout": { "n": 303, "escalation": 0.8877887788778878, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6697247706422018 }, "accuracy": 0.8316831683168316, "ece_hard": 0.20185779947470392, "calibration": 62.50379580566943, "p50_s": 3.024282158911228, "p95_s": 4.98705321252346, "cost_usd_per_1000": 0.025922360044993265, "axes": { "intelligence": 82.34060156675464, "calibration": 62.50379580566943, "speed": 68.21533690332035, "cost": 57.589763848095615 }, "jevbench_score": 67.05589115222297 } }, { "threshold": 0.9, "public": { "n": 231, "escalation": 0.8008658008658008, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "hard": 0.6486486486486487 }, "accuracy": 0.7662337662337663, "ece_hard": 0.21403948716400195, "calibration": 60.46595625889957, "p50_s": 1.4088232532143592, "p95_s": 4.751377658545971, "cost_usd_per_1000": 0.02366659970491431, "axes": { "intelligence": 77.25850850850851, "calibration": 60.46595625889957, "speed": 71.74323937512592, "cost": 58.77592404346074 }, "jevbench_score": 66.62073553589815 }, "heldout": { "n": 303, "escalation": 0.9042904290429042, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6788990825688074 }, "accuracy": 0.834983498349835, "ece_hard": 0.1936376942232804, "calibration": 63.32580633081179, "p50_s": 3.024282158911228, "p95_s": 4.98705321252346, "cost_usd_per_1000": 0.026350598571093065, "axes": { "intelligence": 82.6158309245528, "calibration": 63.32580633081179, "speed": 68.21533690332035, "cost": 57.37628545167945 }, "jevbench_score": 67.26894477877397 } }, { "threshold": 0.91, "public": { "n": 231, "escalation": 0.8138528138528138, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.6486486486486487 }, "accuracy": 0.7705627705627706, "ece_hard": 0.21403948716400195, "calibration": 60.46595625889957, "p50_s": 1.4088232532143592, "p95_s": 4.751377658545971, "cost_usd_per_1000": 0.024003628986494156, "axes": { "intelligence": 77.79863196529865, "calibration": 60.46595625889957, "speed": 71.74323937512592, "cost": 58.591692836569436 }, "jevbench_score": 66.68451217536285 }, "heldout": { "n": 303, "escalation": 0.9108910891089109, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6788990825688074 }, "accuracy": 0.834983498349835, "ece_hard": 0.1936376942232804, "calibration": 63.32580633081179, "p50_s": 3.024282158911228, "p95_s": 4.98705321252346, "cost_usd_per_1000": 0.026521893981532988, "axes": { "intelligence": 82.6158309245528, "calibration": 63.32580633081179, "speed": 68.21533690332035, "cost": 57.291863959649056 }, "jevbench_score": 67.24418681143882 } }, { "threshold": 0.92, "public": { "n": 231, "escalation": 0.8225108225108225, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.6486486486486487 }, "accuracy": 0.7705627705627706, "ece_hard": 0.2102890041435215, "calibration": 61.36439087639471, "p50_s": 1.4088232532143592, "p95_s": 4.751377658545971, "cost_usd_per_1000": 0.024228315174214053, "axes": { "intelligence": 77.79863196529865, "calibration": 61.36439087639471, "speed": 71.74323937512592, "cost": 58.47030356347195 }, "jevbench_score": 66.89615857314998 }, "heldout": { "n": 303, "escalation": 0.9207920792079208, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6788990825688074 }, "accuracy": 0.834983498349835, "ece_hard": 0.19442690228705847, "calibration": 63.24688552443397, "p50_s": 3.024282158911228, "p95_s": 4.98705321252346, "cost_usd_per_1000": 0.026778837097192868, "axes": { "intelligence": 82.6158309245528, "calibration": 63.24688552443397, "speed": 68.21533690332035, "cost": 57.16624860004666 }, "jevbench_score": 67.18634804444935 } }, { "threshold": 0.93, "public": { "n": 231, "escalation": 0.8398268398268398, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.6486486486486487 }, "accuracy": 0.7705627705627706, "ece_hard": 0.2102890041435215, "calibration": 61.36439087639471, "p50_s": 1.4088232532143592, "p95_s": 4.751377658545971, "cost_usd_per_1000": 0.024677687549653844, "axes": { "intelligence": 77.79863196529865, "calibration": 61.36439087639471, "speed": 71.74323937512592, "cost": 58.23086616348804 }, "jevbench_score": 66.82756777935465 }, "heldout": { "n": 303, "escalation": 0.9240924092409241, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6788990825688074 }, "accuracy": 0.834983498349835, "ece_hard": 0.19442690228705847, "calibration": 63.24688552443397, "p50_s": 3.024282158911228, "p95_s": 4.98705321252346, "cost_usd_per_1000": 0.026864484802412825, "axes": { "intelligence": 82.6158309245528, "calibration": 63.24688552443397, "speed": 68.21533690332035, "cost": 57.12464451284149 }, "jevbench_score": 67.1741205940957 } }, { "threshold": 0.94, "public": { "n": 231, "escalation": 0.8528138528138528, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.6486486486486487 }, "accuracy": 0.7705627705627706, "ece_hard": 0.2102890041435215, "calibration": 61.36439087639471, "p50_s": 1.4213839560747146, "p95_s": 4.751377658545971, "cost_usd_per_1000": 0.025014716831233685, "axes": { "intelligence": 77.79863196529865, "calibration": 61.36439087639471, "speed": 71.70469039723565, "cost": 58.0541322701196 }, "jevbench_score": 66.76783171355476 }, "heldout": { "n": 303, "escalation": 0.9405940594059405, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6788990825688074 }, "accuracy": 0.834983498349835, "ece_hard": 0.1935286337386598, "calibration": 64.09002035150621, "p50_s": 3.024282158911228, "p95_s": 4.98705321252346, "cost_usd_per_1000": 0.027292723328512628, "axes": { "intelligence": 82.6158309245528, "calibration": 64.09002035150621, "speed": 68.21533690332035, "cost": 56.918593819017126 }, "jevbench_score": 67.3360239803631 } }, { "threshold": 0.95, "public": { "n": 231, "escalation": 0.8614718614718615, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.6486486486486487 }, "accuracy": 0.7705627705627706, "ece_hard": 0.2102890041435215, "calibration": 61.36439087639471, "p50_s": 1.4213839560747146, "p95_s": 4.751377658545971, "cost_usd_per_1000": 0.025239403018953582, "axes": { "intelligence": 77.79863196529865, "calibration": 61.36439087639471, "speed": 71.70469039723565, "cost": 57.937627646831245 }, "jevbench_score": 66.7343085963098 }, "heldout": { "n": 303, "escalation": 0.9471947194719472, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6788990825688074 }, "accuracy": 0.834983498349835, "ece_hard": 0.1935286337386598, "calibration": 64.09002035150621, "p50_s": 3.024282158911228, "p95_s": 4.98705321252346, "cost_usd_per_1000": 0.027464018738952548, "axes": { "intelligence": 82.6158309245528, "calibration": 64.09002035150621, "speed": 68.21533690332035, "cost": 56.837077397772596 }, "jevbench_score": 67.31190206471899 } }, { "threshold": 0.96, "public": { "n": 231, "escalation": 0.8874458874458875, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6486486486486487 }, "accuracy": 0.7748917748917749, "ece_hard": 0.21041641822205612, "calibration": 61.35164946854124, "p50_s": 1.4213839560747146, "p95_s": 4.751377658545971, "cost_usd_per_1000": 0.02591346158211327, "axes": { "intelligence": 78.33875542208877, "calibration": 61.35164946854124, "speed": 71.70469039723565, "cost": 57.59423707133336 }, "jevbench_score": 66.74709616125327 }, "heldout": { "n": 303, "escalation": 0.9537953795379538, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6788990825688074 }, "accuracy": 0.834983498349835, "ece_hard": 0.1935286337386598, "calibration": 64.09002035150621, "p50_s": 3.024282158911228, "p95_s": 4.98705321252346, "cost_usd_per_1000": 0.027635314149392468, "axes": { "intelligence": 82.6158309245528, "calibration": 64.09002035150621, "speed": 68.21533690332035, "cost": 56.75606782401077 }, "jevbench_score": 67.28790441357782 } }, { "threshold": 0.97, "public": { "n": 231, "escalation": 0.8961038961038961, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6486486486486487 }, "accuracy": 0.7748917748917749, "ece_hard": 0.21041641822205612, "calibration": 61.35164946854124, "p50_s": 1.4213839560747146, "p95_s": 4.751377658545971, "cost_usd_per_1000": 0.026138147769833163, "axes": { "intelligence": 78.33875542208877, "calibration": 61.35164946854124, "speed": 71.70469039723565, "cost": 57.48175573354596 }, "jevbench_score": 66.71448305688234 }, "heldout": { "n": 303, "escalation": 0.9603960396039604, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6788990825688074 }, "accuracy": 0.834983498349835, "ece_hard": 0.1935286337386598, "calibration": 64.09002035150621, "p50_s": 3.024282158911228, "p95_s": 4.98705321252346, "cost_usd_per_1000": 0.02780660955983239, "axes": { "intelligence": 82.6158309245528, "calibration": 64.09002035150621, "speed": 68.21533690332035, "cost": 56.675558833772875 }, "jevbench_score": 67.26402958614796 } }, { "threshold": 0.98, "public": { "n": 231, "escalation": 0.9090909090909091, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6486486486486487 }, "accuracy": 0.7748917748917749, "ece_hard": 0.21041641822205612, "calibration": 61.35164946854124, "p50_s": 1.4213839560747146, "p95_s": 4.751377658545971, "cost_usd_per_1000": 0.02647517705141301, "axes": { "intelligence": 78.33875542208877, "calibration": 61.35164946854124, "speed": 71.70469039723565, "cost": 57.3148338066488 }, "jevbench_score": 66.66599698787292 }, "heldout": { "n": 303, "escalation": 0.9702970297029703, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6788990825688074 }, "accuracy": 0.834983498349835, "ece_hard": 0.1935286337386598, "calibration": 64.09002035150621, "p50_s": 3.024282158911228, "p95_s": 4.98705321252346, "cost_usd_per_1000": 0.02806355267549227, "axes": { "intelligence": 82.6158309245528, "calibration": 64.09002035150621, "speed": 68.21533690332035, "cost": 56.555720523431326 }, "jevbench_score": 67.22844455039794 } }, { "threshold": 0.99, "public": { "n": 231, "escalation": 0.9393939393939394, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6486486486486487 }, "accuracy": 0.7748917748917749, "ece_hard": 0.21041641822205612, "calibration": 61.35164946854124, "p50_s": 1.4749147981405257, "p95_s": 4.751377658545971, "cost_usd_per_1000": 0.027261578708432644, "axes": { "intelligence": 78.33875542208877, "calibration": 61.35164946854124, "speed": 71.5441351572713, "cost": 56.93346993797325 }, "jevbench_score": 66.51753559918282 }, "heldout": { "n": 303, "escalation": 0.9834983498349835, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6788990825688074 }, "accuracy": 0.834983498349835, "ece_hard": 0.1935286337386598, "calibration": 64.09002035150621, "p50_s": 3.024282158911228, "p95_s": 4.98705321252346, "cost_usd_per_1000": 0.028406143496372112, "axes": { "intelligence": 82.6158309245528, "calibration": 64.09002035150621, "speed": 68.21533690332035, "cost": 56.39763170207305 }, "jevbench_score": 67.18141471724296 } }, { "threshold": 1.0, "public": { "n": 231, "escalation": 1.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6486486486486487 }, "accuracy": 0.7748917748917749, "ece_hard": 0.21041641822205612, "calibration": 61.35164946854124, "p50_s": 1.4749147981405257, "p95_s": 4.751377658545971, "cost_usd_per_1000": 0.028834382022471912, "axes": { "intelligence": 78.33875542208877, "calibration": 61.35164946854124, "speed": 71.5441351572713, "cost": 56.20268055815229 }, "jevbench_score": 66.30304814930756 }, "heldout": { "n": 303, "escalation": 1.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6788990825688074 }, "accuracy": 0.834983498349835, "ece_hard": 0.1935286337386598, "calibration": 64.09002035150621, "p50_s": 3.024282158911228, "p95_s": 4.98705321252346, "cost_usd_per_1000": 0.028834382022471912, "axes": { "intelligence": 82.6158309245528, "calibration": 64.09002035150621, "speed": 68.21533690332035, "cost": 56.20268055815229 }, "jevbench_score": 67.123282376547 } } ] }, { "cheap": "openjev-verdict-1.4", "backend": "djev", "chosen": { "threshold": 0.72, "public": { "n": 231, "escalation": 0.8225108225108225, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.6486486486486487 }, "accuracy": 0.7705627705627706, "ece_hard": 0.2084349624406332, "calibration": 61.54979504668353, "p50_s": 0.9340311378240586, "p95_s": 1.9161328352987765, "cost_usd_per_1000": 0.025218109181704692, "axes": { "intelligence": 77.79863196529865, "calibration": 61.54979504668353, "speed": 77.47213032327545, "cost": 57.948624379917845 }, "jevbench_score": 68.0921112690567 }, "heldout": { "n": 303, "escalation": 0.9207920792079208, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6880733944954128 }, "accuracy": 0.8382838283828383, "ece_hard": 0.18853606427239092, "calibration": 64.58927729813311, "p50_s": 1.4824405029416083, "p95_s": 2.227580388635397, "cost_usd_per_1000": 0.027768631104683507, "axes": { "intelligence": 82.89106028235096, "calibration": 64.58927729813311, "speed": 74.81189342065187, "cost": 56.69336586712729 }, "jevbench_score": 69.0307754543251 } }, "full": { "tiers": { "easy": 1.0, "standard": 0.8020833333333334, "judge": 0.9315068493150684, "hard": 0.6681818181818182 }, "accuracy": 0.8089887640449438, "ece_hard": 0.19857596289364077, "calibration": 63.06049121869545, "p50_s": 1.2642089441418647, "p95_s": 2.134927472099662, "cost_usd_per_1000": 0.026665315441372447, "axes": { "intelligence": 82.5859796596098, "calibration": 63.06049121869545, "speed": 75.68798015680628, "cost": 57.22159823195841 }, "jevbench_score": 68.91482273985757 }, "full_escalation": 0.8782771535580525, "backend_heldout_accuracy": 0.834983498349835, "useful": false, "curve": [ { "threshold": 0.0, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.01, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.02, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.03, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.04, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.05, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.06, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.07, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.08, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.09, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.1, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.11, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.12, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.13, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.14, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.15, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.16, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.17, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.18, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.19, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.2, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.21, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.22, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.23, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.24, "public": { "n": 231, "escalation": 0.004329004329004329, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12696379813558564, "calibration": 73.69515575536224, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003985264442174554, "axes": { "intelligence": 53.09371871871872, "calibration": 73.69515575536224, "speed": 79.65074585196557, "cost": 81.98628567119684 }, "jevbench_score": 71.09732910861055 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.25, "public": { "n": 231, "escalation": 0.008658008658008658, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.1334095028256101, "calibration": 73.05058528635979, "p50_s": 0.6819422334432602, "p95_s": 1.6314278408885001, "cost_usd_per_1000": 0.004097607536034503, "axes": { "intelligence": 53.09371871871872, "calibration": 73.05058528635979, "speed": 79.53684542958638, "cost": 81.62408920452208 }, "jevbench_score": 70.83752660085219 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.26, "public": { "n": 231, "escalation": 0.008658008658008658, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.1334095028256101, "calibration": 73.05058528635979, "p50_s": 0.6819422334432602, "p95_s": 1.6314278408885001, "cost_usd_per_1000": 0.004097607536034503, "axes": { "intelligence": 53.09371871871872, "calibration": 73.05058528635979, "speed": 79.53684542958638, "cost": 81.62408920452208 }, "jevbench_score": 70.83752660085219 }, "heldout": { "n": 303, "escalation": 0.0033003300330033004, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5684931506849316, "hard": 0.3577981651376147 }, "accuracy": 0.504950495049505, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003958569053534567, "axes": { "intelligence": 51.151753173306524, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.07385324391491 }, "jevbench_score": 68.83325424240569 } }, { "threshold": 0.27, "public": { "n": 231, "escalation": 0.008658008658008658, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.1334095028256101, "calibration": 73.05058528635979, "p50_s": 0.6819422334432602, "p95_s": 1.6314278408885001, "cost_usd_per_1000": 0.004097607536034503, "axes": { "intelligence": 53.09371871871872, "calibration": 73.05058528635979, "speed": 79.53684542958638, "cost": 81.62408920452208 }, "jevbench_score": 70.83752660085219 }, "heldout": { "n": 303, "escalation": 0.006600660066006601, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5684931506849316, "hard": 0.3669724770642202 }, "accuracy": 0.5082508250825083, "ece_hard": 0.14535053610844778, "calibration": 70.47596383120866, "p50_s": 1.2242849111557006, "p95_s": 1.9938405439257618, "cost_usd_per_1000": 0.004044216758754527, "axes": { "intelligence": 51.42698253110469, "calibration": 70.47596383120866, "speed": 76.12427080036376, "cost": 81.79496726447705 }, "jevbench_score": 68.92399604592556 } }, { "threshold": 0.28, "public": { "n": 231, "escalation": 0.008658008658008658, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.1334095028256101, "calibration": 73.05058528635979, "p50_s": 0.6819422334432602, "p95_s": 1.6314278408885001, "cost_usd_per_1000": 0.004097607536034503, "axes": { "intelligence": 53.09371871871872, "calibration": 73.05058528635979, "speed": 79.53684542958638, "cost": 81.62408920452208 }, "jevbench_score": 70.83752660085219 }, "heldout": { "n": 303, "escalation": 0.013201320132013201, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5684931506849316, "hard": 0.3853211009174312 }, "accuracy": 0.5148514851485149, "ece_hard": 0.1402997214721165, "calibration": 70.98104529484178, "p50_s": 1.2242849111557006, "p95_s": 1.9938405439257618, "cost_usd_per_1000": 0.004215512169194448, "axes": { "intelligence": 51.97744124670103, "calibration": 70.98104529484178, "speed": 76.12427080036376, "cost": 81.2544895796323 }, "jevbench_score": 69.11653373735712 } }, { "threshold": 0.29, "public": { "n": 231, "escalation": 0.017316017316017316, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.3783783783783784 }, "accuracy": 0.5324675324675324, "ece_hard": 0.13273199847676684, "calibration": 73.11833572124412, "p50_s": 0.6819422334432602, "p95_s": 1.6541020438075065, "cost_usd_per_1000": 0.004322293723754398, "axes": { "intelligence": 53.84446946946947, "calibration": 73.11833572124412, "speed": 79.47690114119766, "cost": 80.92857171216721 }, "jevbench_score": 70.93777299901303 }, "heldout": { "n": 303, "escalation": 0.019801980198019802, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5684931506849316, "hard": 0.3944954128440367 }, "accuracy": 0.5181518151815182, "ece_hard": 0.1382660839190954, "calibration": 71.87207144211186, "p50_s": 1.2242849111557006, "p95_s": 1.9938405439257618, "cost_usd_per_1000": 0.004386807579634368, "axes": { "intelligence": 52.25267060449919, "calibration": 71.87207144211186, "speed": 76.12427080036376, "cost": 80.73554244450364 }, "jevbench_score": 69.3129114822165 } }, { "threshold": 0.3, "public": { "n": 231, "escalation": 0.030303030303030304, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36936936936936937 }, "accuracy": 0.5281385281385281, "ece_hard": 0.14438702911582987, "calibration": 71.95283265733781, "p50_s": 0.6859634757041931, "p95_s": 1.6590973153710364, "cost_usd_per_1000": 0.004659323005334242, "axes": { "intelligence": 53.46909409409409, "calibration": 71.95283265733781, "speed": 79.43827147214512, "cost": 79.95031543754003 }, "jevbench_score": 70.30725988479745 }, "heldout": { "n": 303, "escalation": 0.033003300330033, "tiers": { "easy": 0.875, "standard": 0.4583333333333333, "judge": 0.5753424657534246, "hard": 0.4036697247706422 }, "accuracy": 0.528052805280528, "ece_hard": 0.1355320328183975, "calibration": 72.14547655218165, "p50_s": 1.2242849111557006, "p95_s": 1.9947584256529807, "cost_usd_per_1000": 0.004729398400514209, "axes": { "intelligence": 53.3030141175485, "calibration": 72.14547655218165, "speed": 76.12227194823924, "cost": 79.75582299533099 }, "jevbench_score": 69.51183513866458 } }, { "threshold": 0.31, "public": { "n": 231, "escalation": 0.030303030303030304, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36936936936936937 }, "accuracy": 0.5281385281385281, "ece_hard": 0.14438702911582987, "calibration": 71.95283265733781, "p50_s": 0.6859634757041931, "p95_s": 1.6590973153710364, "cost_usd_per_1000": 0.004659323005334242, "axes": { "intelligence": 53.46909409409409, "calibration": 71.95283265733781, "speed": 79.43827147214512, "cost": 79.95031543754003 }, "jevbench_score": 70.30725988479745 }, "heldout": { "n": 303, "escalation": 0.0462046204620462, "tiers": { "easy": 0.875, "standard": 0.4583333333333333, "judge": 0.589041095890411, "hard": 0.4036697247706422 }, "accuracy": 0.5346534653465347, "ece_hard": 0.1379693055086114, "calibration": 71.90174928316026, "p50_s": 1.2242849111557006, "p95_s": 1.9994657054543494, "cost_usd_per_1000": 0.00507198922139405, "axes": { "intelligence": 53.68657576138411, "calibration": 71.90174928316026, "speed": 76.11203543410345, "cost": 78.8446503416489 }, "jevbench_score": 69.37574856601248 } }, { "threshold": 0.32, "public": { "n": 231, "escalation": 0.05194805194805195, "tiers": { "easy": 0.875, "standard": 0.5555555555555556, "hard": 0.36936936936936937 }, "accuracy": 0.5324675324675324, "ece_hard": 0.16268622179872239, "calibration": 70.12291338904856, "p50_s": 0.6935107052326203, "p95_s": 1.6796640172600745, "cost_usd_per_1000": 0.0052210384746339805, "axes": { "intelligence": 54.00921755088422, "calibration": 70.12291338904856, "speed": 79.3372441203802, "cost": 78.46729319184365 }, "jevbench_score": 69.6824488624407 }, "heldout": { "n": 303, "escalation": 0.0627062706270627, "tiers": { "easy": 0.875, "standard": 0.4583333333333333, "judge": 0.6027397260273972, "hard": 0.42201834862385323 }, "accuracy": 0.5478547854785478, "ece_hard": 0.12967030970030013, "calibration": 72.73164886399138, "p50_s": 1.234055870771408, "p95_s": 1.9997007146477699, "cost_usd_per_1000": 0.0055002277474938506, "axes": { "intelligence": 54.62059612081605, "calibration": 72.73164886399138, "speed": 76.07700175752542, "cost": 77.78857982008878 }, "jevbench_score": 69.63254604875131 } }, { "threshold": 0.33, "public": { "n": 231, "escalation": 0.05627705627705628, "tiers": { "easy": 0.875, "standard": 0.5555555555555556, "hard": 0.3783783783783784 }, "accuracy": 0.5367965367965368, "ece_hard": 0.15470015068596232, "calibration": 70.92152050032458, "p50_s": 0.6935107052326203, "p95_s": 1.683034674823284, "cost_usd_per_1000": 0.005333381568493929, "axes": { "intelligence": 54.3845929262596, "calibration": 70.92152050032458, "speed": 79.32853766997002, "cost": 78.18992032845952 }, "jevbench_score": 69.93724732761993 }, "heldout": { "n": 303, "escalation": 0.10561056105610561, "tiers": { "easy": 0.875, "standard": 0.4583333333333333, "judge": 0.6506849315068494, "hard": 0.43119266055045874 }, "accuracy": 0.5742574257425742, "ece_hard": 0.14008033543144988, "calibration": 71.69064629087642, "p50_s": 1.234055870771408, "p95_s": 2.0280852518975725, "cost_usd_per_1000": 0.006613647915353333, "axes": { "intelligence": 56.23829123203888, "calibration": 71.69064629087642, "speed": 76.01578970665761, "cost": 75.38676786819681 }, "jevbench_score": 69.33034907362287 } }, { "threshold": 0.34, "public": { "n": 231, "escalation": 0.06926406926406926, "tiers": { "easy": 0.875, "standard": 0.5694444444444444, "hard": 0.38738738738738737 }, "accuracy": 0.5454545454545454, "ece_hard": 0.15319322744424166, "calibration": 71.07221282449663, "p50_s": 0.6935107052326203, "p95_s": 1.683034674823284, "cost_usd_per_1000": 0.005670410850073772, "axes": { "intelligence": 55.30009175842509, "calibration": 71.07221282449663, "speed": 79.32853766997002, "cost": 77.39156419376468 }, "jevbench_score": 70.0869547697372 }, "heldout": { "n": 303, "escalation": 0.1353135313531353, "tiers": { "easy": 0.875, "standard": 0.4583333333333333, "judge": 0.684931506849315, "hard": 0.44954128440366975 }, "accuracy": 0.5973597359735974, "ece_hard": 0.13646826970798762, "calibration": 72.05185286322265, "p50_s": 1.2708158284425735, "p95_s": 2.0280852518975725, "cost_usd_per_1000": 0.007384477262332975, "axes": { "intelligence": 57.74765405722425, "calibration": 72.05185286322265, "speed": 75.88831177626773, "cost": 73.95040727238647 }, "jevbench_score": 69.51423381366465 } }, { "threshold": 0.35, "public": { "n": 231, "escalation": 0.08225108225108226, "tiers": { "easy": 0.875, "standard": 0.5694444444444444, "hard": 0.4144144144144144 }, "accuracy": 0.5584415584415584, "ece_hard": 0.13586502278545465, "calibration": 72.80503329037535, "p50_s": 0.6935107052326203, "p95_s": 1.7303867317736148, "cost_usd_per_1000": 0.006007440131653615, "axes": { "intelligence": 56.42621788455122, "calibration": 72.80503329037535, "speed": 79.20803654274371, "cost": 76.63931645653102 }, "jevbench_score": 70.66686159934711 }, "heldout": { "n": 303, "escalation": 0.16501650165016502, "tiers": { "easy": 0.875, "standard": 0.4583333333333333, "judge": 0.726027397260274, "hard": 0.45871559633027525 }, "accuracy": 0.6204620462046204, "ece_hard": 0.13197091944249228, "calibration": 74.24966463773225, "p50_s": 1.2708158284425735, "p95_s": 2.0280852518975725, "cost_usd_per_1000": 0.008155306609312617, "axes": { "intelligence": 59.17356834652927, "calibration": 74.24966463773225, "speed": 75.88831177626773, "cost": 72.65679119359993 }, "jevbench_score": 70.15656876332903 } }, { "threshold": 0.36, "public": { "n": 231, "escalation": 0.1038961038961039, "tiers": { "easy": 0.875, "standard": 0.5833333333333334, "hard": 0.43243243243243246 }, "accuracy": 0.5714285714285714, "ece_hard": 0.13499011837093566, "calibration": 72.89252373182724, "p50_s": 0.7019563555717468, "p95_s": 1.745402393490076, "cost_usd_per_1000": 0.006569155600953355, "axes": { "intelligence": 57.7170920920921, "calibration": 72.89252373182724, "speed": 79.11794322286707, "cost": 75.47471353153728 }, "jevbench_score": 70.79718222092738 }, "heldout": { "n": 303, "escalation": 0.1782178217821782, "tiers": { "easy": 0.875, "standard": 0.4583333333333333, "judge": 0.7397260273972602, "hard": 0.46788990825688076 }, "accuracy": 0.6303630363036303, "ece_hard": 0.13447352667936086, "calibration": 73.9994039140454, "p50_s": 1.2708158284425735, "p95_s": 2.0280852518975725, "cost_usd_per_1000": 0.008497897430192459, "axes": { "intelligence": 59.832359348163045, "calibration": 73.9994039140454, "speed": 75.88831177626773, "cost": 72.12065545476247 }, "jevbench_score": 70.16163914459848 } }, { "threshold": 0.37, "public": { "n": 231, "escalation": 0.12554112554112554, "tiers": { "easy": 0.875, "standard": 0.5833333333333334, "hard": 0.44144144144144143 }, "accuracy": 0.5757575757575758, "ece_hard": 0.14007422803762895, "calibration": 73.0073093451322, "p50_s": 0.7169283866882324, "p95_s": 1.745402393490076, "cost_usd_per_1000": 0.007130871070253094, "axes": { "intelligence": 58.09246746746747, "calibration": 73.0073093451322, "speed": 79.02628656254726, "cost": 74.40572247233597 }, "jevbench_score": 70.66689668915492 }, "heldout": { "n": 303, "escalation": 0.2079207920792079, "tiers": { "easy": 0.875, "standard": 0.5, "judge": 0.7808219178082192, "hard": 0.46788990825688076 }, "accuracy": 0.6534653465346535, "ece_hard": 0.1384688403133358, "calibration": 73.5998725506479, "p50_s": 1.2708158284425735, "p95_s": 2.0280852518975725, "cost_usd_per_1000": 0.009268726777172099, "axes": { "intelligence": 62.14971094633657, "calibration": 73.5998725506479, "speed": 75.88831177626773, "cost": 70.98939759257645 }, "jevbench_score": 70.45651202011189 } }, { "threshold": 0.38, "public": { "n": 231, "escalation": 0.12987012987012986, "tiers": { "easy": 0.875, "standard": 0.5833333333333334, "hard": 0.43243243243243246 }, "accuracy": 0.5714285714285714, "ece_hard": 0.1527713783475047, "calibration": 71.73759431414462, "p50_s": 0.7169283866882324, "p95_s": 1.745402393490076, "cost_usd_per_1000": 0.00724321416411304, "axes": { "intelligence": 57.7170920920921, "calibration": 71.73759431414462, "speed": 79.02628656254726, "cost": 74.20206020754007 }, "jevbench_score": 70.19556878078392 }, "heldout": { "n": 303, "escalation": 0.22772277227722773, "tiers": { "easy": 0.875, "standard": 0.5, "judge": 0.821917808219178, "hard": 0.46788990825688076 }, "accuracy": 0.6732673267326733, "ece_hard": 0.1384688403133358, "calibration": 73.5998725506479, "p50_s": 1.2708158284425735, "p95_s": 2.0280852518975725, "cost_usd_per_1000": 0.009782613008491862, "axes": { "intelligence": 63.30039587784341, "calibration": 73.5998725506479, "speed": 75.88831177626773, "cost": 70.28635379319124 }, "jevbench_score": 70.60449458834943 } }, { "threshold": 0.39, "public": { "n": 231, "escalation": 0.1645021645021645, "tiers": { "easy": 0.875, "standard": 0.5833333333333334, "hard": 0.42342342342342343 }, "accuracy": 0.5670995670995671, "ece_hard": 0.179333793749151, "calibration": 67.06770282397669, "p50_s": 0.7303599804639817, "p95_s": 1.7543271757662295, "cost_usd_per_1000": 0.008141958914992625, "axes": { "intelligence": 57.34171671671673, "calibration": 67.06770282397669, "speed": 78.92352440710697, "cost": 72.67813280312382 }, "jevbench_score": 68.5328177826173 }, "heldout": { "n": 303, "escalation": 0.24422442244224424, "tiers": { "easy": 0.875, "standard": 0.5, "judge": 0.8561643835616438, "hard": 0.46788990825688076 }, "accuracy": 0.6897689768976898, "ece_hard": 0.1384688403133358, "calibration": 73.5998725506479, "p50_s": 1.2723965734243392, "p95_s": 2.0280852518975725, "cost_usd_per_1000": 0.010210851534591663, "axes": { "intelligence": 64.25929998743246, "calibration": 73.5998725506479, "speed": 75.8829130222291, "cost": 69.72814115163334 }, "jevbench_score": 70.72798470720188 } }, { "threshold": 0.4, "public": { "n": 231, "escalation": 0.18614718614718614, "tiers": { "easy": 0.875, "standard": 0.5833333333333334, "hard": 0.4594594594594595 }, "accuracy": 0.5844155844155844, "ece_hard": 0.16571562898917955, "calibration": 69.3381140816019, "p50_s": 0.7313418105244637, "p95_s": 1.76125024035573, "cost_usd_per_1000": 0.008703674384292363, "axes": { "intelligence": 58.84321821821823, "calibration": 69.3381140816019, "speed": 78.90058532035746, "cost": 71.80892094579542 }, "jevbench_score": 69.33966944291016 }, "heldout": { "n": 303, "escalation": 0.2739273927392739, "tiers": { "easy": 0.875, "standard": 0.5, "judge": 0.9041095890410958, "hard": 0.47706422018348627 }, "accuracy": 0.7161716171617162, "ece_hard": 0.15173206570493575, "calibration": 72.27355001148791, "p50_s": 1.2723965734243392, "p95_s": 2.0306707464158533, "cost_usd_per_1000": 0.010981680881571303, "axes": { "intelligence": 65.87699509865527, "calibration": 72.27355001148791, "speed": 75.87737996651138, "cost": 68.77993542029165 }, "jevbench_score": 70.60278208365698 } }, { "threshold": 0.41, "public": { "n": 231, "escalation": 0.21645021645021645, "tiers": { "easy": 0.875, "standard": 0.5833333333333334, "hard": 0.4594594594594595 }, "accuracy": 0.5844155844155844, "ece_hard": 0.18953442771637835, "calibration": 67.93566790818623, "p50_s": 0.7336763456463814, "p95_s": 1.76125024035573, "cost_usd_per_1000": 0.009490076041311997, "axes": { "intelligence": 58.84321821821823, "calibration": 67.93566790818623, "speed": 78.8867441734042, "cost": 70.6819092303744 }, "jevbench_score": 68.71105908455631 }, "heldout": { "n": 303, "escalation": 0.29042904290429045, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "judge": 0.910958904109589, "hard": 0.48623853211009177 }, "accuracy": 0.7260726072607261, "ece_hard": 0.15551317581708818, "calibration": 70.02444979348545, "p50_s": 1.2723965734243392, "p95_s": 2.055198547989129, "cost_usd_per_1000": 0.011409919407671106, "axes": { "intelligence": 67.51067194503793, "calibration": 70.02444979348545, "speed": 75.8252372469555, "cost": 68.28152269443166 }, "jevbench_score": 70.33715172337456 } }, { "threshold": 0.42, "public": { "n": 231, "escalation": 0.22943722943722944, "tiers": { "easy": 0.875, "standard": 0.5972222222222222, "hard": 0.46846846846846846 }, "accuracy": 0.5930735930735931, "ece_hard": 0.1856740328406391, "calibration": 68.32170739576016, "p50_s": 0.7448194563388825, "p95_s": 1.7727243207395076, "cost_usd_per_1000": 0.009827105322891841, "axes": { "intelligence": 59.75871705038372, "calibration": 68.32170739576016, "speed": 78.7930778413569, "cost": 70.227231679977 }, "jevbench_score": 68.94271746204515 }, "heldout": { "n": 303, "escalation": 0.3102310231023102, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "judge": 0.9178082191780822, "hard": 0.48623853211009177 }, "accuracy": 0.7293729372937293, "ece_hard": 0.16613097906501884, "calibration": 68.96266946869237, "p50_s": 1.3009985327720641, "p95_s": 2.065838665515184, "cost_usd_per_1000": 0.011923805638990866, "axes": { "intelligence": 67.70245276695573, "calibration": 68.96266946869237, "speed": 75.70626791533968, "cost": 67.70755335070614 }, "jevbench_score": 69.94341874537216 } }, { "threshold": 0.43, "public": { "n": 231, "escalation": 0.24242424242424243, "tiers": { "easy": 0.875, "standard": 0.6111111111111112, "hard": 0.46846846846846846 }, "accuracy": 0.5974025974025974, "ece_hard": 0.18612229699952715, "calibration": 68.27688097987135, "p50_s": 0.7448194563388825, "p95_s": 1.7883843421936034, "cost_usd_per_1000": 0.010164134604471685, "axes": { "intelligence": 60.29884050717385, "calibration": 68.27688097987135, "speed": 78.75488128299175, "cost": 69.78788776557977 }, "jevbench_score": 68.9699708498292 }, "heldout": { "n": 303, "escalation": 0.3234323432343234, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "judge": 0.9178082191780822, "hard": 0.4954128440366973 }, "accuracy": 0.7326732673267327, "ece_hard": 0.16619782939118016, "calibration": 69.52026466759682, "p50_s": 1.3009985327720641, "p95_s": 2.0718681916594504, "cost_usd_per_1000": 0.012266396459870706, "axes": { "intelligence": 67.97768212475388, "calibration": 69.52026466759682, "speed": 75.69361070265236, "cost": 67.33849008016601 }, "jevbench_score": 70.05676696955965 } }, { "threshold": 0.44, "public": { "n": 231, "escalation": 0.2597402597402597, "tiers": { "easy": 0.875, "standard": 0.6111111111111112, "hard": 0.4864864864864865 }, "accuracy": 0.6060606060606061, "ece_hard": 0.18114295861422308, "calibration": 67.89646404444356, "p50_s": 0.7448194563388825, "p95_s": 1.8059252515435218, "cost_usd_per_1000": 0.010613506979911475, "axes": { "intelligence": 61.049591257924604, "calibration": 67.89646404444356, "speed": 78.71249216532021, "cost": 69.2242327043559 }, "jevbench_score": 68.93788146239604 }, "heldout": { "n": 303, "escalation": 0.33003300330033003, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "judge": 0.9178082191780822, "hard": 0.5137614678899083 }, "accuracy": 0.7392739273927392, "ece_hard": 0.15487636624119178, "calibration": 70.65241098259565, "p50_s": 1.3009985327720641, "p95_s": 2.0718681916594504, "cost_usd_per_1000": 0.012437691870310626, "axes": { "intelligence": 68.52814084035022, "calibration": 70.65241098259565, "speed": 75.69361070265236, "cost": 67.1578061962523 }, "jevbench_score": 70.43490202275457 } }, { "threshold": 0.45, "public": { "n": 231, "escalation": 0.27705627705627706, "tiers": { "easy": 0.875, "standard": 0.6111111111111112, "hard": 0.4954954954954955 }, "accuracy": 0.6103896103896104, "ece_hard": 0.1820072788310327, "calibration": 67.8100320227626, "p50_s": 0.7492050841450691, "p95_s": 1.8059252515435218, "cost_usd_per_1000": 0.011062879355351267, "axes": { "intelligence": 61.424966633299974, "calibration": 67.8100320227626, "speed": 78.68699513531801, "cost": 68.68395471157348 }, "jevbench_score": 68.88097458047915 }, "heldout": { "n": 303, "escalation": 0.35313531353135313, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "judge": 0.9178082191780822, "hard": 0.5229357798165137 }, "accuracy": 0.7425742574257426, "ece_hard": 0.16802024745932126, "calibration": 70.81458820974996, "p50_s": 1.3175783544778823, "p95_s": 2.1125313609838474, "cost_usd_per_1000": 0.01303722580685035, "axes": { "intelligence": 68.80337019814837, "calibration": 70.81458820974996, "speed": 75.55420383618264, "cost": 66.54444437074743 }, "jevbench_score": 70.3518821280675 } }, { "threshold": 0.46, "public": { "n": 231, "escalation": 0.30735930735930733, "tiers": { "easy": 0.875, "standard": 0.6111111111111112, "hard": 0.5135135135135135 }, "accuracy": 0.6190476190476191, "ece_hard": 0.1814726388892312, "calibration": 67.86349601694275, "p50_s": 0.7518308401107788, "p95_s": 1.820576274394989, "cost_usd_per_1000": 0.0118492810123709, "axes": { "intelligence": 62.17571738405072, "calibration": 67.86349601694275, "speed": 78.63670984750604, "cost": 67.78924002590257 }, "jevbench_score": 68.86693953904397 }, "heldout": { "n": 303, "escalation": 0.3795379537953795, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "judge": 0.9178082191780822, "hard": 0.5229357798165137 }, "accuracy": 0.7425742574257426, "ece_hard": 0.17780283276041736, "calibration": 69.83632967964036, "p50_s": 1.3324144572019576, "p95_s": 2.1125313609838474, "cost_usd_per_1000": 0.01372240744861003, "axes": { "intelligence": 68.80337019814837, "calibration": 69.83632967964036, "speed": 75.50557498830662, "cost": 65.87709068965003 }, "jevbench_score": 69.9199545290295 } }, { "threshold": 0.47, "public": { "n": 231, "escalation": 0.3203463203463203, "tiers": { "easy": 0.875, "standard": 0.6111111111111112, "hard": 0.5225225225225225 }, "accuracy": 0.6233766233766234, "ece_hard": 0.18494046185473448, "calibration": 67.51671372039243, "p50_s": 0.7686461001634598, "p95_s": 1.8279631301760673, "cost_usd_per_1000": 0.012186310293950744, "axes": { "intelligence": 62.55109275942609, "calibration": 67.51671372039243, "speed": 78.5230614012013, "cost": 67.4238330353989 }, "jevbench_score": 68.76448813769491 }, "heldout": { "n": 303, "escalation": 0.38943894389438943, "tiers": { "easy": 0.875, "standard": 0.5833333333333334, "judge": 0.9178082191780822, "hard": 0.5321100917431193 }, "accuracy": 0.7491749174917491, "ece_hard": 0.17334411613207368, "calibration": 70.28220134247472, "p50_s": 1.3324144572019576, "p95_s": 2.1125313609838474, "cost_usd_per_1000": 0.01397935056426991, "axes": { "intelligence": 70.24526622261322, "calibration": 70.28220134247472, "speed": 75.50557498830662, "cost": 65.63539012145543 }, "jevbench_score": 70.33068982980505 } }, { "threshold": 0.48, "public": { "n": 231, "escalation": 0.354978354978355, "tiers": { "easy": 0.875, "standard": 0.6111111111111112, "hard": 0.5585585585585585 }, "accuracy": 0.6406926406926406, "ece_hard": 0.16461597269915856, "calibration": 69.54916263595001, "p50_s": 0.7781706541776657, "p95_s": 1.8279631301760673, "cost_usd_per_1000": 0.013085055044830329, "axes": { "intelligence": 64.05259426092759, "calibration": 69.54916263595001, "speed": 78.46957719218203, "cost": 66.49673338228024 }, "jevbench_score": 69.4356592801023 }, "heldout": { "n": 303, "escalation": 0.41254125412541254, "tiers": { "easy": 0.875, "standard": 0.625, "judge": 0.9178082191780822, "hard": 0.5504587155963303 }, "accuracy": 0.759075907590759, "ece_hard": 0.17138153373925163, "calibration": 70.47845958175692, "p50_s": 1.3324144572019576, "p95_s": 2.124293764680624, "cost_usd_per_1000": 0.014578884500809634, "axes": { "intelligence": 71.96239160487622, "calibration": 70.47845958175692, "speed": 75.48146089347335, "cost": 65.08827113995788 }, "jevbench_score": 70.65229216471855 } }, { "threshold": 0.49, "public": { "n": 231, "escalation": 0.36363636363636365, "tiers": { "easy": 0.875, "standard": 0.6111111111111112, "hard": 0.5675675675675675 }, "accuracy": 0.645021645021645, "ece_hard": 0.1602325482482137, "calibration": 69.9875050810445, "p50_s": 0.7781706541776657, "p95_s": 1.8319671392440795, "cost_usd_per_1000": 0.013309741232550222, "axes": { "intelligence": 64.42796963630298, "calibration": 69.9875050810445, "speed": 78.46007471749279, "cost": 66.27491163936902 }, "jevbench_score": 69.58621395131622 }, "heldout": { "n": 303, "escalation": 0.42244224422442245, "tiers": { "easy": 0.875, "standard": 0.625, "judge": 0.9178082191780822, "hard": 0.5504587155963303 }, "accuracy": 0.759075907590759, "ece_hard": 0.16642838564960055, "calibration": 70.97377439072203, "p50_s": 1.3324144572019576, "p95_s": 2.1344793170690535, "cost_usd_per_1000": 0.014835827616469514, "axes": { "intelligence": 71.96239160487622, "calibration": 70.97377439072203, "speed": 75.46068712813378, "cost": 64.86064664676347 }, "jevbench_score": 70.7092742779967 } }, { "threshold": 0.5, "public": { "n": 231, "escalation": 0.36363636363636365, "tiers": { "easy": 0.875, "standard": 0.6111111111111112, "hard": 0.5675675675675675 }, "accuracy": 0.645021645021645, "ece_hard": 0.1602325482482137, "calibration": 69.9875050810445, "p50_s": 0.7781706541776657, "p95_s": 1.8319671392440795, "cost_usd_per_1000": 0.013309741232550222, "axes": { "intelligence": 64.42796963630298, "calibration": 69.9875050810445, "speed": 78.46007471749279, "cost": 66.27491163936902 }, "jevbench_score": 69.58621395131622 }, "heldout": { "n": 303, "escalation": 0.43234323432343236, "tiers": { "easy": 0.875, "standard": 0.625, "judge": 0.9178082191780822, "hard": 0.5412844036697247 }, "accuracy": 0.7557755775577558, "ece_hard": 0.17666622925700148, "calibration": 69.94999002998193, "p50_s": 1.3324144572019576, "p95_s": 2.1344793170690535, "cost_usd_per_1000": 0.015092770732129394, "axes": { "intelligence": 71.68716224707805, "calibration": 69.94999002998193, "speed": 75.46068712813378, "cost": 64.63693075262563 }, "jevbench_score": 70.32465849849167 } }, { "threshold": 0.51, "public": { "n": 231, "escalation": 0.38095238095238093, "tiers": { "easy": 0.875, "standard": 0.6111111111111112, "hard": 0.5675675675675675 }, "accuracy": 0.645021645021645, "ece_hard": 0.16835068606854872, "calibration": 69.17569129901099, "p50_s": 0.7781706541776657, "p95_s": 1.8319671392440795, "cost_usd_per_1000": 0.013759113607990012, "axes": { "intelligence": 64.42796963630298, "calibration": 69.17569129901099, "speed": 78.46007471749279, "cost": 65.84228630181966 }, "jevbench_score": 69.2700328145711 }, "heldout": { "n": 303, "escalation": 0.45544554455445546, "tiers": { "easy": 0.875, "standard": 0.625, "judge": 0.9178082191780822, "hard": 0.5412844036697247 }, "accuracy": 0.7557755775577558, "ece_hard": 0.1976889193248734, "calibration": 65.66704801654481, "p50_s": 1.3478259533643722, "p95_s": 2.1344793170690535, "cost_usd_per_1000": 0.015692304668669117, "axes": { "intelligence": 71.68716224707805, "calibration": 65.66704801654481, "speed": 75.41074234347029, "cost": 64.12939805686133 }, "jevbench_score": 69.07482853319902 } }, { "threshold": 0.52, "public": { "n": 231, "escalation": 0.4025974025974026, "tiers": { "easy": 0.8958333333333334, "standard": 0.6111111111111112, "hard": 0.5585585585585585 }, "accuracy": 0.645021645021645, "ece_hard": 0.18329450157738367, "calibration": 66.31880570081344, "p50_s": 0.7942025229334831, "p95_s": 1.8434350915253161, "cost_usd_per_1000": 0.014320829077289755, "axes": { "intelligence": 64.45768685352019, "calibration": 66.31880570081344, "speed": 78.34440887214379, "cost": 65.32095515930325 }, "jevbench_score": 68.39006033627099 }, "heldout": { "n": 303, "escalation": 0.46864686468646866, "tiers": { "easy": 0.875, "standard": 0.625, "judge": 0.9178082191780822, "hard": 0.5504587155963303 }, "accuracy": 0.759075907590759, "ece_hard": 0.1954948851280165, "calibration": 64.52734911393699, "p50_s": 1.3478259533643722, "p95_s": 2.149421533197164, "cost_usd_per_1000": 0.016034895489548957, "axes": { "intelligence": 71.96239160487622, "calibration": 64.52734911393699, "speed": 75.38044589845242, "cost": 63.84801598967165 }, "jevbench_score": 68.75651885599918 } }, { "threshold": 0.53, "public": { "n": 231, "escalation": 0.42424242424242425, "tiers": { "easy": 0.8958333333333334, "standard": 0.6111111111111112, "hard": 0.5855855855855856 }, "accuracy": 0.658008658008658, "ece_hard": 0.16664928450443262, "calibration": 67.98332740810855, "p50_s": 0.8003151804208756, "p95_s": 1.8434350915253161, "cost_usd_per_1000": 0.014882544546589493, "axes": { "intelligence": 65.58381297964632, "calibration": 67.98332740810855, "speed": 78.31111094869527, "cost": 64.81968426518303 }, "jevbench_score": 68.97351263281219 }, "heldout": { "n": 303, "escalation": 0.4884488448844885, "tiers": { "easy": 0.875, "standard": 0.625, "judge": 0.9178082191780822, "hard": 0.5688073394495413 }, "accuracy": 0.7656765676567657, "ece_hard": 0.20143625901309575, "calibration": 63.93321172542907, "p50_s": 1.3478259533643722, "p95_s": 2.149421533197164, "cost_usd_per_1000": 0.016548781720868717, "axes": { "intelligence": 72.51285032047254, "calibration": 63.93321172542907, "speed": 75.38044589845242, "cost": 63.43701917092979 }, "jevbench_score": 68.61763448132167 } }, { "threshold": 0.54, "public": { "n": 231, "escalation": 0.4458874458874459, "tiers": { "easy": 0.8958333333333334, "standard": 0.6111111111111112, "hard": 0.5765765765765766 }, "accuracy": 0.6536796536796536, "ece_hard": 0.18766041297083977, "calibration": 65.88221456146783, "p50_s": 0.8003151804208756, "p95_s": 1.874721498787403, "cost_usd_per_1000": 0.015444260015889232, "axes": { "intelligence": 65.20843760427096, "calibration": 65.88221456146783, "speed": 78.23802185522526, "cost": 64.33698685923429 }, "jevbench_score": 68.19266145504702 }, "heldout": { "n": 303, "escalation": 0.5082508250825083, "tiers": { "easy": 0.875, "standard": 0.625, "judge": 0.9178082191780822, "hard": 0.6055045871559633 }, "accuracy": 0.7788778877887789, "ece_hard": 0.18951912051486616, "calibration": 65.12492557525202, "p50_s": 1.3478259533643722, "p95_s": 2.149421533197164, "cost_usd_per_1000": 0.01706266795218848, "axes": { "intelligence": 73.61376775166522, "calibration": 65.12492557525202, "speed": 75.38044589845242, "cost": 63.038591821686374 }, "jevbench_score": 69.08644934826816 } }, { "threshold": 0.55, "public": { "n": 231, "escalation": 0.4675324675324675, "tiers": { "easy": 0.8958333333333334, "standard": 0.6111111111111112, "hard": 0.5675675675675675 }, "accuracy": 0.6493506493506493, "ece_hard": 0.19897942090339166, "calibration": 64.75031376821265, "p50_s": 0.8056192338466645, "p95_s": 1.874721498787403, "cost_usd_per_1000": 0.01600597548518897, "axes": { "intelligence": 64.83306222889557, "calibration": 64.75031376821265, "speed": 78.2093341393382, "cost": 63.871535578257394 }, "jevbench_score": 67.67076553150838 }, "heldout": { "n": 303, "escalation": 0.528052805280528, "tiers": { "easy": 0.875, "standard": 0.7083333333333334, "judge": 0.9178082191780822, "hard": 0.5963302752293578 }, "accuracy": 0.7821782178217822, "ece_hard": 0.2012975396252404, "calibration": 64.36417800228985, "p50_s": 1.3536114186048507, "p95_s": 2.149421533197164, "cost_usd_per_1000": 0.017576554183508238, "axes": { "intelligence": 75.67187172720038, "calibration": 64.36417800228985, "speed": 75.36184395345342, "cost": 62.65198788088774 }, "jevbench_score": 69.24944047992281 } }, { "threshold": 0.56, "public": { "n": 231, "escalation": 0.48484848484848486, "tiers": { "easy": 0.8958333333333334, "standard": 0.6111111111111112, "hard": 0.5675675675675675 }, "accuracy": 0.6493506493506493, "ece_hard": 0.21277593020124685, "calibration": 63.37066283842713, "p50_s": 0.8084952920675278, "p95_s": 1.909963024407625, "cost_usd_per_1000": 0.01645534786062876, "axes": { "intelligence": 64.83306222889557, "calibration": 63.37066283842713, "speed": 78.11297544999903, "cost": 63.51078797291192 }, "jevbench_score": 67.1914273149568 }, "heldout": { "n": 303, "escalation": 0.5445544554455446, "tiers": { "easy": 0.875, "standard": 0.7083333333333334, "judge": 0.9178082191780822, "hard": 0.6238532110091743 }, "accuracy": 0.7920792079207921, "ece_hard": 0.1931661825632655, "calibration": 65.17731370848735, "p50_s": 1.3536114186048507, "p95_s": 2.149421533197164, "cost_usd_per_1000": 0.01800479270960804, "axes": { "intelligence": 76.49755980059487, "calibration": 65.17731370848735, "speed": 75.36184395345342, "cost": 62.338356229766624 }, "jevbench_score": 69.56851376430649 } }, { "threshold": 0.57, "public": { "n": 231, "escalation": 0.5108225108225108, "tiers": { "easy": 0.8958333333333334, "standard": 0.6111111111111112, "hard": 0.5765765765765766 }, "accuracy": 0.6536796536796536, "ece_hard": 0.21510512705902146, "calibration": 63.13774315264966, "p50_s": 0.823828735947609, "p95_s": 1.909963024407625, "cost_usd_per_1000": 0.01712940642378845, "axes": { "intelligence": 65.20843760427096, "calibration": 63.13774315264966, "speed": 78.03138103819781, "cost": 62.98773058448423 }, "jevbench_score": 67.07018832861856 }, "heldout": { "n": 303, "escalation": 0.5742574257425742, "tiers": { "easy": 0.875, "standard": 0.75, "judge": 0.9246575342465754, "hard": 0.6238532110091743 }, "accuracy": 0.7986798679867987, "ece_hard": 0.19578340890426432, "calibration": 64.91559107438746, "p50_s": 1.3673696160316466, "p95_s": 2.149421533197164, "cost_usd_per_1000": 0.01877562205658768, "axes": { "intelligence": 77.85600728917935, "calibration": 64.91559107438746, "speed": 75.31792479290742, "cost": 61.7921699632401 }, "jevbench_score": 69.6415187971267 } }, { "threshold": 0.58, "public": { "n": 231, "escalation": 0.5238095238095238, "tiers": { "easy": 0.8958333333333334, "standard": 0.6111111111111112, "hard": 0.5855855855855856 }, "accuracy": 0.658008658008658, "ece_hard": 0.2129664954520247, "calibration": 62.42146660675448, "p50_s": 0.823828735947609, "p95_s": 1.909963024407625, "cost_usd_per_1000": 0.01746643570536829, "axes": { "intelligence": 65.58381297964632, "calibration": 62.42146660675448, "speed": 78.03138103819781, "cost": 62.73387131317069 }, "jevbench_score": 66.90760791329936 }, "heldout": { "n": 303, "escalation": 0.5841584158415841, "tiers": { "easy": 0.875, "standard": 0.75, "judge": 0.9246575342465754, "hard": 0.6238532110091743 }, "accuracy": 0.7986798679867987, "ece_hard": 0.19493644457674802, "calibration": 65.00028750713909, "p50_s": 1.3673696160316466, "p95_s": 2.1689073584973806, "cost_usd_per_1000": 0.01903256517224756, "axes": { "intelligence": 77.85600728917935, "calibration": 65.00028750713909, "speed": 75.27873072477286, "cost": 61.61508023198615 }, "jevbench_score": 69.60519875539329 } }, { "threshold": 0.59, "public": { "n": 231, "escalation": 0.5584415584415584, "tiers": { "easy": 0.9166666666666666, "standard": 0.6111111111111112, "hard": 0.5765765765765766 }, "accuracy": 0.658008658008658, "ece_hard": 0.2315789137425565, "calibration": 58.13758071968016, "p50_s": 0.825623007118702, "p95_s": 1.9158442258834838, "cost_usd_per_1000": 0.01836518045624787, "axes": { "intelligence": 65.61353019686354, "calibration": 58.13758071968016, "speed": 78.00858018922213, "cost": 62.080143997615124 }, "jevbench_score": 65.55961014279737 }, "heldout": { "n": 303, "escalation": 0.6171617161716172, "tiers": { "easy": 0.875, "standard": 0.75, "judge": 0.9315068493150684, "hard": 0.6422018348623854 }, "accuracy": 0.8085808580858086, "ece_hard": 0.1942440950522303, "calibration": 65.06952245959087, "p50_s": 1.3742529690265655, "p95_s": 2.191360459476709, "cost_usd_per_1000": 0.01988904222444717, "axes": { "intelligence": 78.59824682669347, "calibration": 65.06952245959087, "speed": 75.21219495032491, "cost": 61.04158390694978 }, "jevbench_score": 69.61072130395871 } }, { "threshold": 0.6, "public": { "n": 231, "escalation": 0.5670995670995671, "tiers": { "easy": 0.9166666666666666, "standard": 0.625, "hard": 0.5855855855855856 }, "accuracy": 0.6666666666666666, "ece_hard": 0.226035614714947, "calibration": 58.69191062244111, "p50_s": 0.825623007118702, "p95_s": 1.9158442258834838, "cost_usd_per_1000": 0.018589866643967768, "axes": { "intelligence": 66.52902902902903, "calibration": 58.69191062244111, "speed": 78.00858018922213, "cost": 61.921711769989884 }, "jevbench_score": 65.90125571142067 }, "heldout": { "n": 303, "escalation": 0.6303630363036303, "tiers": { "easy": 0.875, "standard": 0.75, "judge": 0.9315068493150684, "hard": 0.6422018348623854 }, "accuracy": 0.8085808580858086, "ece_hard": 0.20450112555075473, "calibration": 64.40842517058749, "p50_s": 1.3831777215003966, "p95_s": 2.191360459476709, "cost_usd_per_1000": 0.020231633045327005, "axes": { "intelligence": 78.59824682669347, "calibration": 64.40842517058749, "speed": 75.18408193700681, "cost": 60.81907182045944 }, "jevbench_score": 69.36338949270962 } }, { "threshold": 0.61, "public": { "n": 231, "escalation": 0.5887445887445888, "tiers": { "easy": 0.9166666666666666, "standard": 0.625, "hard": 0.6036036036036037 }, "accuracy": 0.6753246753246753, "ece_hard": 0.22226398844000483, "calibration": 59.06907324993532, "p50_s": 0.825623007118702, "p95_s": 1.9158442258834838, "cost_usd_per_1000": 0.01915158211326751, "axes": { "intelligence": 67.27977977977979, "calibration": 59.06907324993532, "speed": 78.00858018922213, "cost": 61.533860293659345 }, "jevbench_score": 66.08841149588544 }, "heldout": { "n": 303, "escalation": 0.66996699669967, "tiers": { "easy": 0.875, "standard": 0.75, "judge": 0.9315068493150684, "hard": 0.6422018348623854 }, "accuracy": 0.8085808580858086, "ece_hard": 0.21140389980288216, "calibration": 63.718147745374736, "p50_s": 1.3833439737558364, "p95_s": 2.1991213545203205, "cost_usd_per_1000": 0.02125940550796653, "axes": { "intelligence": 78.59824682669347, "calibration": 63.718147745374736, "speed": 75.16820621708223, "cost": 60.17346652394828 }, "jevbench_score": 68.98883400196223 } }, { "threshold": 0.62, "public": { "n": 231, "escalation": 0.6017316017316018, "tiers": { "easy": 0.9375, "standard": 0.625, "hard": 0.6126126126126126 }, "accuracy": 0.683982683982684, "ece_hard": 0.2162947653668733, "calibration": 59.66599555724848, "p50_s": 0.825623007118702, "p95_s": 1.9158442258834838, "cost_usd_per_1000": 0.019488611394847352, "axes": { "intelligence": 68.06024774774775, "calibration": 59.66599555724848, "speed": 78.00858018922213, "cost": 61.30657312566009 }, "jevbench_score": 66.38461759595798 }, "heldout": { "n": 303, "escalation": 0.7227722772277227, "tiers": { "easy": 0.9166666666666666, "standard": 0.75, "judge": 0.9246575342465754, "hard": 0.6422018348623854 }, "accuracy": 0.8085808580858086, "ece_hard": 0.21140389980288216, "calibration": 63.718147745374736, "p50_s": 1.4152396112680434, "p95_s": 2.2004228018224237, "cost_usd_per_1000": 0.022629768791485892, "axes": { "intelligence": 78.98979933810901, "calibration": 63.718147745374736, "speed": 75.0666388775513, "cost": 59.35959649637556 }, "jevbench_score": 68.81656897821253 } }, { "threshold": 0.63, "public": { "n": 231, "escalation": 0.6190476190476191, "tiers": { "easy": 0.9375, "standard": 0.6388888888888888, "hard": 0.6126126126126126 }, "accuracy": 0.6883116883116883, "ece_hard": 0.21922783651902067, "calibration": 60.470507638844786, "p50_s": 0.8354266524314881, "p95_s": 1.9158442258834838, "cost_usd_per_1000": 0.019937983770287146, "axes": { "intelligence": 68.60037120453786, "calibration": 60.470507638844786, "speed": 77.95731479235215, "cost": 61.009562855723104 }, "jevbench_score": 66.64709377607447 }, "heldout": { "n": 303, "escalation": 0.768976897689769, "tiers": { "easy": 0.9583333333333334, "standard": 0.75, "judge": 0.9246575342465754, "hard": 0.6422018348623854 }, "accuracy": 0.8118811881188119, "ece_hard": 0.2142461373539342, "calibration": 63.43392399026954, "p50_s": 1.4460218638181686, "p95_s": 2.2218950860202304, "cost_usd_per_1000": 0.023828836664565336, "axes": { "intelligence": 79.57313267144235, "calibration": 63.43392399026954, "speed": 74.93101592001962, "cost": 58.68691478813649 }, "jevbench_score": 68.63928221381444 } }, { "threshold": 0.64, "public": { "n": 231, "escalation": 0.6536796536796536, "tiers": { "easy": 0.9375, "standard": 0.6805555555555556, "hard": 0.6126126126126126 }, "accuracy": 0.7012987012987013, "ece_hard": 0.21843648223732418, "calibration": 60.549643067014436, "p50_s": 0.8506983011960983, "p95_s": 1.9158442258834838, "cost_usd_per_1000": 0.020836728521166727, "axes": { "intelligence": 70.22074157490825, "calibration": 60.549643067014436, "speed": 77.87864240045596, "cost": 60.43511399793344 }, "jevbench_score": 66.88383690632155 }, "heldout": { "n": 303, "escalation": 0.8085808580858086, "tiers": { "easy": 0.9583333333333334, "standard": 0.75, "judge": 0.9246575342465754, "hard": 0.6513761467889908 }, "accuracy": 0.8151815181518152, "ece_hard": 0.20331457804661565, "calibration": 64.52707992100139, "p50_s": 1.4600932538509368, "p95_s": 2.2218950860202304, "cost_usd_per_1000": 0.02485660912720486, "axes": { "intelligence": 79.84836202924052, "calibration": 64.52707992100139, "speed": 74.88895857159795, "cost": 58.13674350870633 }, "jevbench_score": 68.82070688347761 } }, { "threshold": 0.65, "public": { "n": 231, "escalation": 0.6926406926406926, "tiers": { "easy": 0.9583333333333334, "standard": 0.7361111111111112, "hard": 0.6216216216216216 }, "accuracy": 0.7272727272727273, "ece_hard": 0.22209625533654356, "calibration": 60.1836657570925, "p50_s": 0.8543005079030991, "p95_s": 1.9158442258834838, "cost_usd_per_1000": 0.021847816365906256, "axes": { "intelligence": 73.1617033700367, "calibration": 60.1836657570925, "speed": 77.86029141024002, "cost": 59.81775889484972 }, "jevbench_score": 67.294125223676 }, "heldout": { "n": 303, "escalation": 0.8283828382838284, "tiers": { "easy": 0.9583333333333334, "standard": 0.75, "judge": 0.9246575342465754, "hard": 0.6513761467889908 }, "accuracy": 0.8151815181518152, "ece_hard": 0.20331457804661565, "calibration": 64.52707992100139, "p50_s": 1.4600932538509368, "p95_s": 2.2218950860202304, "cost_usd_per_1000": 0.02537049535852462, "axes": { "intelligence": 79.84836202924052, "calibration": 64.52707992100139, "speed": 74.88895857159795, "cost": 57.87013159307582 }, "jevbench_score": 68.74166883290212 } }, { "threshold": 0.66, "public": { "n": 231, "escalation": 0.7056277056277056, "tiers": { "easy": 0.9583333333333334, "standard": 0.7361111111111112, "hard": 0.6306306306306306 }, "accuracy": 0.7316017316017316, "ece_hard": 0.2161615404751408, "calibration": 60.77713724323277, "p50_s": 0.8733458980917931, "p95_s": 1.9158442258834838, "cost_usd_per_1000": 0.022184845647486098, "axes": { "intelligence": 73.53707874541207, "calibration": 60.77713724323277, "speed": 77.76453520780574, "cost": 59.61830765707493 }, "jevbench_score": 67.46864054732627 }, "heldout": { "n": 303, "escalation": 0.8646864686468647, "tiers": { "easy": 1.0, "standard": 0.75, "judge": 0.9315068493150684, "hard": 0.6513761467889908 }, "accuracy": 0.8217821782178217, "ece_hard": 0.20912141571978177, "calibration": 62.53074215339402, "p50_s": 1.4824405029416083, "p95_s": 2.227580388635397, "cost_usd_per_1000": 0.026312620115944183, "axes": { "intelligence": 80.62347618449165, "calibration": 62.53074215339402, "speed": 74.81189342065187, "cost": 57.39507712842031 }, "jevbench_score": 68.21032176650597 } }, { "threshold": 0.67, "public": { "n": 231, "escalation": 0.7272727272727273, "tiers": { "easy": 0.9791666666666666, "standard": 0.7361111111111112, "hard": 0.6396396396396397 }, "accuracy": 0.7402597402597403, "ece_hard": 0.2079976519221129, "calibration": 61.59352609853556, "p50_s": 0.8733458980917931, "p95_s": 1.9158442258834838, "cost_usd_per_1000": 0.02274656111678584, "axes": { "intelligence": 74.31754671338007, "calibration": 61.59352609853556, "speed": 77.76453520780574, "cost": 59.29252755380096 }, "jevbench_score": 67.78006705894502 }, "heldout": { "n": 303, "escalation": 0.8811881188118812, "tiers": { "easy": 1.0, "standard": 0.75, "judge": 0.9315068493150684, "hard": 0.6605504587155964 }, "accuracy": 0.8250825082508251, "ece_hard": 0.20887596442112477, "calibration": 62.555287283259716, "p50_s": 1.4824405029416083, "p95_s": 2.227580388635397, "cost_usd_per_1000": 0.026740858642043983, "axes": { "intelligence": 80.89870554228982, "calibration": 62.555287283259716, "speed": 74.81189342065187, "cost": 57.184739553004896 }, "jevbench_score": 68.21252050593006 } }, { "threshold": 0.68, "public": { "n": 231, "escalation": 0.7402597402597403, "tiers": { "easy": 1.0, "standard": 0.75, "hard": 0.6396396396396397 }, "accuracy": 0.7489177489177489, "ece_hard": 0.2079976519221129, "calibration": 61.59352609853556, "p50_s": 0.8734878540039063, "p95_s": 1.9161328352987765, "cost_usd_per_1000": 0.023083590398365682, "axes": { "intelligence": 75.26276276276278, "calibration": 61.59352609853556, "speed": 77.76317516464306, "cost": 59.10089921610459 }, "jevbench_score": 67.93926190398965 }, "heldout": { "n": 303, "escalation": 0.900990099009901, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6788990825688074 }, "accuracy": 0.834983498349835, "ece_hard": 0.19938304931752704, "calibration": 63.504578793619494, "p50_s": 1.4824405029416083, "p95_s": 2.227580388635397, "cost_usd_per_1000": 0.027254744873363743, "axes": { "intelligence": 82.6158309245528, "calibration": 63.504578793619494, "speed": 74.81189342065187, "cost": 56.93673636903228 }, "jevbench_score": 68.75556936098779 } }, { "threshold": 0.69, "public": { "n": 231, "escalation": 0.7619047619047619, "tiers": { "easy": 1.0, "standard": 0.75, "hard": 0.6396396396396397 }, "accuracy": 0.7489177489177489, "ece_hard": 0.21208770515859382, "calibration": 61.184520774887474, "p50_s": 0.9017323344945908, "p95_s": 1.9161328352987765, "cost_usd_per_1000": 0.02364530586766542, "axes": { "intelligence": 75.26276276276278, "calibration": 61.184520774887474, "speed": 77.62496743916336, "cost": 58.78765191177834 }, "jevbench_score": 67.70602481572038 }, "heldout": { "n": 303, "escalation": 0.9075907590759076, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6880733944954128 }, "accuracy": 0.8382838283828383, "ece_hard": 0.19344261553335568, "calibration": 64.09862217203663, "p50_s": 1.4824405029416083, "p95_s": 2.227580388635397, "cost_usd_per_1000": 0.027426040283803663, "axes": { "intelligence": 82.89106028235096, "calibration": 64.09862217203663, "speed": 74.81189342065187, "cost": 56.85510671283769 }, "jevbench_score": 68.94838986423457 } }, { "threshold": 0.7, "public": { "n": 231, "escalation": 0.7965367965367965, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "hard": 0.6396396396396397 }, "accuracy": 0.7619047619047619, "ece_hard": 0.2148708721832008, "calibration": 60.906204072426775, "p50_s": 0.92564936876297, "p95_s": 1.9161328352987765, "cost_usd_per_1000": 0.024544050618545005, "axes": { "intelligence": 76.88313313313314, "calibration": 60.906204072426775, "speed": 77.51127877579611, "cost": 58.30161286182407 }, "jevbench_score": 67.82417494857657 }, "heldout": { "n": 303, "escalation": 0.9141914191419142, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6880733944954128 }, "accuracy": 0.8382838283828383, "ece_hard": 0.19344261553335568, "calibration": 64.09862217203663, "p50_s": 1.4824405029416083, "p95_s": 2.227580388635397, "cost_usd_per_1000": 0.027597335694243583, "axes": { "intelligence": 82.89106028235096, "calibration": 64.09862217203663, "speed": 74.81189342065187, "cost": 56.7739853088483 }, "jevbench_score": 68.92378264044396 } }, { "threshold": 0.71, "public": { "n": 231, "escalation": 0.8051948051948052, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.6396396396396397 }, "accuracy": 0.7662337662337663, "ece_hard": 0.2148708721832008, "calibration": 60.906204072426775, "p50_s": 0.92564936876297, "p95_s": 1.9161328352987765, "cost_usd_per_1000": 0.0247687368062649, "axes": { "intelligence": 77.42325658992328, "calibration": 60.906204072426775, "speed": 77.51127877579611, "cost": 58.182884249809966 }, "jevbench_score": 67.90836575071133 }, "heldout": { "n": 303, "escalation": 0.9174917491749175, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6880733944954128 }, "accuracy": 0.8382838283828383, "ece_hard": 0.19599581397385438, "calibration": 63.84330232798676, "p50_s": 1.4824405029416083, "p95_s": 2.227580388635397, "cost_usd_per_1000": 0.027682983399463547, "axes": { "intelligence": 82.89106028235096, "calibration": 63.84330232798676, "speed": 74.81189342065187, "cost": 56.733613231258616 }, "jevbench_score": 68.84280098287385 } }, { "threshold": 0.72, "public": { "n": 231, "escalation": 0.8225108225108225, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.6486486486486487 }, "accuracy": 0.7705627705627706, "ece_hard": 0.2084349624406332, "calibration": 61.54979504668353, "p50_s": 0.9340311378240586, "p95_s": 1.9161328352987765, "cost_usd_per_1000": 0.025218109181704692, "axes": { "intelligence": 77.79863196529865, "calibration": 61.54979504668353, "speed": 77.47213032327545, "cost": 57.948624379917845 }, "jevbench_score": 68.0921112690567 }, "heldout": { "n": 303, "escalation": 0.9207920792079208, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6880733944954128 }, "accuracy": 0.8382838283828383, "ece_hard": 0.18853606427239092, "calibration": 64.58927729813311, "p50_s": 1.4824405029416083, "p95_s": 2.227580388635397, "cost_usd_per_1000": 0.027768631104683507, "axes": { "intelligence": 82.89106028235096, "calibration": 64.58927729813311, "speed": 74.81189342065187, "cost": 56.69336586712729 }, "jevbench_score": 69.0307754543251 } }, { "threshold": 0.73, "public": { "n": 231, "escalation": 0.8225108225108225, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.6486486486486487 }, "accuracy": 0.7705627705627706, "ece_hard": 0.2084349624406332, "calibration": 61.54979504668353, "p50_s": 0.9340311378240586, "p95_s": 1.9161328352987765, "cost_usd_per_1000": 0.025218109181704692, "axes": { "intelligence": 77.79863196529865, "calibration": 61.54979504668353, "speed": 77.47213032327545, "cost": 57.948624379917845 }, "jevbench_score": 68.0921112690567 }, "heldout": { "n": 303, "escalation": 0.9273927392739274, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6880733944954128 }, "accuracy": 0.8382838283828383, "ece_hard": 0.19107510867374006, "calibration": 64.3353728579982, "p50_s": 1.4824405029416083, "p95_s": 2.227580388635397, "cost_usd_per_1000": 0.027939926515123427, "axes": { "intelligence": 82.89106028235096, "calibration": 64.3353728579982, "speed": 74.81189342065187, "cost": 56.61324221378292 }, "jevbench_score": 68.93845526674833 } }, { "threshold": 0.74, "public": { "n": 231, "escalation": 0.8268398268398268, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.6486486486486487 }, "accuracy": 0.7705627705627706, "ece_hard": 0.2084349624406332, "calibration": 61.54979504668353, "p50_s": 0.9340311378240586, "p95_s": 1.9161328352987765, "cost_usd_per_1000": 0.025330452275564637, "axes": { "intelligence": 77.79863196529865, "calibration": 61.54979504668353, "speed": 77.47213032327545, "cost": 57.89071167419301 }, "jevbench_score": 68.0750924132846 }, "heldout": { "n": 303, "escalation": 0.9273927392739274, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6880733944954128 }, "accuracy": 0.8382838283828383, "ece_hard": 0.19107510867374006, "calibration": 64.3353728579982, "p50_s": 1.4824405029416083, "p95_s": 2.227580388635397, "cost_usd_per_1000": 0.027939926515123427, "axes": { "intelligence": 82.89106028235096, "calibration": 64.3353728579982, "speed": 74.81189342065187, "cost": 56.61324221378292 }, "jevbench_score": 68.93845526674833 } }, { "threshold": 0.75, "public": { "n": 231, "escalation": 0.8311688311688312, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.6486486486486487 }, "accuracy": 0.7705627705627706, "ece_hard": 0.21041641822205612, "calibration": 61.35164946854124, "p50_s": 0.9340311378240586, "p95_s": 1.9177926644682883, "cost_usd_per_1000": 0.02544279536942459, "axes": { "intelligence": 77.79863196529865, "calibration": 61.35164946854124, "speed": 77.46836992316177, "cost": 57.833055249644524 }, "jevbench_score": 68.00247029279659 }, "heldout": { "n": 303, "escalation": 0.933993399339934, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6880733944954128 }, "accuracy": 0.8382838283828383, "ece_hard": 0.1888257594066398, "calibration": 64.56030778470821, "p50_s": 1.4824405029416083, "p95_s": 2.227580388635397, "cost_usd_per_1000": 0.028111221925563347, "axes": { "intelligence": 82.89106028235096, "calibration": 64.56030778470821, "speed": 74.81189342065187, "cost": 56.53360828813141 }, "jevbench_score": 68.97435687103052 } }, { "threshold": 0.76, "public": { "n": 231, "escalation": 0.8311688311688312, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.6486486486486487 }, "accuracy": 0.7705627705627706, "ece_hard": 0.21041641822205612, "calibration": 61.35164946854124, "p50_s": 0.9340311378240586, "p95_s": 1.9177926644682883, "cost_usd_per_1000": 0.02544279536942459, "axes": { "intelligence": 77.79863196529865, "calibration": 61.35164946854124, "speed": 77.46836992316177, "cost": 57.833055249644524 }, "jevbench_score": 68.00247029279659 }, "heldout": { "n": 303, "escalation": 0.933993399339934, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6880733944954128 }, "accuracy": 0.8382838283828383, "ece_hard": 0.1888257594066398, "calibration": 64.56030778470821, "p50_s": 1.4824405029416083, "p95_s": 2.227580388635397, "cost_usd_per_1000": 0.028111221925563347, "axes": { "intelligence": 82.89106028235096, "calibration": 64.56030778470821, "speed": 74.81189342065187, "cost": 56.53360828813141 }, "jevbench_score": 68.97435687103052 } }, { "threshold": 0.77, "public": { "n": 231, "escalation": 0.8441558441558441, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.6486486486486487 }, "accuracy": 0.7705627705627706, "ece_hard": 0.21041641822205612, "calibration": 61.35164946854124, "p50_s": 0.9340311378240586, "p95_s": 1.9177926644682883, "cost_usd_per_1000": 0.025779824651004427, "axes": { "intelligence": 77.79863196529865, "calibration": 61.35164946854124, "speed": 77.46836992316177, "cost": 57.66160122859427 }, "jevbench_score": 67.95201350178469 }, "heldout": { "n": 303, "escalation": 0.9372937293729373, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6788990825688074 }, "accuracy": 0.834983498349835, "ece_hard": 0.19718167237937023, "calibration": 63.724716487435174, "p50_s": 1.4824405029416083, "p95_s": 2.227580388635397, "cost_usd_per_1000": 0.028196869630783307, "axes": { "intelligence": 82.6158309245528, "calibration": 63.724716487435174, "speed": 74.81189342065187, "cost": 56.49397310950338 }, "jevbench_score": 68.68090167309917 } }, { "threshold": 0.78, "public": { "n": 231, "escalation": 0.8441558441558441, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.6486486486486487 }, "accuracy": 0.7705627705627706, "ece_hard": 0.21041641822205612, "calibration": 61.35164946854124, "p50_s": 0.9340311378240586, "p95_s": 1.9177926644682883, "cost_usd_per_1000": 0.025779824651004427, "axes": { "intelligence": 77.79863196529865, "calibration": 61.35164946854124, "speed": 77.46836992316177, "cost": 57.66160122859427 }, "jevbench_score": 67.95201350178469 }, "heldout": { "n": 303, "escalation": 0.9405940594059405, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6788990825688074 }, "accuracy": 0.834983498349835, "ece_hard": 0.19718167237937023, "calibration": 63.724716487435174, "p50_s": 1.4824405029416083, "p95_s": 2.227580388635397, "cost_usd_per_1000": 0.028282517336003267, "axes": { "intelligence": 82.6158309245528, "calibration": 63.724716487435174, "speed": 74.81189342065187, "cost": 56.45445813994595 }, "jevbench_score": 68.66888872848806 } }, { "threshold": 0.79, "public": { "n": 231, "escalation": 0.8441558441558441, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.6486486486486487 }, "accuracy": 0.7705627705627706, "ece_hard": 0.21041641822205612, "calibration": 61.35164946854124, "p50_s": 0.9340311378240586, "p95_s": 1.9177926644682883, "cost_usd_per_1000": 0.025779824651004427, "axes": { "intelligence": 77.79863196529865, "calibration": 61.35164946854124, "speed": 77.46836992316177, "cost": 57.66160122859427 }, "jevbench_score": 67.95201350178469 }, "heldout": { "n": 303, "escalation": 0.9438943894389439, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6788990825688074 }, "accuracy": 0.834983498349835, "ece_hard": 0.19718167237937023, "calibration": 63.724716487435174, "p50_s": 1.4824405029416083, "p95_s": 2.227580388635397, "cost_usd_per_1000": 0.028368165041223227, "axes": { "intelligence": 82.6158309245528, "calibration": 63.724716487435174, "speed": 74.81189342065187, "cost": 56.415062652501966 }, "jevbench_score": 68.6569058282703 } }, { "threshold": 0.8, "public": { "n": 231, "escalation": 0.8484848484848485, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.6486486486486487 }, "accuracy": 0.7705627705627706, "ece_hard": 0.21041641822205612, "calibration": 61.35164946854124, "p50_s": 0.9340311378240586, "p95_s": 1.9177926644682883, "cost_usd_per_1000": 0.02589216774486438, "axes": { "intelligence": 77.79863196529865, "calibration": 61.35164946854124, "speed": 77.46836992316177, "cost": 57.60494764081009 }, "jevbench_score": 67.93531632158458 }, "heldout": { "n": 303, "escalation": 0.9438943894389439, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6788990825688074 }, "accuracy": 0.834983498349835, "ece_hard": 0.19718167237937023, "calibration": 63.724716487435174, "p50_s": 1.4824405029416083, "p95_s": 2.227580388635397, "cost_usd_per_1000": 0.028368165041223227, "axes": { "intelligence": 82.6158309245528, "calibration": 63.724716487435174, "speed": 74.81189342065187, "cost": 56.415062652501966 }, "jevbench_score": 68.6569058282703 } }, { "threshold": 0.81, "public": { "n": 231, "escalation": 0.8484848484848485, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.6486486486486487 }, "accuracy": 0.7705627705627706, "ece_hard": 0.21041641822205612, "calibration": 61.35164946854124, "p50_s": 0.9340311378240586, "p95_s": 1.9177926644682883, "cost_usd_per_1000": 0.02589216774486438, "axes": { "intelligence": 77.79863196529865, "calibration": 61.35164946854124, "speed": 77.46836992316177, "cost": 57.60494764081009 }, "jevbench_score": 67.93531632158458 }, "heldout": { "n": 303, "escalation": 0.9471947194719472, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6788990825688074 }, "accuracy": 0.834983498349835, "ece_hard": 0.19718167237937023, "calibration": 63.724716487435174, "p50_s": 1.4824405029416083, "p95_s": 2.227580388635397, "cost_usd_per_1000": 0.028453812746443187, "axes": { "intelligence": 82.6158309245528, "calibration": 63.724716487435174, "speed": 74.81189342065187, "cost": 56.37578592678875 }, "jevbench_score": 68.64495280257353 } }, { "threshold": 0.82, "public": { "n": 231, "escalation": 0.8484848484848485, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.6486486486486487 }, "accuracy": 0.7705627705627706, "ece_hard": 0.21041641822205612, "calibration": 61.35164946854124, "p50_s": 0.9340311378240586, "p95_s": 1.9177926644682883, "cost_usd_per_1000": 0.02589216774486438, "axes": { "intelligence": 77.79863196529865, "calibration": 61.35164946854124, "speed": 77.46836992316177, "cost": 57.60494764081009 }, "jevbench_score": 67.93531632158458 }, "heldout": { "n": 303, "escalation": 0.9504950495049505, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6788990825688074 }, "accuracy": 0.834983498349835, "ece_hard": 0.19804228405620955, "calibration": 63.638655319751244, "p50_s": 1.4908051952719688, "p95_s": 2.227580388635397, "cost_usd_per_1000": 0.028539460451663147, "axes": { "intelligence": 82.6158309245528, "calibration": 63.638655319751244, "speed": 74.78745716697648, "cost": 56.33662724891906 }, "jevbench_score": 68.60424196220856 } }, { "threshold": 0.83, "public": { "n": 231, "escalation": 0.8528138528138528, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.6486486486486487 }, "accuracy": 0.7705627705627706, "ece_hard": 0.21041641822205612, "calibration": 61.35164946854124, "p50_s": 0.9340311378240586, "p95_s": 1.9177926644682883, "cost_usd_per_1000": 0.026004510838724324, "axes": { "intelligence": 77.79863196529865, "calibration": 61.35164946854124, "speed": 77.46836992316177, "cost": 57.548539334975764 }, "jevbench_score": 67.91867919121276 }, "heldout": { "n": 303, "escalation": 0.9504950495049505, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6788990825688074 }, "accuracy": 0.834983498349835, "ece_hard": 0.19804228405620955, "calibration": 63.638655319751244, "p50_s": 1.4908051952719688, "p95_s": 2.227580388635397, "cost_usd_per_1000": 0.028539460451663147, "axes": { "intelligence": 82.6158309245528, "calibration": 63.638655319751244, "speed": 74.78745716697648, "cost": 56.33662724891906 }, "jevbench_score": 68.60424196220856 } }, { "threshold": 0.84, "public": { "n": 231, "escalation": 0.8571428571428571, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.6486486486486487 }, "accuracy": 0.7705627705627706, "ece_hard": 0.21041641822205612, "calibration": 61.35164946854124, "p50_s": 0.9340311378240586, "p95_s": 1.9177926644682883, "cost_usd_per_1000": 0.026116853932584273, "axes": { "intelligence": 77.79863196529865, "calibration": 61.35164946854124, "speed": 77.46836992316177, "cost": 57.49237419634532 }, "jevbench_score": 67.90210162476819 }, "heldout": { "n": 303, "escalation": 0.9504950495049505, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6788990825688074 }, "accuracy": 0.834983498349835, "ece_hard": 0.19804228405620955, "calibration": 63.638655319751244, "p50_s": 1.4908051952719688, "p95_s": 2.227580388635397, "cost_usd_per_1000": 0.028539460451663147, "axes": { "intelligence": 82.6158309245528, "calibration": 63.638655319751244, "speed": 74.78745716697648, "cost": 56.33662724891906 }, "jevbench_score": 68.60424196220856 } }, { "threshold": 0.85, "public": { "n": 231, "escalation": 0.8571428571428571, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.6486486486486487 }, "accuracy": 0.7705627705627706, "ece_hard": 0.21041641822205612, "calibration": 61.35164946854124, "p50_s": 0.9340311378240586, "p95_s": 1.9177926644682883, "cost_usd_per_1000": 0.026116853932584273, "axes": { "intelligence": 77.79863196529865, "calibration": 61.35164946854124, "speed": 77.46836992316177, "cost": 57.49237419634532 }, "jevbench_score": 67.90210162476819 }, "heldout": { "n": 303, "escalation": 0.9537953795379538, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6788990825688074 }, "accuracy": 0.834983498349835, "ece_hard": 0.19804228405620955, "calibration": 63.638655319751244, "p50_s": 1.4908051952719688, "p95_s": 2.227580388635397, "cost_usd_per_1000": 0.028625108156883107, "axes": { "intelligence": 82.6158309245528, "calibration": 63.638655319751244, "speed": 74.78745716697648, "cost": 56.29758591142321 }, "jevbench_score": 68.59235317055132 } }, { "threshold": 0.86, "public": { "n": 231, "escalation": 0.8658008658008658, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.6486486486486487 }, "accuracy": 0.7705627705627706, "ece_hard": 0.21041641822205612, "calibration": 61.35164946854124, "p50_s": 0.9340311378240586, "p95_s": 1.9177926644682883, "cost_usd_per_1000": 0.02634154012030417, "axes": { "intelligence": 77.79863196529865, "calibration": 61.35164946854124, "speed": 77.46836992316177, "cost": 57.380765097404975 }, "jevbench_score": 67.86912327004615 }, "heldout": { "n": 303, "escalation": 0.9537953795379538, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6788990825688074 }, "accuracy": 0.834983498349835, "ece_hard": 0.19804228405620955, "calibration": 63.638655319751244, "p50_s": 1.4908051952719688, "p95_s": 2.227580388635397, "cost_usd_per_1000": 0.028625108156883107, "axes": { "intelligence": 82.6158309245528, "calibration": 63.638655319751244, "speed": 74.78745716697648, "cost": 56.29758591142321 }, "jevbench_score": 68.59235317055132 } }, { "threshold": 0.87, "public": { "n": 231, "escalation": 0.8701298701298701, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.6486486486486487 }, "accuracy": 0.7705627705627706, "ece_hard": 0.21041641822205612, "calibration": 61.35164946854124, "p50_s": 0.9340311378240586, "p95_s": 1.9177926644682883, "cost_usd_per_1000": 0.026453883214164114, "axes": { "intelligence": 77.79863196529865, "calibration": 61.35164946854124, "speed": 77.46836992316177, "cost": 57.32531704190927 }, "jevbench_score": 67.852721539658 }, "heldout": { "n": 303, "escalation": 0.9537953795379538, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6788990825688074 }, "accuracy": 0.834983498349835, "ece_hard": 0.19804228405620955, "calibration": 63.638655319751244, "p50_s": 1.4908051952719688, "p95_s": 2.227580388635397, "cost_usd_per_1000": 0.028625108156883107, "axes": { "intelligence": 82.6158309245528, "calibration": 63.638655319751244, "speed": 74.78745716697648, "cost": 56.29758591142321 }, "jevbench_score": 68.59235317055132 } }, { "threshold": 0.88, "public": { "n": 231, "escalation": 0.8831168831168831, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.6486486486486487 }, "accuracy": 0.7705627705627706, "ece_hard": 0.21041641822205612, "calibration": 61.35164946854124, "p50_s": 0.9340311378240586, "p95_s": 1.9177926644682883, "cost_usd_per_1000": 0.02679091249574396, "axes": { "intelligence": 77.79863196529865, "calibration": 61.35164946854124, "speed": 77.46836992316177, "cost": 57.160374823425116 }, "jevbench_score": 67.80386060393734 }, "heldout": { "n": 303, "escalation": 0.9537953795379538, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6788990825688074 }, "accuracy": 0.834983498349835, "ece_hard": 0.19804228405620955, "calibration": 63.638655319751244, "p50_s": 1.4908051952719688, "p95_s": 2.227580388635397, "cost_usd_per_1000": 0.028625108156883107, "axes": { "intelligence": 82.6158309245528, "calibration": 63.638655319751244, "speed": 74.78745716697648, "cost": 56.29758591142321 }, "jevbench_score": 68.59235317055132 } }, { "threshold": 0.89, "public": { "n": 231, "escalation": 0.8961038961038961, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6486486486486487 }, "accuracy": 0.7748917748917749, "ece_hard": 0.21041641822205612, "calibration": 61.35164946854124, "p50_s": 0.9340311378240586, "p95_s": 1.9177926644682883, "cost_usd_per_1000": 0.0271279417773238, "axes": { "intelligence": 78.33875542208877, "calibration": 61.35164946854124, "speed": 77.46836992316177, "cost": 56.997494659683824 }, "jevbench_score": 67.87280125509729 }, "heldout": { "n": 303, "escalation": 0.9570957095709571, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6788990825688074 }, "accuracy": 0.834983498349835, "ece_hard": 0.19804228405620955, "calibration": 63.638655319751244, "p50_s": 1.4908051952719688, "p95_s": 2.227580388635397, "cost_usd_per_1000": 0.02871075586210307, "axes": { "intelligence": 82.6158309245528, "calibration": 63.638655319751244, "speed": 74.78745716697648, "cost": 56.258661213172424 }, "jevbench_score": 68.58049374024388 } }, { "threshold": 0.9, "public": { "n": 231, "escalation": 0.8961038961038961, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6486486486486487 }, "accuracy": 0.7748917748917749, "ece_hard": 0.21041641822205612, "calibration": 61.35164946854124, "p50_s": 0.9340311378240586, "p95_s": 1.9177926644682883, "cost_usd_per_1000": 0.0271279417773238, "axes": { "intelligence": 78.33875542208877, "calibration": 61.35164946854124, "speed": 77.46836992316177, "cost": 56.997494659683824 }, "jevbench_score": 67.87280125509729 }, "heldout": { "n": 303, "escalation": 0.9570957095709571, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6788990825688074 }, "accuracy": 0.834983498349835, "ece_hard": 0.19804228405620955, "calibration": 63.638655319751244, "p50_s": 1.4908051952719688, "p95_s": 2.227580388635397, "cost_usd_per_1000": 0.02871075586210307, "axes": { "intelligence": 82.6158309245528, "calibration": 63.638655319751244, "speed": 74.78745716697648, "cost": 56.258661213172424 }, "jevbench_score": 68.58049374024388 } }, { "threshold": 0.91, "public": { "n": 231, "escalation": 0.9090909090909091, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6486486486486487 }, "accuracy": 0.7748917748917749, "ece_hard": 0.21041641822205612, "calibration": 61.35164946854124, "p50_s": 0.9340311378240586, "p95_s": 1.9177926644682883, "cost_usd_per_1000": 0.027464971058903647, "axes": { "intelligence": 78.33875542208877, "calibration": 61.35164946854124, "speed": 77.46836992316177, "cost": 56.83662562835029 }, "jevbench_score": 67.82485964314604 }, "heldout": { "n": 303, "escalation": 0.9603960396039604, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6788990825688074 }, "accuracy": 0.834983498349835, "ece_hard": 0.19804228405620955, "calibration": 63.638655319751244, "p50_s": 1.4908051952719688, "p95_s": 2.227580388635397, "cost_usd_per_1000": 0.028796403567323027, "axes": { "intelligence": 82.6158309245528, "calibration": 63.638655319751244, "speed": 74.78745716697648, "cost": 56.21985245930325 }, "jevbench_score": 68.56866350727046 } }, { "threshold": 0.92, "public": { "n": 231, "escalation": 0.922077922077922, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6486486486486487 }, "accuracy": 0.7748917748917749, "ece_hard": 0.21041641822205612, "calibration": 61.35164946854124, "p50_s": 0.9340311378240586, "p95_s": 1.9177926644682883, "cost_usd_per_1000": 0.02780200034048349, "axes": { "intelligence": 78.33875542208877, "calibration": 61.35164946854124, "speed": 77.46836992316177, "cost": 56.67771867039236 }, "jevbench_score": 67.77740274169238 }, "heldout": { "n": 303, "escalation": 0.9636963696369637, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6788990825688074 }, "accuracy": 0.834983498349835, "ece_hard": 0.19804228405620955, "calibration": 63.638655319751244, "p50_s": 1.4908051952719688, "p95_s": 2.227580388635397, "cost_usd_per_1000": 0.028882051272542987, "axes": { "intelligence": 82.6158309245528, "calibration": 63.638655319751244, "speed": 74.78745716697648, "cost": 56.181158961143105 }, "jevbench_score": 68.55686230901983 } }, { "threshold": 0.93, "public": { "n": 231, "escalation": 0.922077922077922, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6486486486486487 }, "accuracy": 0.7748917748917749, "ece_hard": 0.21041641822205612, "calibration": 61.35164946854124, "p50_s": 0.9340311378240586, "p95_s": 1.9177926644682883, "cost_usd_per_1000": 0.02780200034048349, "axes": { "intelligence": 78.33875542208877, "calibration": 61.35164946854124, "speed": 77.46836992316177, "cost": 56.67771867039236 }, "jevbench_score": 67.77740274169238 }, "heldout": { "n": 303, "escalation": 0.9702970297029703, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6788990825688074 }, "accuracy": 0.834983498349835, "ece_hard": 0.19804228405620955, "calibration": 63.638655319751244, "p50_s": 1.4908051952719688, "p95_s": 2.227580388635397, "cost_usd_per_1000": 0.029053346682982907, "axes": { "intelligence": 82.6158309245528, "calibration": 63.638655319751244, "speed": 74.78745716697648, "cost": 56.104115007775405 }, "jevbench_score": 68.53334637316854 } }, { "threshold": 0.94, "public": { "n": 231, "escalation": 0.9264069264069265, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6486486486486487 }, "accuracy": 0.7748917748917749, "ece_hard": 0.21041641822205612, "calibration": 61.35164946854124, "p50_s": 0.9340311378240586, "p95_s": 1.9177926644682883, "cost_usd_per_1000": 0.02791434343434344, "axes": { "intelligence": 78.33875542208877, "calibration": 61.35164946854124, "speed": 77.46836992316177, "cost": 56.62517747846352 }, "jevbench_score": 67.76168958213455 }, "heldout": { "n": 303, "escalation": 0.9735973597359736, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6788990825688074 }, "accuracy": 0.834983498349835, "ece_hard": 0.19804228405620955, "calibration": 63.638655319751244, "p50_s": 1.4908051952719688, "p95_s": 2.227580388635397, "cost_usd_per_1000": 0.02913899438820287, "axes": { "intelligence": 82.6158309245528, "calibration": 63.638655319751244, "speed": 74.78745716697648, "cost": 56.06576320552273 }, "jevbench_score": 68.52163131722422 } }, { "threshold": 0.95, "public": { "n": 231, "escalation": 0.935064935064935, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6486486486486487 }, "accuracy": 0.7748917748917749, "ece_hard": 0.21041641822205612, "calibration": 61.35164946854124, "p50_s": 0.9340311378240586, "p95_s": 1.9177926644682883, "cost_usd_per_1000": 0.028139029622063334, "axes": { "intelligence": 78.33875542208877, "calibration": 61.35164946854124, "speed": 77.46836992316177, "cost": 56.520726500268154 }, "jevbench_score": 67.73041958779125 }, "heldout": { "n": 303, "escalation": 0.976897689768977, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6788990825688074 }, "accuracy": 0.834983498349835, "ece_hard": 0.19804228405620955, "calibration": 63.638655319751244, "p50_s": 1.4908051952719688, "p95_s": 2.227580388635397, "cost_usd_per_1000": 0.02922464209342283, "axes": { "intelligence": 82.6158309245528, "calibration": 63.638655319751244, "speed": 74.78745716697648, "cost": 56.02752396474753 }, "jevbench_score": 68.50994465928417 } }, { "threshold": 0.96, "public": { "n": 231, "escalation": 0.9523809523809523, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6486486486486487 }, "accuracy": 0.7748917748917749, "ece_hard": 0.21041641822205612, "calibration": 61.35164946854124, "p50_s": 0.9340311378240586, "p95_s": 1.9177926644682883, "cost_usd_per_1000": 0.028588401997503125, "axes": { "intelligence": 78.33875542208877, "calibration": 61.35164946854124, "speed": 77.46836992316177, "cost": 56.31430359011282 }, "jevbench_score": 67.66849407804497 }, "heldout": { "n": 303, "escalation": 0.976897689768977, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6788990825688074 }, "accuracy": 0.834983498349835, "ece_hard": 0.19804228405620955, "calibration": 63.638655319751244, "p50_s": 1.4908051952719688, "p95_s": 2.227580388635397, "cost_usd_per_1000": 0.02922464209342283, "axes": { "intelligence": 82.6158309245528, "calibration": 63.638655319751244, "speed": 74.78745716697648, "cost": 56.02752396474753 }, "jevbench_score": 68.50994465928417 } }, { "threshold": 0.97, "public": { "n": 231, "escalation": 0.9696969696969697, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6486486486486487 }, "accuracy": 0.7748917748917749, "ece_hard": 0.21041641822205612, "calibration": 61.35164946854124, "p50_s": 0.9368793085217476, "p95_s": 1.9177926644682883, "cost_usd_per_1000": 0.02903777437294292, "axes": { "intelligence": 78.33875542208877, "calibration": 61.35164946854124, "speed": 77.45514699444816, "cost": 56.1111002079505 }, "jevbench_score": 67.60448277263121 }, "heldout": { "n": 303, "escalation": 0.9933993399339934, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6788990825688074 }, "accuracy": 0.834983498349835, "ece_hard": 0.1935286337386598, "calibration": 64.09002035150621, "p50_s": 1.4908051952719688, "p95_s": 2.234320732206106, "cost_usd_per_1000": 0.02965288061952263, "axes": { "intelligence": 82.6158309245528, "calibration": 64.09002035150621, "speed": 74.77433587106985, "cost": 55.83799332558983 }, "jevbench_score": 68.56997843753382 } }, { "threshold": 0.98, "public": { "n": 231, "escalation": 0.9826839826839827, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6486486486486487 }, "accuracy": 0.7748917748917749, "ece_hard": 0.21041641822205612, "calibration": 61.35164946854124, "p50_s": 0.9368793085217476, "p95_s": 1.9177926644682883, "cost_usd_per_1000": 0.02937480365452276, "axes": { "intelligence": 78.33875542208877, "calibration": 61.35164946854124, "speed": 77.45514699444816, "cost": 55.96075082794728 }, "jevbench_score": 67.559150738452 }, "heldout": { "n": 303, "escalation": 0.9966996699669967, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6788990825688074 }, "accuracy": 0.834983498349835, "ece_hard": 0.1935286337386598, "calibration": 64.09002035150621, "p50_s": 1.4908051952719688, "p95_s": 2.234320732206106, "cost_usd_per_1000": 0.02973852832474259, "axes": { "intelligence": 82.6158309245528, "calibration": 64.09002035150621, "speed": 74.77433587106985, "cost": 55.800415818461424 }, "jevbench_score": 68.55843907507659 } }, { "threshold": 0.99, "public": { "n": 231, "escalation": 1.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6486486486486487 }, "accuracy": 0.7748917748917749, "ece_hard": 0.21041641822205612, "calibration": 61.35164946854124, "p50_s": 0.9368793085217476, "p95_s": 1.9177926644682883, "cost_usd_per_1000": 0.02982417602996255, "axes": { "intelligence": 78.33875542208877, "calibration": 61.35164946854124, "speed": 77.45514699444816, "cost": 55.762946380022335 }, "jevbench_score": 67.49937111427766 }, "heldout": { "n": 303, "escalation": 1.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6788990825688074 }, "accuracy": 0.834983498349835, "ece_hard": 0.1935286337386598, "calibration": 64.09002035150621, "p50_s": 1.4908051952719688, "p95_s": 2.234320732206106, "cost_usd_per_1000": 0.02982417602996255, "axes": { "intelligence": 82.6158309245528, "calibration": 64.09002035150621, "speed": 74.77433587106985, "cost": 55.762946380022335 }, "jevbench_score": 68.54692709399879 } }, { "threshold": 1.0, "public": { "n": 231, "escalation": 1.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6486486486486487 }, "accuracy": 0.7748917748917749, "ece_hard": 0.21041641822205612, "calibration": 61.35164946854124, "p50_s": 0.9368793085217476, "p95_s": 1.9177926644682883, "cost_usd_per_1000": 0.02982417602996255, "axes": { "intelligence": 78.33875542208877, "calibration": 61.35164946854124, "speed": 77.45514699444816, "cost": 55.762946380022335 }, "jevbench_score": 67.49937111427766 }, "heldout": { "n": 303, "escalation": 1.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6788990825688074 }, "accuracy": 0.834983498349835, "ece_hard": 0.1935286337386598, "calibration": 64.09002035150621, "p50_s": 1.4908051952719688, "p95_s": 2.234320732206106, "cost_usd_per_1000": 0.02982417602996255, "axes": { "intelligence": 82.6158309245528, "calibration": 64.09002035150621, "speed": 74.77433587106985, "cost": 55.762946380022335 }, "jevbench_score": 68.54692709399879 } } ] }, { "cheap": "semif-qwen3.5-4b", "backend": "jev-1.13.0", "chosen": { "threshold": 0.61, "public": { "n": 231, "escalation": 0.20346320346320346, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.7027027027027027 }, "accuracy": 0.8008658008658008, "ece_hard": 0.12322321771956599, "calibration": 72.81320697295129, "p50_s": 0.5404984593391419, "p95_s": 1.7742935769259929, "cost_usd_per_1000": 0.030565685052774936, "axes": { "intelligence": 80.59100767434103, "calibration": 72.81320697295129, "speed": 80.18180059269342, "cost": 55.44297598447476 }, "jevbench_score": 71.46687148557807 }, "heldout": { "n": 303, "escalation": 0.15841584158415842, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9657534246575342, "hard": 0.6055045871559633 }, "accuracy": 0.8250825082508251, "ece_hard": 0.18494530682510557, "calibration": 69.71544213255548, "p50_s": 0.5717611402273178, "p95_s": 1.4723677232861514, "cost_usd_per_1000": 0.02876765402158193, "axes": { "intelligence": 81.37290017175654, "calibration": 69.71544213255548, "speed": 80.74769076387742, "cost": 56.23286659065571 }, "jevbench_score": 71.24137266342876 } }, "full": { "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9657534246575342, "hard": 0.6545454545454545 }, "accuracy": 0.8146067415730337, "ece_hard": 0.1371510670291043, "calibration": 72.95764407707654, "p50_s": 0.560910464823246, "p95_s": 1.6781534798443314, "cost_usd_per_1000": 0.029545453962041827, "axes": { "intelligence": 83.4274595267746, "calibration": 72.95764407707654, "speed": 80.26274779518857, "cost": 55.88527998265149 }, "jevbench_score": 72.28493898995923 }, "full_escalation": 0.17790262172284643, "backend_heldout_accuracy": 0.8514851485148515, "useful": false, "curve": [ { "threshold": 0.0, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6036036036036037 }, "accuracy": 0.7532467532467533, "ece_hard": 0.12914387221294593, "calibration": 72.24478620042561, "p50_s": 0.5384091228246689, "p95_s": 1.2262836933135985, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 76.46187854521189, "calibration": 72.24478620042561, "speed": 81.80296636812625, "cost": 59.46663998783557 }, "jevbench_score": 71.99850550126745 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7986798679867987, "ece_hard": 0.17685813516393573, "calibration": 66.51802582183663, "p50_s": 0.5651323288679123, "p95_s": 1.115284267067909, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 79.33796238113192, "calibration": 66.51802582183663, "speed": 82.00464272011534, "cost": 59.46663998783557 }, "jevbench_score": 71.22509854109671 } }, { "threshold": 0.01, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6036036036036037 }, "accuracy": 0.7532467532467533, "ece_hard": 0.12914387221294593, "calibration": 72.24478620042561, "p50_s": 0.5384091228246689, "p95_s": 1.2262836933135985, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 76.46187854521189, "calibration": 72.24478620042561, "speed": 81.80296636812625, "cost": 59.46663998783557 }, "jevbench_score": 71.99850550126745 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7986798679867987, "ece_hard": 0.17685813516393573, "calibration": 66.51802582183663, "p50_s": 0.5651323288679123, "p95_s": 1.115284267067909, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 79.33796238113192, "calibration": 66.51802582183663, "speed": 82.00464272011534, "cost": 59.46663998783557 }, "jevbench_score": 71.22509854109671 } }, { "threshold": 0.02, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6036036036036037 }, "accuracy": 0.7532467532467533, "ece_hard": 0.12914387221294593, "calibration": 72.24478620042561, "p50_s": 0.5384091228246689, "p95_s": 1.2262836933135985, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 76.46187854521189, "calibration": 72.24478620042561, "speed": 81.80296636812625, "cost": 59.46663998783557 }, "jevbench_score": 71.99850550126745 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7986798679867987, "ece_hard": 0.17685813516393573, "calibration": 66.51802582183663, "p50_s": 0.5651323288679123, "p95_s": 1.115284267067909, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 79.33796238113192, "calibration": 66.51802582183663, "speed": 82.00464272011534, "cost": 59.46663998783557 }, "jevbench_score": 71.22509854109671 } }, { "threshold": 0.03, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6036036036036037 }, "accuracy": 0.7532467532467533, "ece_hard": 0.12914387221294593, "calibration": 72.24478620042561, "p50_s": 0.5384091228246689, "p95_s": 1.2262836933135985, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 76.46187854521189, "calibration": 72.24478620042561, "speed": 81.80296636812625, "cost": 59.46663998783557 }, "jevbench_score": 71.99850550126745 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7986798679867987, "ece_hard": 0.17685813516393573, "calibration": 66.51802582183663, "p50_s": 0.5651323288679123, "p95_s": 1.115284267067909, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 79.33796238113192, "calibration": 66.51802582183663, "speed": 82.00464272011534, "cost": 59.46663998783557 }, "jevbench_score": 71.22509854109671 } }, { "threshold": 0.04, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6036036036036037 }, "accuracy": 0.7532467532467533, "ece_hard": 0.12914387221294593, "calibration": 72.24478620042561, "p50_s": 0.5384091228246689, "p95_s": 1.2262836933135985, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 76.46187854521189, "calibration": 72.24478620042561, "speed": 81.80296636812625, "cost": 59.46663998783557 }, "jevbench_score": 71.99850550126745 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7986798679867987, "ece_hard": 0.17685813516393573, "calibration": 66.51802582183663, "p50_s": 0.5651323288679123, "p95_s": 1.115284267067909, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 79.33796238113192, "calibration": 66.51802582183663, "speed": 82.00464272011534, "cost": 59.46663998783557 }, "jevbench_score": 71.22509854109671 } }, { "threshold": 0.05, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6036036036036037 }, "accuracy": 0.7532467532467533, "ece_hard": 0.12914387221294593, "calibration": 72.24478620042561, "p50_s": 0.5384091228246689, "p95_s": 1.2262836933135985, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 76.46187854521189, "calibration": 72.24478620042561, "speed": 81.80296636812625, "cost": 59.46663998783557 }, "jevbench_score": 71.99850550126745 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7986798679867987, "ece_hard": 0.17685813516393573, "calibration": 66.51802582183663, "p50_s": 0.5651323288679123, "p95_s": 1.115284267067909, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 79.33796238113192, "calibration": 66.51802582183663, "speed": 82.00464272011534, "cost": 59.46663998783557 }, "jevbench_score": 71.22509854109671 } }, { "threshold": 0.06, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6036036036036037 }, "accuracy": 0.7532467532467533, "ece_hard": 0.12914387221294593, "calibration": 72.24478620042561, "p50_s": 0.5384091228246689, "p95_s": 1.2262836933135985, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 76.46187854521189, "calibration": 72.24478620042561, "speed": 81.80296636812625, "cost": 59.46663998783557 }, "jevbench_score": 71.99850550126745 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7986798679867987, "ece_hard": 0.17685813516393573, "calibration": 66.51802582183663, "p50_s": 0.5651323288679123, "p95_s": 1.115284267067909, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 79.33796238113192, "calibration": 66.51802582183663, "speed": 82.00464272011534, "cost": 59.46663998783557 }, "jevbench_score": 71.22509854109671 } }, { "threshold": 0.07, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6036036036036037 }, "accuracy": 0.7532467532467533, "ece_hard": 0.12914387221294593, "calibration": 72.24478620042561, "p50_s": 0.5384091228246689, "p95_s": 1.2262836933135985, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 76.46187854521189, "calibration": 72.24478620042561, "speed": 81.80296636812625, "cost": 59.46663998783557 }, "jevbench_score": 71.99850550126745 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7986798679867987, "ece_hard": 0.17685813516393573, "calibration": 66.51802582183663, "p50_s": 0.5651323288679123, "p95_s": 1.115284267067909, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 79.33796238113192, "calibration": 66.51802582183663, "speed": 82.00464272011534, "cost": 59.46663998783557 }, "jevbench_score": 71.22509854109671 } }, { "threshold": 0.08, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6036036036036037 }, "accuracy": 0.7532467532467533, "ece_hard": 0.12914387221294593, "calibration": 72.24478620042561, "p50_s": 0.5384091228246689, "p95_s": 1.2262836933135985, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 76.46187854521189, "calibration": 72.24478620042561, "speed": 81.80296636812625, "cost": 59.46663998783557 }, "jevbench_score": 71.99850550126745 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7986798679867987, "ece_hard": 0.17685813516393573, "calibration": 66.51802582183663, "p50_s": 0.5651323288679123, "p95_s": 1.115284267067909, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 79.33796238113192, "calibration": 66.51802582183663, "speed": 82.00464272011534, "cost": 59.46663998783557 }, "jevbench_score": 71.22509854109671 } }, { "threshold": 0.09, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6036036036036037 }, "accuracy": 0.7532467532467533, "ece_hard": 0.12914387221294593, "calibration": 72.24478620042561, "p50_s": 0.5384091228246689, "p95_s": 1.2262836933135985, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 76.46187854521189, "calibration": 72.24478620042561, "speed": 81.80296636812625, "cost": 59.46663998783557 }, "jevbench_score": 71.99850550126745 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7986798679867987, "ece_hard": 0.17685813516393573, "calibration": 66.51802582183663, "p50_s": 0.5651323288679123, "p95_s": 1.115284267067909, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 79.33796238113192, "calibration": 66.51802582183663, "speed": 82.00464272011534, "cost": 59.46663998783557 }, "jevbench_score": 71.22509854109671 } }, { "threshold": 0.1, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6036036036036037 }, "accuracy": 0.7532467532467533, "ece_hard": 0.12914387221294593, "calibration": 72.24478620042561, "p50_s": 0.5384091228246689, "p95_s": 1.2262836933135985, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 76.46187854521189, "calibration": 72.24478620042561, "speed": 81.80296636812625, "cost": 59.46663998783557 }, "jevbench_score": 71.99850550126745 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7986798679867987, "ece_hard": 0.17685813516393573, "calibration": 66.51802582183663, "p50_s": 0.5651323288679123, "p95_s": 1.115284267067909, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 79.33796238113192, "calibration": 66.51802582183663, "speed": 82.00464272011534, "cost": 59.46663998783557 }, "jevbench_score": 71.22509854109671 } }, { "threshold": 0.11, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6036036036036037 }, "accuracy": 0.7532467532467533, "ece_hard": 0.12914387221294593, "calibration": 72.24478620042561, "p50_s": 0.5384091228246689, "p95_s": 1.2262836933135985, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 76.46187854521189, "calibration": 72.24478620042561, "speed": 81.80296636812625, "cost": 59.46663998783557 }, "jevbench_score": 71.99850550126745 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7986798679867987, "ece_hard": 0.17685813516393573, "calibration": 66.51802582183663, "p50_s": 0.5651323288679123, "p95_s": 1.115284267067909, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 79.33796238113192, "calibration": 66.51802582183663, "speed": 82.00464272011534, "cost": 59.46663998783557 }, "jevbench_score": 71.22509854109671 } }, { "threshold": 0.12, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6036036036036037 }, "accuracy": 0.7532467532467533, "ece_hard": 0.12914387221294593, "calibration": 72.24478620042561, "p50_s": 0.5384091228246689, "p95_s": 1.2262836933135985, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 76.46187854521189, "calibration": 72.24478620042561, "speed": 81.80296636812625, "cost": 59.46663998783557 }, "jevbench_score": 71.99850550126745 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7986798679867987, "ece_hard": 0.17685813516393573, "calibration": 66.51802582183663, "p50_s": 0.5651323288679123, "p95_s": 1.115284267067909, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 79.33796238113192, "calibration": 66.51802582183663, "speed": 82.00464272011534, "cost": 59.46663998783557 }, "jevbench_score": 71.22509854109671 } }, { "threshold": 0.13, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6036036036036037 }, "accuracy": 0.7532467532467533, "ece_hard": 0.12914387221294593, "calibration": 72.24478620042561, "p50_s": 0.5384091228246689, "p95_s": 1.2262836933135985, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 76.46187854521189, "calibration": 72.24478620042561, "speed": 81.80296636812625, "cost": 59.46663998783557 }, "jevbench_score": 71.99850550126745 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7986798679867987, "ece_hard": 0.17685813516393573, "calibration": 66.51802582183663, "p50_s": 0.5651323288679123, "p95_s": 1.115284267067909, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 79.33796238113192, "calibration": 66.51802582183663, "speed": 82.00464272011534, "cost": 59.46663998783557 }, "jevbench_score": 71.22509854109671 } }, { "threshold": 0.14, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6036036036036037 }, "accuracy": 0.7532467532467533, "ece_hard": 0.12914387221294593, "calibration": 72.24478620042561, "p50_s": 0.5384091228246689, "p95_s": 1.2262836933135985, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 76.46187854521189, "calibration": 72.24478620042561, "speed": 81.80296636812625, "cost": 59.46663998783557 }, "jevbench_score": 71.99850550126745 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7986798679867987, "ece_hard": 0.17685813516393573, "calibration": 66.51802582183663, "p50_s": 0.5651323288679123, "p95_s": 1.115284267067909, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 79.33796238113192, "calibration": 66.51802582183663, "speed": 82.00464272011534, "cost": 59.46663998783557 }, "jevbench_score": 71.22509854109671 } }, { "threshold": 0.15, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6036036036036037 }, "accuracy": 0.7532467532467533, "ece_hard": 0.12914387221294593, "calibration": 72.24478620042561, "p50_s": 0.5384091228246689, "p95_s": 1.2262836933135985, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 76.46187854521189, "calibration": 72.24478620042561, "speed": 81.80296636812625, "cost": 59.46663998783557 }, "jevbench_score": 71.99850550126745 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7986798679867987, "ece_hard": 0.17685813516393573, "calibration": 66.51802582183663, "p50_s": 0.5651323288679123, "p95_s": 1.115284267067909, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 79.33796238113192, "calibration": 66.51802582183663, "speed": 82.00464272011534, "cost": 59.46663998783557 }, "jevbench_score": 71.22509854109671 } }, { "threshold": 0.16, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6036036036036037 }, "accuracy": 0.7532467532467533, "ece_hard": 0.12914387221294593, "calibration": 72.24478620042561, "p50_s": 0.5384091228246689, "p95_s": 1.2262836933135985, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 76.46187854521189, "calibration": 72.24478620042561, "speed": 81.80296636812625, "cost": 59.46663998783557 }, "jevbench_score": 71.99850550126745 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7986798679867987, "ece_hard": 0.17685813516393573, "calibration": 66.51802582183663, "p50_s": 0.5651323288679123, "p95_s": 1.115284267067909, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 79.33796238113192, "calibration": 66.51802582183663, "speed": 82.00464272011534, "cost": 59.46663998783557 }, "jevbench_score": 71.22509854109671 } }, { "threshold": 0.17, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6036036036036037 }, "accuracy": 0.7532467532467533, "ece_hard": 0.12914387221294593, "calibration": 72.24478620042561, "p50_s": 0.5384091228246689, "p95_s": 1.2262836933135985, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 76.46187854521189, "calibration": 72.24478620042561, "speed": 81.80296636812625, "cost": 59.46663998783557 }, "jevbench_score": 71.99850550126745 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7986798679867987, "ece_hard": 0.17685813516393573, "calibration": 66.51802582183663, "p50_s": 0.5651323288679123, "p95_s": 1.115284267067909, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 79.33796238113192, "calibration": 66.51802582183663, "speed": 82.00464272011534, "cost": 59.46663998783557 }, "jevbench_score": 71.22509854109671 } }, { "threshold": 0.18, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6036036036036037 }, "accuracy": 0.7532467532467533, "ece_hard": 0.12914387221294593, "calibration": 72.24478620042561, "p50_s": 0.5384091228246689, "p95_s": 1.2262836933135985, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 76.46187854521189, "calibration": 72.24478620042561, "speed": 81.80296636812625, "cost": 59.46663998783557 }, "jevbench_score": 71.99850550126745 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7986798679867987, "ece_hard": 0.17685813516393573, "calibration": 66.51802582183663, "p50_s": 0.5651323288679123, "p95_s": 1.115284267067909, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 79.33796238113192, "calibration": 66.51802582183663, "speed": 82.00464272011534, "cost": 59.46663998783557 }, "jevbench_score": 71.22509854109671 } }, { "threshold": 0.19, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6036036036036037 }, "accuracy": 0.7532467532467533, "ece_hard": 0.12914387221294593, "calibration": 72.24478620042561, "p50_s": 0.5384091228246689, "p95_s": 1.2262836933135985, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 76.46187854521189, "calibration": 72.24478620042561, "speed": 81.80296636812625, "cost": 59.46663998783557 }, "jevbench_score": 71.99850550126745 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7986798679867987, "ece_hard": 0.17685813516393573, "calibration": 66.51802582183663, "p50_s": 0.5651323288679123, "p95_s": 1.115284267067909, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 79.33796238113192, "calibration": 66.51802582183663, "speed": 82.00464272011534, "cost": 59.46663998783557 }, "jevbench_score": 71.22509854109671 } }, { "threshold": 0.2, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6036036036036037 }, "accuracy": 0.7532467532467533, "ece_hard": 0.12914387221294593, "calibration": 72.24478620042561, "p50_s": 0.5384091228246689, "p95_s": 1.2262836933135985, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 76.46187854521189, "calibration": 72.24478620042561, "speed": 81.80296636812625, "cost": 59.46663998783557 }, "jevbench_score": 71.99850550126745 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7986798679867987, "ece_hard": 0.17685813516393573, "calibration": 66.51802582183663, "p50_s": 0.5651323288679123, "p95_s": 1.115284267067909, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 79.33796238113192, "calibration": 66.51802582183663, "speed": 82.00464272011534, "cost": 59.46663998783557 }, "jevbench_score": 71.22509854109671 } }, { "threshold": 0.21, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6036036036036037 }, "accuracy": 0.7532467532467533, "ece_hard": 0.12914387221294593, "calibration": 72.24478620042561, "p50_s": 0.5384091228246689, "p95_s": 1.2262836933135985, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 76.46187854521189, "calibration": 72.24478620042561, "speed": 81.80296636812625, "cost": 59.46663998783557 }, "jevbench_score": 71.99850550126745 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7986798679867987, "ece_hard": 0.17685813516393573, "calibration": 66.51802582183663, "p50_s": 0.5651323288679123, "p95_s": 1.115284267067909, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 79.33796238113192, "calibration": 66.51802582183663, "speed": 82.00464272011534, "cost": 59.46663998783557 }, "jevbench_score": 71.22509854109671 } }, { "threshold": 0.22, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6036036036036037 }, "accuracy": 0.7532467532467533, "ece_hard": 0.12914387221294593, "calibration": 72.24478620042561, "p50_s": 0.5384091228246689, "p95_s": 1.2262836933135985, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 76.46187854521189, "calibration": 72.24478620042561, "speed": 81.80296636812625, "cost": 59.46663998783557 }, "jevbench_score": 71.99850550126745 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7986798679867987, "ece_hard": 0.17685813516393573, "calibration": 66.51802582183663, "p50_s": 0.5651323288679123, "p95_s": 1.115284267067909, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 79.33796238113192, "calibration": 66.51802582183663, "speed": 82.00464272011534, "cost": 59.46663998783557 }, "jevbench_score": 71.22509854109671 } }, { "threshold": 0.23, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6036036036036037 }, "accuracy": 0.7532467532467533, "ece_hard": 0.12914387221294593, "calibration": 72.24478620042561, "p50_s": 0.5384091228246689, "p95_s": 1.2262836933135985, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 76.46187854521189, "calibration": 72.24478620042561, "speed": 81.80296636812625, "cost": 59.46663998783557 }, "jevbench_score": 71.99850550126745 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7986798679867987, "ece_hard": 0.17685813516393573, "calibration": 66.51802582183663, "p50_s": 0.5651323288679123, "p95_s": 1.115284267067909, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 79.33796238113192, "calibration": 66.51802582183663, "speed": 82.00464272011534, "cost": 59.46663998783557 }, "jevbench_score": 71.22509854109671 } }, { "threshold": 0.24, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6036036036036037 }, "accuracy": 0.7532467532467533, "ece_hard": 0.12914387221294593, "calibration": 72.24478620042561, "p50_s": 0.5384091228246689, "p95_s": 1.2262836933135985, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 76.46187854521189, "calibration": 72.24478620042561, "speed": 81.80296636812625, "cost": 59.46663998783557 }, "jevbench_score": 71.99850550126745 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7986798679867987, "ece_hard": 0.17685813516393573, "calibration": 66.51802582183663, "p50_s": 0.5651323288679123, "p95_s": 1.115284267067909, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 79.33796238113192, "calibration": 66.51802582183663, "speed": 82.00464272011534, "cost": 59.46663998783557 }, "jevbench_score": 71.22509854109671 } }, { "threshold": 0.25, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6036036036036037 }, "accuracy": 0.7532467532467533, "ece_hard": 0.12914387221294593, "calibration": 72.24478620042561, "p50_s": 0.5384091228246689, "p95_s": 1.2262836933135985, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 76.46187854521189, "calibration": 72.24478620042561, "speed": 81.80296636812625, "cost": 59.46663998783557 }, "jevbench_score": 71.99850550126745 }, "heldout": { "n": 303, "escalation": 0.0033003300330033004, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5412844036697247 }, "accuracy": 0.7953795379537953, "ece_hard": 0.17375644816052943, "calibration": 66.82819452217726, "p50_s": 0.5651323288679123, "p95_s": 1.1195790365338323, "cost_usd_per_1000": 0.022576336893239884, "axes": { "intelligence": 79.06273302333375, "calibration": 66.82819452217726, "speed": 81.98795089853374, "cost": 59.39039568469827 }, "jevbench_score": 71.21958693803471 } }, { "threshold": 0.26, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6036036036036037 }, "accuracy": 0.7532467532467533, "ece_hard": 0.12914387221294593, "calibration": 72.24478620042561, "p50_s": 0.5384091228246689, "p95_s": 1.2262836933135985, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 76.46187854521189, "calibration": 72.24478620042561, "speed": 81.80296636812625, "cost": 59.46663998783557 }, "jevbench_score": 71.99850550126745 }, "heldout": { "n": 303, "escalation": 0.0033003300330033004, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5412844036697247 }, "accuracy": 0.7953795379537953, "ece_hard": 0.17375644816052943, "calibration": 66.82819452217726, "p50_s": 0.5651323288679123, "p95_s": 1.1195790365338323, "cost_usd_per_1000": 0.022576336893239884, "axes": { "intelligence": 79.06273302333375, "calibration": 66.82819452217726, "speed": 81.98795089853374, "cost": 59.39039568469827 }, "jevbench_score": 71.21958693803471 } }, { "threshold": 0.27, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6036036036036037 }, "accuracy": 0.7532467532467533, "ece_hard": 0.12914387221294593, "calibration": 72.24478620042561, "p50_s": 0.5384091228246689, "p95_s": 1.2262836933135985, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 76.46187854521189, "calibration": 72.24478620042561, "speed": 81.80296636812625, "cost": 59.46663998783557 }, "jevbench_score": 71.99850550126745 }, "heldout": { "n": 303, "escalation": 0.0033003300330033004, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5412844036697247 }, "accuracy": 0.7953795379537953, "ece_hard": 0.17375644816052943, "calibration": 66.82819452217726, "p50_s": 0.5651323288679123, "p95_s": 1.1195790365338323, "cost_usd_per_1000": 0.022576336893239884, "axes": { "intelligence": 79.06273302333375, "calibration": 66.82819452217726, "speed": 81.98795089853374, "cost": 59.39039568469827 }, "jevbench_score": 71.21958693803471 } }, { "threshold": 0.28, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6036036036036037 }, "accuracy": 0.7532467532467533, "ece_hard": 0.12914387221294593, "calibration": 72.24478620042561, "p50_s": 0.5384091228246689, "p95_s": 1.2262836933135985, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 76.46187854521189, "calibration": 72.24478620042561, "speed": 81.80296636812625, "cost": 59.46663998783557 }, "jevbench_score": 71.99850550126745 }, "heldout": { "n": 303, "escalation": 0.0033003300330033004, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5412844036697247 }, "accuracy": 0.7953795379537953, "ece_hard": 0.17375644816052943, "calibration": 66.82819452217726, "p50_s": 0.5651323288679123, "p95_s": 1.1195790365338323, "cost_usd_per_1000": 0.022576336893239884, "axes": { "intelligence": 79.06273302333375, "calibration": 66.82819452217726, "speed": 81.98795089853374, "cost": 59.39039568469827 }, "jevbench_score": 71.21958693803471 } }, { "threshold": 0.29, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6036036036036037 }, "accuracy": 0.7532467532467533, "ece_hard": 0.12914387221294593, "calibration": 72.24478620042561, "p50_s": 0.5384091228246689, "p95_s": 1.2262836933135985, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 76.46187854521189, "calibration": 72.24478620042561, "speed": 81.80296636812625, "cost": 59.46663998783557 }, "jevbench_score": 71.99850550126745 }, "heldout": { "n": 303, "escalation": 0.0033003300330033004, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5412844036697247 }, "accuracy": 0.7953795379537953, "ece_hard": 0.17375644816052943, "calibration": 66.82819452217726, "p50_s": 0.5651323288679123, "p95_s": 1.1195790365338323, "cost_usd_per_1000": 0.022576336893239884, "axes": { "intelligence": 79.06273302333375, "calibration": 66.82819452217726, "speed": 81.98795089853374, "cost": 59.39039568469827 }, "jevbench_score": 71.21958693803471 } }, { "threshold": 0.3, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6036036036036037 }, "accuracy": 0.7532467532467533, "ece_hard": 0.12914387221294593, "calibration": 72.24478620042561, "p50_s": 0.5384091228246689, "p95_s": 1.2262836933135985, "cost_usd_per_1000": 0.02244460674157303, "axes": { "intelligence": 76.46187854521189, "calibration": 72.24478620042561, "speed": 81.80296636812625, "cost": 59.46663998783557 }, "jevbench_score": 71.99850550126745 }, "heldout": { "n": 303, "escalation": 0.0033003300330033004, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5412844036697247 }, "accuracy": 0.7953795379537953, "ece_hard": 0.17375644816052943, "calibration": 66.82819452217726, "p50_s": 0.5651323288679123, "p95_s": 1.1195790365338323, "cost_usd_per_1000": 0.022576336893239884, "axes": { "intelligence": 79.06273302333375, "calibration": 66.82819452217726, "speed": 81.98795089853374, "cost": 59.39039568469827 }, "jevbench_score": 71.21958693803471 } }, { "threshold": 0.31, "public": { "n": 231, "escalation": 0.004329004329004329, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6126126126126126 }, "accuracy": 0.7575757575757576, "ece_hard": 0.12032603551264726, "calibration": 73.12656987045547, "p50_s": 0.5384091228246689, "p95_s": 1.2311300501227378, "cost_usd_per_1000": 0.02261739564181137, "axes": { "intelligence": 76.83725392058726, "calibration": 73.12656987045547, "speed": 81.78583658073207, "cost": 59.366722146120004 }, "jevbench_score": 72.2714972677882 }, "heldout": { "n": 303, "escalation": 0.009900990099009901, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5321100917431193 }, "accuracy": 0.7920792079207921, "ece_hard": 0.18254547445513858, "calibration": 65.94929189271635, "p50_s": 0.5651323288679123, "p95_s": 1.1299098640680312, "cost_usd_per_1000": 0.022839797196573586, "axes": { "intelligence": 78.7875036655356, "calibration": 65.94929189271635, "speed": 81.94806047779304, "cost": 59.23923270035141 }, "jevbench_score": 70.86860683070951 } }, { "threshold": 0.32, "public": { "n": 231, "escalation": 0.004329004329004329, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6126126126126126 }, "accuracy": 0.7575757575757576, "ece_hard": 0.12032603551264726, "calibration": 73.12656987045547, "p50_s": 0.5384091228246689, "p95_s": 1.2311300501227378, "cost_usd_per_1000": 0.02261739564181137, "axes": { "intelligence": 76.83725392058726, "calibration": 73.12656987045547, "speed": 81.78583658073207, "cost": 59.366722146120004 }, "jevbench_score": 72.2714972677882 }, "heldout": { "n": 303, "escalation": 0.009900990099009901, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5321100917431193 }, "accuracy": 0.7920792079207921, "ece_hard": 0.18254547445513858, "calibration": 65.94929189271635, "p50_s": 0.5651323288679123, "p95_s": 1.1299098640680312, "cost_usd_per_1000": 0.022839797196573586, "axes": { "intelligence": 78.7875036655356, "calibration": 65.94929189271635, "speed": 81.94806047779304, "cost": 59.23923270035141 }, "jevbench_score": 70.86860683070951 } }, { "threshold": 0.33, "public": { "n": 231, "escalation": 0.008658008658008658, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6126126126126126 }, "accuracy": 0.7575757575757576, "ece_hard": 0.12239935107084304, "calibration": 72.91923831463589, "p50_s": 0.5384091228246689, "p95_s": 1.2382635936141013, "cost_usd_per_1000": 0.02279018454204971, "axes": { "intelligence": 76.83725392058726, "calibration": 72.91923831463589, "speed": 81.76074485669321, "cost": 59.267564744014734 }, "jevbench_score": 72.18450293079682 }, "heldout": { "n": 303, "escalation": 0.013201320132013201, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5412844036697247 }, "accuracy": 0.7953795379537953, "ece_hard": 0.17608879118024806, "calibration": 66.5949602202054, "p50_s": 0.5651678681373596, "p95_s": 1.1334389865398407, "cost_usd_per_1000": 0.02297152734824044, "axes": { "intelligence": 79.06273302333375, "calibration": 66.5949602202054, "speed": 81.93424390501681, "cost": 59.16430384437475 }, "jevbench_score": 71.07790178156247 } }, { "threshold": 0.34, "public": { "n": 231, "escalation": 0.008658008658008658, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6126126126126126 }, "accuracy": 0.7575757575757576, "ece_hard": 0.12239935107084304, "calibration": 72.91923831463589, "p50_s": 0.5384091228246689, "p95_s": 1.2382635936141013, "cost_usd_per_1000": 0.02279018454204971, "axes": { "intelligence": 76.83725392058726, "calibration": 72.91923831463589, "speed": 81.76074485669321, "cost": 59.267564744014734 }, "jevbench_score": 72.18450293079682 }, "heldout": { "n": 303, "escalation": 0.0165016501650165, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5412844036697247 }, "accuracy": 0.7953795379537953, "ece_hard": 0.17525960102620924, "calibration": 66.67787923560928, "p50_s": 0.5651678681373596, "p95_s": 1.136622895300388, "cost_usd_per_1000": 0.02310325749990729, "axes": { "intelligence": 79.06273302333375, "calibration": 66.67787923560928, "speed": 81.92206137445794, "cost": 59.08980344155474 }, "jevbench_score": 71.07498137010995 } }, { "threshold": 0.35, "public": { "n": 231, "escalation": 0.017316017316017316, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6126126126126126 }, "accuracy": 0.7575757575757576, "ece_hard": 0.12357199707886896, "calibration": 72.80197371383329, "p50_s": 0.5384091228246689, "p95_s": 1.27316669896245, "cost_usd_per_1000": 0.023135762342526386, "axes": { "intelligence": 76.83725392058726, "calibration": 72.80197371383329, "speed": 81.64002319358937, "cost": 59.07148556696561 }, "jevbench_score": 72.06908368045228 }, "heldout": { "n": 303, "escalation": 0.019801980198019802, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5412844036697247 }, "accuracy": 0.7953795379537953, "ece_hard": 0.17966038353651292, "calibration": 67.4296141854937, "p50_s": 0.5651678681373596, "p95_s": 1.1976596251130087, "cost_usd_per_1000": 0.023234987651574144, "axes": { "intelligence": 79.06273302333375, "calibration": 67.4296141854937, "speed": 81.69489128263746, "cost": 59.015726619833096 }, "jevbench_score": 71.20267155561595 } }, { "threshold": 0.36, "public": { "n": 231, "escalation": 0.017316017316017316, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6126126126126126 }, "accuracy": 0.7575757575757576, "ece_hard": 0.12357199707886896, "calibration": 72.80197371383329, "p50_s": 0.5384091228246689, "p95_s": 1.27316669896245, "cost_usd_per_1000": 0.023135762342526386, "axes": { "intelligence": 76.83725392058726, "calibration": 72.80197371383329, "speed": 81.64002319358937, "cost": 59.07148556696561 }, "jevbench_score": 72.06908368045228 }, "heldout": { "n": 303, "escalation": 0.0297029702970297, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7986798679867987, "ece_hard": 0.17789321912296585, "calibration": 68.45500530008174, "p50_s": 0.5654387563467026, "p95_s": 1.2154198020696638, "cost_usd_per_1000": 0.0236301781065747, "axes": { "intelligence": 79.33796238113192, "calibration": 68.45500530008174, "speed": 81.62888119625009, "cost": 58.79599014906819 }, "jevbench_score": 71.45283271039058 } }, { "threshold": 0.37, "public": { "n": 231, "escalation": 0.021645021645021644, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6126126126126126 }, "accuracy": 0.7575757575757576, "ece_hard": 0.12417368825816263, "calibration": 72.74180459590393, "p50_s": 0.538740599155426, "p95_s": 1.27316669896245, "cost_usd_per_1000": 0.023308551242764722, "axes": { "intelligence": 76.83725392058726, "calibration": 72.74180459590393, "speed": 81.63735024399034, "cost": 58.97454158437788 }, "jevbench_score": 72.0240178898587 }, "heldout": { "n": 303, "escalation": 0.033003300330033, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7986798679867987, "ece_hard": 0.1821683720802459, "calibration": 68.02749000435374, "p50_s": 0.5654387563467026, "p95_s": 1.2254433177411552, "cost_usd_per_1000": 0.02376190825824155, "axes": { "intelligence": 79.33796238113192, "calibration": 68.02749000435374, "speed": 81.59321199398755, "cost": 58.72356055624777 }, "jevbench_score": 71.3112378015368 } }, { "threshold": 0.38, "public": { "n": 231, "escalation": 0.030303030303030304, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6216216216216216 }, "accuracy": 0.7619047619047619, "ece_hard": 0.11480296728453682, "calibration": 73.67887669326652, "p50_s": 0.538740599155426, "p95_s": 1.3788734540343284, "cost_usd_per_1000": 0.0236541290432414, "axes": { "intelligence": 77.21262929596264, "calibration": 73.67887669326652, "speed": 81.2909588438514, "cost": 58.782791147774915 }, "jevbench_score": 72.20727353454373 }, "heldout": { "n": 303, "escalation": 0.033003300330033, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7986798679867987, "ece_hard": 0.1821683720802459, "calibration": 68.02749000435374, "p50_s": 0.5654387563467026, "p95_s": 1.2254433177411552, "cost_usd_per_1000": 0.02376190825824155, "axes": { "intelligence": 79.33796238113192, "calibration": 68.02749000435374, "speed": 81.59321199398755, "cost": 58.72356055624777 }, "jevbench_score": 71.3112378015368 } }, { "threshold": 0.39, "public": { "n": 231, "escalation": 0.04329004329004329, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6306306306306306 }, "accuracy": 0.7662337662337663, "ece_hard": 0.12359219475131261, "calibration": 72.79995394658894, "p50_s": 0.538740599155426, "p95_s": 1.4451277270913123, "cost_usd_per_1000": 0.024172495743956416, "axes": { "intelligence": 77.58800467133801, "calibration": 72.79995394658894, "speed": 81.08714061309959, "cost": 58.50035522686147 }, "jevbench_score": 71.9463960867971 }, "heldout": { "n": 303, "escalation": 0.039603960396039604, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7986798679867987, "ece_hard": 0.19224589238416673, "calibration": 66.8514793180089, "p50_s": 0.5654387563467026, "p95_s": 1.2386606723070144, "cost_usd_per_1000": 0.024025368561575257, "axes": { "intelligence": 79.33796238113192, "calibration": 66.8514793180089, "speed": 81.54662078493027, "cost": 58.57989823919275 }, "jevbench_score": 70.94742891754129 } }, { "threshold": 0.4, "public": { "n": 231, "escalation": 0.04329004329004329, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6306306306306306 }, "accuracy": 0.7662337662337663, "ece_hard": 0.12359219475131261, "calibration": 72.79995394658894, "p50_s": 0.538740599155426, "p95_s": 1.4451277270913123, "cost_usd_per_1000": 0.024172495743956416, "axes": { "intelligence": 77.58800467133801, "calibration": 72.79995394658894, "speed": 81.08714061309959, "cost": 58.50035522686147 }, "jevbench_score": 71.9463960867971 }, "heldout": { "n": 303, "escalation": 0.0462046204620462, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5596330275229358 }, "accuracy": 0.801980198019802, "ece_hard": 0.18830897299397226, "calibration": 67.24517125702835, "p50_s": 0.5671136170625687, "p95_s": 1.267179660499095, "cost_usd_per_1000": 0.02428882886490896, "axes": { "intelligence": 79.61319173893008, "calibration": 67.24517125702835, "speed": 81.43491731228755, "cost": 58.4378027523501 }, "jevbench_score": 71.04567900914743 } }, { "threshold": 0.41, "public": { "n": 231, "escalation": 0.04329004329004329, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6306306306306306 }, "accuracy": 0.7662337662337663, "ece_hard": 0.12359219475131261, "calibration": 72.79995394658894, "p50_s": 0.538740599155426, "p95_s": 1.4451277270913123, "cost_usd_per_1000": 0.024172495743956416, "axes": { "intelligence": 77.58800467133801, "calibration": 72.79995394658894, "speed": 81.08714061309959, "cost": 58.50035522686147 }, "jevbench_score": 71.9463960867971 }, "heldout": { "n": 303, "escalation": 0.0462046204620462, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5596330275229358 }, "accuracy": 0.801980198019802, "ece_hard": 0.18830897299397226, "calibration": 67.24517125702835, "p50_s": 0.5671136170625687, "p95_s": 1.267179660499095, "cost_usd_per_1000": 0.02428882886490896, "axes": { "intelligence": 79.61319173893008, "calibration": 67.24517125702835, "speed": 81.43491731228755, "cost": 58.4378027523501 }, "jevbench_score": 71.04567900914743 } }, { "threshold": 0.42, "public": { "n": 231, "escalation": 0.05194805194805195, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6396396396396397 }, "accuracy": 0.7705627705627706, "ece_hard": 0.11900929325691689, "calibration": 73.2582440960285, "p50_s": 0.538740599155426, "p95_s": 1.4451277270913123, "cost_usd_per_1000": 0.02451807354443309, "axes": { "intelligence": 77.96338004671338, "calibration": 73.2582440960285, "speed": 81.08714061309959, "cost": 58.3154096973947 }, "jevbench_score": 72.08926878225934 }, "heldout": { "n": 303, "escalation": 0.04950495049504951, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5596330275229358 }, "accuracy": 0.801980198019802, "ece_hard": 0.18642885473896476, "calibration": 67.4331830825291, "p50_s": 0.5671136170625687, "p95_s": 1.267179660499095, "cost_usd_per_1000": 0.02442055901657581, "axes": { "intelligence": 79.61319173893008, "calibration": 67.4331830825291, "speed": 81.43491731228755, "cost": 58.367331962304846 }, "jevbench_score": 71.07384311194976 } }, { "threshold": 0.43, "public": { "n": 231, "escalation": 0.05627705627705628, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6396396396396397 }, "accuracy": 0.7705627705627706, "ece_hard": 0.11762727470115306, "calibration": 73.39644595160489, "p50_s": 0.538740599155426, "p95_s": 1.4451277270913123, "cost_usd_per_1000": 0.02469086244467143, "axes": { "intelligence": 77.96338004671338, "calibration": 73.39644595160489, "speed": 81.08714061309959, "cost": 58.223912200681 }, "jevbench_score": 72.09493670700641 }, "heldout": { "n": 303, "escalation": 0.052805280528052806, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5596330275229358 }, "accuracy": 0.801980198019802, "ece_hard": 0.18033774455575358, "calibration": 68.04229410085021, "p50_s": 0.5671136170625687, "p95_s": 1.2837200626730914, "cost_usd_per_1000": 0.024552289168242664, "axes": { "intelligence": 79.61319173893008, "calibration": 68.04229410085021, "speed": 81.37859596023084, "cost": 58.29724028734495 }, "jevbench_score": 71.20009006949569 } }, { "threshold": 0.44, "public": { "n": 231, "escalation": 0.06493506493506493, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6396396396396397 }, "accuracy": 0.7705627705627706, "ece_hard": 0.12109874851227515, "calibration": 73.04929857049268, "p50_s": 0.538740599155426, "p95_s": 1.4995943695306777, "cost_usd_per_1000": 0.025036440245148107, "axes": { "intelligence": 77.96338004671338, "calibration": 73.04929857049268, "speed": 80.9264649393582, "cost": 58.04282261024204 }, "jevbench_score": 71.91780946496482 }, "heldout": { "n": 303, "escalation": 0.056105610561056105, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5688073394495413 }, "accuracy": 0.8052805280528053, "ece_hard": 0.18420498936842572, "calibration": 67.655569619583, "p50_s": 0.5671136170625687, "p95_s": 1.306520934402942, "cost_usd_per_1000": 0.024684019319909517, "axes": { "intelligence": 79.88842109672825, "calibration": 67.655569619583, "speed": 81.30213551843764, "cost": 58.2275236702006 }, "jevbench_score": 71.12207464968895 } }, { "threshold": 0.45, "public": { "n": 231, "escalation": 0.06493506493506493, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6396396396396397 }, "accuracy": 0.7705627705627706, "ece_hard": 0.12109874851227515, "calibration": 73.04929857049268, "p50_s": 0.538740599155426, "p95_s": 1.4995943695306777, "cost_usd_per_1000": 0.025036440245148107, "axes": { "intelligence": 77.96338004671338, "calibration": 73.04929857049268, "speed": 80.9264649393582, "cost": 58.04282261024204 }, "jevbench_score": 71.91780946496482 }, "heldout": { "n": 303, "escalation": 0.0627062706270627, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5688073394495413 }, "accuracy": 0.8052805280528053, "ece_hard": 0.18002269785582384, "calibration": 68.0737987708432, "p50_s": 0.5671136170625687, "p95_s": 1.3082165434956547, "cost_usd_per_1000": 0.02494747962324322, "axes": { "intelligence": 79.88842109672825, "calibration": 68.0737987708432, "speed": 81.29650287820603, "cost": 58.08919970282991 }, "jevbench_score": 71.1881604165434 } }, { "threshold": 0.46, "public": { "n": 231, "escalation": 0.06493506493506493, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6396396396396397 }, "accuracy": 0.7705627705627706, "ece_hard": 0.12109874851227515, "calibration": 73.04929857049268, "p50_s": 0.538740599155426, "p95_s": 1.4995943695306777, "cost_usd_per_1000": 0.025036440245148107, "axes": { "intelligence": 77.96338004671338, "calibration": 73.04929857049268, "speed": 80.9264649393582, "cost": 58.04282261024204 }, "jevbench_score": 71.91780946496482 }, "heldout": { "n": 303, "escalation": 0.066006600660066, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5688073394495413 }, "accuracy": 0.8052805280528053, "ece_hard": 0.17404654226007302, "calibration": 68.59744034201333, "p50_s": 0.5671136170625687, "p95_s": 1.3086190357804295, "cost_usd_per_1000": 0.025079209774910072, "axes": { "intelligence": 79.88842109672825, "calibration": 68.59744034201333, "speed": 81.29516691211745, "cost": 58.02058455653483 }, "jevbench_score": 71.30330228837093 } }, { "threshold": 0.47, "public": { "n": 231, "escalation": 0.08225108225108226, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6306306306306306 }, "accuracy": 0.7662337662337663, "ece_hard": 0.13601437209102038, "calibration": 70.87855837628553, "p50_s": 0.5388893276453018, "p95_s": 1.5448087468743323, "cost_usd_per_1000": 0.025727595846101462, "axes": { "intelligence": 77.58800467133801, "calibration": 70.87855837628553, "speed": 80.7962569743955, "cost": 57.6880238558441 }, "jevbench_score": 71.1535529337317 }, "heldout": { "n": 303, "escalation": 0.06930693069306931, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5779816513761468 }, "accuracy": 0.8085808580858086, "ece_hard": 0.17402186817941728, "calibration": 69.42029240210483, "p50_s": 0.5671136170625687, "p95_s": 1.3247340187430379, "cost_usd_per_1000": 0.025210939926576924, "axes": { "intelligence": 80.16365045452642, "calibration": 69.42029240210483, "speed": 81.24201235670412, "cost": 57.95232887328114 }, "jevbench_score": 71.54493154034917 } }, { "threshold": 0.48, "public": { "n": 231, "escalation": 0.08225108225108226, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6306306306306306 }, "accuracy": 0.7662337662337663, "ece_hard": 0.13601437209102038, "calibration": 70.87855837628553, "p50_s": 0.5388893276453018, "p95_s": 1.5448087468743323, "cost_usd_per_1000": 0.025727595846101462, "axes": { "intelligence": 77.58800467133801, "calibration": 70.87855837628553, "speed": 80.7962569743955, "cost": 57.6880238558441 }, "jevbench_score": 71.1535529337317 }, "heldout": { "n": 303, "escalation": 0.06930693069306931, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5779816513761468 }, "accuracy": 0.8085808580858086, "ece_hard": 0.17402186817941728, "calibration": 69.42029240210483, "p50_s": 0.5671136170625687, "p95_s": 1.3247340187430379, "cost_usd_per_1000": 0.025210939926576924, "axes": { "intelligence": 80.16365045452642, "calibration": 69.42029240210483, "speed": 81.24201235670412, "cost": 57.95232887328114 }, "jevbench_score": 71.54493154034917 } }, { "threshold": 0.49, "public": { "n": 231, "escalation": 0.09090909090909091, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6396396396396397 }, "accuracy": 0.7705627705627706, "ece_hard": 0.13342225650604939, "calibration": 71.27733276112914, "p50_s": 0.5388893276453018, "p95_s": 1.620636561512947, "cost_usd_per_1000": 0.026073173646578136, "axes": { "intelligence": 77.96338004671338, "calibration": 71.27733276112914, "speed": 80.5881478488792, "cost": 57.514183088759914 }, "jevbench_score": 71.23969596515916 }, "heldout": { "n": 303, "escalation": 0.07260726072607261, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5779816513761468 }, "accuracy": 0.8085808580858086, "ece_hard": 0.1819191475815931, "calibration": 68.63056446188725, "p50_s": 0.5671136170625687, "p95_s": 1.3247340187430379, "cost_usd_per_1000": 0.025342670078243777, "axes": { "intelligence": 80.16365045452642, "calibration": 68.63056446188725, "speed": 81.24201235670412, "cost": 57.88442890635765 }, "jevbench_score": 71.31967745684581 } }, { "threshold": 0.5, "public": { "n": 231, "escalation": 0.09956709956709957, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6486486486486487 }, "accuracy": 0.7748917748917749, "ece_hard": 0.13420269921783784, "calibration": 71.19928848995029, "p50_s": 0.5388893276453018, "p95_s": 1.704740324616432, "cost_usd_per_1000": 0.026418751447054814, "axes": { "intelligence": 78.33875542208877, "calibration": 71.19928848995029, "speed": 80.36842182987962, "cost": 57.342631331168676 }, "jevbench_score": 71.20391037423025 }, "heldout": { "n": 303, "escalation": 0.07590759075907591, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5688073394495413 }, "accuracy": 0.8052805280528053, "ece_hard": 0.17635972475201214, "calibration": 69.18650674484535, "p50_s": 0.5671136170625687, "p95_s": 1.3878645688295348, "cost_usd_per_1000": 0.025474400229910626, "axes": { "intelligence": 79.88842109672825, "calibration": 69.18650674484535, "speed": 81.03982835985413, "cost": 57.81688096732797 }, "jevbench_score": 71.33696085143794 } }, { "threshold": 0.51, "public": { "n": 231, "escalation": 0.11255411255411256, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6486486486486487 }, "accuracy": 0.7748917748917749, "ece_hard": 0.13748065842998125, "calibration": 70.87149256873596, "p50_s": 0.5397706717252732, "p95_s": 1.704740324616432, "cost_usd_per_1000": 0.02693711814776983, "axes": { "intelligence": 78.33875542208877, "calibration": 70.87149256873596, "speed": 80.36132482144339, "cost": 57.08946606626207 }, "jevbench_score": 71.04161552282552 }, "heldout": { "n": 303, "escalation": 0.08250825082508251, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.958904109589041, "hard": 0.5688073394495413 }, "accuracy": 0.8085808580858086, "ece_hard": 0.1759501517580138, "calibration": 69.22746404424518, "p50_s": 0.567900562286377, "p95_s": 1.3878645688295348, "cost_usd_per_1000": 0.025737860533244332, "axes": { "intelligence": 80.08020191864607, "calibration": 69.22746404424518, "speed": 81.03380612631305, "cost": 57.68282670339583 }, "jevbench_score": 71.34755397704691 } }, { "threshold": 0.52, "public": { "n": 231, "escalation": 0.12121212121212122, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6576576576576577 }, "accuracy": 0.7792207792207793, "ece_hard": 0.13248670980516475, "calibration": 71.3708874312176, "p50_s": 0.5397706717252732, "p95_s": 1.705234532803297, "cost_usd_per_1000": 0.027282695948246505, "axes": { "intelligence": 78.71413079746414, "calibration": 71.3708874312176, "speed": 80.36006597409732, "cost": 56.92338150844681 }, "jevbench_score": 71.19937710858242 }, "heldout": { "n": 303, "escalation": 0.0858085808580858, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.958904109589041, "hard": 0.5688073394495413 }, "accuracy": 0.8085808580858086, "ece_hard": 0.17752980971135998, "calibration": 69.06949824891056, "p50_s": 0.567900562286377, "p95_s": 1.3878645688295348, "cost_usd_per_1000": 0.025869590684911185, "axes": { "intelligence": 80.08020191864607, "calibration": 69.06949824891056, "speed": 81.03380612631305, "cost": 57.6163132823275 }, "jevbench_score": 71.28625356180753 } }, { "threshold": 0.53, "public": { "n": 231, "escalation": 0.12121212121212122, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6576576576576577 }, "accuracy": 0.7792207792207793, "ece_hard": 0.13248670980516475, "calibration": 71.3708874312176, "p50_s": 0.5397706717252732, "p95_s": 1.705234532803297, "cost_usd_per_1000": 0.027282695948246505, "axes": { "intelligence": 78.71413079746414, "calibration": 71.3708874312176, "speed": 80.36006597409732, "cost": 56.92338150844681 }, "jevbench_score": 71.19937710858242 }, "heldout": { "n": 303, "escalation": 0.09570957095709572, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.958904109589041, "hard": 0.5779816513761468 }, "accuracy": 0.8118811881188119, "ece_hard": 0.175743837883937, "calibration": 69.24809543165287, "p50_s": 0.567900562286377, "p95_s": 1.4324895538389675, "cost_usd_per_1000": 0.02626478113991174, "axes": { "intelligence": 80.35543127644422, "calibration": 69.24809543165287, "speed": 80.8963623767519, "cost": 57.41878641250197 }, "jevbench_score": 71.3019681771959 } }, { "threshold": 0.54, "public": { "n": 231, "escalation": 0.1341991341991342, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6666666666666666 }, "accuracy": 0.7835497835497836, "ece_hard": 0.12447204548253311, "calibration": 72.17235386348077, "p50_s": 0.5404984593391419, "p95_s": 1.745522206276655, "cost_usd_per_1000": 0.02780106264896152, "axes": { "intelligence": 79.0895061728395, "calibration": 72.17235386348077, "speed": 80.25280161425387, "cost": 56.678158107608795 }, "jevbench_score": 71.38244446657156 }, "heldout": { "n": 303, "escalation": 0.11551155115511551, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9657534246575342, "hard": 0.5871559633027523 }, "accuracy": 0.8184818481848185, "ece_hard": 0.19204126861836368, "calibration": 67.61835235821019, "p50_s": 0.5697284132242203, "p95_s": 1.452318468689918, "cost_usd_per_1000": 0.027055162049912852, "axes": { "intelligence": 80.82244145616019, "calibration": 67.61835235821019, "speed": 80.82270260889744, "cost": 57.03249581405009 }, "jevbench_score": 70.845631175423 } }, { "threshold": 0.55, "public": { "n": 231, "escalation": 0.13852813852813853, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6666666666666666 }, "accuracy": 0.7835497835497836, "ece_hard": 0.12719236211949114, "calibration": 71.90032219978497, "p50_s": 0.5404984593391419, "p95_s": 1.7550475500524043, "cost_usd_per_1000": 0.02797385154919986, "axes": { "intelligence": 79.0895061728395, "calibration": 71.90032219978497, "speed": 80.22916652203696, "cost": 56.59743202450673 }, "jevbench_score": 71.28442927416924 }, "heldout": { "n": 303, "escalation": 0.12211221122112212, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9657534246575342, "hard": 0.5871559633027523 }, "accuracy": 0.8184818481848185, "ece_hard": 0.18638503184797686, "calibration": 68.18397603524888, "p50_s": 0.5702764481306076, "p95_s": 1.452318468689918, "cost_usd_per_1000": 0.027318622353246558, "axes": { "intelligence": 80.82244145616019, "calibration": 68.18397603524888, "speed": 80.81852703803565, "cost": 56.90623616230887 }, "jevbench_score": 70.9530828996721 } }, { "threshold": 0.56, "public": { "n": 231, "escalation": 0.13852813852813853, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6666666666666666 }, "accuracy": 0.7835497835497836, "ece_hard": 0.12719236211949114, "calibration": 71.90032219978497, "p50_s": 0.5404984593391419, "p95_s": 1.7550475500524043, "cost_usd_per_1000": 0.02797385154919986, "axes": { "intelligence": 79.0895061728395, "calibration": 71.90032219978497, "speed": 80.22916652203696, "cost": 56.59743202450673 }, "jevbench_score": 71.28442927416924 }, "heldout": { "n": 303, "escalation": 0.12211221122112212, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9657534246575342, "hard": 0.5871559633027523 }, "accuracy": 0.8184818481848185, "ece_hard": 0.18638503184797686, "calibration": 68.18397603524888, "p50_s": 0.5702764481306076, "p95_s": 1.452318468689918, "cost_usd_per_1000": 0.027318622353246558, "axes": { "intelligence": 80.82244145616019, "calibration": 68.18397603524888, "speed": 80.81852703803565, "cost": 56.90623616230887 }, "jevbench_score": 70.9530828996721 } }, { "threshold": 0.57, "public": { "n": 231, "escalation": 0.15584415584415584, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6666666666666666 }, "accuracy": 0.7835497835497836, "ece_hard": 0.1207343914273618, "calibration": 71.47159304815308, "p50_s": 0.5404984593391419, "p95_s": 1.7550475500524043, "cost_usd_per_1000": 0.028665007150153216, "axes": { "intelligence": 79.0895061728395, "calibration": 71.47159304815308, "speed": 80.22916652203696, "cost": 56.27943836678729 }, "jevbench_score": 71.07773671137724 }, "heldout": { "n": 303, "escalation": 0.13861386138613863, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9657534246575342, "hard": 0.6055045871559633 }, "accuracy": 0.8250825082508251, "ece_hard": 0.18175722142354345, "calibration": 70.03425067271169, "p50_s": 0.5712133586406708, "p95_s": 1.4723677232861514, "cost_usd_per_1000": 0.027977273111580818, "axes": { "intelligence": 81.37290017175654, "calibration": 70.03425067271169, "speed": 80.75185356083955, "cost": 56.59583852767635 }, "jevbench_score": 71.43841658036341 } }, { "threshold": 0.58, "public": { "n": 231, "escalation": 0.16017316017316016, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6666666666666666 }, "accuracy": 0.7835497835497836, "ece_hard": 0.12408720639630622, "calibration": 71.13631155125864, "p50_s": 0.5404984593391419, "p95_s": 1.7742935769259929, "cost_usd_per_1000": 0.02883779605039155, "axes": { "intelligence": 79.0895061728395, "calibration": 71.13631155125864, "speed": 80.18180059269342, "cost": 56.201138018711774 }, "jevbench_score": 70.95904800796207 }, "heldout": { "n": 303, "escalation": 0.13861386138613863, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9657534246575342, "hard": 0.6055045871559633 }, "accuracy": 0.8250825082508251, "ece_hard": 0.18175722142354345, "calibration": 70.03425067271169, "p50_s": 0.5712133586406708, "p95_s": 1.4723677232861514, "cost_usd_per_1000": 0.027977273111580818, "axes": { "intelligence": 81.37290017175654, "calibration": 70.03425067271169, "speed": 80.75185356083955, "cost": 56.59583852767635 }, "jevbench_score": 71.43841658036341 } }, { "threshold": 0.59, "public": { "n": 231, "escalation": 0.1645021645021645, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6666666666666666 }, "accuracy": 0.7835497835497836, "ece_hard": 0.12408902924624483, "calibration": 71.37662582028341, "p50_s": 0.5404984593391419, "p95_s": 1.7742935769259929, "cost_usd_per_1000": 0.02901058495062989, "axes": { "intelligence": 79.0895061728395, "calibration": 71.37662582028341, "speed": 80.18180059269342, "cost": 56.123305428299005 }, "jevbench_score": 70.99429991222513 }, "heldout": { "n": 303, "escalation": 0.1419141914191419, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9657534246575342, "hard": 0.6055045871559633 }, "accuracy": 0.8250825082508251, "ece_hard": 0.18250913453915957, "calibration": 69.95905936115008, "p50_s": 0.5712133586406708, "p95_s": 1.4723677232861514, "cost_usd_per_1000": 0.028109003263247667, "axes": { "intelligence": 81.37290017175654, "calibration": 69.95905936115008, "speed": 80.75185356083955, "cost": 56.53463662211425 }, "jevbench_score": 71.39991837531825 } }, { "threshold": 0.6, "public": { "n": 231, "escalation": 0.19913419913419914, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6936936936936937 }, "accuracy": 0.7965367965367965, "ece_hard": 0.11418126216700523, "calibration": 73.71740252820737, "p50_s": 0.5404984593391419, "p95_s": 1.7742935769259929, "cost_usd_per_1000": 0.030392896152536597, "axes": { "intelligence": 80.21563229896564, "calibration": 73.71740252820737, "speed": 80.18180059269342, "cost": 55.51683741503889 }, "jevbench_score": 71.62792818852131 }, "heldout": { "n": 303, "escalation": 0.15841584158415842, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9657534246575342, "hard": 0.6055045871559633 }, "accuracy": 0.8250825082508251, "ece_hard": 0.18494530682510557, "calibration": 69.71544213255548, "p50_s": 0.5717611402273178, "p95_s": 1.4723677232861514, "cost_usd_per_1000": 0.02876765402158193, "axes": { "intelligence": 81.37290017175654, "calibration": 69.71544213255548, "speed": 80.74769076387742, "cost": 56.23286659065571 }, "jevbench_score": 71.24137266342876 } }, { "threshold": 0.61, "public": { "n": 231, "escalation": 0.20346320346320346, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.7027027027027027 }, "accuracy": 0.8008658008658008, "ece_hard": 0.12322321771956599, "calibration": 72.81320697295129, "p50_s": 0.5404984593391419, "p95_s": 1.7742935769259929, "cost_usd_per_1000": 0.030565685052774936, "axes": { "intelligence": 80.59100767434103, "calibration": 72.81320697295129, "speed": 80.18180059269342, "cost": 55.44297598447476 }, "jevbench_score": 71.46687148557807 }, "heldout": { "n": 303, "escalation": 0.15841584158415842, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9657534246575342, "hard": 0.6055045871559633 }, "accuracy": 0.8250825082508251, "ece_hard": 0.18494530682510557, "calibration": 69.71544213255548, "p50_s": 0.5717611402273178, "p95_s": 1.4723677232861514, "cost_usd_per_1000": 0.02876765402158193, "axes": { "intelligence": 81.37290017175654, "calibration": 69.71544213255548, "speed": 80.74769076387742, "cost": 56.23286659065571 }, "jevbench_score": 71.24137266342876 } }, { "threshold": 0.62, "public": { "n": 231, "escalation": 0.2077922077922078, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.7027027027027027 }, "accuracy": 0.8008658008658008, "ece_hard": 0.11979513803594462, "calibration": 73.15601494131344, "p50_s": 0.5404984593391419, "p95_s": 1.7742935769259929, "cost_usd_per_1000": 0.030738473953013275, "axes": { "intelligence": 80.59100767434103, "calibration": 73.15601494131344, "speed": 80.18180059269342, "cost": 55.369530920543745 }, "jevbench_score": 71.52713317061813 }, "heldout": { "n": 303, "escalation": 0.1617161716171617, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9657534246575342, "hard": 0.6055045871559633 }, "accuracy": 0.8250825082508251, "ece_hard": 0.18494530682510557, "calibration": 69.71544213255548, "p50_s": 0.5717611402273178, "p95_s": 1.4723677232861514, "cost_usd_per_1000": 0.02889938417324878, "axes": { "intelligence": 81.37290017175654, "calibration": 69.71544213255548, "speed": 80.74769076387742, "cost": 56.17334235018533 }, "jevbench_score": 71.2225123723153 } }, { "threshold": 0.63, "public": { "n": 231, "escalation": 0.21645021645021645, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.7027027027027027 }, "accuracy": 0.8008658008658008, "ece_hard": 0.11102736767633255, "calibration": 74.03279197727466, "p50_s": 0.5404984593391419, "p95_s": 1.7742935769259929, "cost_usd_per_1000": 0.031084051753489952, "axes": { "intelligence": 80.59100767434103, "calibration": 74.03279197727466, "speed": 80.18180059269342, "cost": 55.223871298938946 }, "jevbench_score": 71.69326213238534 }, "heldout": { "n": 303, "escalation": 0.16831683168316833, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9726027397260274, "hard": 0.6055045871559633 }, "accuracy": 0.8283828382838284, "ece_hard": 0.18494530682510557, "calibration": 69.71544213255548, "p50_s": 0.5727051436901093, "p95_s": 1.4723677232861514, "cost_usd_per_1000": 0.029162844476582486, "axes": { "intelligence": 81.56468099367433, "calibration": 69.71544213255548, "speed": 80.74052627867543, "cost": 56.05510354618198 }, "jevbench_score": 71.22532928863514 } }, { "threshold": 0.64, "public": { "n": 231, "escalation": 0.21645021645021645, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.7027027027027027 }, "accuracy": 0.8008658008658008, "ece_hard": 0.11102736767633255, "calibration": 74.03279197727466, "p50_s": 0.5404984593391419, "p95_s": 1.7742935769259929, "cost_usd_per_1000": 0.031084051753489952, "axes": { "intelligence": 80.59100767434103, "calibration": 74.03279197727466, "speed": 80.18180059269342, "cost": 55.223871298938946 }, "jevbench_score": 71.69326213238534 }, "heldout": { "n": 303, "escalation": 0.17491749174917492, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9726027397260274, "hard": 0.6146788990825688 }, "accuracy": 0.8316831683168316, "ece_hard": 0.18139761799542853, "calibration": 71.08979907035896, "p50_s": 0.5727051436901093, "p95_s": 1.4723677232861514, "cost_usd_per_1000": 0.02942630477991619, "axes": { "intelligence": 81.8399103514725, "calibration": 71.08979907035896, "speed": 80.74052627867543, "cost": 55.93792813543181 }, "jevbench_score": 71.59663250398403 } }, { "threshold": 0.65, "public": { "n": 231, "escalation": 0.22077922077922077, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.7027027027027027 }, "accuracy": 0.8008658008658008, "ece_hard": 0.11102736767633255, "calibration": 74.03279197727466, "p50_s": 0.5407783031463623, "p95_s": 1.7742935769259929, "cost_usd_per_1000": 0.03125684065372829, "axes": { "intelligence": 80.59100767434103, "calibration": 74.03279197727466, "speed": 80.17955260904667, "cost": 55.151647637866816 }, "jevbench_score": 71.66930756398585 }, "heldout": { "n": 303, "escalation": 0.17491749174917492, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9726027397260274, "hard": 0.6146788990825688 }, "accuracy": 0.8316831683168316, "ece_hard": 0.18139761799542853, "calibration": 71.08979907035896, "p50_s": 0.5727051436901093, "p95_s": 1.4723677232861514, "cost_usd_per_1000": 0.02942630477991619, "axes": { "intelligence": 81.8399103514725, "calibration": 71.08979907035896, "speed": 80.74052627867543, "cost": 55.93792813543181 }, "jevbench_score": 71.59663250398403 } }, { "threshold": 0.66, "public": { "n": 231, "escalation": 0.24242424242424243, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6846846846846847 }, "accuracy": 0.7922077922077922, "ece_hard": 0.11716950710141005, "calibration": 73.4185780347669, "p50_s": 0.5409701555967331, "p95_s": 1.7742935769259929, "cost_usd_per_1000": 0.03212078515491998, "axes": { "intelligence": 79.84025692359026, "calibration": 73.4185780347669, "speed": 80.17801213148488, "cost": 54.79641542333094 }, "jevbench_score": 71.2375266042696 }, "heldout": { "n": 303, "escalation": 0.1848184818481848, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9726027397260274, "hard": 0.6146788990825688 }, "accuracy": 0.8316831683168316, "ece_hard": 0.18110301343193905, "calibration": 71.1192595267079, "p50_s": 0.5745811700820923, "p95_s": 1.4813224613666531, "cost_usd_per_1000": 0.029821495234916746, "axes": { "intelligence": 81.8399103514725, "calibration": 71.1192595267079, "speed": 80.6999900250097, "cost": 55.764117550843224 }, "jevbench_score": 71.5393799300775 } }, { "threshold": 0.67, "public": { "n": 231, "escalation": 0.24242424242424243, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6846846846846847 }, "accuracy": 0.7922077922077922, "ece_hard": 0.11716950710141005, "calibration": 73.4185780347669, "p50_s": 0.5409701555967331, "p95_s": 1.7742935769259929, "cost_usd_per_1000": 0.03212078515491998, "axes": { "intelligence": 79.84025692359026, "calibration": 73.4185780347669, "speed": 80.17801213148488, "cost": 54.79641542333094 }, "jevbench_score": 71.2375266042696 }, "heldout": { "n": 303, "escalation": 0.1848184818481848, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9726027397260274, "hard": 0.6146788990825688 }, "accuracy": 0.8316831683168316, "ece_hard": 0.18110301343193905, "calibration": 71.1192595267079, "p50_s": 0.5745811700820923, "p95_s": 1.4813224613666531, "cost_usd_per_1000": 0.029821495234916746, "axes": { "intelligence": 81.8399103514725, "calibration": 71.1192595267079, "speed": 80.6999900250097, "cost": 55.764117550843224 }, "jevbench_score": 71.5393799300775 } }, { "threshold": 0.68, "public": { "n": 231, "escalation": 0.26406926406926406, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "hard": 0.6846846846846847 }, "accuracy": 0.7965367965367965, "ece_hard": 0.1298176951296409, "calibration": 72.15375923194381, "p50_s": 0.5424583941698075, "p95_s": 1.778119371831417, "cost_usd_per_1000": 0.032984729656111676, "axes": { "intelligence": 80.3803803803804, "calibration": 72.15375923194381, "speed": 80.15672652572965, "cost": 54.45061213180786 }, "jevbench_score": 70.93130332330036 }, "heldout": { "n": 303, "escalation": 0.19141914191419143, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9726027397260274, "hard": 0.6238532110091743 }, "accuracy": 0.834983498349835, "ece_hard": 0.1720355143645006, "calibration": 72.02600943345175, "p50_s": 0.5745811700820923, "p95_s": 1.4835964404046533, "cost_usd_per_1000": 0.03008495553825045, "axes": { "intelligence": 82.11513970927066, "calibration": 72.02600943345175, "speed": 80.69332827961613, "cost": 55.64951878012082 }, "jevbench_score": 71.78817418644385 } }, { "threshold": 0.69, "public": { "n": 231, "escalation": 0.2683982683982684, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "hard": 0.6846846846846847 }, "accuracy": 0.7965367965367965, "ece_hard": 0.13176564625828036, "calibration": 71.95896411907987, "p50_s": 0.5424583941698075, "p95_s": 1.778119371831417, "cost_usd_per_1000": 0.03315751855635001, "axes": { "intelligence": 80.3803803803804, "calibration": 71.95896411907987, "speed": 80.15672652572965, "cost": 54.38253935958383 }, "jevbench_score": 70.86121652991366 }, "heldout": { "n": 303, "escalation": 0.19141914191419143, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9726027397260274, "hard": 0.6238532110091743 }, "accuracy": 0.834983498349835, "ece_hard": 0.1720355143645006, "calibration": 72.02600943345175, "p50_s": 0.5745811700820923, "p95_s": 1.4835964404046533, "cost_usd_per_1000": 0.03008495553825045, "axes": { "intelligence": 82.11513970927066, "calibration": 72.02600943345175, "speed": 80.69332827961613, "cost": 55.64951878012082 }, "jevbench_score": 71.78817418644385 } }, { "threshold": 0.7, "public": { "n": 231, "escalation": 0.2727272727272727, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "hard": 0.6846846846846847 }, "accuracy": 0.7965367965367965, "ece_hard": 0.13176564625828036, "calibration": 71.95896411907987, "p50_s": 0.543133819103241, "p95_s": 1.778119371831417, "cost_usd_per_1000": 0.033330307456588354, "axes": { "intelligence": 80.3803803803804, "calibration": 71.95896411907987, "speed": 80.15132240886527, "cost": 54.31482040469005 }, "jevbench_score": 70.83795251977932 }, "heldout": { "n": 303, "escalation": 0.19141914191419143, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9726027397260274, "hard": 0.6238532110091743 }, "accuracy": 0.834983498349835, "ece_hard": 0.1720355143645006, "calibration": 72.02600943345175, "p50_s": 0.5745811700820923, "p95_s": 1.4835964404046533, "cost_usd_per_1000": 0.03008495553825045, "axes": { "intelligence": 82.11513970927066, "calibration": 72.02600943345175, "speed": 80.69332827961613, "cost": 55.64951878012082 }, "jevbench_score": 71.78817418644385 } }, { "threshold": 0.71, "public": { "n": 231, "escalation": 0.27705627705627706, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "hard": 0.6846846846846847 }, "accuracy": 0.7965367965367965, "ece_hard": 0.13154758675217695, "calibration": 72.55393952723018, "p50_s": 0.543133819103241, "p95_s": 1.778119371831417, "cost_usd_per_1000": 0.03350309635682669, "axes": { "intelligence": 80.3803803803804, "calibration": 72.55393952723018, "speed": 80.15132240886527, "cost": 54.24745160811085 }, "jevbench_score": 70.96190605449407 }, "heldout": { "n": 303, "escalation": 0.2079207920792079, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9726027397260274, "hard": 0.6330275229357798 }, "accuracy": 0.8382838283828383, "ece_hard": 0.16709124703252765, "calibration": 72.52043616664906, "p50_s": 0.5753569543361664, "p95_s": 1.5273265570402135, "cost_usd_per_1000": 0.030743606296584708, "axes": { "intelligence": 82.39036906706883, "calibration": 72.52043616664906, "speed": 80.56130729920844, "cost": 55.36735570283306 }, "jevbench_score": 71.8504158114182 } }, { "threshold": 0.72, "public": { "n": 231, "escalation": 0.2813852813852814, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "hard": 0.6936936936936937 }, "accuracy": 0.8008658008658008, "ece_hard": 0.12410149160552987, "calibration": 73.29854904189489, "p50_s": 0.543133819103241, "p95_s": 1.778119371831417, "cost_usd_per_1000": 0.033675885257065025, "axes": { "intelligence": 80.75575575575577, "calibration": 73.29854904189489, "speed": 80.15132240886527, "cost": 54.18042936729846 }, "jevbench_score": 71.20418166420036 }, "heldout": { "n": 303, "escalation": 0.2079207920792079, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9726027397260274, "hard": 0.6330275229357798 }, "accuracy": 0.8382838283828383, "ece_hard": 0.16709124703252765, "calibration": 72.52043616664906, "p50_s": 0.5753569543361664, "p95_s": 1.5273265570402135, "cost_usd_per_1000": 0.030743606296584708, "axes": { "intelligence": 82.39036906706883, "calibration": 72.52043616664906, "speed": 80.56130729920844, "cost": 55.36735570283306 }, "jevbench_score": 71.8504158114182 } }, { "threshold": 0.73, "public": { "n": 231, "escalation": 0.2813852813852814, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "hard": 0.6936936936936937 }, "accuracy": 0.8008658008658008, "ece_hard": 0.12410149160552987, "calibration": 73.29854904189489, "p50_s": 0.543133819103241, "p95_s": 1.778119371831417, "cost_usd_per_1000": 0.033675885257065025, "axes": { "intelligence": 80.75575575575577, "calibration": 73.29854904189489, "speed": 80.15132240886527, "cost": 54.18042936729846 }, "jevbench_score": 71.20418166420036 }, "heldout": { "n": 303, "escalation": 0.2079207920792079, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9726027397260274, "hard": 0.6330275229357798 }, "accuracy": 0.8382838283828383, "ece_hard": 0.16709124703252765, "calibration": 72.52043616664906, "p50_s": 0.5753569543361664, "p95_s": 1.5273265570402135, "cost_usd_per_1000": 0.030743606296584708, "axes": { "intelligence": 82.39036906706883, "calibration": 72.52043616664906, "speed": 80.56130729920844, "cost": 55.36735570283306 }, "jevbench_score": 71.8504158114182 } }, { "threshold": 0.74, "public": { "n": 231, "escalation": 0.2987012987012987, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "hard": 0.6936936936936937 }, "accuracy": 0.8008658008658008, "ece_hard": 0.1274747954458646, "calibration": 72.96121865786142, "p50_s": 0.545120233297348, "p95_s": 1.778119371831417, "cost_usd_per_1000": 0.03436704085801838, "axes": { "intelligence": 80.75575575575577, "calibration": 72.96121865786142, "speed": 80.13546784476827, "cost": 53.915735823221084 }, "jevbench_score": 71.03157911515022 }, "heldout": { "n": 303, "escalation": 0.21122112211221122, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9726027397260274, "hard": 0.6330275229357798 }, "accuracy": 0.8382838283828383, "ece_hard": 0.16709124703252765, "calibration": 72.52043616664906, "p50_s": 0.5755759567022324, "p95_s": 1.5273265570402135, "cost_usd_per_1000": 0.030875336448251564, "axes": { "intelligence": 82.39036906706883, "calibration": 72.52043616664906, "speed": 80.55965452658646, "cost": 55.31164903610795 }, "jevbench_score": 71.83196787766292 } }, { "threshold": 0.75, "public": { "n": 231, "escalation": 0.2987012987012987, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "hard": 0.6936936936936937 }, "accuracy": 0.8008658008658008, "ece_hard": 0.1274747954458646, "calibration": 72.96121865786142, "p50_s": 0.545120233297348, "p95_s": 1.778119371831417, "cost_usd_per_1000": 0.03436704085801838, "axes": { "intelligence": 80.75575575575577, "calibration": 72.96121865786142, "speed": 80.13546784476827, "cost": 53.915735823221084 }, "jevbench_score": 71.03157911515022 }, "heldout": { "n": 303, "escalation": 0.21122112211221122, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9726027397260274, "hard": 0.6330275229357798 }, "accuracy": 0.8382838283828383, "ece_hard": 0.16709124703252765, "calibration": 72.52043616664906, "p50_s": 0.5755759567022324, "p95_s": 1.5273265570402135, "cost_usd_per_1000": 0.030875336448251564, "axes": { "intelligence": 82.39036906706883, "calibration": 72.52043616664906, "speed": 80.55965452658646, "cost": 55.31164903610795 }, "jevbench_score": 71.83196787766292 } }, { "threshold": 0.76, "public": { "n": 231, "escalation": 0.30303030303030304, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "hard": 0.7027027027027027 }, "accuracy": 0.8051948051948052, "ece_hard": 0.12712528964005615, "calibration": 74.4527441727804, "p50_s": 0.545120233297348, "p95_s": 1.778119371831417, "cost_usd_per_1000": 0.03453982975825672, "axes": { "intelligence": 81.13113113113114, "calibration": 74.4527441727804, "speed": 80.13546784476827, "cost": 53.85039422010825 }, "jevbench_score": 71.4530013855504 }, "heldout": { "n": 303, "escalation": 0.22112211221122113, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9726027397260274, "hard": 0.6330275229357798 }, "accuracy": 0.8382838283828383, "ece_hard": 0.16557826550163196, "calibration": 72.67173431973862, "p50_s": 0.5755759567022324, "p95_s": 1.5273265570402135, "cost_usd_per_1000": 0.03127052690325212, "axes": { "intelligence": 82.39036906706883, "calibration": 72.67173431973862, "speed": 80.55965452658646, "cost": 55.14594402713765 }, "jevbench_score": 71.81551611725236 } }, { "threshold": 0.77, "public": { "n": 231, "escalation": 0.30303030303030304, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "hard": 0.7027027027027027 }, "accuracy": 0.8051948051948052, "ece_hard": 0.12712528964005615, "calibration": 74.4527441727804, "p50_s": 0.545120233297348, "p95_s": 1.778119371831417, "cost_usd_per_1000": 0.03453982975825672, "axes": { "intelligence": 81.13113113113114, "calibration": 74.4527441727804, "speed": 80.13546784476827, "cost": 53.85039422010825 }, "jevbench_score": 71.4530013855504 }, "heldout": { "n": 303, "escalation": 0.22442244224422442, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9726027397260274, "hard": 0.6330275229357798 }, "accuracy": 0.8382838283828383, "ece_hard": 0.16562877819179128, "calibration": 72.6666830507227, "p50_s": 0.5755759567022324, "p95_s": 1.6215087994933093, "cost_usd_per_1000": 0.03140225705491897, "axes": { "intelligence": 82.39036906706883, "calibration": 72.6666830507227, "speed": 80.299780461463, "cost": 55.09117406933032 }, "jevbench_score": 71.73845888122682 } }, { "threshold": 0.78, "public": { "n": 231, "escalation": 0.30735930735930733, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "hard": 0.7027027027027027 }, "accuracy": 0.8051948051948052, "ece_hard": 0.1308020745232018, "calibration": 74.08506568446583, "p50_s": 0.545120233297348, "p95_s": 1.778119371831417, "cost_usd_per_1000": 0.03471261865849506, "axes": { "intelligence": 81.13113113113114, "calibration": 74.08506568446583, "speed": 80.13546784476827, "cost": 53.785378680578006 }, "jevbench_score": 71.34307138139856 }, "heldout": { "n": 303, "escalation": 0.23432343234323433, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9726027397260274, "hard": 0.6330275229357798 }, "accuracy": 0.8382838283828383, "ece_hard": 0.16562877819179128, "calibration": 72.6666830507227, "p50_s": 0.5812294721603394, "p95_s": 1.6215087994933093, "cost_usd_per_1000": 0.03179744750991953, "axes": { "intelligence": 82.39036906706883, "calibration": 72.6666830507227, "speed": 80.25733062411537, "cost": 54.92823222768279 }, "jevbench_score": 71.6758793157993 } }, { "threshold": 0.79, "public": { "n": 231, "escalation": 0.30735930735930733, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "hard": 0.7027027027027027 }, "accuracy": 0.8051948051948052, "ece_hard": 0.1308020745232018, "calibration": 74.08506568446583, "p50_s": 0.545120233297348, "p95_s": 1.778119371831417, "cost_usd_per_1000": 0.03471261865849506, "axes": { "intelligence": 81.13113113113114, "calibration": 74.08506568446583, "speed": 80.13546784476827, "cost": 53.785378680578006 }, "jevbench_score": 71.34307138139856 }, "heldout": { "n": 303, "escalation": 0.24092409240924093, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9726027397260274, "hard": 0.6422018348623854 }, "accuracy": 0.8415841584158416, "ece_hard": 0.16321428103788252, "calibration": 72.90813276611357, "p50_s": 0.5812294721603394, "p95_s": 1.67167922630906, "cost_usd_per_1000": 0.03206090781325323, "axes": { "intelligence": 82.665598424867, "calibration": 72.90813276611357, "speed": 80.12499426640917, "cost": 54.82072553927095 }, "jevbench_score": 71.73042351951042 } }, { "threshold": 0.8, "public": { "n": 231, "escalation": 0.3203463203463203, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "hard": 0.7027027027027027 }, "accuracy": 0.8051948051948052, "ece_hard": 0.12588150656607916, "calibration": 74.5771224801781, "p50_s": 0.5452094167470932, "p95_s": 1.778119371831417, "cost_usd_per_1000": 0.03523098535921007, "axes": { "intelligence": 81.13113113113114, "calibration": 74.5771224801781, "speed": 80.13475738293343, "cost": 53.59225630199401 }, "jevbench_score": 71.39684655730561 }, "heldout": { "n": 303, "escalation": 0.26732673267326734, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9657534246575342, "hard": 0.6513761467889908 }, "accuracy": 0.8415841584158416, "ece_hard": 0.1607079774259918, "calibration": 73.15876312730263, "p50_s": 0.5844195425510407, "p95_s": 1.6846594184637067, "cost_usd_per_1000": 0.03311474902658805, "axes": { "intelligence": 82.74904696074735, "calibration": 73.15876312730263, "speed": 80.06763155685644, "cost": 54.39935596238803 }, "jevbench_score": 71.65888127827549 } }, { "threshold": 0.81, "public": { "n": 231, "escalation": 0.3246753246753247, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "hard": 0.7027027027027027 }, "accuracy": 0.8051948051948052, "ece_hard": 0.12721120021428467, "calibration": 74.44415311535755, "p50_s": 0.5452094167470932, "p95_s": 1.778119371831417, "cost_usd_per_1000": 0.03540377425944841, "axes": { "intelligence": 81.13113113113114, "calibration": 74.44415311535755, "speed": 80.13475738293343, "cost": 53.52851311153364 }, "jevbench_score": 71.34377046468121 }, "heldout": { "n": 303, "escalation": 0.26732673267326734, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9657534246575342, "hard": 0.6513761467889908 }, "accuracy": 0.8415841584158416, "ece_hard": 0.1607079774259918, "calibration": 73.15876312730263, "p50_s": 0.5844195425510407, "p95_s": 1.6846594184637067, "cost_usd_per_1000": 0.03311474902658805, "axes": { "intelligence": 82.74904696074735, "calibration": 73.15876312730263, "speed": 80.06763155685644, "cost": 54.39935596238803 }, "jevbench_score": 71.65888127827549 } }, { "threshold": 0.82, "public": { "n": 231, "escalation": 0.35064935064935066, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "hard": 0.6936936936936937 }, "accuracy": 0.8008658008658008, "ece_hard": 0.1452805574797152, "calibration": 71.75771922156068, "p50_s": 0.5457840502262116, "p95_s": 1.8344867430627345, "cost_usd_per_1000": 0.036440507660878446, "axes": { "intelligence": 80.75575575575577, "calibration": 71.75771922156068, "speed": 79.9946458245834, "cost": 53.152467442094306 }, "jevbench_score": 70.45415582345872 }, "heldout": { "n": 303, "escalation": 0.29042904290429045, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6513761467889908 }, "accuracy": 0.8481848184818482, "ece_hard": 0.16075248244996165, "calibration": 72.88218500587388, "p50_s": 0.5914658546447754, "p95_s": 1.6846594184637067, "cost_usd_per_1000": 0.034036860088256016, "axes": { "intelligence": 84.10749444933184, "calibration": 72.88218500587388, "speed": 80.01558207828266, "cost": 54.041515316531154 }, "jevbench_score": 71.75291376211362 } }, { "threshold": 0.83, "public": { "n": 231, "escalation": 0.354978354978355, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "hard": 0.7027027027027027 }, "accuracy": 0.8051948051948052, "ece_hard": 0.13631921048107604, "calibration": 72.6538539214246, "p50_s": 0.5457840502262116, "p95_s": 1.8344867430627345, "cost_usd_per_1000": 0.03661329656111678, "axes": { "intelligence": 81.13113113113114, "calibration": 72.6538539214246, "speed": 79.9946458245834, "cost": 53.09083500010845 }, "jevbench_score": 70.73456153257037 }, "heldout": { "n": 303, "escalation": 0.29372937293729373, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6513761467889908 }, "accuracy": 0.8481848184818482, "ece_hard": 0.16109553497788975, "calibration": 72.84787975308106, "p50_s": 0.5914658546447754, "p95_s": 1.6980811275541778, "cost_usd_per_1000": 0.034168590239922865, "axes": { "intelligence": 84.10749444933184, "calibration": 72.84787975308106, "speed": 79.98111886891405, "cost": 53.99118818094717 }, "jevbench_score": 71.72003502873983 } }, { "threshold": 0.84, "public": { "n": 231, "escalation": 0.36796536796536794, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "hard": 0.7027027027027027 }, "accuracy": 0.8051948051948052, "ece_hard": 0.13725208785665882, "calibration": 72.56056618386631, "p50_s": 0.5469208598136902, "p95_s": 1.8344867430627345, "cost_usd_per_1000": 0.037131663261831795, "axes": { "intelligence": 81.13113113113114, "calibration": 72.56056618386631, "speed": 79.98560934447968, "cost": 52.90766790140777 }, "jevbench_score": 70.64878015492614 }, "heldout": { "n": 303, "escalation": 0.30363036303630364, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6605504587155964 }, "accuracy": 0.8514851485148515, "ece_hard": 0.15176392750527865, "calibration": 73.78104050034217, "p50_s": 0.5918420493602753, "p95_s": 1.731509952247142, "cost_usd_per_1000": 0.03456378069492342, "axes": { "intelligence": 84.38272380713, "calibration": 73.78104050034217, "speed": 79.89369190103852, "cost": 53.84136277299943 }, "jevbench_score": 71.93772717152842 } }, { "threshold": 0.85, "public": { "n": 231, "escalation": 0.36796536796536794, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "hard": 0.7027027027027027 }, "accuracy": 0.8051948051948052, "ece_hard": 0.13725208785665882, "calibration": 72.56056618386631, "p50_s": 0.5469208598136902, "p95_s": 1.8344867430627345, "cost_usd_per_1000": 0.037131663261831795, "axes": { "intelligence": 81.13113113113114, "calibration": 72.56056618386631, "speed": 79.98560934447968, "cost": 52.90766790140777 }, "jevbench_score": 70.64878015492614 }, "heldout": { "n": 303, "escalation": 0.3069306930693069, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6605504587155964 }, "accuracy": 0.8514851485148515, "ece_hard": 0.15176392750527865, "calibration": 73.78104050034217, "p50_s": 0.5918420493602753, "p95_s": 1.731509952247142, "cost_usd_per_1000": 0.03469551084659027, "axes": { "intelligence": 84.38272380713, "calibration": 73.78104050034217, "speed": 79.89369190103852, "cost": 53.79180141103722 }, "jevbench_score": 71.92116665499317 } }, { "threshold": 0.86, "public": { "n": 231, "escalation": 0.38095238095238093, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "hard": 0.7117117117117117 }, "accuracy": 0.8095238095238095, "ece_hard": 0.12920133839495493, "calibration": 73.3656411300367, "p50_s": 0.5478601396083832, "p95_s": 1.8344867430627345, "cost_usd_per_1000": 0.03765002996254681, "axes": { "intelligence": 81.50650650650651, "calibration": 73.3656411300367, "speed": 79.97815718496825, "cost": 52.72704021532187 }, "jevbench_score": 70.86347503311501 }, "heldout": { "n": 303, "escalation": 0.3333333333333333, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6880733944954128 }, "accuracy": 0.8613861386138614, "ece_hard": 0.11212255243922566, "calibration": 77.74517800694747, "p50_s": 0.6031645923852921, "p95_s": 1.7430013202130792, "cost_usd_per_1000": 0.03574935205992509, "axes": { "intelligence": 85.2084118805245, "calibration": 77.74517800694747, "speed": 79.7826644474973, "cost": 53.40195475520104 }, "jevbench_score": 72.8878946549661 } }, { "threshold": 0.87, "public": { "n": 231, "escalation": 0.3939393939393939, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "hard": 0.7117117117117117 }, "accuracy": 0.8095238095238095, "ece_hard": 0.1296784937120525, "calibration": 73.31792559832695, "p50_s": 0.550955080986023, "p95_s": 1.8344867430627345, "cost_usd_per_1000": 0.03816839666326183, "axes": { "intelligence": 81.50650650650651, "calibration": 73.31792559832695, "speed": 79.95369229785373, "cost": 52.548882491222834 }, "jevbench_score": 70.78660999678297 }, "heldout": { "n": 303, "escalation": 0.34983498349834985, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6880733944954128 }, "accuracy": 0.8613861386138614, "ece_hard": 0.11376208545558322, "calibration": 77.58122470531171, "p50_s": 0.6054746598005295, "p95_s": 1.7688356354832642, "cost_usd_per_1000": 0.03640800281825935, "axes": { "intelligence": 85.2084118805245, "calibration": 77.58122470531171, "speed": 79.70216549059643, "cost": 53.164094316517904 }, "jevbench_score": 72.74981793538701 } }, { "threshold": 0.88, "public": { "n": 231, "escalation": 0.39826839826839827, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "hard": 0.7117117117117117 }, "accuracy": 0.8095238095238095, "ece_hard": 0.13021187802566417, "calibration": 73.26458716696578, "p50_s": 0.550955080986023, "p95_s": 1.8667128473520278, "cost_usd_per_1000": 0.03834118556350016, "axes": { "intelligence": 81.50650650650651, "calibration": 73.26458716696578, "speed": 79.8780629100679, "cost": 52.49003386758413 }, "jevbench_score": 70.73717161326503 }, "heldout": { "n": 303, "escalation": 0.34983498349834985, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6880733944954128 }, "accuracy": 0.8613861386138614, "ece_hard": 0.11376208545558322, "calibration": 77.58122470531171, "p50_s": 0.6054746598005295, "p95_s": 1.7688356354832642, "cost_usd_per_1000": 0.03640800281825935, "axes": { "intelligence": 85.2084118805245, "calibration": 77.58122470531171, "speed": 79.70216549059643, "cost": 53.164094316517904 }, "jevbench_score": 72.74981793538701 } }, { "threshold": 0.89, "public": { "n": 231, "escalation": 0.4025974025974026, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "hard": 0.7117117117117117 }, "accuracy": 0.8095238095238095, "ece_hard": 0.12954920096330266, "calibration": 73.33085487320193, "p50_s": 0.550955080986023, "p95_s": 1.8667128473520278, "cost_usd_per_1000": 0.038513974463738505, "axes": { "intelligence": 81.50650650650651, "calibration": 73.33085487320193, "speed": 79.8780629100679, "cost": 52.43144985658296 }, "jevbench_score": 70.73341147640848 }, "heldout": { "n": 303, "escalation": 0.36303630363036304, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6880733944954128 }, "accuracy": 0.8613861386138614, "ece_hard": 0.11312933488342455, "calibration": 77.64449976252757, "p50_s": 0.6071688771247864, "p95_s": 1.7731440924108026, "cost_usd_per_1000": 0.036934923424926755, "axes": { "intelligence": 85.2084118805245, "calibration": 77.64449976252757, "speed": 79.67946468596028, "cost": 52.97688391120231 }, "jevbench_score": 72.69532728350676 } }, { "threshold": 0.9, "public": { "n": 231, "escalation": 0.4155844155844156, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "hard": 0.7117117117117117 }, "accuracy": 0.8095238095238095, "ece_hard": 0.13035742512636503, "calibration": 73.2500324568957, "p50_s": 0.5544787734746933, "p95_s": 1.8667128473520278, "cost_usd_per_1000": 0.03903234116445352, "axes": { "intelligence": 81.50650650650651, "calibration": 73.2500324568957, "speed": 79.85037558385235, "cost": 52.25726196708887 }, "jevbench_score": 70.64898530441937 }, "heldout": { "n": 303, "escalation": 0.36633663366336633, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6880733944954128 }, "accuracy": 0.8613861386138614, "ece_hard": 0.114008745013904, "calibration": 77.55655874947963, "p50_s": 0.6071688771247864, "p95_s": 1.7988253034651274, "cost_usd_per_1000": 0.03706665357659361, "axes": { "intelligence": 85.2084118805245, "calibration": 77.55655874947963, "speed": 79.61701510035024, "cost": 52.93049862533529 }, "jevbench_score": 72.64458040751718 } }, { "threshold": 0.91, "public": { "n": 231, "escalation": 0.4329004329004329, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "hard": 0.7117117117117117 }, "accuracy": 0.8095238095238095, "ece_hard": 0.13184154780612295, "calibration": 73.10162018891991, "p50_s": 0.5592252552509308, "p95_s": 1.8667128473520278, "cost_usd_per_1000": 0.039723496765406874, "axes": { "intelligence": 81.50650650650651, "calibration": 73.10162018891991, "speed": 79.81335707136535, "cost": 52.028575857355406 }, "jevbench_score": 70.52761529386373 }, "heldout": { "n": 303, "escalation": 0.37293729372937295, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6880733944954128 }, "accuracy": 0.8613861386138614, "ece_hard": 0.11470513461377888, "calibration": 77.48691978949216, "p50_s": 0.6102951854467392, "p95_s": 1.7988253034651274, "cost_usd_per_1000": 0.03733011387992731, "axes": { "intelligence": 85.2084118805245, "calibration": 77.48691978949216, "speed": 79.59471068120507, "cost": 52.83822055614985 }, "jevbench_score": 72.59150743586983 } }, { "threshold": 0.92, "public": { "n": 231, "escalation": 0.45021645021645024, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "hard": 0.7117117117117117 }, "accuracy": 0.8095238095238095, "ece_hard": 0.13164228547069412, "calibration": 73.12154642246279, "p50_s": 0.5608406305313111, "p95_s": 1.8667128473520278, "cost_usd_per_1000": 0.04041465236636023, "axes": { "intelligence": 81.50650650650651, "calibration": 73.12154642246279, "speed": 79.8008301465517, "cost": 51.80383457525955 }, "jevbench_score": 70.45336502701703 }, "heldout": { "n": 303, "escalation": 0.3795379537953795, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6880733944954128 }, "accuracy": 0.8613861386138614, "ece_hard": 0.11493540108173321, "calibration": 77.46389314269672, "p50_s": 0.6141898572444916, "p95_s": 1.7988253034651274, "cost_usd_per_1000": 0.037593574183261015, "axes": { "intelligence": 85.2084118805245, "calibration": 77.46389314269672, "speed": 79.56708371777947, "cost": 52.74659146220433 }, "jevbench_score": 72.5483280161543 } }, { "threshold": 0.93, "public": { "n": 231, "escalation": 0.45454545454545453, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "hard": 0.7117117117117117 }, "accuracy": 0.8095238095238095, "ece_hard": 0.13052389069611076, "calibration": 73.23338589992113, "p50_s": 0.5718684434890747, "p95_s": 1.8667128473520278, "cost_usd_per_1000": 0.040587441266598565, "axes": { "intelligence": 81.50650650650651, "calibration": 73.23338589992113, "speed": 79.71626351328592, "cost": 51.748249804771135 }, "jevbench_score": 70.44270073888902 }, "heldout": { "n": 303, "escalation": 0.39933993399339934, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6972477064220184 }, "accuracy": 0.8646864686468647, "ece_hard": 0.09973882862684202, "calibration": 78.98355038818585, "p50_s": 0.6410244047641754, "p95_s": 1.8178120583295816, "cost_usd_per_1000": 0.03838395509326213, "axes": { "intelligence": 85.48364123832266, "calibration": 78.98355038818585, "speed": 79.3357645596291, "cost": 52.47550832416968 }, "jevbench_score": 72.81340365939371 } }, { "threshold": 0.94, "public": { "n": 231, "escalation": 0.46320346320346323, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "hard": 0.7117117117117117 }, "accuracy": 0.8095238095238095, "ece_hard": 0.12985821021220825, "calibration": 73.29995394831138, "p50_s": 0.577006408572197, "p95_s": 1.8667128473520278, "cost_usd_per_1000": 0.04093301906707524, "axes": { "intelligence": 81.50650650650651, "calibration": 73.29995394831138, "speed": 79.67741846839432, "cost": 51.63778666688418 }, "jevbench_score": 70.4124917878959 }, "heldout": { "n": 303, "escalation": 0.42244224422442245, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6972477064220184 }, "accuracy": 0.8646864686468647, "ece_hard": 0.09822609504946603, "calibration": 80.1843904950534, "p50_s": 0.6549608945846558, "p95_s": 1.8178120583295816, "cost_usd_per_1000": 0.0393060661549301, "axes": { "intelligence": 85.48364123832266, "calibration": 80.1843904950534, "speed": 79.24235649474033, "cost": 52.16621257705066 }, "jevbench_score": 72.95916980191905 } }, { "threshold": 0.95, "public": { "n": 231, "escalation": 0.48917748917748916, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.7117117117117117 }, "accuracy": 0.8051948051948052, "ece_hard": 0.13078962952340845, "calibration": 73.20681201719137, "p50_s": 1.1270293906331061, "p95_s": 1.8667128473520278, "cost_usd_per_1000": 0.041969752468505275, "axes": { "intelligence": 80.9663830497164, "calibration": 73.20681201719137, "speed": 76.76987241841518, "cost": 51.311907765599024 }, "jevbench_score": 69.51304348168172 }, "heldout": { "n": 303, "escalation": 0.44554455445544555, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.7064220183486238 }, "accuracy": 0.8679867986798679, "ece_hard": 0.08907448918048061, "calibration": 81.09955108195194, "p50_s": 0.6615447193384171, "p95_s": 1.8178120583295816, "cost_usd_per_1000": 0.04022817721659806, "axes": { "intelligence": 85.75887059612081, "calibration": 81.09955108195194, "speed": 79.19891812930476, "cost": 51.864089360451715 }, "jevbench_score": 73.10900567407666 } }, { "threshold": 0.96, "public": { "n": 231, "escalation": 0.49783549783549785, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.7117117117117117 }, "accuracy": 0.8051948051948052, "ece_hard": 0.13067634910158463, "calibration": 73.21814005937374, "p50_s": 1.1309201285243033, "p95_s": 1.8667128473520278, "cost_usd_per_1000": 0.042315330268981946, "axes": { "intelligence": 80.9663830497164, "calibration": 73.21814005937374, "speed": 76.75490549683693, "cost": 51.20506795388718 }, "jevbench_score": 69.47613171412839 }, "heldout": { "n": 303, "escalation": 0.49504950495049505, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9726027397260274, "hard": 0.6972477064220184 }, "accuracy": 0.8646864686468647, "ece_hard": 0.09124690691725382, "calibration": 80.88230930827461, "p50_s": 1.0916826367378234, "p95_s": 1.8232153564691542, "cost_usd_per_1000": 0.042204129491600836, "axes": { "intelligence": 85.48364123832266, "calibration": 80.88230930827461, "speed": 77.01065626975446, "cost": 51.23935159365679 }, "jevbench_score": 72.27244566783516 } }, { "threshold": 0.97, "public": { "n": 231, "escalation": 0.5151515151515151, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.7117117117117117 }, "accuracy": 0.8051948051948052, "ece_hard": 0.1282198345731534, "calibration": 73.46379151221686, "p50_s": 1.1483509629964828, "p95_s": 1.9664708875119685, "cost_usd_per_1000": 0.0430064858699353, "axes": { "intelligence": 80.9663830497164, "calibration": 73.46379151221686, "speed": 76.46237839758734, "cost": 50.993981287185825 }, "jevbench_score": 69.3962815217201 }, "heldout": { "n": 303, "escalation": 0.5544554455445545, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9657534246575342, "hard": 0.6972477064220184 }, "accuracy": 0.8613861386138614, "ece_hard": 0.08621849110565537, "calibration": 81.38515088943447, "p50_s": 1.1864204198122024, "p95_s": 1.8232153564691542, "cost_usd_per_1000": 0.04457527222160418, "axes": { "intelligence": 85.29186041640486, "calibration": 81.38515088943447, "speed": 76.64923417014805, "cost": 50.52717987712144 }, "jevbench_score": 72.00645160814258 } }, { "threshold": 0.98, "public": { "n": 231, "escalation": 0.5584415584415584, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.7117117117117117 }, "accuracy": 0.8051948051948052, "ece_hard": 0.12391913145607261, "calibration": 73.89386182392494, "p50_s": 1.1671251818537711, "p95_s": 2.048865792900324, "cost_usd_per_1000": 0.04473437487231868, "axes": { "intelligence": 80.9663830497164, "calibration": 73.89386182392494, "speed": 76.21369048826406, "cost": 50.48075881605569 }, "jevbench_score": 69.26566248536997 }, "heldout": { "n": 303, "escalation": 0.6204620462046204, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.958904109589041, "hard": 0.6972477064220184 }, "accuracy": 0.858085808580858, "ece_hard": 0.08612979709962473, "calibration": 81.39402029003753, "p50_s": 1.20046097189188, "p95_s": 1.8232153564691542, "cost_usd_per_1000": 0.04720987525494122, "axes": { "intelligence": 85.10007959448704, "calibration": 81.39402029003753, "speed": 76.59813984559645, "cost": 49.779014413769985 }, "jevbench_score": 71.68804646122192 } }, { "threshold": 0.99, "public": { "n": 231, "escalation": 0.6103896103896104, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.7117117117117117 }, "accuracy": 0.8051948051948052, "ece_hard": 0.12383646164655294, "calibration": 73.90212880487691, "p50_s": 1.1832777500152587, "p95_s": 2.048865792900324, "cost_usd_per_1000": 0.04680784167517875, "axes": { "intelligence": 80.9663830497164, "calibration": 73.90212880487691, "speed": 76.15399790055322, "cost": 49.89044151612573 }, "jevbench_score": 69.05067615907639 }, "heldout": { "n": 303, "escalation": 0.7194719471947195, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6972477064220184 }, "accuracy": 0.8514851485148515, "ece_hard": 0.08548460053014867, "calibration": 81.45853994698513, "p50_s": 1.2129713013768195, "p95_s": 1.8232153564691542, "cost_usd_per_1000": 0.05116177980494678, "axes": { "intelligence": 84.71651795065142, "calibration": 81.45853994698513, "speed": 76.5531150418256, "cost": 48.73163067363831 }, "jevbench_score": 71.23109695458257 } }, { "threshold": 1.0, "public": { "n": 231, "escalation": 1.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.7117117117117117 }, "accuracy": 0.8051948051948052, "ece_hard": 0.09203203203203206, "calibration": 78.2177967967968, "p50_s": 1.2106154575943946, "p95_s": 2.048865792900324, "cost_usd_per_1000": 0.0623588426966292, "axes": { "intelligence": 80.9663830497164, "calibration": 78.2177967967968, "speed": 76.05480273462686, "cost": 46.153058601384124 }, "jevbench_score": 68.66483207501612 }, "heldout": { "n": 303, "escalation": 1.0, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6972477064220184 }, "accuracy": 0.8514851485148515, "ece_hard": 0.08064220183486236, "calibration": 81.94277981651376, "p50_s": 1.23065178245306, "p95_s": 1.8311943441629408, "cost_usd_per_1000": 0.0623588426966292, "axes": { "intelligence": 84.71651795065142, "calibration": 81.94277981651376, "speed": 76.47130376638181, "cost": 46.153058601384124 }, "jevbench_score": 70.35491504015424 } } ] }, { "cheap": "djev", "backend": "jev-1.13.0", "chosen": { "threshold": 0.79, "public": { "n": 231, "escalation": 0.16017316017316016, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "hard": 0.6936936936936937 }, "accuracy": 0.8008658008658008, "ece_hard": 0.17468406799995964, "calibration": 66.41821287200715, "p50_s": 0.24066395312547684, "p95_s": 0.9882492944598198, "cost_usd_per_1000": 0.032344443990466466, "axes": { "intelligence": 80.75575575575577, "calibration": 66.41821287200715, "speed": 86.23722440932626, "cost": 54.70600930288466 }, "jevbench_score": 70.92471824094812 }, "heldout": { "n": 303, "escalation": 0.12211221122112212, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.958904109589041, "hard": 0.6880733944954128 }, "accuracy": 0.8514851485148515, "ece_hard": 0.18456589139739973, "calibration": 68.41422928950267, "p50_s": 0.2428498938679695, "p95_s": 0.9318535953760148, "cost_usd_per_1000": 0.03082527029332147, "axes": { "intelligence": 83.6581835700222, "calibration": 68.41422928950267, "speed": 86.45314396797306, "cost": 55.332793197311595 }, "jevbench_score": 72.33601233598237 } }, "full": { "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.958904109589041, "hard": 0.6909090909090909 }, "accuracy": 0.8295880149812734, "ece_hard": 0.17794369868323717, "calibration": 67.58434918229915, "p50_s": 0.2419801652431488, "p95_s": 0.971130575239658, "cost_usd_per_1000": 0.03148244093759206, "axes": { "intelligence": 84.61825446243255, "calibration": 67.58434918229915, "speed": 86.28942603299521, "cost": 55.05794808167331 }, "jevbench_score": 72.1974652065132 }, "full_escalation": 0.13857677902621723, "backend_heldout_accuracy": 0.8514851485148515, "useful": false, "curve": [ { "threshold": 0.0, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6486486486486487 }, "accuracy": 0.7748917748917749, "ece_hard": 0.21041641822205612, "calibration": 61.35164946854124, "p50_s": 0.23927149176597595, "p95_s": 0.35407450795173645, "cost_usd_per_1000": 0.025951254681647943, "axes": { "intelligence": 78.33875542208877, "calibration": 61.35164946854124, "speed": 90.72014382446159, "cost": 57.57524920597589 }, "jevbench_score": 70.78408408803388 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6788990825688074 }, "accuracy": 0.834983498349835, "ece_hard": 0.1935286337386598, "calibration": 64.09002035150621, "p50_s": 0.24110446870326996, "p95_s": 0.3450261786580081, "cost_usd_per_1000": 0.025951254681647943, "axes": { "intelligence": 82.6158309245528, "calibration": 64.09002035150621, "speed": 90.7994269204109, "cost": 57.57524920597589 }, "jevbench_score": 72.534254981433 } }, { "threshold": 0.01, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6486486486486487 }, "accuracy": 0.7748917748917749, "ece_hard": 0.21041641822205612, "calibration": 61.35164946854124, "p50_s": 0.23927149176597595, "p95_s": 0.35407450795173645, "cost_usd_per_1000": 0.025951254681647943, "axes": { "intelligence": 78.33875542208877, "calibration": 61.35164946854124, "speed": 90.72014382446159, "cost": 57.57524920597589 }, "jevbench_score": 70.78408408803388 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6788990825688074 }, "accuracy": 0.834983498349835, "ece_hard": 0.1935286337386598, "calibration": 64.09002035150621, "p50_s": 0.24110446870326996, "p95_s": 0.3450261786580081, "cost_usd_per_1000": 0.025951254681647943, "axes": { "intelligence": 82.6158309245528, "calibration": 64.09002035150621, "speed": 90.7994269204109, "cost": 57.57524920597589 }, "jevbench_score": 72.534254981433 } }, { "threshold": 0.02, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6486486486486487 }, "accuracy": 0.7748917748917749, "ece_hard": 0.21041641822205612, "calibration": 61.35164946854124, "p50_s": 0.23927149176597595, "p95_s": 0.35407450795173645, "cost_usd_per_1000": 0.025951254681647943, "axes": { "intelligence": 78.33875542208877, "calibration": 61.35164946854124, "speed": 90.72014382446159, "cost": 57.57524920597589 }, "jevbench_score": 70.78408408803388 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6788990825688074 }, "accuracy": 0.834983498349835, "ece_hard": 0.1935286337386598, "calibration": 64.09002035150621, "p50_s": 0.24110446870326996, "p95_s": 0.3450261786580081, "cost_usd_per_1000": 0.025951254681647943, "axes": { "intelligence": 82.6158309245528, "calibration": 64.09002035150621, "speed": 90.7994269204109, "cost": 57.57524920597589 }, "jevbench_score": 72.534254981433 } }, { "threshold": 0.03, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6486486486486487 }, "accuracy": 0.7748917748917749, "ece_hard": 0.21041641822205612, "calibration": 61.35164946854124, "p50_s": 0.23927149176597595, "p95_s": 0.35407450795173645, "cost_usd_per_1000": 0.025951254681647943, "axes": { "intelligence": 78.33875542208877, "calibration": 61.35164946854124, "speed": 90.72014382446159, "cost": 57.57524920597589 }, "jevbench_score": 70.78408408803388 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6788990825688074 }, "accuracy": 0.834983498349835, "ece_hard": 0.1935286337386598, "calibration": 64.09002035150621, "p50_s": 0.24110446870326996, "p95_s": 0.3450261786580081, "cost_usd_per_1000": 0.025951254681647943, "axes": { "intelligence": 82.6158309245528, "calibration": 64.09002035150621, "speed": 90.7994269204109, "cost": 57.57524920597589 }, "jevbench_score": 72.534254981433 } }, { "threshold": 0.04, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6486486486486487 }, "accuracy": 0.7748917748917749, "ece_hard": 0.21041641822205612, "calibration": 61.35164946854124, "p50_s": 0.23927149176597595, "p95_s": 0.35407450795173645, "cost_usd_per_1000": 0.025951254681647943, "axes": { "intelligence": 78.33875542208877, "calibration": 61.35164946854124, "speed": 90.72014382446159, "cost": 57.57524920597589 }, "jevbench_score": 70.78408408803388 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6788990825688074 }, "accuracy": 0.834983498349835, "ece_hard": 0.1935286337386598, "calibration": 64.09002035150621, "p50_s": 0.24110446870326996, "p95_s": 0.3450261786580081, "cost_usd_per_1000": 0.025951254681647943, "axes": { "intelligence": 82.6158309245528, "calibration": 64.09002035150621, "speed": 90.7994269204109, "cost": 57.57524920597589 }, "jevbench_score": 72.534254981433 } }, { "threshold": 0.05, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6486486486486487 }, "accuracy": 0.7748917748917749, "ece_hard": 0.21041641822205612, "calibration": 61.35164946854124, "p50_s": 0.23927149176597595, "p95_s": 0.35407450795173645, "cost_usd_per_1000": 0.025951254681647943, "axes": { "intelligence": 78.33875542208877, "calibration": 61.35164946854124, "speed": 90.72014382446159, "cost": 57.57524920597589 }, "jevbench_score": 70.78408408803388 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6788990825688074 }, "accuracy": 0.834983498349835, "ece_hard": 0.1935286337386598, "calibration": 64.09002035150621, "p50_s": 0.24110446870326996, "p95_s": 0.3450261786580081, "cost_usd_per_1000": 0.025951254681647943, "axes": { "intelligence": 82.6158309245528, "calibration": 64.09002035150621, "speed": 90.7994269204109, "cost": 57.57524920597589 }, "jevbench_score": 72.534254981433 } }, { "threshold": 0.06, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6486486486486487 }, "accuracy": 0.7748917748917749, "ece_hard": 0.21041641822205612, "calibration": 61.35164946854124, "p50_s": 0.23927149176597595, "p95_s": 0.35407450795173645, "cost_usd_per_1000": 0.025951254681647943, "axes": { "intelligence": 78.33875542208877, "calibration": 61.35164946854124, "speed": 90.72014382446159, "cost": 57.57524920597589 }, "jevbench_score": 70.78408408803388 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6788990825688074 }, "accuracy": 0.834983498349835, "ece_hard": 0.1935286337386598, "calibration": 64.09002035150621, "p50_s": 0.24110446870326996, "p95_s": 0.3450261786580081, "cost_usd_per_1000": 0.025951254681647943, "axes": { "intelligence": 82.6158309245528, "calibration": 64.09002035150621, "speed": 90.7994269204109, "cost": 57.57524920597589 }, "jevbench_score": 72.534254981433 } }, { "threshold": 0.07, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6486486486486487 }, "accuracy": 0.7748917748917749, "ece_hard": 0.21041641822205612, "calibration": 61.35164946854124, "p50_s": 0.23927149176597595, "p95_s": 0.35407450795173645, "cost_usd_per_1000": 0.025951254681647943, "axes": { "intelligence": 78.33875542208877, "calibration": 61.35164946854124, "speed": 90.72014382446159, "cost": 57.57524920597589 }, "jevbench_score": 70.78408408803388 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6788990825688074 }, "accuracy": 0.834983498349835, "ece_hard": 0.1935286337386598, "calibration": 64.09002035150621, "p50_s": 0.24110446870326996, "p95_s": 0.3450261786580081, "cost_usd_per_1000": 0.025951254681647943, "axes": { "intelligence": 82.6158309245528, "calibration": 64.09002035150621, "speed": 90.7994269204109, "cost": 57.57524920597589 }, "jevbench_score": 72.534254981433 } }, { "threshold": 0.08, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6486486486486487 }, "accuracy": 0.7748917748917749, "ece_hard": 0.21041641822205612, "calibration": 61.35164946854124, "p50_s": 0.23927149176597595, "p95_s": 0.35407450795173645, "cost_usd_per_1000": 0.025951254681647943, "axes": { "intelligence": 78.33875542208877, "calibration": 61.35164946854124, "speed": 90.72014382446159, "cost": 57.57524920597589 }, "jevbench_score": 70.78408408803388 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6788990825688074 }, "accuracy": 0.834983498349835, "ece_hard": 0.1935286337386598, "calibration": 64.09002035150621, "p50_s": 0.24110446870326996, "p95_s": 0.3450261786580081, "cost_usd_per_1000": 0.025951254681647943, "axes": { "intelligence": 82.6158309245528, "calibration": 64.09002035150621, "speed": 90.7994269204109, "cost": 57.57524920597589 }, "jevbench_score": 72.534254981433 } }, { "threshold": 0.09, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6486486486486487 }, "accuracy": 0.7748917748917749, "ece_hard": 0.21041641822205612, "calibration": 61.35164946854124, "p50_s": 0.23927149176597595, "p95_s": 0.35407450795173645, "cost_usd_per_1000": 0.025951254681647943, "axes": { "intelligence": 78.33875542208877, "calibration": 61.35164946854124, "speed": 90.72014382446159, "cost": 57.57524920597589 }, "jevbench_score": 70.78408408803388 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6788990825688074 }, "accuracy": 0.834983498349835, "ece_hard": 0.1935286337386598, "calibration": 64.09002035150621, "p50_s": 0.24110446870326996, "p95_s": 0.3450261786580081, "cost_usd_per_1000": 0.025951254681647943, "axes": { "intelligence": 82.6158309245528, "calibration": 64.09002035150621, "speed": 90.7994269204109, "cost": 57.57524920597589 }, "jevbench_score": 72.534254981433 } }, { "threshold": 0.1, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6486486486486487 }, "accuracy": 0.7748917748917749, "ece_hard": 0.21041641822205612, "calibration": 61.35164946854124, "p50_s": 0.23927149176597595, "p95_s": 0.35407450795173645, "cost_usd_per_1000": 0.025951254681647943, "axes": { "intelligence": 78.33875542208877, "calibration": 61.35164946854124, "speed": 90.72014382446159, "cost": 57.57524920597589 }, "jevbench_score": 70.78408408803388 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6788990825688074 }, "accuracy": 0.834983498349835, "ece_hard": 0.1935286337386598, "calibration": 64.09002035150621, "p50_s": 0.24110446870326996, "p95_s": 0.3450261786580081, "cost_usd_per_1000": 0.025951254681647943, "axes": { "intelligence": 82.6158309245528, "calibration": 64.09002035150621, "speed": 90.7994269204109, "cost": 57.57524920597589 }, "jevbench_score": 72.534254981433 } }, { "threshold": 0.11, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6486486486486487 }, "accuracy": 0.7748917748917749, "ece_hard": 0.21041641822205612, "calibration": 61.35164946854124, "p50_s": 0.23927149176597595, "p95_s": 0.35407450795173645, "cost_usd_per_1000": 0.025951254681647943, "axes": { "intelligence": 78.33875542208877, "calibration": 61.35164946854124, "speed": 90.72014382446159, "cost": 57.57524920597589 }, "jevbench_score": 70.78408408803388 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6788990825688074 }, "accuracy": 0.834983498349835, "ece_hard": 0.1935286337386598, "calibration": 64.09002035150621, "p50_s": 0.24110446870326996, "p95_s": 0.3450261786580081, "cost_usd_per_1000": 0.025951254681647943, "axes": { "intelligence": 82.6158309245528, "calibration": 64.09002035150621, "speed": 90.7994269204109, "cost": 57.57524920597589 }, "jevbench_score": 72.534254981433 } }, { "threshold": 0.12, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6486486486486487 }, "accuracy": 0.7748917748917749, "ece_hard": 0.21041641822205612, "calibration": 61.35164946854124, "p50_s": 0.23927149176597595, "p95_s": 0.35407450795173645, "cost_usd_per_1000": 0.025951254681647943, "axes": { "intelligence": 78.33875542208877, "calibration": 61.35164946854124, "speed": 90.72014382446159, "cost": 57.57524920597589 }, "jevbench_score": 70.78408408803388 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6788990825688074 }, "accuracy": 0.834983498349835, "ece_hard": 0.1935286337386598, "calibration": 64.09002035150621, "p50_s": 0.24110446870326996, "p95_s": 0.3450261786580081, "cost_usd_per_1000": 0.025951254681647943, "axes": { "intelligence": 82.6158309245528, "calibration": 64.09002035150621, "speed": 90.7994269204109, "cost": 57.57524920597589 }, "jevbench_score": 72.534254981433 } }, { "threshold": 0.13, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6486486486486487 }, "accuracy": 0.7748917748917749, "ece_hard": 0.21041641822205612, "calibration": 61.35164946854124, "p50_s": 0.23927149176597595, "p95_s": 0.35407450795173645, "cost_usd_per_1000": 0.025951254681647943, "axes": { "intelligence": 78.33875542208877, "calibration": 61.35164946854124, "speed": 90.72014382446159, "cost": 57.57524920597589 }, "jevbench_score": 70.78408408803388 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6788990825688074 }, "accuracy": 0.834983498349835, "ece_hard": 0.1935286337386598, "calibration": 64.09002035150621, "p50_s": 0.24110446870326996, "p95_s": 0.3450261786580081, "cost_usd_per_1000": 0.025951254681647943, "axes": { "intelligence": 82.6158309245528, "calibration": 64.09002035150621, "speed": 90.7994269204109, "cost": 57.57524920597589 }, "jevbench_score": 72.534254981433 } }, { "threshold": 0.14, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6486486486486487 }, "accuracy": 0.7748917748917749, "ece_hard": 0.21041641822205612, "calibration": 61.35164946854124, "p50_s": 0.23927149176597595, "p95_s": 0.35407450795173645, "cost_usd_per_1000": 0.025951254681647943, "axes": { "intelligence": 78.33875542208877, "calibration": 61.35164946854124, "speed": 90.72014382446159, "cost": 57.57524920597589 }, "jevbench_score": 70.78408408803388 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6788990825688074 }, "accuracy": 0.834983498349835, "ece_hard": 0.1935286337386598, "calibration": 64.09002035150621, "p50_s": 0.24110446870326996, "p95_s": 0.3450261786580081, "cost_usd_per_1000": 0.025951254681647943, "axes": { "intelligence": 82.6158309245528, "calibration": 64.09002035150621, "speed": 90.7994269204109, "cost": 57.57524920597589 }, "jevbench_score": 72.534254981433 } }, { "threshold": 0.15, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6486486486486487 }, "accuracy": 0.7748917748917749, "ece_hard": 0.21041641822205612, "calibration": 61.35164946854124, "p50_s": 0.23927149176597595, "p95_s": 0.35407450795173645, "cost_usd_per_1000": 0.025951254681647943, "axes": { "intelligence": 78.33875542208877, "calibration": 61.35164946854124, "speed": 90.72014382446159, "cost": 57.57524920597589 }, "jevbench_score": 70.78408408803388 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6788990825688074 }, "accuracy": 0.834983498349835, "ece_hard": 0.1935286337386598, "calibration": 64.09002035150621, "p50_s": 0.24110446870326996, "p95_s": 0.3450261786580081, "cost_usd_per_1000": 0.025951254681647943, "axes": { "intelligence": 82.6158309245528, "calibration": 64.09002035150621, "speed": 90.7994269204109, "cost": 57.57524920597589 }, "jevbench_score": 72.534254981433 } }, { "threshold": 0.16, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6486486486486487 }, "accuracy": 0.7748917748917749, "ece_hard": 0.21041641822205612, "calibration": 61.35164946854124, "p50_s": 0.23927149176597595, "p95_s": 0.35407450795173645, "cost_usd_per_1000": 0.025951254681647943, "axes": { "intelligence": 78.33875542208877, "calibration": 61.35164946854124, "speed": 90.72014382446159, "cost": 57.57524920597589 }, "jevbench_score": 70.78408408803388 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6788990825688074 }, "accuracy": 0.834983498349835, "ece_hard": 0.1935286337386598, "calibration": 64.09002035150621, "p50_s": 0.24110446870326996, "p95_s": 0.3450261786580081, "cost_usd_per_1000": 0.025951254681647943, "axes": { "intelligence": 82.6158309245528, "calibration": 64.09002035150621, "speed": 90.7994269204109, "cost": 57.57524920597589 }, "jevbench_score": 72.534254981433 } }, { "threshold": 0.17, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6486486486486487 }, "accuracy": 0.7748917748917749, "ece_hard": 0.21041641822205612, "calibration": 61.35164946854124, "p50_s": 0.23927149176597595, "p95_s": 0.35407450795173645, "cost_usd_per_1000": 0.025951254681647943, "axes": { "intelligence": 78.33875542208877, "calibration": 61.35164946854124, "speed": 90.72014382446159, "cost": 57.57524920597589 }, "jevbench_score": 70.78408408803388 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6788990825688074 }, "accuracy": 0.834983498349835, "ece_hard": 0.1935286337386598, "calibration": 64.09002035150621, "p50_s": 0.24110446870326996, "p95_s": 0.3450261786580081, "cost_usd_per_1000": 0.025951254681647943, "axes": { "intelligence": 82.6158309245528, "calibration": 64.09002035150621, "speed": 90.7994269204109, "cost": 57.57524920597589 }, "jevbench_score": 72.534254981433 } }, { "threshold": 0.18, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6486486486486487 }, "accuracy": 0.7748917748917749, "ece_hard": 0.21041641822205612, "calibration": 61.35164946854124, "p50_s": 0.23927149176597595, "p95_s": 0.35407450795173645, "cost_usd_per_1000": 0.025951254681647943, "axes": { "intelligence": 78.33875542208877, "calibration": 61.35164946854124, "speed": 90.72014382446159, "cost": 57.57524920597589 }, "jevbench_score": 70.78408408803388 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6788990825688074 }, "accuracy": 0.834983498349835, "ece_hard": 0.1935286337386598, "calibration": 64.09002035150621, "p50_s": 0.24110446870326996, "p95_s": 0.3450261786580081, "cost_usd_per_1000": 0.025951254681647943, "axes": { "intelligence": 82.6158309245528, "calibration": 64.09002035150621, "speed": 90.7994269204109, "cost": 57.57524920597589 }, "jevbench_score": 72.534254981433 } }, { "threshold": 0.19, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6486486486486487 }, "accuracy": 0.7748917748917749, "ece_hard": 0.21041641822205612, "calibration": 61.35164946854124, "p50_s": 0.23927149176597595, "p95_s": 0.35407450795173645, "cost_usd_per_1000": 0.025951254681647943, "axes": { "intelligence": 78.33875542208877, "calibration": 61.35164946854124, "speed": 90.72014382446159, "cost": 57.57524920597589 }, "jevbench_score": 70.78408408803388 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6788990825688074 }, "accuracy": 0.834983498349835, "ece_hard": 0.1935286337386598, "calibration": 64.09002035150621, "p50_s": 0.24110446870326996, "p95_s": 0.3450261786580081, "cost_usd_per_1000": 0.025951254681647943, "axes": { "intelligence": 82.6158309245528, "calibration": 64.09002035150621, "speed": 90.7994269204109, "cost": 57.57524920597589 }, "jevbench_score": 72.534254981433 } }, { "threshold": 0.2, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6486486486486487 }, "accuracy": 0.7748917748917749, "ece_hard": 0.21041641822205612, "calibration": 61.35164946854124, "p50_s": 0.23927149176597595, "p95_s": 0.35407450795173645, "cost_usd_per_1000": 0.025951254681647943, "axes": { "intelligence": 78.33875542208877, "calibration": 61.35164946854124, "speed": 90.72014382446159, "cost": 57.57524920597589 }, "jevbench_score": 70.78408408803388 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6788990825688074 }, "accuracy": 0.834983498349835, "ece_hard": 0.1935286337386598, "calibration": 64.09002035150621, "p50_s": 0.24110446870326996, "p95_s": 0.3450261786580081, "cost_usd_per_1000": 0.025951254681647943, "axes": { "intelligence": 82.6158309245528, "calibration": 64.09002035150621, "speed": 90.7994269204109, "cost": 57.57524920597589 }, "jevbench_score": 72.534254981433 } }, { "threshold": 0.21, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6486486486486487 }, "accuracy": 0.7748917748917749, "ece_hard": 0.21041641822205612, "calibration": 61.35164946854124, "p50_s": 0.23927149176597595, "p95_s": 0.35407450795173645, "cost_usd_per_1000": 0.025951254681647943, "axes": { "intelligence": 78.33875542208877, "calibration": 61.35164946854124, "speed": 90.72014382446159, "cost": 57.57524920597589 }, "jevbench_score": 70.78408408803388 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6788990825688074 }, "accuracy": 0.834983498349835, "ece_hard": 0.1935286337386598, "calibration": 64.09002035150621, "p50_s": 0.24110446870326996, "p95_s": 0.3450261786580081, "cost_usd_per_1000": 0.025951254681647943, "axes": { "intelligence": 82.6158309245528, "calibration": 64.09002035150621, "speed": 90.7994269204109, "cost": 57.57524920597589 }, "jevbench_score": 72.534254981433 } }, { "threshold": 0.22, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6486486486486487 }, "accuracy": 0.7748917748917749, "ece_hard": 0.21041641822205612, "calibration": 61.35164946854124, "p50_s": 0.23927149176597595, "p95_s": 0.35407450795173645, "cost_usd_per_1000": 0.025951254681647943, "axes": { "intelligence": 78.33875542208877, "calibration": 61.35164946854124, "speed": 90.72014382446159, "cost": 57.57524920597589 }, "jevbench_score": 70.78408408803388 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6788990825688074 }, "accuracy": 0.834983498349835, "ece_hard": 0.1935286337386598, "calibration": 64.09002035150621, "p50_s": 0.24110446870326996, "p95_s": 0.3450261786580081, "cost_usd_per_1000": 0.025951254681647943, "axes": { "intelligence": 82.6158309245528, "calibration": 64.09002035150621, "speed": 90.7994269204109, "cost": 57.57524920597589 }, "jevbench_score": 72.534254981433 } }, { "threshold": 0.23, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6486486486486487 }, "accuracy": 0.7748917748917749, "ece_hard": 0.21041641822205612, "calibration": 61.35164946854124, "p50_s": 0.23927149176597595, "p95_s": 0.35407450795173645, "cost_usd_per_1000": 0.025951254681647943, "axes": { "intelligence": 78.33875542208877, "calibration": 61.35164946854124, "speed": 90.72014382446159, "cost": 57.57524920597589 }, "jevbench_score": 70.78408408803388 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6788990825688074 }, "accuracy": 0.834983498349835, "ece_hard": 0.1935286337386598, "calibration": 64.09002035150621, "p50_s": 0.24110446870326996, "p95_s": 0.3450261786580081, "cost_usd_per_1000": 0.025951254681647943, "axes": { "intelligence": 82.6158309245528, "calibration": 64.09002035150621, "speed": 90.7994269204109, "cost": 57.57524920597589 }, "jevbench_score": 72.534254981433 } }, { "threshold": 0.24, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6486486486486487 }, "accuracy": 0.7748917748917749, "ece_hard": 0.21041641822205612, "calibration": 61.35164946854124, "p50_s": 0.23927149176597595, "p95_s": 0.35407450795173645, "cost_usd_per_1000": 0.025951254681647943, "axes": { "intelligence": 78.33875542208877, "calibration": 61.35164946854124, "speed": 90.72014382446159, "cost": 57.57524920597589 }, "jevbench_score": 70.78408408803388 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6788990825688074 }, "accuracy": 0.834983498349835, "ece_hard": 0.1935286337386598, "calibration": 64.09002035150621, "p50_s": 0.24110446870326996, "p95_s": 0.3450261786580081, "cost_usd_per_1000": 0.025951254681647943, "axes": { "intelligence": 82.6158309245528, "calibration": 64.09002035150621, "speed": 90.7994269204109, "cost": 57.57524920597589 }, "jevbench_score": 72.534254981433 } }, { "threshold": 0.25, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6486486486486487 }, "accuracy": 0.7748917748917749, "ece_hard": 0.21041641822205612, "calibration": 61.35164946854124, "p50_s": 0.23927149176597595, "p95_s": 0.35407450795173645, "cost_usd_per_1000": 0.025951254681647943, "axes": { "intelligence": 78.33875542208877, "calibration": 61.35164946854124, "speed": 90.72014382446159, "cost": 57.57524920597589 }, "jevbench_score": 70.78408408803388 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6788990825688074 }, "accuracy": 0.834983498349835, "ece_hard": 0.1935286337386598, "calibration": 64.09002035150621, "p50_s": 0.24110446870326996, "p95_s": 0.3450261786580081, "cost_usd_per_1000": 0.025951254681647943, "axes": { "intelligence": 82.6158309245528, "calibration": 64.09002035150621, "speed": 90.7994269204109, "cost": 57.57524920597589 }, "jevbench_score": 72.534254981433 } }, { "threshold": 0.26, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6486486486486487 }, "accuracy": 0.7748917748917749, "ece_hard": 0.21041641822205612, "calibration": 61.35164946854124, "p50_s": 0.23927149176597595, "p95_s": 0.35407450795173645, "cost_usd_per_1000": 0.025951254681647943, "axes": { "intelligence": 78.33875542208877, "calibration": 61.35164946854124, "speed": 90.72014382446159, "cost": 57.57524920597589 }, "jevbench_score": 70.78408408803388 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6788990825688074 }, "accuracy": 0.834983498349835, "ece_hard": 0.1935286337386598, "calibration": 64.09002035150621, "p50_s": 0.24110446870326996, "p95_s": 0.3450261786580081, "cost_usd_per_1000": 0.025951254681647943, "axes": { "intelligence": 82.6158309245528, "calibration": 64.09002035150621, "speed": 90.7994269204109, "cost": 57.57524920597589 }, "jevbench_score": 72.534254981433 } }, { "threshold": 0.27, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6486486486486487 }, "accuracy": 0.7748917748917749, "ece_hard": 0.21041641822205612, "calibration": 61.35164946854124, "p50_s": 0.23927149176597595, "p95_s": 0.35407450795173645, "cost_usd_per_1000": 0.025951254681647943, "axes": { "intelligence": 78.33875542208877, "calibration": 61.35164946854124, "speed": 90.72014382446159, "cost": 57.57524920597589 }, "jevbench_score": 70.78408408803388 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6788990825688074 }, "accuracy": 0.834983498349835, "ece_hard": 0.1935286337386598, "calibration": 64.09002035150621, "p50_s": 0.24110446870326996, "p95_s": 0.3450261786580081, "cost_usd_per_1000": 0.025951254681647943, "axes": { "intelligence": 82.6158309245528, "calibration": 64.09002035150621, "speed": 90.7994269204109, "cost": 57.57524920597589 }, "jevbench_score": 72.534254981433 } }, { "threshold": 0.28, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6486486486486487 }, "accuracy": 0.7748917748917749, "ece_hard": 0.21041641822205612, "calibration": 61.35164946854124, "p50_s": 0.23927149176597595, "p95_s": 0.35407450795173645, "cost_usd_per_1000": 0.025951254681647943, "axes": { "intelligence": 78.33875542208877, "calibration": 61.35164946854124, "speed": 90.72014382446159, "cost": 57.57524920597589 }, "jevbench_score": 70.78408408803388 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6788990825688074 }, "accuracy": 0.834983498349835, "ece_hard": 0.1935286337386598, "calibration": 64.09002035150621, "p50_s": 0.24110446870326996, "p95_s": 0.3450261786580081, "cost_usd_per_1000": 0.025951254681647943, "axes": { "intelligence": 82.6158309245528, "calibration": 64.09002035150621, "speed": 90.7994269204109, "cost": 57.57524920597589 }, "jevbench_score": 72.534254981433 } }, { "threshold": 0.29, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6486486486486487 }, "accuracy": 0.7748917748917749, "ece_hard": 0.21041641822205612, "calibration": 61.35164946854124, "p50_s": 0.23927149176597595, "p95_s": 0.35407450795173645, "cost_usd_per_1000": 0.025951254681647943, "axes": { "intelligence": 78.33875542208877, "calibration": 61.35164946854124, "speed": 90.72014382446159, "cost": 57.57524920597589 }, "jevbench_score": 70.78408408803388 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6788990825688074 }, "accuracy": 0.834983498349835, "ece_hard": 0.1935286337386598, "calibration": 64.09002035150621, "p50_s": 0.24110446870326996, "p95_s": 0.3450261786580081, "cost_usd_per_1000": 0.025951254681647943, "axes": { "intelligence": 82.6158309245528, "calibration": 64.09002035150621, "speed": 90.7994269204109, "cost": 57.57524920597589 }, "jevbench_score": 72.534254981433 } }, { "threshold": 0.3, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6486486486486487 }, "accuracy": 0.7748917748917749, "ece_hard": 0.21041641822205612, "calibration": 61.35164946854124, "p50_s": 0.23927149176597595, "p95_s": 0.35407450795173645, "cost_usd_per_1000": 0.025951254681647943, "axes": { "intelligence": 78.33875542208877, "calibration": 61.35164946854124, "speed": 90.72014382446159, "cost": 57.57524920597589 }, "jevbench_score": 70.78408408803388 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6788990825688074 }, "accuracy": 0.834983498349835, "ece_hard": 0.1935286337386598, "calibration": 64.09002035150621, "p50_s": 0.24110446870326996, "p95_s": 0.3450261786580081, "cost_usd_per_1000": 0.025951254681647943, "axes": { "intelligence": 82.6158309245528, "calibration": 64.09002035150621, "speed": 90.7994269204109, "cost": 57.57524920597589 }, "jevbench_score": 72.534254981433 } }, { "threshold": 0.31, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6486486486486487 }, "accuracy": 0.7748917748917749, "ece_hard": 0.21041641822205612, "calibration": 61.35164946854124, "p50_s": 0.23927149176597595, "p95_s": 0.35407450795173645, "cost_usd_per_1000": 0.025951254681647943, "axes": { "intelligence": 78.33875542208877, "calibration": 61.35164946854124, "speed": 90.72014382446159, "cost": 57.57524920597589 }, "jevbench_score": 70.78408408803388 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6788990825688074 }, "accuracy": 0.834983498349835, "ece_hard": 0.1935286337386598, "calibration": 64.09002035150621, "p50_s": 0.24110446870326996, "p95_s": 0.3450261786580081, "cost_usd_per_1000": 0.025951254681647943, "axes": { "intelligence": 82.6158309245528, "calibration": 64.09002035150621, "speed": 90.7994269204109, "cost": 57.57524920597589 }, "jevbench_score": 72.534254981433 } }, { "threshold": 0.32, "public": { "n": 231, "escalation": 0.004329004329004329, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6486486486486487 }, "accuracy": 0.7748917748917749, "ece_hard": 0.2108745807410753, "calibration": 61.30583321663932, "p50_s": 0.23927149176597595, "p95_s": 0.3553285673260689, "cost_usd_per_1000": 0.026124043581886282, "axes": { "intelligence": 78.33875542208877, "calibration": 61.30583321663932, "speed": 90.70478917905459, "cost": 57.4887880117425 }, "jevbench_score": 70.74128745994828 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6788990825688074 }, "accuracy": 0.834983498349835, "ece_hard": 0.1935286337386598, "calibration": 64.09002035150621, "p50_s": 0.24110446870326996, "p95_s": 0.3450261786580081, "cost_usd_per_1000": 0.025951254681647943, "axes": { "intelligence": 82.6158309245528, "calibration": 64.09002035150621, "speed": 90.7994269204109, "cost": 57.57524920597589 }, "jevbench_score": 72.534254981433 } }, { "threshold": 0.33, "public": { "n": 231, "escalation": 0.004329004329004329, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6486486486486487 }, "accuracy": 0.7748917748917749, "ece_hard": 0.2108745807410753, "calibration": 61.30583321663932, "p50_s": 0.23927149176597595, "p95_s": 0.3553285673260689, "cost_usd_per_1000": 0.026124043581886282, "axes": { "intelligence": 78.33875542208877, "calibration": 61.30583321663932, "speed": 90.70478917905459, "cost": 57.4887880117425 }, "jevbench_score": 70.74128745994828 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6788990825688074 }, "accuracy": 0.834983498349835, "ece_hard": 0.1935286337386598, "calibration": 64.09002035150621, "p50_s": 0.24110446870326996, "p95_s": 0.3450261786580081, "cost_usd_per_1000": 0.025951254681647943, "axes": { "intelligence": 82.6158309245528, "calibration": 64.09002035150621, "speed": 90.7994269204109, "cost": 57.57524920597589 }, "jevbench_score": 72.534254981433 } }, { "threshold": 0.34, "public": { "n": 231, "escalation": 0.004329004329004329, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6486486486486487 }, "accuracy": 0.7748917748917749, "ece_hard": 0.2108745807410753, "calibration": 61.30583321663932, "p50_s": 0.23927149176597595, "p95_s": 0.3553285673260689, "cost_usd_per_1000": 0.026124043581886282, "axes": { "intelligence": 78.33875542208877, "calibration": 61.30583321663932, "speed": 90.70478917905459, "cost": 57.4887880117425 }, "jevbench_score": 70.74128745994828 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6788990825688074 }, "accuracy": 0.834983498349835, "ece_hard": 0.1935286337386598, "calibration": 64.09002035150621, "p50_s": 0.24110446870326996, "p95_s": 0.3450261786580081, "cost_usd_per_1000": 0.025951254681647943, "axes": { "intelligence": 82.6158309245528, "calibration": 64.09002035150621, "speed": 90.7994269204109, "cost": 57.57524920597589 }, "jevbench_score": 72.534254981433 } }, { "threshold": 0.35, "public": { "n": 231, "escalation": 0.004329004329004329, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6486486486486487 }, "accuracy": 0.7748917748917749, "ece_hard": 0.2108745807410753, "calibration": 61.30583321663932, "p50_s": 0.23927149176597595, "p95_s": 0.3553285673260689, "cost_usd_per_1000": 0.026124043581886282, "axes": { "intelligence": 78.33875542208877, "calibration": 61.30583321663932, "speed": 90.70478917905459, "cost": 57.4887880117425 }, "jevbench_score": 70.74128745994828 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6788990825688074 }, "accuracy": 0.834983498349835, "ece_hard": 0.1935286337386598, "calibration": 64.09002035150621, "p50_s": 0.24110446870326996, "p95_s": 0.3450261786580081, "cost_usd_per_1000": 0.025951254681647943, "axes": { "intelligence": 82.6158309245528, "calibration": 64.09002035150621, "speed": 90.7994269204109, "cost": 57.57524920597589 }, "jevbench_score": 72.534254981433 } }, { "threshold": 0.36, "public": { "n": 231, "escalation": 0.004329004329004329, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6486486486486487 }, "accuracy": 0.7748917748917749, "ece_hard": 0.2108745807410753, "calibration": 61.30583321663932, "p50_s": 0.23927149176597595, "p95_s": 0.3553285673260689, "cost_usd_per_1000": 0.026124043581886282, "axes": { "intelligence": 78.33875542208877, "calibration": 61.30583321663932, "speed": 90.70478917905459, "cost": 57.4887880117425 }, "jevbench_score": 70.74128745994828 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6788990825688074 }, "accuracy": 0.834983498349835, "ece_hard": 0.1935286337386598, "calibration": 64.09002035150621, "p50_s": 0.24110446870326996, "p95_s": 0.3450261786580081, "cost_usd_per_1000": 0.025951254681647943, "axes": { "intelligence": 82.6158309245528, "calibration": 64.09002035150621, "speed": 90.7994269204109, "cost": 57.57524920597589 }, "jevbench_score": 72.534254981433 } }, { "threshold": 0.37, "public": { "n": 231, "escalation": 0.004329004329004329, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6486486486486487 }, "accuracy": 0.7748917748917749, "ece_hard": 0.2108745807410753, "calibration": 61.30583321663932, "p50_s": 0.23927149176597595, "p95_s": 0.3553285673260689, "cost_usd_per_1000": 0.026124043581886282, "axes": { "intelligence": 78.33875542208877, "calibration": 61.30583321663932, "speed": 90.70478917905459, "cost": 57.4887880117425 }, "jevbench_score": 70.74128745994828 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6788990825688074 }, "accuracy": 0.834983498349835, "ece_hard": 0.1935286337386598, "calibration": 64.09002035150621, "p50_s": 0.24110446870326996, "p95_s": 0.3450261786580081, "cost_usd_per_1000": 0.025951254681647943, "axes": { "intelligence": 82.6158309245528, "calibration": 64.09002035150621, "speed": 90.7994269204109, "cost": 57.57524920597589 }, "jevbench_score": 72.534254981433 } }, { "threshold": 0.38, "public": { "n": 231, "escalation": 0.004329004329004329, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6486486486486487 }, "accuracy": 0.7748917748917749, "ece_hard": 0.2108745807410753, "calibration": 61.30583321663932, "p50_s": 0.23927149176597595, "p95_s": 0.3553285673260689, "cost_usd_per_1000": 0.026124043581886282, "axes": { "intelligence": 78.33875542208877, "calibration": 61.30583321663932, "speed": 90.70478917905459, "cost": 57.4887880117425 }, "jevbench_score": 70.74128745994828 }, "heldout": { "n": 303, "escalation": 0.006600660066006601, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6697247706422018 }, "accuracy": 0.8316831683168316, "ece_hard": 0.20194184154595135, "calibration": 63.24869957077706, "p50_s": 0.24110446870326996, "p95_s": 0.35049599558114997, "cost_usd_per_1000": 0.02621471498498165, "axes": { "intelligence": 82.34060156675464, "calibration": 63.24869957077706, "speed": 90.73111679688483, "cost": 57.44364579151855 }, "jevbench_score": 72.17984948213862 } }, { "threshold": 0.39, "public": { "n": 231, "escalation": 0.004329004329004329, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6486486486486487 }, "accuracy": 0.7748917748917749, "ece_hard": 0.2108745807410753, "calibration": 61.30583321663932, "p50_s": 0.23927149176597595, "p95_s": 0.3553285673260689, "cost_usd_per_1000": 0.026124043581886282, "axes": { "intelligence": 78.33875542208877, "calibration": 61.30583321663932, "speed": 90.70478917905459, "cost": 57.4887880117425 }, "jevbench_score": 70.74128745994828 }, "heldout": { "n": 303, "escalation": 0.006600660066006601, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6697247706422018 }, "accuracy": 0.8316831683168316, "ece_hard": 0.20194184154595135, "calibration": 63.24869957077706, "p50_s": 0.24110446870326996, "p95_s": 0.35049599558114997, "cost_usd_per_1000": 0.02621471498498165, "axes": { "intelligence": 82.34060156675464, "calibration": 63.24869957077706, "speed": 90.73111679688483, "cost": 57.44364579151855 }, "jevbench_score": 72.17984948213862 } }, { "threshold": 0.4, "public": { "n": 231, "escalation": 0.004329004329004329, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6486486486486487 }, "accuracy": 0.7748917748917749, "ece_hard": 0.2108745807410753, "calibration": 61.30583321663932, "p50_s": 0.23927149176597595, "p95_s": 0.3553285673260689, "cost_usd_per_1000": 0.026124043581886282, "axes": { "intelligence": 78.33875542208877, "calibration": 61.30583321663932, "speed": 90.70478917905459, "cost": 57.4887880117425 }, "jevbench_score": 70.74128745994828 }, "heldout": { "n": 303, "escalation": 0.009900990099009901, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6697247706422018 }, "accuracy": 0.8316831683168316, "ece_hard": 0.20249800676672494, "calibration": 63.193083048699705, "p50_s": 0.24146977066993713, "p95_s": 0.3535130485892295, "cost_usd_per_1000": 0.026346445136648498, "axes": { "intelligence": 82.34060156675464, "calibration": 63.193083048699705, "speed": 90.68731781591922, "cost": 57.37833924468609 }, "jevbench_score": 72.13474945923463 } }, { "threshold": 0.41, "public": { "n": 231, "escalation": 0.004329004329004329, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6486486486486487 }, "accuracy": 0.7748917748917749, "ece_hard": 0.2108745807410753, "calibration": 61.30583321663932, "p50_s": 0.23927149176597595, "p95_s": 0.3553285673260689, "cost_usd_per_1000": 0.026124043581886282, "axes": { "intelligence": 78.33875542208877, "calibration": 61.30583321663932, "speed": 90.70478917905459, "cost": 57.4887880117425 }, "jevbench_score": 70.74128745994828 }, "heldout": { "n": 303, "escalation": 0.009900990099009901, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6697247706422018 }, "accuracy": 0.8316831683168316, "ece_hard": 0.20249800676672494, "calibration": 63.193083048699705, "p50_s": 0.24146977066993713, "p95_s": 0.3535130485892295, "cost_usd_per_1000": 0.026346445136648498, "axes": { "intelligence": 82.34060156675464, "calibration": 63.193083048699705, "speed": 90.68731781591922, "cost": 57.37833924468609 }, "jevbench_score": 72.13474945923463 } }, { "threshold": 0.42, "public": { "n": 231, "escalation": 0.008658008658008658, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6486486486486487 }, "accuracy": 0.7748917748917749, "ece_hard": 0.21070112410193095, "calibration": 61.323178880553755, "p50_s": 0.23927149176597595, "p95_s": 0.358288049697876, "cost_usd_per_1000": 0.02629683248212462, "axes": { "intelligence": 78.33875542208877, "calibration": 61.323178880553755, "speed": 90.66876719746664, "cost": 57.40289680581747 }, "jevbench_score": 70.71282892159891 }, "heldout": { "n": 303, "escalation": 0.009900990099009901, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6697247706422018 }, "accuracy": 0.8316831683168316, "ece_hard": 0.20249800676672494, "calibration": 63.193083048699705, "p50_s": 0.24146977066993713, "p95_s": 0.3535130485892295, "cost_usd_per_1000": 0.026346445136648498, "axes": { "intelligence": 82.34060156675464, "calibration": 63.193083048699705, "speed": 90.68731781591922, "cost": 57.37833924468609 }, "jevbench_score": 72.13474945923463 } }, { "threshold": 0.43, "public": { "n": 231, "escalation": 0.008658008658008658, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6486486486486487 }, "accuracy": 0.7748917748917749, "ece_hard": 0.21070112410193095, "calibration": 61.323178880553755, "p50_s": 0.23927149176597595, "p95_s": 0.358288049697876, "cost_usd_per_1000": 0.02629683248212462, "axes": { "intelligence": 78.33875542208877, "calibration": 61.323178880553755, "speed": 90.66876719746664, "cost": 57.40289680581747 }, "jevbench_score": 70.71282892159891 }, "heldout": { "n": 303, "escalation": 0.009900990099009901, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6697247706422018 }, "accuracy": 0.8316831683168316, "ece_hard": 0.20249800676672494, "calibration": 63.193083048699705, "p50_s": 0.24146977066993713, "p95_s": 0.3535130485892295, "cost_usd_per_1000": 0.026346445136648498, "axes": { "intelligence": 82.34060156675464, "calibration": 63.193083048699705, "speed": 90.68731781591922, "cost": 57.37833924468609 }, "jevbench_score": 72.13474945923463 } }, { "threshold": 0.44, "public": { "n": 231, "escalation": 0.017316017316017316, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6396396396396397 }, "accuracy": 0.7705627705627706, "ece_hard": 0.222799887420075, "calibration": 60.113302548739355, "p50_s": 0.23927149176597595, "p95_s": 0.3747003823518753, "cost_usd_per_1000": 0.026642410282601295, "axes": { "intelligence": 77.96338004671338, "calibration": 60.113302548739355, "speed": 90.47424906484702, "cost": 57.23279464058982 }, "jevbench_score": 70.18718053587314 }, "heldout": { "n": 303, "escalation": 0.013201320132013201, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6788990825688074 }, "accuracy": 0.834983498349835, "ece_hard": 0.1976414463153635, "calibration": 63.678739093835844, "p50_s": 0.24146977066993713, "p95_s": 0.3535130485892295, "cost_usd_per_1000": 0.02647817528831535, "axes": { "intelligence": 82.6158309245528, "calibration": 63.678739093835844, "speed": 90.68731781591922, "cost": 57.313358412530036 }, "jevbench_score": 72.31277641663073 } }, { "threshold": 0.45, "public": { "n": 231, "escalation": 0.017316017316017316, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6396396396396397 }, "accuracy": 0.7705627705627706, "ece_hard": 0.222799887420075, "calibration": 60.113302548739355, "p50_s": 0.23927149176597595, "p95_s": 0.3747003823518753, "cost_usd_per_1000": 0.026642410282601295, "axes": { "intelligence": 77.96338004671338, "calibration": 60.113302548739355, "speed": 90.47424906484702, "cost": 57.23279464058982 }, "jevbench_score": 70.18718053587314 }, "heldout": { "n": 303, "escalation": 0.019801980198019802, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6697247706422018 }, "accuracy": 0.8316831683168316, "ece_hard": 0.20768659948996832, "calibration": 62.67422377637537, "p50_s": 0.24170594662427902, "p95_s": 0.3617294557392597, "cost_usd_per_1000": 0.026741635591649056, "axes": { "intelligence": 82.34060156675464, "calibration": 62.67422377637537, "speed": 90.58328789701463, "cost": 57.184361008663174 }, "jevbench_score": 71.90466844170005 } }, { "threshold": 0.46, "public": { "n": 231, "escalation": 0.017316017316017316, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6396396396396397 }, "accuracy": 0.7705627705627706, "ece_hard": 0.222799887420075, "calibration": 60.113302548739355, "p50_s": 0.23927149176597595, "p95_s": 0.3747003823518753, "cost_usd_per_1000": 0.026642410282601295, "axes": { "intelligence": 77.96338004671338, "calibration": 60.113302548739355, "speed": 90.47424906484702, "cost": 57.23279464058982 }, "jevbench_score": 70.18718053587314 }, "heldout": { "n": 303, "escalation": 0.019801980198019802, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6697247706422018 }, "accuracy": 0.8316831683168316, "ece_hard": 0.20768659948996832, "calibration": 62.67422377637537, "p50_s": 0.24170594662427902, "p95_s": 0.3617294557392597, "cost_usd_per_1000": 0.026741635591649056, "axes": { "intelligence": 82.34060156675464, "calibration": 62.67422377637537, "speed": 90.58328789701463, "cost": 57.184361008663174 }, "jevbench_score": 71.90466844170005 } }, { "threshold": 0.47, "public": { "n": 231, "escalation": 0.021645021645021644, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6306306306306306 }, "accuracy": 0.7662337662337663, "ece_hard": 0.2316463369326548, "calibration": 59.22865759748137, "p50_s": 0.23927149176597595, "p95_s": 0.3791080117225647, "cost_usd_per_1000": 0.026815199182839634, "axes": { "intelligence": 77.58800467133801, "calibration": 59.22865759748137, "speed": 90.42346080235178, "cost": 57.14856918270906 }, "jevbench_score": 69.80768562333616 }, "heldout": { "n": 303, "escalation": 0.026402640264026403, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6697247706422018 }, "accuracy": 0.8316831683168316, "ece_hard": 0.2108291774415955, "calibration": 62.35996598121265, "p50_s": 0.24170594662427902, "p95_s": 0.3672341391444205, "cost_usd_per_1000": 0.027005095894982758, "axes": { "intelligence": 82.34060156675464, "calibration": 62.35996598121265, "speed": 90.51769616961215, "cost": 57.056628286064736 }, "jevbench_score": 71.7612302031271 } }, { "threshold": 0.48, "public": { "n": 231, "escalation": 0.025974025974025976, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6306306306306306 }, "accuracy": 0.7662337662337663, "ece_hard": 0.22521950051899137, "calibration": 59.87134123884772, "p50_s": 0.23927149176597595, "p95_s": 0.3791080117225647, "cost_usd_per_1000": 0.026987988083077973, "axes": { "intelligence": 77.58800467133801, "calibration": 59.87134123884772, "speed": 90.42346080235178, "cost": 57.06488470856434 }, "jevbench_score": 69.97065025113558 }, "heldout": { "n": 303, "escalation": 0.026402640264026403, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6697247706422018 }, "accuracy": 0.8316831683168316, "ece_hard": 0.2108291774415955, "calibration": 62.35996598121265, "p50_s": 0.24170594662427902, "p95_s": 0.3672341391444205, "cost_usd_per_1000": 0.027005095894982758, "axes": { "intelligence": 82.34060156675464, "calibration": 62.35996598121265, "speed": 90.51769616961215, "cost": 57.056628286064736 }, "jevbench_score": 71.7612302031271 } }, { "threshold": 0.49, "public": { "n": 231, "escalation": 0.03463203463203463, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6306306306306306 }, "accuracy": 0.7662337662337663, "ece_hard": 0.22568212123386092, "calibration": 59.82507916736076, "p50_s": 0.23927149176597595, "p95_s": 0.3934786021709442, "cost_usd_per_1000": 0.02733356588355465, "axes": { "intelligence": 77.58800467133801, "calibration": 59.82507916736076, "speed": 90.26187922848986, "cost": 56.89911122207419 }, "jevbench_score": 69.87501751826409 }, "heldout": { "n": 303, "escalation": 0.026402640264026403, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6697247706422018 }, "accuracy": 0.8316831683168316, "ece_hard": 0.2108291774415955, "calibration": 62.35996598121265, "p50_s": 0.24170594662427902, "p95_s": 0.3672341391444205, "cost_usd_per_1000": 0.027005095894982758, "axes": { "intelligence": 82.34060156675464, "calibration": 62.35996598121265, "speed": 90.51769616961215, "cost": 57.056628286064736 }, "jevbench_score": 71.7612302031271 } }, { "threshold": 0.5, "public": { "n": 231, "escalation": 0.04329004329004329, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6306306306306306 }, "accuracy": 0.7662337662337663, "ece_hard": 0.22612069968783655, "calibration": 59.966335006516104, "p50_s": 0.2396840751171112, "p95_s": 0.8906489871442318, "cost_usd_per_1000": 0.027679143684031328, "axes": { "intelligence": 77.58800467133801, "calibration": 59.966335006516104, "speed": 86.7065424167208, "cost": 56.735420496240806 }, "jevbench_score": 69.16749686053565 }, "heldout": { "n": 303, "escalation": 0.0297029702970297, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6697247706422018 }, "accuracy": 0.8316831683168316, "ece_hard": 0.20380814639242636, "calibration": 63.062069086129554, "p50_s": 0.24171222746372223, "p95_s": 0.38975498601794184, "cost_usd_per_1000": 0.02713682604664961, "axes": { "intelligence": 82.34060156675464, "calibration": 63.062069086129554, "speed": 90.25909699745802, "cost": 56.9932284781449 }, "jevbench_score": 71.8909336275323 } }, { "threshold": 0.51, "public": { "n": 231, "escalation": 0.047619047619047616, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6306306306306306 }, "accuracy": 0.7662337662337663, "ece_hard": 0.22688508852997796, "calibration": 59.88989612230196, "p50_s": 0.2396840751171112, "p95_s": 0.8913721702992916, "cost_usd_per_1000": 0.027851932584269667, "axes": { "intelligence": 77.58800467133801, "calibration": 59.88989612230196, "speed": 86.70301749256849, "cost": 56.65433994120902 }, "jevbench_score": 69.1200247340135 }, "heldout": { "n": 303, "escalation": 0.0297029702970297, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6697247706422018 }, "accuracy": 0.8316831683168316, "ece_hard": 0.20380814639242636, "calibration": 63.062069086129554, "p50_s": 0.24171222746372223, "p95_s": 0.38975498601794184, "cost_usd_per_1000": 0.02713682604664961, "axes": { "intelligence": 82.34060156675464, "calibration": 63.062069086129554, "speed": 90.25909699745802, "cost": 56.9932284781449 }, "jevbench_score": 71.8909336275323 } }, { "threshold": 0.52, "public": { "n": 231, "escalation": 0.047619047619047616, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6306306306306306 }, "accuracy": 0.7662337662337663, "ece_hard": 0.22688508852997796, "calibration": 59.88989612230196, "p50_s": 0.2396840751171112, "p95_s": 0.8913721702992916, "cost_usd_per_1000": 0.027851932584269667, "axes": { "intelligence": 77.58800467133801, "calibration": 59.88989612230196, "speed": 86.70301749256849, "cost": 56.65433994120902 }, "jevbench_score": 69.1200247340135 }, "heldout": { "n": 303, "escalation": 0.0297029702970297, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6697247706422018 }, "accuracy": 0.8316831683168316, "ece_hard": 0.20380814639242636, "calibration": 63.062069086129554, "p50_s": 0.24171222746372223, "p95_s": 0.38975498601794184, "cost_usd_per_1000": 0.02713682604664961, "axes": { "intelligence": 82.34060156675464, "calibration": 63.062069086129554, "speed": 90.25909699745802, "cost": 56.9932284781449 }, "jevbench_score": 71.8909336275323 } }, { "threshold": 0.53, "public": { "n": 231, "escalation": 0.05194805194805195, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6306306306306306 }, "accuracy": 0.7662337662337663, "ece_hard": 0.22758866654719923, "calibration": 59.81953832057983, "p50_s": 0.2396840751171112, "p95_s": 0.9016663990914822, "cost_usd_per_1000": 0.028024721484508006, "axes": { "intelligence": 77.58800467133801, "calibration": 59.81953832057983, "speed": 86.65314934184414, "cost": 56.573760844604145 }, "jevbench_score": 69.06519787958457 }, "heldout": { "n": 303, "escalation": 0.036303630363036306, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9383561643835616, "hard": 0.6697247706422018 }, "accuracy": 0.834983498349835, "ece_hard": 0.2028079440908321, "calibration": 63.162089316288984, "p50_s": 0.24171222746372223, "p95_s": 0.44981941133737485, "cost_usd_per_1000": 0.027400286349983316, "axes": { "intelligence": 82.53238238867245, "calibration": 63.162089316288984, "speed": 89.63663186556069, "cost": 56.867346955277604 }, "jevbench_score": 71.79717253389707 } }, { "threshold": 0.54, "public": { "n": 231, "escalation": 0.05194805194805195, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6306306306306306 }, "accuracy": 0.7662337662337663, "ece_hard": 0.22758866654719923, "calibration": 59.81953832057983, "p50_s": 0.2396840751171112, "p95_s": 0.9016663990914822, "cost_usd_per_1000": 0.028024721484508006, "axes": { "intelligence": 77.58800467133801, "calibration": 59.81953832057983, "speed": 86.65314934184414, "cost": 56.573760844604145 }, "jevbench_score": 69.06519787958457 }, "heldout": { "n": 303, "escalation": 0.039603960396039604, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9383561643835616, "hard": 0.6697247706422018 }, "accuracy": 0.834983498349835, "ece_hard": 0.20590775088184343, "calibration": 62.852108637187854, "p50_s": 0.24171222746372223, "p95_s": 0.5316799387335758, "cost_usd_per_1000": 0.02753201650165017, "axes": { "intelligence": 82.53238238867245, "calibration": 62.852108637187854, "speed": 88.9105110555038, "cost": 56.80485936455582 }, "jevbench_score": 71.54358646125938 } }, { "threshold": 0.55, "public": { "n": 231, "escalation": 0.05627705627705628, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6396396396396397 }, "accuracy": 0.7705627705627706, "ece_hard": 0.22004114368408306, "calibration": 60.57429060689145, "p50_s": 0.2396840751171112, "p95_s": 0.9150901921093464, "cost_usd_per_1000": 0.02819751038474634, "axes": { "intelligence": 77.96338004671338, "calibration": 60.57429060689145, "speed": 86.58896919398418, "cost": 56.4936770417991 }, "jevbench_score": 69.3282682580919 }, "heldout": { "n": 303, "escalation": 0.0429042904290429, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9383561643835616, "hard": 0.6697247706422018 }, "accuracy": 0.834983498349835, "ece_hard": 0.20326534622384781, "calibration": 63.116349102987414, "p50_s": 0.24171222746372223, "p95_s": 0.6294243253767468, "cost_usd_per_1000": 0.027663746653317018, "axes": { "intelligence": 82.53238238867245, "calibration": 63.116349102987414, "speed": 88.17757856349868, "cost": 56.742670040609035 }, "jevbench_score": 71.45103874860833 } }, { "threshold": 0.56, "public": { "n": 231, "escalation": 0.06060606060606061, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6486486486486487 }, "accuracy": 0.7748917748917749, "ece_hard": 0.21337434193633092, "calibration": 61.24097078166666, "p50_s": 0.2396840751171112, "p95_s": 0.9243743121623993, "cost_usd_per_1000": 0.02837029928498468, "axes": { "intelligence": 78.33875542208877, "calibration": 61.24097078166666, "speed": 86.54512951850187, "cost": 56.41408248114899 }, "jevbench_score": 69.568433429639 }, "heldout": { "n": 303, "escalation": 0.052805280528052806, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9383561643835616, "hard": 0.6788990825688074 }, "accuracy": 0.8382838283828383, "ece_hard": 0.19594457342344276, "calibration": 63.84842638302793, "p50_s": 0.24179083853960037, "p95_s": 0.8597023494541642, "cost_usd_per_1000": 0.028058937108317576, "axes": { "intelligence": 82.80761174647063, "calibration": 63.84842638302793, "speed": 86.82212044981416, "cost": 56.55786353102104 }, "jevbench_score": 71.38154773752193 } }, { "threshold": 0.57, "public": { "n": 231, "escalation": 0.06060606060606061, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6486486486486487 }, "accuracy": 0.7748917748917749, "ece_hard": 0.21337434193633092, "calibration": 61.24097078166666, "p50_s": 0.2396840751171112, "p95_s": 0.9243743121623993, "cost_usd_per_1000": 0.02837029928498468, "axes": { "intelligence": 78.33875542208877, "calibration": 61.24097078166666, "speed": 86.54512951850187, "cost": 56.41408248114899 }, "jevbench_score": 69.568433429639 }, "heldout": { "n": 303, "escalation": 0.0594059405940594, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9383561643835616, "hard": 0.6788990825688074 }, "accuracy": 0.8382838283828383, "ece_hard": 0.19244002233151858, "calibration": 64.19888149222034, "p50_s": 0.2421077936887741, "p95_s": 0.8677682131528853, "cost_usd_per_1000": 0.02832239741165128, "axes": { "intelligence": 82.80761174647063, "calibration": 64.19888149222034, "speed": 86.77587492336625, "cost": 56.436099628426575 }, "jevbench_score": 71.43127933267712 } }, { "threshold": 0.58, "public": { "n": 231, "escalation": 0.06926406926406926, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6396396396396397 }, "accuracy": 0.7705627705627706, "ece_hard": 0.2193187054445353, "calibration": 60.64653443084622, "p50_s": 0.2396840751171112, "p95_s": 0.9289816468954086, "cost_usd_per_1000": 0.028715877085461358, "axes": { "intelligence": 77.96338004671338, "calibration": 60.64653443084622, "speed": 86.52353685938058, "cost": 56.256337428262704 }, "jevbench_score": 69.26288727992625 }, "heldout": { "n": 303, "escalation": 0.0594059405940594, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9383561643835616, "hard": 0.6788990825688074 }, "accuracy": 0.8382838283828383, "ece_hard": 0.19244002233151858, "calibration": 64.19888149222034, "p50_s": 0.2421077936887741, "p95_s": 0.8677682131528853, "cost_usd_per_1000": 0.02832239741165128, "axes": { "intelligence": 82.80761174647063, "calibration": 64.19888149222034, "speed": 86.77587492336625, "cost": 56.436099628426575 }, "jevbench_score": 71.43127933267712 } }, { "threshold": 0.59, "public": { "n": 231, "escalation": 0.06926406926406926, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6396396396396397 }, "accuracy": 0.7705627705627706, "ece_hard": 0.2193187054445353, "calibration": 60.64653443084622, "p50_s": 0.2396840751171112, "p95_s": 0.9289816468954086, "cost_usd_per_1000": 0.028715877085461358, "axes": { "intelligence": 77.96338004671338, "calibration": 60.64653443084622, "speed": 86.52353685938058, "cost": 56.256337428262704 }, "jevbench_score": 69.26288727992625 }, "heldout": { "n": 303, "escalation": 0.0627062706270627, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9452054794520548, "hard": 0.6788990825688074 }, "accuracy": 0.8415841584158416, "ece_hard": 0.19244002233151858, "calibration": 64.19888149222034, "p50_s": 0.2422117367386818, "p95_s": 0.8738573908805847, "cost_usd_per_1000": 0.02845412756331813, "axes": { "intelligence": 82.99939256838842, "calibration": 64.19888149222034, "speed": 86.74364252534744, "cost": 56.37564177478024 }, "jevbench_score": 71.44681649182336 } }, { "threshold": 0.6, "public": { "n": 231, "escalation": 0.0735930735930736, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6396396396396397 }, "accuracy": 0.7705627705627706, "ece_hard": 0.2205665635018917, "calibration": 61.01430984694335, "p50_s": 0.2396840751171112, "p95_s": 0.930261418223381, "cost_usd_per_1000": 0.028888665985699696, "axes": { "intelligence": 77.96338004671338, "calibration": 61.01430984694335, "speed": 86.51755810680066, "cost": 56.178175373361384 }, "jevbench_score": 69.34235089662091 }, "heldout": { "n": 303, "escalation": 0.0627062706270627, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9452054794520548, "hard": 0.6788990825688074 }, "accuracy": 0.8415841584158416, "ece_hard": 0.19244002233151858, "calibration": 64.19888149222034, "p50_s": 0.2422117367386818, "p95_s": 0.8738573908805847, "cost_usd_per_1000": 0.02845412756331813, "axes": { "intelligence": 82.99939256838842, "calibration": 64.19888149222034, "speed": 86.74364252534744, "cost": 56.37564177478024 }, "jevbench_score": 71.44681649182336 } }, { "threshold": 0.61, "public": { "n": 231, "escalation": 0.0735930735930736, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6396396396396397 }, "accuracy": 0.7705627705627706, "ece_hard": 0.2205665635018917, "calibration": 61.01430984694335, "p50_s": 0.2396840751171112, "p95_s": 0.930261418223381, "cost_usd_per_1000": 0.028888665985699696, "axes": { "intelligence": 77.96338004671338, "calibration": 61.01430984694335, "speed": 86.51755810680066, "cost": 56.178175373361384 }, "jevbench_score": 69.34235089662091 }, "heldout": { "n": 303, "escalation": 0.066006600660066, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9452054794520548, "hard": 0.6788990825688074 }, "accuracy": 0.8415841584158416, "ece_hard": 0.19244002233151858, "calibration": 64.19888149222034, "p50_s": 0.2422117367386818, "p95_s": 0.8860593914985654, "cost_usd_per_1000": 0.028585857714984984, "axes": { "intelligence": 82.99939256838842, "calibration": 64.19888149222034, "speed": 86.6834198318129, "cost": 56.31546316912371 }, "jevbench_score": 71.41534164768552 } }, { "threshold": 0.62, "public": { "n": 231, "escalation": 0.0735930735930736, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6396396396396397 }, "accuracy": 0.7705627705627706, "ece_hard": 0.2205665635018917, "calibration": 61.01430984694335, "p50_s": 0.2396840751171112, "p95_s": 0.930261418223381, "cost_usd_per_1000": 0.028888665985699696, "axes": { "intelligence": 77.96338004671338, "calibration": 61.01430984694335, "speed": 86.51755810680066, "cost": 56.178175373361384 }, "jevbench_score": 69.34235089662091 }, "heldout": { "n": 303, "escalation": 0.07260726072607261, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9452054794520548, "hard": 0.6788990825688074 }, "accuracy": 0.8415841584158416, "ece_hard": 0.19244002233151858, "calibration": 64.19888149222034, "p50_s": 0.24259134382009506, "p95_s": 0.8860593914985654, "cost_usd_per_1000": 0.02884931801831869, "axes": { "intelligence": 82.99939256838842, "calibration": 64.19888149222034, "speed": 86.67661866651288, "cost": 56.195933465939035 }, "jevbench_score": 71.37601653468577 } }, { "threshold": 0.63, "public": { "n": 231, "escalation": 0.08658008658008658, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6396396396396397 }, "accuracy": 0.7705627705627706, "ece_hard": 0.22166176588758177, "calibration": 60.904789608374344, "p50_s": 0.2401008978486061, "p95_s": 0.9425243772566319, "cost_usd_per_1000": 0.02940703268641471, "axes": { "intelligence": 77.96338004671338, "calibration": 60.904789608374344, "speed": 86.45313624385217, "cost": 55.9464638716544 }, "jevbench_score": 69.22673918121457 }, "heldout": { "n": 303, "escalation": 0.08250825082508251, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9452054794520548, "hard": 0.6788990825688074 }, "accuracy": 0.8415841584158416, "ece_hard": 0.19447724629227553, "calibration": 63.99515909614465, "p50_s": 0.24259134382009506, "p95_s": 0.90349033921957, "cost_usd_per_1000": 0.029244508473319244, "axes": { "intelligence": 82.99939256838842, "calibration": 63.99515909614465, "speed": 86.5920118662527, "cost": 56.0186702091385 }, "jevbench_score": 71.24561924729441 } }, { "threshold": 0.64, "public": { "n": 231, "escalation": 0.08658008658008658, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6396396396396397 }, "accuracy": 0.7705627705627706, "ece_hard": 0.22166176588758177, "calibration": 60.904789608374344, "p50_s": 0.2401008978486061, "p95_s": 0.9425243772566319, "cost_usd_per_1000": 0.02940703268641471, "axes": { "intelligence": 77.96338004671338, "calibration": 60.904789608374344, "speed": 86.45313624385217, "cost": 55.9464638716544 }, "jevbench_score": 69.22673918121457 }, "heldout": { "n": 303, "escalation": 0.0858085808580858, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9452054794520548, "hard": 0.6697247706422018 }, "accuracy": 0.8382838283828383, "ece_hard": 0.20139367396509109, "calibration": 63.30351632886308, "p50_s": 0.24259134382009506, "p95_s": 0.9052535198628902, "cost_usd_per_1000": 0.029376238624986097, "axes": { "intelligence": 82.72416321059026, "calibration": 63.30351632886308, "speed": 86.58354477609291, "cost": 55.96011437991519 }, "jevbench_score": 70.97306212445503 } }, { "threshold": 0.65, "public": { "n": 231, "escalation": 0.08658008658008658, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6396396396396397 }, "accuracy": 0.7705627705627706, "ece_hard": 0.22166176588758177, "calibration": 60.904789608374344, "p50_s": 0.2401008978486061, "p95_s": 0.9425243772566319, "cost_usd_per_1000": 0.02940703268641471, "axes": { "intelligence": 77.96338004671338, "calibration": 60.904789608374344, "speed": 86.45313624385217, "cost": 55.9464638716544 }, "jevbench_score": 69.22673918121457 }, "heldout": { "n": 303, "escalation": 0.0891089108910891, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9452054794520548, "hard": 0.6788990825688074 }, "accuracy": 0.8415841584158416, "ece_hard": 0.1944415467933511, "calibration": 65.7374717221399, "p50_s": 0.24259134382009506, "p95_s": 0.9172920644283292, "cost_usd_per_1000": 0.02950796877665295, "axes": { "intelligence": 82.99939256838842, "calibration": 65.7374717221399, "speed": 86.52617063347022, "cost": 55.9018205426384 }, "jevbench_score": 71.67460968272168 } }, { "threshold": 0.66, "public": { "n": 231, "escalation": 0.08658008658008658, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6396396396396397 }, "accuracy": 0.7705627705627706, "ece_hard": 0.22166176588758177, "calibration": 60.904789608374344, "p50_s": 0.2401008978486061, "p95_s": 0.9425243772566319, "cost_usd_per_1000": 0.02940703268641471, "axes": { "intelligence": 77.96338004671338, "calibration": 60.904789608374344, "speed": 86.45313624385217, "cost": 55.9464638716544 }, "jevbench_score": 69.22673918121457 }, "heldout": { "n": 303, "escalation": 0.0891089108910891, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9452054794520548, "hard": 0.6788990825688074 }, "accuracy": 0.8415841584158416, "ece_hard": 0.1944415467933511, "calibration": 65.7374717221399, "p50_s": 0.24259134382009506, "p95_s": 0.9172920644283292, "cost_usd_per_1000": 0.02950796877665295, "axes": { "intelligence": 82.99939256838842, "calibration": 65.7374717221399, "speed": 86.52617063347022, "cost": 55.9018205426384 }, "jevbench_score": 71.67460968272168 } }, { "threshold": 0.67, "public": { "n": 231, "escalation": 0.09956709956709957, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6576576576576577 }, "accuracy": 0.7792207792207793, "ece_hard": 0.20271635226990178, "calibration": 62.799330970142336, "p50_s": 0.2401016280055046, "p95_s": 0.9646848775446415, "cost_usd_per_1000": 0.029925399387129726, "axes": { "intelligence": 78.71413079746414, "calibration": 62.799330970142336, "speed": 86.35219424653269, "cost": 55.71880134246447 }, "jevbench_score": 69.83461061715178 }, "heldout": { "n": 303, "escalation": 0.0891089108910891, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9452054794520548, "hard": 0.6788990825688074 }, "accuracy": 0.8415841584158416, "ece_hard": 0.1944415467933511, "calibration": 65.7374717221399, "p50_s": 0.24259134382009506, "p95_s": 0.9172920644283292, "cost_usd_per_1000": 0.02950796877665295, "axes": { "intelligence": 82.99939256838842, "calibration": 65.7374717221399, "speed": 86.52617063347022, "cost": 55.9018205426384 }, "jevbench_score": 71.67460968272168 } }, { "threshold": 0.68, "public": { "n": 231, "escalation": 0.1038961038961039, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6576576576576577 }, "accuracy": 0.7792207792207793, "ece_hard": 0.20271635226990178, "calibration": 62.799330970142336, "p50_s": 0.24027424305677414, "p95_s": 0.9646848775446415, "cost_usd_per_1000": 0.030098188287368065, "axes": { "intelligence": 78.71413079746414, "calibration": 62.799330970142336, "speed": 86.34907311693388, "cost": 55.643789358575695 }, "jevbench_score": 69.81046403442245 }, "heldout": { "n": 303, "escalation": 0.0924092409240924, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9452054794520548, "hard": 0.6788990825688074 }, "accuracy": 0.8415841584158416, "ece_hard": 0.19530427470143885, "calibration": 65.65119893133114, "p50_s": 0.24259134382009506, "p95_s": 0.9201007060706615, "cost_usd_per_1000": 0.029639698928319802, "axes": { "intelligence": 82.99939256838842, "calibration": 65.65119893133114, "speed": 86.51289335779407, "cost": 55.84378636332526 }, "jevbench_score": 71.62973053519664 } }, { "threshold": 0.69, "public": { "n": 231, "escalation": 0.1038961038961039, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6576576576576577 }, "accuracy": 0.7792207792207793, "ece_hard": 0.20271635226990178, "calibration": 62.799330970142336, "p50_s": 0.24027424305677414, "p95_s": 0.9646848775446415, "cost_usd_per_1000": 0.030098188287368065, "axes": { "intelligence": 78.71413079746414, "calibration": 62.799330970142336, "speed": 86.34907311693388, "cost": 55.643789358575695 }, "jevbench_score": 69.81046403442245 }, "heldout": { "n": 303, "escalation": 0.09570957095709572, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9452054794520548, "hard": 0.6788990825688074 }, "accuracy": 0.8415841584158416, "ece_hard": 0.19530427470143885, "calibration": 65.65119893133114, "p50_s": 0.24261443316936493, "p95_s": 0.9201007060706615, "cost_usd_per_1000": 0.02977142907998665, "axes": { "intelligence": 82.99939256838842, "calibration": 65.65119893133114, "speed": 86.51248002483467, "cost": 55.78600953904344 }, "jevbench_score": 71.61111050810212 } }, { "threshold": 0.7, "public": { "n": 231, "escalation": 0.11255411255411256, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6666666666666666 }, "accuracy": 0.7835497835497836, "ece_hard": 0.198655730954682, "calibration": 64.0210465765349, "p50_s": 0.24027424305677414, "p95_s": 0.9646848775446415, "cost_usd_per_1000": 0.030443766087844742, "axes": { "intelligence": 79.0895061728395, "calibration": 64.0210465765349, "speed": 86.34907311693388, "cost": 55.495048707505205 }, "jevbench_score": 70.18404383355698 }, "heldout": { "n": 303, "escalation": 0.09900990099009901, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.6788990825688074 }, "accuracy": 0.8448844884488449, "ece_hard": 0.19530427470143885, "calibration": 65.65119893133114, "p50_s": 0.24261443316936493, "p95_s": 0.9308271415531633, "cost_usd_per_1000": 0.029903159231653504, "axes": { "intelligence": 83.19117339030623, "calibration": 65.65119893133114, "speed": 86.46214328267482, "cost": 55.72848779736281 }, "jevbench_score": 71.62354156964993 } }, { "threshold": 0.71, "public": { "n": 231, "escalation": 0.12554112554112554, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6756756756756757 }, "accuracy": 0.7878787878787878, "ece_hard": 0.1924092345341295, "calibration": 64.64569621859016, "p50_s": 0.24037498235702515, "p95_s": 0.9804389961063862, "cost_usd_per_1000": 0.030962132788559756, "axes": { "intelligence": 79.46488154821488, "calibration": 64.64569621859016, "speed": 86.27690158021686, "cost": 55.27507393347304 }, "jevbench_score": 70.35333387356823 }, "heldout": { "n": 303, "escalation": 0.09900990099009901, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.6788990825688074 }, "accuracy": 0.8448844884488449, "ece_hard": 0.19530427470143885, "calibration": 65.65119893133114, "p50_s": 0.24261443316936493, "p95_s": 0.9308271415531633, "cost_usd_per_1000": 0.029903159231653504, "axes": { "intelligence": 83.19117339030623, "calibration": 65.65119893133114, "speed": 86.46214328267482, "cost": 55.72848779736281 }, "jevbench_score": 71.62354156964993 } }, { "threshold": 0.72, "public": { "n": 231, "escalation": 0.12987012987012986, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6756756756756757 }, "accuracy": 0.7878787878787878, "ece_hard": 0.19497405951883795, "calibration": 64.38921372011932, "p50_s": 0.24037498235702515, "p95_s": 0.9804389961063862, "cost_usd_per_1000": 0.031134921688798094, "axes": { "intelligence": 79.46488154821488, "calibration": 64.38921372011932, "speed": 86.27690158021686, "cost": 55.20256666863134 }, "jevbench_score": 70.26038792281665 }, "heldout": { "n": 303, "escalation": 0.10561056105610561, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.6788990825688074 }, "accuracy": 0.8448844884488449, "ece_hard": 0.1938927025123651, "calibration": 65.7923561502385, "p50_s": 0.24261443316936493, "p95_s": 0.9308271415531633, "cost_usd_per_1000": 0.03016661953498721, "axes": { "intelligence": 83.19117339030623, "calibration": 65.7923561502385, "speed": 86.46214328267482, "cost": 55.614200621389436 }, "jevbench_score": 71.62524102551043 } }, { "threshold": 0.73, "public": { "n": 231, "escalation": 0.14285714285714285, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "hard": 0.6846846846846847 }, "accuracy": 0.7965367965367965, "ece_hard": 0.18291705904877348, "calibration": 65.59491376712576, "p50_s": 0.24047942459583282, "p95_s": 0.9804389961063862, "cost_usd_per_1000": 0.03165328838951311, "axes": { "intelligence": 80.3803803803804, "calibration": 65.59491376712576, "speed": 86.2750149930067, "cost": 54.987434962541215 }, "jevbench_score": 70.71998857070967 }, "heldout": { "n": 303, "escalation": 0.10891089108910891, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.6788990825688074 }, "accuracy": 0.8448844884488449, "ece_hard": 0.1938927025123651, "calibration": 65.7923561502385, "p50_s": 0.24268721789121628, "p95_s": 0.9308271415531633, "cost_usd_per_1000": 0.030298349686654062, "axes": { "intelligence": 83.19117339030623, "calibration": 65.7923561502385, "speed": 86.4608405876094, "cost": 55.557430789979044 }, "jevbench_score": 71.60668591072684 } }, { "threshold": 0.74, "public": { "n": 231, "escalation": 0.14285714285714285, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "hard": 0.6846846846846847 }, "accuracy": 0.7965367965367965, "ece_hard": 0.18291705904877348, "calibration": 65.59491376712576, "p50_s": 0.24047942459583282, "p95_s": 0.9804389961063862, "cost_usd_per_1000": 0.03165328838951311, "axes": { "intelligence": 80.3803803803804, "calibration": 65.59491376712576, "speed": 86.2750149930067, "cost": 54.987434962541215 }, "jevbench_score": 70.71998857070967 }, "heldout": { "n": 303, "escalation": 0.11221122112211221, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.6788990825688074 }, "accuracy": 0.8448844884488449, "ece_hard": 0.1938927025123651, "calibration": 65.7923561502385, "p50_s": 0.24268721789121628, "p95_s": 0.9318535953760148, "cost_usd_per_1000": 0.03043007983832091, "axes": { "intelligence": 83.19117339030623, "calibration": 65.7923561502385, "speed": 86.45605411757748, "cost": 55.5009072459194 }, "jevbench_score": 71.58747518173226 } }, { "threshold": 0.75, "public": { "n": 231, "escalation": 0.14285714285714285, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "hard": 0.6846846846846847 }, "accuracy": 0.7965367965367965, "ece_hard": 0.18291705904877348, "calibration": 65.59491376712576, "p50_s": 0.24047942459583282, "p95_s": 0.9804389961063862, "cost_usd_per_1000": 0.03165328838951311, "axes": { "intelligence": 80.3803803803804, "calibration": 65.59491376712576, "speed": 86.2750149930067, "cost": 54.987434962541215 }, "jevbench_score": 70.71998857070967 }, "heldout": { "n": 303, "escalation": 0.11221122112211221, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.6788990825688074 }, "accuracy": 0.8448844884488449, "ece_hard": 0.1938927025123651, "calibration": 65.7923561502385, "p50_s": 0.24268721789121628, "p95_s": 0.9318535953760148, "cost_usd_per_1000": 0.03043007983832091, "axes": { "intelligence": 83.19117339030623, "calibration": 65.7923561502385, "speed": 86.45605411757748, "cost": 55.5009072459194 }, "jevbench_score": 71.58747518173226 } }, { "threshold": 0.76, "public": { "n": 231, "escalation": 0.1471861471861472, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "hard": 0.6846846846846847 }, "accuracy": 0.7965367965367965, "ece_hard": 0.18388996910958733, "calibration": 65.49762276104437, "p50_s": 0.24047942459583282, "p95_s": 0.9846451729536057, "cost_usd_per_1000": 0.031826077289751446, "axes": { "intelligence": 80.3803803803804, "calibration": 65.49762276104437, "speed": 86.2564231971624, "cost": 54.91650660611987 }, "jevbench_score": 70.66713519918933 }, "heldout": { "n": 303, "escalation": 0.11551155115511551, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.6788990825688074 }, "accuracy": 0.8448844884488449, "ece_hard": 0.19472220704467977, "calibration": 65.70940569700704, "p50_s": 0.24284886568784714, "p95_s": 0.9318535953760148, "cost_usd_per_1000": 0.030561809989987764, "axes": { "intelligence": 83.19117339030623, "calibration": 65.70940569700704, "speed": 86.45316235521184, "cost": 55.44462786147699 }, "jevbench_score": 71.54615292836175 } }, { "threshold": 0.77, "public": { "n": 231, "escalation": 0.15151515151515152, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "hard": 0.6846846846846847 }, "accuracy": 0.7965367965367965, "ece_hard": 0.18509003360488752, "calibration": 65.37761631151436, "p50_s": 0.24066395312547684, "p95_s": 0.9846451729536057, "cost_usd_per_1000": 0.03199886618998979, "axes": { "intelligence": 80.3803803803804, "calibration": 65.37761631151436, "speed": 86.25309197699735, "cost": 54.84596229054771 }, "jevbench_score": 70.61136680119272 }, "heldout": { "n": 303, "escalation": 0.11551155115511551, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.6788990825688074 }, "accuracy": 0.8448844884488449, "ece_hard": 0.19472220704467977, "calibration": 65.70940569700704, "p50_s": 0.24284886568784714, "p95_s": 0.9318535953760148, "cost_usd_per_1000": 0.030561809989987764, "axes": { "intelligence": 83.19117339030623, "calibration": 65.70940569700704, "speed": 86.45316235521184, "cost": 55.44462786147699 }, "jevbench_score": 71.54615292836175 } }, { "threshold": 0.78, "public": { "n": 231, "escalation": 0.15151515151515152, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "hard": 0.6846846846846847 }, "accuracy": 0.7965367965367965, "ece_hard": 0.18509003360488752, "calibration": 65.37761631151436, "p50_s": 0.24066395312547684, "p95_s": 0.9846451729536057, "cost_usd_per_1000": 0.03199886618998979, "axes": { "intelligence": 80.3803803803804, "calibration": 65.37761631151436, "speed": 86.25309197699735, "cost": 54.84596229054771 }, "jevbench_score": 70.61136680119272 }, "heldout": { "n": 303, "escalation": 0.11551155115511551, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.6788990825688074 }, "accuracy": 0.8448844884488449, "ece_hard": 0.19472220704467977, "calibration": 65.70940569700704, "p50_s": 0.24284886568784714, "p95_s": 0.9318535953760148, "cost_usd_per_1000": 0.030561809989987764, "axes": { "intelligence": 83.19117339030623, "calibration": 65.70940569700704, "speed": 86.45316235521184, "cost": 55.44462786147699 }, "jevbench_score": 71.54615292836175 } }, { "threshold": 0.79, "public": { "n": 231, "escalation": 0.16017316017316016, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "hard": 0.6936936936936937 }, "accuracy": 0.8008658008658008, "ece_hard": 0.17468406799995964, "calibration": 66.41821287200715, "p50_s": 0.24066395312547684, "p95_s": 0.9882492944598198, "cost_usd_per_1000": 0.032344443990466466, "axes": { "intelligence": 80.75575575575577, "calibration": 66.41821287200715, "speed": 86.23722440932626, "cost": 54.70600930288466 }, "jevbench_score": 70.92471824094812 }, "heldout": { "n": 303, "escalation": 0.12211221122112212, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.958904109589041, "hard": 0.6880733944954128 }, "accuracy": 0.8514851485148515, "ece_hard": 0.18456589139739973, "calibration": 68.41422928950267, "p50_s": 0.2428498938679695, "p95_s": 0.9318535953760148, "cost_usd_per_1000": 0.03082527029332147, "axes": { "intelligence": 83.6581835700222, "calibration": 68.41422928950267, "speed": 86.45314396797306, "cost": 55.332793197311595 }, "jevbench_score": 72.33601233598237 } }, { "threshold": 0.8, "public": { "n": 231, "escalation": 0.1774891774891775, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "hard": 0.6936936936936937 }, "accuracy": 0.8008658008658008, "ece_hard": 0.17255249012922866, "calibration": 66.28684306994082, "p50_s": 0.24139812588691711, "p95_s": 0.9882492944598198, "cost_usd_per_1000": 0.033035599591419815, "axes": { "intelligence": 80.75575575575577, "calibration": 66.28684306994082, "speed": 86.22399592944953, "cost": 54.43053419203717 }, "jevbench_score": 70.79749493226547 }, "heldout": { "n": 303, "escalation": 0.12871287128712872, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.958904109589041, "hard": 0.6880733944954128 }, "accuracy": 0.8514851485148515, "ece_hard": 0.18728613353482795, "calibration": 67.883671379226, "p50_s": 0.24302451312541962, "p95_s": 0.9443351678550242, "cost_usd_per_1000": 0.03108873059665517, "axes": { "intelligence": 83.6581835700222, "calibration": 67.883671379226, "speed": 86.3922375477315, "cost": 55.2219103163076 }, "jevbench_score": 72.14645151931651 } }, { "threshold": 0.81, "public": { "n": 231, "escalation": 0.1774891774891775, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "hard": 0.6936936936936937 }, "accuracy": 0.8008658008658008, "ece_hard": 0.17255249012922866, "calibration": 66.28684306994082, "p50_s": 0.24139812588691711, "p95_s": 0.9882492944598198, "cost_usd_per_1000": 0.033035599591419815, "axes": { "intelligence": 80.75575575575577, "calibration": 66.28684306994082, "speed": 86.22399592944953, "cost": 54.43053419203717 }, "jevbench_score": 70.79749493226547 }, "heldout": { "n": 303, "escalation": 0.13861386138613863, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.958904109589041, "hard": 0.6972477064220184 }, "accuracy": 0.8547854785478548, "ece_hard": 0.1761160233154957, "calibration": 69.00068240115922, "p50_s": 0.2430928722023964, "p95_s": 0.9477042384445666, "cost_usd_per_1000": 0.03148392105165573, "axes": { "intelligence": 83.93341292782037, "calibration": 69.00068240115922, "speed": 86.37554952109522, "cost": 55.057335559034 }, "jevbench_score": 72.44335757522127 } }, { "threshold": 0.82, "public": { "n": 231, "escalation": 0.1774891774891775, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "hard": 0.6936936936936937 }, "accuracy": 0.8008658008658008, "ece_hard": 0.17255249012922866, "calibration": 66.28684306994082, "p50_s": 0.24139812588691711, "p95_s": 0.9882492944598198, "cost_usd_per_1000": 0.033035599591419815, "axes": { "intelligence": 80.75575575575577, "calibration": 66.28684306994082, "speed": 86.22399592944953, "cost": 54.43053419203717 }, "jevbench_score": 70.79749493226547 }, "heldout": { "n": 303, "escalation": 0.1419141914191419, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.958904109589041, "hard": 0.6972477064220184 }, "accuracy": 0.8547854785478548, "ece_hard": 0.1761160233154957, "calibration": 69.00068240115922, "p50_s": 0.2430928722023964, "p95_s": 0.9477042384445666, "cost_usd_per_1000": 0.03161565120332258, "axes": { "intelligence": 83.93341292782037, "calibration": 69.00068240115922, "speed": 86.37554952109522, "cost": 55.00293605136321 }, "jevbench_score": 72.42545649100529 } }, { "threshold": 0.83, "public": { "n": 231, "escalation": 0.18181818181818182, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "hard": 0.6936936936936937 }, "accuracy": 0.8008658008658008, "ece_hard": 0.1710703264966994, "calibration": 66.43505943319374, "p50_s": 0.24139812588691711, "p95_s": 0.9985015355050564, "cost_usd_per_1000": 0.03320838849165816, "axes": { "intelligence": 80.75575575575577, "calibration": 66.43505943319374, "speed": 86.17917368835117, "cost": 54.362565968869994 }, "jevbench_score": 70.80570832930582 }, "heldout": { "n": 303, "escalation": 0.15181518151815182, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.958904109589041, "hard": 0.6972477064220184 }, "accuracy": 0.8547854785478548, "ece_hard": 0.1760387469998087, "calibration": 69.02896655527545, "p50_s": 0.24334686994552612, "p95_s": 0.9479597538709641, "cost_usd_per_1000": 0.03201084165832314, "axes": { "intelligence": 83.93341292782037, "calibration": 69.02896655527545, "speed": 86.36984335977667, "cost": 54.84108720514187 }, "jevbench_score": 72.37833870969538 } }, { "threshold": 0.84, "public": { "n": 231, "escalation": 0.18181818181818182, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "hard": 0.6936936936936937 }, "accuracy": 0.8008658008658008, "ece_hard": 0.1710703264966994, "calibration": 66.43505943319374, "p50_s": 0.24139812588691711, "p95_s": 0.9985015355050564, "cost_usd_per_1000": 0.03320838849165816, "axes": { "intelligence": 80.75575575575577, "calibration": 66.43505943319374, "speed": 86.17917368835117, "cost": 54.362565968869994 }, "jevbench_score": 70.80570832930582 }, "heldout": { "n": 303, "escalation": 0.15181518151815182, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.958904109589041, "hard": 0.6972477064220184 }, "accuracy": 0.8547854785478548, "ece_hard": 0.1760387469998087, "calibration": 69.02896655527545, "p50_s": 0.24334686994552612, "p95_s": 0.9479597538709641, "cost_usd_per_1000": 0.03201084165832314, "axes": { "intelligence": 83.93341292782037, "calibration": 69.02896655527545, "speed": 86.36984335977667, "cost": 54.84108720514187 }, "jevbench_score": 72.37833870969538 } }, { "threshold": 0.85, "public": { "n": 231, "escalation": 0.18181818181818182, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "hard": 0.6936936936936937 }, "accuracy": 0.8008658008658008, "ece_hard": 0.1710703264966994, "calibration": 66.43505943319374, "p50_s": 0.24139812588691711, "p95_s": 0.9985015355050564, "cost_usd_per_1000": 0.03320838849165816, "axes": { "intelligence": 80.75575575575577, "calibration": 66.43505943319374, "speed": 86.17917368835117, "cost": 54.362565968869994 }, "jevbench_score": 70.80570832930582 }, "heldout": { "n": 303, "escalation": 0.15841584158415842, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.958904109589041, "hard": 0.7064220183486238 }, "accuracy": 0.858085808580858, "ece_hard": 0.16016392470347968, "calibration": 72.82817420001533, "p50_s": 0.24364512413740158, "p95_s": 0.9529628425836562, "cost_usd_per_1000": 0.03227430196165684, "axes": { "intelligence": 84.20864228561852, "calibration": 72.82817420001533, "speed": 86.34166308170923, "cost": 54.734294260362695 }, "jevbench_score": 73.37261624280784 } }, { "threshold": 0.86, "public": { "n": 231, "escalation": 0.18614718614718614, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "hard": 0.6936936936936937 }, "accuracy": 0.8008658008658008, "ece_hard": 0.17211320809342143, "calibration": 65.75197198734082, "p50_s": 0.24139812588691711, "p95_s": 0.9985015355050564, "cost_usd_per_1000": 0.03338117739189649, "axes": { "intelligence": 80.75575575575577, "calibration": 65.75197198734082, "speed": 86.17917368835117, "cost": 54.294950479865626 }, "jevbench_score": 70.60102541300809 }, "heldout": { "n": 303, "escalation": 0.1617161716171617, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.958904109589041, "hard": 0.7064220183486238 }, "accuracy": 0.858085808580858, "ece_hard": 0.16016392470347968, "calibration": 72.82817420001533, "p50_s": 0.24364512413740158, "p95_s": 0.9529628425836562, "cost_usd_per_1000": 0.03240603211332369, "axes": { "intelligence": 84.20864228561852, "calibration": 72.82817420001533, "speed": 86.34166308170923, "cost": 54.68122425889735 }, "jevbench_score": 73.35482437432704 } }, { "threshold": 0.87, "public": { "n": 231, "escalation": 0.18614718614718614, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "hard": 0.6936936936936937 }, "accuracy": 0.8008658008658008, "ece_hard": 0.17211320809342143, "calibration": 65.75197198734082, "p50_s": 0.24139812588691711, "p95_s": 0.9985015355050564, "cost_usd_per_1000": 0.03338117739189649, "axes": { "intelligence": 80.75575575575577, "calibration": 65.75197198734082, "speed": 86.17917368835117, "cost": 54.294950479865626 }, "jevbench_score": 70.60102541300809 }, "heldout": { "n": 303, "escalation": 0.1716171617161716, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.958904109589041, "hard": 0.7064220183486238 }, "accuracy": 0.858085808580858, "ece_hard": 0.16115690007417827, "calibration": 72.18770508591467, "p50_s": 0.24370402842760086, "p95_s": 0.9691216707229611, "cost_usd_per_1000": 0.03280122256832425, "axes": { "intelligence": 84.20864228561852, "calibration": 72.18770508591467, "speed": 86.26758986843757, "cost": 54.52329906842222 }, "jevbench_score": 73.1244180599122 } }, { "threshold": 0.88, "public": { "n": 231, "escalation": 0.19047619047619047, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "hard": 0.6936936936936937 }, "accuracy": 0.8008658008658008, "ece_hard": 0.1714847809546806, "calibration": 65.81481470121491, "p50_s": 0.24139812588691711, "p95_s": 0.9985015355050564, "cost_usd_per_1000": 0.033553966292134835, "axes": { "intelligence": 80.75575575575577, "calibration": 65.81481470121491, "speed": 86.17917368835117, "cost": 54.22768408275468 }, "jevbench_score": 70.59600625529981 }, "heldout": { "n": 303, "escalation": 0.1848184818481848, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.958904109589041, "hard": 0.7064220183486238 }, "accuracy": 0.858085808580858, "ece_hard": 0.15959863196825455, "calibration": 72.34353189650705, "p50_s": 0.24378608912229538, "p95_s": 0.9774567693471906, "cost_usd_per_1000": 0.03332814317499166, "axes": { "intelligence": 84.20864228561852, "calibration": 72.34353189650705, "speed": 86.2289352099186, "cost": 54.3156664509568 }, "jevbench_score": 73.08590468889274 } }, { "threshold": 0.89, "public": { "n": 231, "escalation": 0.19047619047619047, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "hard": 0.6936936936936937 }, "accuracy": 0.8008658008658008, "ece_hard": 0.1714847809546806, "calibration": 65.81481470121491, "p50_s": 0.24139812588691711, "p95_s": 0.9985015355050564, "cost_usd_per_1000": 0.033553966292134835, "axes": { "intelligence": 80.75575575575577, "calibration": 65.81481470121491, "speed": 86.17917368835117, "cost": 54.22768408275468 }, "jevbench_score": 70.59600625529981 }, "heldout": { "n": 303, "escalation": 0.20132013201320131, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.958904109589041, "hard": 0.7155963302752294 }, "accuracy": 0.8613861386138614, "ece_hard": 0.14008930310052325, "calibration": 74.29446478328018, "p50_s": 0.24394351989030838, "p95_s": 0.9789580598473548, "cost_usd_per_1000": 0.03398679393332592, "axes": { "intelligence": 84.4838716434167, "calibration": 74.29446478328018, "speed": 86.21946627665488, "cost": 54.06069404991903 }, "jevbench_score": 73.54519493136381 } }, { "threshold": 0.9, "public": { "n": 231, "escalation": 0.19480519480519481, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "hard": 0.7027027027027027 }, "accuracy": 0.8051948051948052, "ece_hard": 0.16231288579507808, "calibration": 68.90440603075456, "p50_s": 0.24161306023597717, "p95_s": 0.9985015355050564, "cost_usd_per_1000": 0.03372675519237317, "axes": { "intelligence": 81.13113113113114, "calibration": 68.90440603075456, "speed": 86.17530856842802, "cost": 54.16076319139222 }, "jevbench_score": 71.47028875369368 }, "heldout": { "n": 303, "escalation": 0.20462046204620463, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.958904109589041, "hard": 0.7155963302752294 }, "accuracy": 0.8613861386138614, "ece_hard": 0.14008930310052325, "calibration": 74.29446478328018, "p50_s": 0.24419748783111572, "p95_s": 0.9789580598473548, "cost_usd_per_1000": 0.034118524084992774, "axes": { "intelligence": 84.4838716434167, "calibration": 74.29446478328018, "speed": 86.21494721828935, "cost": 54.01029292033515 }, "jevbench_score": 73.52708378832621 } }, { "threshold": 0.91, "public": { "n": 231, "escalation": 0.19480519480519481, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "hard": 0.7027027027027027 }, "accuracy": 0.8051948051948052, "ece_hard": 0.16231288579507808, "calibration": 68.90440603075456, "p50_s": 0.24161306023597717, "p95_s": 0.9985015355050564, "cost_usd_per_1000": 0.03372675519237317, "axes": { "intelligence": 81.13113113113114, "calibration": 68.90440603075456, "speed": 86.17530856842802, "cost": 54.16076319139222 }, "jevbench_score": 71.47028875369368 }, "heldout": { "n": 303, "escalation": 0.21782178217821782, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.958904109589041, "hard": 0.7247706422018348 }, "accuracy": 0.8646864686468647, "ece_hard": 0.12606332645839652, "calibration": 76.35573882550396, "p50_s": 0.24425430595874786, "p95_s": 0.9789580598473548, "cost_usd_per_1000": 0.03464544469166018, "axes": { "intelligence": 84.75910100121486, "calibration": 76.35573882550396, "speed": 86.21393685044026, "cost": 53.81061579721861 }, "jevbench_score": 74.02328702380224 } }, { "threshold": 0.92, "public": { "n": 231, "escalation": 0.19913419913419914, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "hard": 0.7027027027027027 }, "accuracy": 0.8051948051948052, "ece_hard": 0.16228197751166465, "calibration": 68.90749685909589, "p50_s": 0.24161306023597717, "p95_s": 0.9985015355050564, "cost_usd_per_1000": 0.033899544092611505, "axes": { "intelligence": 81.13113113113114, "calibration": 68.90749685909589, "speed": 86.17530856842802, "cost": 54.094184274610626 }, "jevbench_score": 71.44911553629461 }, "heldout": { "n": 303, "escalation": 0.22772277227722773, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.7247706422018348 }, "accuracy": 0.8613861386138614, "ece_hard": 0.124487662637938, "calibration": 76.51330520754982, "p50_s": 0.24440407007932663, "p95_s": 0.9789580598473548, "cost_usd_per_1000": 0.03504063514666073, "axes": { "intelligence": 84.56732017929706, "calibration": 76.51330520754982, "speed": 86.21127479709665, "cost": 53.66284091228481 }, "jevbench_score": 73.96807471415974 } }, { "threshold": 0.93, "public": { "n": 231, "escalation": 0.2077922077922078, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.7027027027027027 }, "accuracy": 0.8008658008658008, "ece_hard": 0.16228197751166465, "calibration": 68.90749685909589, "p50_s": 0.24170870333909988, "p95_s": 0.9985015355050564, "cost_usd_per_1000": 0.03424512189308819, "axes": { "intelligence": 80.59100767434103, "calibration": 68.90749685909589, "speed": 86.17358974359576, "cost": 53.96203850832564 }, "jevbench_score": 71.28594284797344 }, "heldout": { "n": 303, "escalation": 0.23102310231023102, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.7247706422018348 }, "accuracy": 0.8613861386138614, "ece_hard": 0.124487662637938, "calibration": 76.51330520754982, "p50_s": 0.24440407007932663, "p95_s": 0.9789580598473548, "cost_usd_per_1000": 0.03517236529832759, "axes": { "intelligence": 84.56732017929706, "calibration": 76.51330520754982, "speed": 86.21127479709665, "cost": 53.61395274792922 }, "jevbench_score": 73.95122227309335 } }, { "threshold": 0.94, "public": { "n": 231, "escalation": 0.22077922077922077, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6936936936936937 }, "accuracy": 0.7965367965367965, "ece_hard": 0.16386668389601405, "calibration": 68.74902622066095, "p50_s": 0.24170870333909988, "p95_s": 1.0133583433926105, "cost_usd_per_1000": 0.0347634885938032, "axes": { "intelligence": 80.21563229896564, "calibration": 68.74902622066095, "speed": 86.10944663858905, "cost": 53.766299428425725 }, "jevbench_score": 71.08396211410809 }, "heldout": { "n": 303, "escalation": 0.24752475247524752, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.7247706422018348 }, "accuracy": 0.8613861386138614, "ece_hard": 0.12100431319371868, "calibration": 77.90656868062813, "p50_s": 0.24452945590019226, "p95_s": 0.9799205616116523, "cost_usd_per_1000": 0.03583101605666185, "axes": { "intelligence": 84.56732017929706, "calibration": 77.90656868062813, "speed": 86.20477947948251, "cost": 53.372226292377846 }, "jevbench_score": 74.20032838367894 } }, { "threshold": 0.95, "public": { "n": 231, "escalation": 0.22510822510822512, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6936936936936937 }, "accuracy": 0.7965367965367965, "ece_hard": 0.16386668389601405, "calibration": 68.74902622066095, "p50_s": 0.24192951619625092, "p95_s": 1.0133583433926105, "cost_usd_per_1000": 0.03493627749404154, "axes": { "intelligence": 80.21563229896564, "calibration": 68.74902622066095, "speed": 86.10548095481207, "cost": 53.70170114595243 }, "jevbench_score": 71.06178308590597 }, "heldout": { "n": 303, "escalation": 0.25412541254125415, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.7247706422018348 }, "accuracy": 0.8613861386138614, "ece_hard": 0.12100431319371868, "calibration": 77.90656868062813, "p50_s": 0.2446764037013054, "p95_s": 0.9899539016187189, "cost_usd_per_1000": 0.036094476359995555, "axes": { "intelligence": 84.56732017929706, "calibration": 77.90656868062813, "speed": 86.15792939838215, "cost": 53.27677762974338 }, "jevbench_score": 74.15705288317697 } }, { "threshold": 0.96, "public": { "n": 231, "escalation": 0.24675324675324675, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6936936936936937 }, "accuracy": 0.7965367965367965, "ece_hard": 0.1565731205628086, "calibration": 69.20304513096423, "p50_s": 0.2420308142900467, "p95_s": 1.0133583433926105, "cost_usd_per_1000": 0.035800221995233236, "axes": { "intelligence": 80.21563229896564, "calibration": 69.20304513096423, "speed": 86.10366290481771, "cost": 53.38342840933806 }, "jevbench_score": 71.07274287122176 }, "heldout": { "n": 303, "escalation": 0.25742574257425743, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.7247706422018348 }, "accuracy": 0.8613861386138614, "ece_hard": 0.12100431319371868, "calibration": 77.90656868062813, "p50_s": 0.2446764037013054, "p95_s": 0.9899539016187189, "cost_usd_per_1000": 0.036226206511662404, "axes": { "intelligence": 84.56732017929706, "calibration": 77.90656868062813, "speed": 86.15792939838215, "cost": 53.229314243917585 }, "jevbench_score": 74.14053104448215 } }, { "threshold": 0.97, "public": { "n": 231, "escalation": 0.26406926406926406, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.7117117117117117 }, "accuracy": 0.8051948051948052, "ece_hard": 0.148110642814895, "calibration": 70.04929290575558, "p50_s": 0.24286006391048431, "p95_s": 1.0248290821909904, "cost_usd_per_1000": 0.036491377596186585, "axes": { "intelligence": 80.9663830497164, "calibration": 70.04929290575558, "speed": 86.0399245388415, "cost": 53.13429223460808 }, "jevbench_score": 71.35852325414889 }, "heldout": { "n": 303, "escalation": 0.2838283828382838, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.958904109589041, "hard": 0.7247706422018348 }, "accuracy": 0.8646864686468647, "ece_hard": 0.11992362025968886, "calibration": 78.01463797403112, "p50_s": 0.24588020145893097, "p95_s": 0.9957831218838691, "cost_usd_per_1000": 0.03728004772499722, "axes": { "intelligence": 84.75910100121486, "calibration": 78.01463797403112, "speed": 86.11111678828966, "cost": 52.855706210338624 }, "jevbench_score": 74.06761921755397 } }, { "threshold": 0.98, "public": { "n": 231, "escalation": 0.27705627705627706, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.7117117117117117 }, "accuracy": 0.8051948051948052, "ece_hard": 0.14402695876047758, "calibration": 70.45766131119733, "p50_s": 0.24302303791046143, "p95_s": 1.0248290821909904, "cost_usd_per_1000": 0.0370097442969016, "axes": { "intelligence": 80.9663830497164, "calibration": 70.45766131119733, "speed": 86.0370111338981, "cost": 52.95051746402987 }, "jevbench_score": 71.39982075219325 }, "heldout": { "n": 303, "escalation": 0.3069306930693069, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.7247706422018348 }, "accuracy": 0.8613861386138614, "ece_hard": 0.11593172785054309, "calibration": 78.41382721494568, "p50_s": 0.24659743160009384, "p95_s": 1.0055367924273013, "cost_usd_per_1000": 0.038202158786665184, "axes": { "intelligence": 84.56732017929706, "calibration": 78.41382721494568, "speed": 86.05613485060358, "cost": 52.53736283831976 }, "jevbench_score": 73.9965260764635 } }, { "threshold": 0.99, "public": { "n": 231, "escalation": 0.329004329004329, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.7117117117117117 }, "accuracy": 0.8051948051948052, "ece_hard": 0.13413195901311326, "calibration": 72.38971679695737, "p50_s": 0.2440943792462349, "p95_s": 1.0567090064287186, "cost_usd_per_1000": 0.039083211099761664, "axes": { "intelligence": 80.9663830497164, "calibration": 72.38971679695737, "speed": 85.88486811982104, "cost": 52.24029284786863 }, "jevbench_score": 71.61037831303452 }, "heldout": { "n": 303, "escalation": 0.35313531353135313, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.952054794520548, "hard": 0.7247706422018348 }, "accuracy": 0.8646864686468647, "ece_hard": 0.10360148153424434, "calibration": 79.64685184657557, "p50_s": 0.24838463217020035, "p95_s": 1.008035872876644, "cost_usd_per_1000": 0.04004638091000111, "axes": { "intelligence": 85.73398684596373, "calibration": 79.64685184657557, "speed": 86.0139929035959, "cost": 51.923101782025554 }, "jevbench_score": 74.3126661368054 } }, { "threshold": 1.0, "public": { "n": 231, "escalation": 1.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.7117117117117117 }, "accuracy": 0.8051948051948052, "ece_hard": 0.09203203203203206, "calibration": 78.2177967967968, "p50_s": 0.9088969975709915, "p95_s": 1.1663119532167912, "cost_usd_per_1000": 0.06586549063670412, "axes": { "intelligence": 80.9663830497164, "calibration": 78.2177967967968, "speed": 79.7467060459708, "cost": 45.44026206212592 }, "jevbench_score": 69.2135260302178 }, "heldout": { "n": 303, "escalation": 1.0, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6972477064220184 }, "accuracy": 0.8514851485148515, "ece_hard": 0.08064220183486236, "calibration": 81.94277981651376, "p50_s": 0.9065034464001656, "p95_s": 1.0630968414247035, "cost_usd_per_1000": 0.06586549063670412, "axes": { "intelligence": 84.71651795065142, "calibration": 81.94277981651376, "speed": 80.16057712621512, "cost": 45.44026206212592 }, "jevbench_score": 70.91206183879923 } } ] }, { "cheap": "system-one-open", "backend": "semif-qwen3.5-4b", "chosen": { "threshold": 0.99, "public": { "n": 231, "escalation": 0.7316017316017316, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.5945945945945946 }, "accuracy": 0.7489177489177489, "ece_hard": 0.14632171565234448, "calibration": 70.52700185648575, "p50_s": 1.8318715035915374, "p95_s": 3.197047398984432, "cost_usd_per_1000": 0.03130144948684274, "axes": { "intelligence": 76.0865031698365, "calibration": 70.52700185648575, "speed": 72.32355918024906, "cost": 55.13306652897859 }, "jevbench_score": 68.01250120356407 }, "heldout": { "n": 303, "escalation": 0.8547854785478548, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7986798679867987, "ece_hard": 0.19046608330777925, "calibration": 65.15723100745228, "p50_s": 1.8709805101156234, "p95_s": 3.0801832318305964, "cost_usd_per_1000": 0.034066260244001924, "axes": { "intelligence": 79.33796238113192, "calibration": 65.15723100745228, "speed": 72.39354184212674, "cost": 54.03026620535524 }, "jevbench_score": 67.0571588153839 } }, "full": { "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.952054794520548, "hard": 0.5727272727272728 }, "accuracy": 0.7771535580524345, "ece_hard": 0.1543920308464603, "calibration": 69.24230329532918, "p50_s": 1.8587638452649116, "p95_s": 3.1388589933514592, "cost_usd_per_1000": 0.032870246601860026, "axes": { "intelligence": 80.58935242839354, "calibration": 69.24230329532918, "speed": 72.34003977183498, "cost": 54.49591113119414 }, "jevbench_score": 68.48539395025396 }, "full_escalation": 0.8014981273408239, "backend_heldout_accuracy": 0.7986798679867987, "useful": false, "curve": [ { "threshold": 0.0, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 1.29877570271492, "p95_s": 2.095727913081646, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 68.67075408742076, "calibration": 50.471910537865476, "speed": 75.65130946864079, "cost": 64.82109224519658 }, "jevbench_score": 64.2079181671165 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28023325293453055, "calibration": 55.17142992768658, "p50_s": 1.3129856139421463, "p95_s": 1.9534294441342355, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 74.75130911985254, "calibration": 55.17142992768658, "speed": 75.90942302768094, "cost": 64.82109224519658 }, "jevbench_score": 67.11757697107404 } }, { "threshold": 0.01, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 1.29877570271492, "p95_s": 2.095727913081646, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 68.67075408742076, "calibration": 50.471910537865476, "speed": 75.65130946864079, "cost": 64.82109224519658 }, "jevbench_score": 64.2079181671165 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28023325293453055, "calibration": 55.17142992768658, "p50_s": 1.3129856139421463, "p95_s": 1.9534294441342355, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 74.75130911985254, "calibration": 55.17142992768658, "speed": 75.90942302768094, "cost": 64.82109224519658 }, "jevbench_score": 67.11757697107404 } }, { "threshold": 0.02, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 1.29877570271492, "p95_s": 2.095727913081646, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 68.67075408742076, "calibration": 50.471910537865476, "speed": 75.65130946864079, "cost": 64.82109224519658 }, "jevbench_score": 64.2079181671165 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28023325293453055, "calibration": 55.17142992768658, "p50_s": 1.3129856139421463, "p95_s": 1.9534294441342355, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 74.75130911985254, "calibration": 55.17142992768658, "speed": 75.90942302768094, "cost": 64.82109224519658 }, "jevbench_score": 67.11757697107404 } }, { "threshold": 0.03, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 1.29877570271492, "p95_s": 2.095727913081646, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 68.67075408742076, "calibration": 50.471910537865476, "speed": 75.65130946864079, "cost": 64.82109224519658 }, "jevbench_score": 64.2079181671165 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28023325293453055, "calibration": 55.17142992768658, "p50_s": 1.3129856139421463, "p95_s": 1.9534294441342355, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 74.75130911985254, "calibration": 55.17142992768658, "speed": 75.90942302768094, "cost": 64.82109224519658 }, "jevbench_score": 67.11757697107404 } }, { "threshold": 0.04, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 1.29877570271492, "p95_s": 2.095727913081646, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 68.67075408742076, "calibration": 50.471910537865476, "speed": 75.65130946864079, "cost": 64.82109224519658 }, "jevbench_score": 64.2079181671165 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28023325293453055, "calibration": 55.17142992768658, "p50_s": 1.3129856139421463, "p95_s": 1.9534294441342355, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 74.75130911985254, "calibration": 55.17142992768658, "speed": 75.90942302768094, "cost": 64.82109224519658 }, "jevbench_score": 67.11757697107404 } }, { "threshold": 0.05, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 1.29877570271492, "p95_s": 2.095727913081646, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 68.67075408742076, "calibration": 50.471910537865476, "speed": 75.65130946864079, "cost": 64.82109224519658 }, "jevbench_score": 64.2079181671165 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28023325293453055, "calibration": 55.17142992768658, "p50_s": 1.3129856139421463, "p95_s": 1.9534294441342355, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 74.75130911985254, "calibration": 55.17142992768658, "speed": 75.90942302768094, "cost": 64.82109224519658 }, "jevbench_score": 67.11757697107404 } }, { "threshold": 0.06, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 1.29877570271492, "p95_s": 2.095727913081646, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 68.67075408742076, "calibration": 50.471910537865476, "speed": 75.65130946864079, "cost": 64.82109224519658 }, "jevbench_score": 64.2079181671165 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28023325293453055, "calibration": 55.17142992768658, "p50_s": 1.3129856139421463, "p95_s": 1.9534294441342355, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 74.75130911985254, "calibration": 55.17142992768658, "speed": 75.90942302768094, "cost": 64.82109224519658 }, "jevbench_score": 67.11757697107404 } }, { "threshold": 0.07, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 1.29877570271492, "p95_s": 2.095727913081646, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 68.67075408742076, "calibration": 50.471910537865476, "speed": 75.65130946864079, "cost": 64.82109224519658 }, "jevbench_score": 64.2079181671165 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28023325293453055, "calibration": 55.17142992768658, "p50_s": 1.3129856139421463, "p95_s": 1.9534294441342355, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 74.75130911985254, "calibration": 55.17142992768658, "speed": 75.90942302768094, "cost": 64.82109224519658 }, "jevbench_score": 67.11757697107404 } }, { "threshold": 0.08, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 1.29877570271492, "p95_s": 2.095727913081646, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 68.67075408742076, "calibration": 50.471910537865476, "speed": 75.65130946864079, "cost": 64.82109224519658 }, "jevbench_score": 64.2079181671165 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28023325293453055, "calibration": 55.17142992768658, "p50_s": 1.3129856139421463, "p95_s": 1.9534294441342355, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 74.75130911985254, "calibration": 55.17142992768658, "speed": 75.90942302768094, "cost": 64.82109224519658 }, "jevbench_score": 67.11757697107404 } }, { "threshold": 0.09, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 1.29877570271492, "p95_s": 2.095727913081646, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 68.67075408742076, "calibration": 50.471910537865476, "speed": 75.65130946864079, "cost": 64.82109224519658 }, "jevbench_score": 64.2079181671165 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28023325293453055, "calibration": 55.17142992768658, "p50_s": 1.3129856139421463, "p95_s": 1.9534294441342355, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 74.75130911985254, "calibration": 55.17142992768658, "speed": 75.90942302768094, "cost": 64.82109224519658 }, "jevbench_score": 67.11757697107404 } }, { "threshold": 0.1, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 1.29877570271492, "p95_s": 2.095727913081646, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 68.67075408742076, "calibration": 50.471910537865476, "speed": 75.65130946864079, "cost": 64.82109224519658 }, "jevbench_score": 64.2079181671165 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28023325293453055, "calibration": 55.17142992768658, "p50_s": 1.3129856139421463, "p95_s": 1.9534294441342355, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 74.75130911985254, "calibration": 55.17142992768658, "speed": 75.90942302768094, "cost": 64.82109224519658 }, "jevbench_score": 67.11757697107404 } }, { "threshold": 0.11, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 1.29877570271492, "p95_s": 2.095727913081646, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 68.67075408742076, "calibration": 50.471910537865476, "speed": 75.65130946864079, "cost": 64.82109224519658 }, "jevbench_score": 64.2079181671165 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28023325293453055, "calibration": 55.17142992768658, "p50_s": 1.3129856139421463, "p95_s": 1.9534294441342355, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 74.75130911985254, "calibration": 55.17142992768658, "speed": 75.90942302768094, "cost": 64.82109224519658 }, "jevbench_score": 67.11757697107404 } }, { "threshold": 0.12, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 1.29877570271492, "p95_s": 2.095727913081646, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 68.67075408742076, "calibration": 50.471910537865476, "speed": 75.65130946864079, "cost": 64.82109224519658 }, "jevbench_score": 64.2079181671165 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28023325293453055, "calibration": 55.17142992768658, "p50_s": 1.3129856139421463, "p95_s": 1.9534294441342355, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 74.75130911985254, "calibration": 55.17142992768658, "speed": 75.90942302768094, "cost": 64.82109224519658 }, "jevbench_score": 67.11757697107404 } }, { "threshold": 0.13, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 1.29877570271492, "p95_s": 2.095727913081646, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 68.67075408742076, "calibration": 50.471910537865476, "speed": 75.65130946864079, "cost": 64.82109224519658 }, "jevbench_score": 64.2079181671165 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28023325293453055, "calibration": 55.17142992768658, "p50_s": 1.3129856139421463, "p95_s": 1.9534294441342355, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 74.75130911985254, "calibration": 55.17142992768658, "speed": 75.90942302768094, "cost": 64.82109224519658 }, "jevbench_score": 67.11757697107404 } }, { "threshold": 0.14, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 1.29877570271492, "p95_s": 2.095727913081646, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 68.67075408742076, "calibration": 50.471910537865476, "speed": 75.65130946864079, "cost": 64.82109224519658 }, "jevbench_score": 64.2079181671165 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28023325293453055, "calibration": 55.17142992768658, "p50_s": 1.3129856139421463, "p95_s": 1.9534294441342355, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 74.75130911985254, "calibration": 55.17142992768658, "speed": 75.90942302768094, "cost": 64.82109224519658 }, "jevbench_score": 67.11757697107404 } }, { "threshold": 0.15, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 1.29877570271492, "p95_s": 2.095727913081646, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 68.67075408742076, "calibration": 50.471910537865476, "speed": 75.65130946864079, "cost": 64.82109224519658 }, "jevbench_score": 64.2079181671165 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28023325293453055, "calibration": 55.17142992768658, "p50_s": 1.3129856139421463, "p95_s": 1.9534294441342355, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 74.75130911985254, "calibration": 55.17142992768658, "speed": 75.90942302768094, "cost": 64.82109224519658 }, "jevbench_score": 67.11757697107404 } }, { "threshold": 0.16, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 1.29877570271492, "p95_s": 2.095727913081646, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 68.67075408742076, "calibration": 50.471910537865476, "speed": 75.65130946864079, "cost": 64.82109224519658 }, "jevbench_score": 64.2079181671165 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28023325293453055, "calibration": 55.17142992768658, "p50_s": 1.3129856139421463, "p95_s": 1.9534294441342355, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 74.75130911985254, "calibration": 55.17142992768658, "speed": 75.90942302768094, "cost": 64.82109224519658 }, "jevbench_score": 67.11757697107404 } }, { "threshold": 0.17, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 1.29877570271492, "p95_s": 2.095727913081646, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 68.67075408742076, "calibration": 50.471910537865476, "speed": 75.65130946864079, "cost": 64.82109224519658 }, "jevbench_score": 64.2079181671165 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28023325293453055, "calibration": 55.17142992768658, "p50_s": 1.3129856139421463, "p95_s": 1.9534294441342355, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 74.75130911985254, "calibration": 55.17142992768658, "speed": 75.90942302768094, "cost": 64.82109224519658 }, "jevbench_score": 67.11757697107404 } }, { "threshold": 0.18, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 1.29877570271492, "p95_s": 2.095727913081646, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 68.67075408742076, "calibration": 50.471910537865476, "speed": 75.65130946864079, "cost": 64.82109224519658 }, "jevbench_score": 64.2079181671165 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28023325293453055, "calibration": 55.17142992768658, "p50_s": 1.3129856139421463, "p95_s": 1.9534294441342355, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 74.75130911985254, "calibration": 55.17142992768658, "speed": 75.90942302768094, "cost": 64.82109224519658 }, "jevbench_score": 67.11757697107404 } }, { "threshold": 0.19, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 1.29877570271492, "p95_s": 2.095727913081646, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 68.67075408742076, "calibration": 50.471910537865476, "speed": 75.65130946864079, "cost": 64.82109224519658 }, "jevbench_score": 64.2079181671165 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28023325293453055, "calibration": 55.17142992768658, "p50_s": 1.3129856139421463, "p95_s": 1.9534294441342355, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 74.75130911985254, "calibration": 55.17142992768658, "speed": 75.90942302768094, "cost": 64.82109224519658 }, "jevbench_score": 67.11757697107404 } }, { "threshold": 0.2, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 1.29877570271492, "p95_s": 2.095727913081646, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 68.67075408742076, "calibration": 50.471910537865476, "speed": 75.65130946864079, "cost": 64.82109224519658 }, "jevbench_score": 64.2079181671165 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28023325293453055, "calibration": 55.17142992768658, "p50_s": 1.3129856139421463, "p95_s": 1.9534294441342355, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 74.75130911985254, "calibration": 55.17142992768658, "speed": 75.90942302768094, "cost": 64.82109224519658 }, "jevbench_score": 67.11757697107404 } }, { "threshold": 0.21, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 1.29877570271492, "p95_s": 2.095727913081646, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 68.67075408742076, "calibration": 50.471910537865476, "speed": 75.65130946864079, "cost": 64.82109224519658 }, "jevbench_score": 64.2079181671165 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28023325293453055, "calibration": 55.17142992768658, "p50_s": 1.3129856139421463, "p95_s": 1.9534294441342355, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 74.75130911985254, "calibration": 55.17142992768658, "speed": 75.90942302768094, "cost": 64.82109224519658 }, "jevbench_score": 67.11757697107404 } }, { "threshold": 0.22, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 1.29877570271492, "p95_s": 2.095727913081646, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 68.67075408742076, "calibration": 50.471910537865476, "speed": 75.65130946864079, "cost": 64.82109224519658 }, "jevbench_score": 64.2079181671165 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28023325293453055, "calibration": 55.17142992768658, "p50_s": 1.3129856139421463, "p95_s": 1.9534294441342355, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 74.75130911985254, "calibration": 55.17142992768658, "speed": 75.90942302768094, "cost": 64.82109224519658 }, "jevbench_score": 67.11757697107404 } }, { "threshold": 0.23, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 1.29877570271492, "p95_s": 2.095727913081646, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 68.67075408742076, "calibration": 50.471910537865476, "speed": 75.65130946864079, "cost": 64.82109224519658 }, "jevbench_score": 64.2079181671165 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28023325293453055, "calibration": 55.17142992768658, "p50_s": 1.3129856139421463, "p95_s": 1.9534294441342355, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 74.75130911985254, "calibration": 55.17142992768658, "speed": 75.90942302768094, "cost": 64.82109224519658 }, "jevbench_score": 67.11757697107404 } }, { "threshold": 0.24, "public": { "n": 231, "escalation": 0.004329004329004329, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.30645345813656805, "calibration": 50.97493237454572, "p50_s": 1.29877570271492, "p95_s": 2.1450010761618614, "cost_usd_per_1000": 0.014978099129335084, "axes": { "intelligence": 68.67075408742076, "calibration": 50.97493237454572, "speed": 75.5503833027139, "cost": 64.73629898365505 }, "jevbench_score": 64.32477169052788 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28023325293453055, "calibration": 55.17142992768658, "p50_s": 1.3129856139421463, "p95_s": 1.9534294441342355, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 74.75130911985254, "calibration": 55.17142992768658, "speed": 75.90942302768094, "cost": 64.82109224519658 }, "jevbench_score": 67.11757697107404 } }, { "threshold": 0.25, "public": { "n": 231, "escalation": 0.004329004329004329, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.30645345813656805, "calibration": 50.97493237454572, "p50_s": 1.29877570271492, "p95_s": 2.1450010761618614, "cost_usd_per_1000": 0.014978099129335084, "axes": { "intelligence": 68.67075408742076, "calibration": 50.97493237454572, "speed": 75.5503833027139, "cost": 64.73629898365505 }, "jevbench_score": 64.32477169052788 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28023325293453055, "calibration": 55.17142992768658, "p50_s": 1.3129856139421463, "p95_s": 1.9534294441342355, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 74.75130911985254, "calibration": 55.17142992768658, "speed": 75.90942302768094, "cost": 64.82109224519658 }, "jevbench_score": 67.11757697107404 } }, { "threshold": 0.26, "public": { "n": 231, "escalation": 0.004329004329004329, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.30645345813656805, "calibration": 50.97493237454572, "p50_s": 1.29877570271492, "p95_s": 2.1450010761618614, "cost_usd_per_1000": 0.014978099129335084, "axes": { "intelligence": 68.67075408742076, "calibration": 50.97493237454572, "speed": 75.5503833027139, "cost": 64.73629898365505 }, "jevbench_score": 64.32477169052788 }, "heldout": { "n": 303, "escalation": 0.0033003300330033004, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.2811976969300108, "calibration": 55.074985528138555, "p50_s": 1.3129856139421463, "p95_s": 1.9764527156949037, "cost_usd_per_1000": 0.014955010939296175, "axes": { "intelligence": 74.75130911985254, "calibration": 55.074985528138555, "speed": 75.85853603522497, "cost": 64.75639794864043 }, "jevbench_score": 67.06023696938036 } }, { "threshold": 0.27, "public": { "n": 231, "escalation": 0.004329004329004329, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.30645345813656805, "calibration": 50.97493237454572, "p50_s": 1.29877570271492, "p95_s": 2.1450010761618614, "cost_usd_per_1000": 0.014978099129335084, "axes": { "intelligence": 68.67075408742076, "calibration": 50.97493237454572, "speed": 75.5503833027139, "cost": 64.73629898365505 }, "jevbench_score": 64.32477169052788 }, "heldout": { "n": 303, "escalation": 0.0033003300330033004, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.2811976969300108, "calibration": 55.074985528138555, "p50_s": 1.3129856139421463, "p95_s": 1.9764527156949037, "cost_usd_per_1000": 0.014955010939296175, "axes": { "intelligence": 74.75130911985254, "calibration": 55.074985528138555, "speed": 75.85853603522497, "cost": 64.75639794864043 }, "jevbench_score": 67.06023696938036 } }, { "threshold": 0.28, "public": { "n": 231, "escalation": 0.004329004329004329, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.30645345813656805, "calibration": 50.97493237454572, "p50_s": 1.29877570271492, "p95_s": 2.1450010761618614, "cost_usd_per_1000": 0.014978099129335084, "axes": { "intelligence": 68.67075408742076, "calibration": 50.97493237454572, "speed": 75.5503833027139, "cost": 64.73629898365505 }, "jevbench_score": 64.32477169052788 }, "heldout": { "n": 303, "escalation": 0.0033003300330033004, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.2811976969300108, "calibration": 55.074985528138555, "p50_s": 1.3129856139421463, "p95_s": 1.9764527156949037, "cost_usd_per_1000": 0.014955010939296175, "axes": { "intelligence": 74.75130911985254, "calibration": 55.074985528138555, "speed": 75.85853603522497, "cost": 64.75639794864043 }, "jevbench_score": 67.06023696938036 } }, { "threshold": 0.29, "public": { "n": 231, "escalation": 0.004329004329004329, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.30645345813656805, "calibration": 50.97493237454572, "p50_s": 1.29877570271492, "p95_s": 2.1450010761618614, "cost_usd_per_1000": 0.014978099129335084, "axes": { "intelligence": 68.67075408742076, "calibration": 50.97493237454572, "speed": 75.5503833027139, "cost": 64.73629898365505 }, "jevbench_score": 64.32477169052788 }, "heldout": { "n": 303, "escalation": 0.0033003300330033004, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.2811976969300108, "calibration": 55.074985528138555, "p50_s": 1.3129856139421463, "p95_s": 1.9764527156949037, "cost_usd_per_1000": 0.014955010939296175, "axes": { "intelligence": 74.75130911985254, "calibration": 55.074985528138555, "speed": 75.85853603522497, "cost": 64.75639794864043 }, "jevbench_score": 67.06023696938036 } }, { "threshold": 0.3, "public": { "n": 231, "escalation": 0.004329004329004329, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.30645345813656805, "calibration": 50.97493237454572, "p50_s": 1.29877570271492, "p95_s": 2.1450010761618614, "cost_usd_per_1000": 0.014978099129335084, "axes": { "intelligence": 68.67075408742076, "calibration": 50.97493237454572, "speed": 75.5503833027139, "cost": 64.73629898365505 }, "jevbench_score": 64.32477169052788 }, "heldout": { "n": 303, "escalation": 0.0033003300330033004, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.2811976969300108, "calibration": 55.074985528138555, "p50_s": 1.3129856139421463, "p95_s": 1.9764527156949037, "cost_usd_per_1000": 0.014955010939296175, "axes": { "intelligence": 74.75130911985254, "calibration": 55.074985528138555, "speed": 75.85853603522497, "cost": 64.75639794864043 }, "jevbench_score": 67.06023696938036 } }, { "threshold": 0.31, "public": { "n": 231, "escalation": 0.004329004329004329, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.30645345813656805, "calibration": 50.97493237454572, "p50_s": 1.29877570271492, "p95_s": 2.1450010761618614, "cost_usd_per_1000": 0.014978099129335084, "axes": { "intelligence": 68.67075408742076, "calibration": 50.97493237454572, "speed": 75.5503833027139, "cost": 64.73629898365505 }, "jevbench_score": 64.32477169052788 }, "heldout": { "n": 303, "escalation": 0.006600660066006601, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.47706422018348627 }, "accuracy": 0.735973597359736, "ece_hard": 0.28533328294100435, "calibration": 54.6614269270392, "p50_s": 1.3150212317705154, "p95_s": 1.9764527156949037, "cost_usd_per_1000": 0.015029085549004338, "axes": { "intelligence": 75.0265384776507, "calibration": 54.6614269270392, "speed": 75.8518080636569, "cost": 64.69202330315773 }, "jevbench_score": 66.9773771624812 } }, { "threshold": 0.32, "public": { "n": 231, "escalation": 0.004329004329004329, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.30645345813656805, "calibration": 50.97493237454572, "p50_s": 1.29877570271492, "p95_s": 2.1450010761618614, "cost_usd_per_1000": 0.014978099129335084, "axes": { "intelligence": 68.67075408742076, "calibration": 50.97493237454572, "speed": 75.5503833027139, "cost": 64.73629898365505 }, "jevbench_score": 64.32477169052788 }, "heldout": { "n": 303, "escalation": 0.0165016501650165, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.48623853211009177 }, "accuracy": 0.7392739273927392, "ece_hard": 0.2788366223090667, "calibration": 55.311092990232964, "p50_s": 1.315921351313591, "p95_s": 1.9786796250939367, "cost_usd_per_1000": 0.015251309378128823, "axes": { "intelligence": 75.30176783544887, "calibration": 55.311092990232964, "speed": 75.8439458466255, "cost": 64.50078607059487 }, "jevbench_score": 67.18554351979924 } }, { "threshold": 0.33, "public": { "n": 231, "escalation": 0.012987012987012988, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.4594594594594595 }, "accuracy": 0.6666666666666666, "ece_hard": 0.30379371553240264, "calibration": 51.24090663496226, "p50_s": 1.29877570271492, "p95_s": 2.182773359119892, "cost_usd_per_1000": 0.015172424728829223, "axes": { "intelligence": 68.29537871204539, "calibration": 51.24090663496226, "speed": 75.47457200127542, "cost": 64.56835025167493 }, "jevbench_score": 64.26242618335999 }, "heldout": { "n": 303, "escalation": 0.0165016501650165, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.48623853211009177 }, "accuracy": 0.7392739273927392, "ece_hard": 0.2788366223090667, "calibration": 55.311092990232964, "p50_s": 1.315921351313591, "p95_s": 1.9786796250939367, "cost_usd_per_1000": 0.015251309378128823, "axes": { "intelligence": 75.30176783544887, "calibration": 55.311092990232964, "speed": 75.8439458466255, "cost": 64.50078607059487 }, "jevbench_score": 67.18554351979924 } }, { "threshold": 0.34, "public": { "n": 231, "escalation": 0.017316017316017316, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.4594594594594595 }, "accuracy": 0.6666666666666666, "ece_hard": 0.2927329068587152, "calibration": 52.346987502331004, "p50_s": 1.29877570271492, "p95_s": 2.194945380091667, "cost_usd_per_1000": 0.015269587528576292, "axes": { "intelligence": 68.29537871204539, "calibration": 52.346987502331004, "speed": 75.45042127127616, "cost": 64.48518082638728 }, "jevbench_score": 64.58046244478156 }, "heldout": { "n": 303, "escalation": 0.019801980198019802, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.47706422018348627 }, "accuracy": 0.735973597359736, "ece_hard": 0.28875836087364554, "calibration": 54.31891913377508, "p50_s": 1.31616772711277, "p95_s": 1.9786796250939367, "cost_usd_per_1000": 0.015325383987836986, "axes": { "intelligence": 75.0265384776507, "calibration": 54.31891913377508, "speed": 75.84313280689028, "cost": 64.43765905395784 }, "jevbench_score": 66.8044683244841 } }, { "threshold": 0.35, "public": { "n": 231, "escalation": 0.017316017316017316, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.4594594594594595 }, "accuracy": 0.6666666666666666, "ece_hard": 0.2927329068587152, "calibration": 52.346987502331004, "p50_s": 1.29877570271492, "p95_s": 2.194945380091667, "cost_usd_per_1000": 0.015269587528576292, "axes": { "intelligence": 68.29537871204539, "calibration": 52.346987502331004, "speed": 75.45042127127616, "cost": 64.48518082638728 }, "jevbench_score": 64.58046244478156 }, "heldout": { "n": 303, "escalation": 0.019801980198019802, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.47706422018348627 }, "accuracy": 0.735973597359736, "ece_hard": 0.28875836087364554, "calibration": 54.31891913377508, "p50_s": 1.31616772711277, "p95_s": 1.9786796250939367, "cost_usd_per_1000": 0.015325383987836986, "axes": { "intelligence": 75.0265384776507, "calibration": 54.31891913377508, "speed": 75.84313280689028, "cost": 64.43765905395784 }, "jevbench_score": 66.8044683244841 } }, { "threshold": 0.36, "public": { "n": 231, "escalation": 0.021645021645021644, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.4594594594594595 }, "accuracy": 0.6666666666666666, "ece_hard": 0.30360415765844373, "calibration": 51.259862422358154, "p50_s": 1.29877570271492, "p95_s": 2.194945380091667, "cost_usd_per_1000": 0.015366750328323362, "axes": { "intelligence": 68.29537871204539, "calibration": 51.259862422358154, "speed": 75.45042127127616, "cost": 64.40253894643568 }, "jevbench_score": 64.22193001117387 }, "heldout": { "n": 303, "escalation": 0.026402640264026403, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.47706422018348627 }, "accuracy": 0.735973597359736, "ece_hard": 0.2861687449463405, "calibration": 54.57788072650559, "p50_s": 1.3169443011283875, "p95_s": 1.9786796250939367, "cost_usd_per_1000": 0.01547353320725331, "axes": { "intelligence": 75.0265384776507, "calibration": 54.57788072650559, "speed": 75.84057110943166, "cost": 64.31231526152177 }, "jevbench_score": 66.85083393124116 } }, { "threshold": 0.37, "public": { "n": 231, "escalation": 0.030303030303030304, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.45045045045045046 }, "accuracy": 0.6623376623376623, "ece_hard": 0.3027281828654965, "calibration": 51.347459901652876, "p50_s": 1.2999516874551773, "p95_s": 2.194945380091667, "cost_usd_per_1000": 0.0155610759278175, "axes": { "intelligence": 67.92000333667002, "calibration": 51.347459901652876, "speed": 75.44649070341212, "cost": 64.23881134630315 }, "jevbench_score": 64.11923036683245 }, "heldout": { "n": 303, "escalation": 0.033003300330033, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8835616438356164, "hard": 0.47706422018348627 }, "accuracy": 0.7392739273927392, "ece_hard": 0.2864496580803158, "calibration": 54.54978941310806, "p50_s": 1.3169443011283875, "p95_s": 1.9910572722554205, "cost_usd_per_1000": 0.015621682426669631, "axes": { "intelligence": 75.21831929956852, "calibration": 54.54978941310806, "speed": 75.81348840133028, "cost": 64.1881658565178 }, "jevbench_score": 66.84663305006384 } }, { "threshold": 0.38, "public": { "n": 231, "escalation": 0.03463203463203463, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.44144144144144143 }, "accuracy": 0.658008658008658, "ece_hard": 0.313825519804187, "calibration": 50.23772620778382, "p50_s": 1.3007893413305283, "p95_s": 2.194945380091667, "cost_usd_per_1000": 0.01565823872756457, "axes": { "intelligence": 67.54462796129464, "calibration": 50.23772620778382, "speed": 75.44369312791835, "cost": 64.15771269730325 }, "jevbench_score": 63.66095513135597 }, "heldout": { "n": 303, "escalation": 0.036303630363036306, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8835616438356164, "hard": 0.47706422018348627 }, "accuracy": 0.7392739273927392, "ece_hard": 0.2878520630839997, "calibration": 54.409548912739666, "p50_s": 1.3169443011283875, "p95_s": 2.017643444240093, "cost_usd_per_1000": 0.015695757036377794, "axes": { "intelligence": 75.21831929956852, "calibration": 54.409548912739666, "speed": 75.75588172078325, "cost": 64.1265319781614 }, "jevbench_score": 66.77489523863359 } }, { "threshold": 0.39, "public": { "n": 231, "escalation": 0.03896103896103896, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.44144144144144143 }, "accuracy": 0.658008658008658, "ece_hard": 0.31772654222384566, "calibration": 49.84762396581796, "p50_s": 1.3007893413305283, "p95_s": 2.2221655368804933, "cost_usd_per_1000": 0.015755401527311638, "axes": { "intelligence": 67.54462796129464, "calibration": 49.84762396581796, "speed": 75.39016622420591, "cost": 64.07711572987488 }, "jevbench_score": 63.50577683814492 }, "heldout": { "n": 303, "escalation": 0.0429042904290429, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8835616438356164, "hard": 0.48623853211009177 }, "accuracy": 0.7425742574257426, "ece_hard": 0.27960700182147996, "calibration": 55.23405503899164, "p50_s": 1.3177975565195084, "p95_s": 2.017643444240093, "cost_usd_per_1000": 0.015843906255794116, "axes": { "intelligence": 75.49354865736669, "calibration": 55.23405503899164, "speed": 75.75306881431277, "cost": 64.00413207597434 }, "jevbench_score": 67.05501332389426 } }, { "threshold": 0.4, "public": { "n": 231, "escalation": 0.04329004329004329, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.44144144144144143 }, "accuracy": 0.658008658008658, "ece_hard": 0.3169847735018237, "calibration": 49.92180083802016, "p50_s": 1.3007893413305283, "p95_s": 2.2221655368804933, "cost_usd_per_1000": 0.01585256432705871, "axes": { "intelligence": 67.54462796129464, "calibration": 49.92180083802016, "speed": 75.39016622420591, "cost": 63.99701427528831 }, "jevbench_score": 63.50952546181638 }, "heldout": { "n": 303, "escalation": 0.0429042904290429, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8835616438356164, "hard": 0.48623853211009177 }, "accuracy": 0.7425742574257426, "ece_hard": 0.27960700182147996, "calibration": 55.23405503899164, "p50_s": 1.3177975565195084, "p95_s": 2.017643444240093, "cost_usd_per_1000": 0.015843906255794116, "axes": { "intelligence": 75.49354865736669, "calibration": 55.23405503899164, "speed": 75.75306881431277, "cost": 64.00413207597434 }, "jevbench_score": 67.05501332389426 } }, { "threshold": 0.41, "public": { "n": 231, "escalation": 0.04329004329004329, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.44144144144144143 }, "accuracy": 0.658008658008658, "ece_hard": 0.3169847735018237, "calibration": 49.92180083802016, "p50_s": 1.3007893413305283, "p95_s": 2.2221655368804933, "cost_usd_per_1000": 0.01585256432705871, "axes": { "intelligence": 67.54462796129464, "calibration": 49.92180083802016, "speed": 75.39016622420591, "cost": 63.99701427528831 }, "jevbench_score": 63.50952546181638 }, "heldout": { "n": 303, "escalation": 0.0429042904290429, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8835616438356164, "hard": 0.48623853211009177 }, "accuracy": 0.7425742574257426, "ece_hard": 0.27960700182147996, "calibration": 55.23405503899164, "p50_s": 1.3177975565195084, "p95_s": 2.017643444240093, "cost_usd_per_1000": 0.015843906255794116, "axes": { "intelligence": 75.49354865736669, "calibration": 55.23405503899164, "speed": 75.75306881431277, "cost": 64.00413207597434 }, "jevbench_score": 67.05501332389426 } }, { "threshold": 0.42, "public": { "n": 231, "escalation": 0.047619047619047616, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.44144144144144143 }, "accuracy": 0.658008658008658, "ece_hard": 0.3167292798257121, "calibration": 49.94735020563132, "p50_s": 1.3007893413305283, "p95_s": 2.2221655368804933, "cost_usd_per_1000": 0.015949727126805778, "axes": { "intelligence": 67.54462796129464, "calibration": 49.94735020563132, "speed": 75.39016622420591, "cost": 63.91740227789598 }, "jevbench_score": 63.49788661313886 }, "heldout": { "n": 303, "escalation": 0.0429042904290429, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8835616438356164, "hard": 0.48623853211009177 }, "accuracy": 0.7425742574257426, "ece_hard": 0.27960700182147996, "calibration": 55.23405503899164, "p50_s": 1.3177975565195084, "p95_s": 2.017643444240093, "cost_usd_per_1000": 0.015843906255794116, "axes": { "intelligence": 75.49354865736669, "calibration": 55.23405503899164, "speed": 75.75306881431277, "cost": 64.00413207597434 }, "jevbench_score": 67.05501332389426 } }, { "threshold": 0.43, "public": { "n": 231, "escalation": 0.05194805194805195, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.44144144144144143 }, "accuracy": 0.658008658008658, "ece_hard": 0.3196440103518768, "calibration": 49.65587715301485, "p50_s": 1.3012879490852356, "p95_s": 2.2221655368804933, "cost_usd_per_1000": 0.016046889926552847, "axes": { "intelligence": 67.54462796129464, "calibration": 49.65587715301485, "speed": 75.38850184167623, "cost": 63.8382737923853 }, "jevbench_score": 63.385063528896175 }, "heldout": { "n": 303, "escalation": 0.0429042904290429, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8835616438356164, "hard": 0.48623853211009177 }, "accuracy": 0.7425742574257426, "ece_hard": 0.27960700182147996, "calibration": 55.23405503899164, "p50_s": 1.3177975565195084, "p95_s": 2.017643444240093, "cost_usd_per_1000": 0.015843906255794116, "axes": { "intelligence": 75.49354865736669, "calibration": 55.23405503899164, "speed": 75.75306881431277, "cost": 64.00413207597434 }, "jevbench_score": 67.05501332389426 } }, { "threshold": 0.44, "public": { "n": 231, "escalation": 0.05194805194805195, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.44144144144144143 }, "accuracy": 0.658008658008658, "ece_hard": 0.3196440103518768, "calibration": 49.65587715301485, "p50_s": 1.3012879490852356, "p95_s": 2.2221655368804933, "cost_usd_per_1000": 0.016046889926552847, "axes": { "intelligence": 67.54462796129464, "calibration": 49.65587715301485, "speed": 75.38850184167623, "cost": 63.8382737923853 }, "jevbench_score": 63.385063528896175 }, "heldout": { "n": 303, "escalation": 0.0429042904290429, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8835616438356164, "hard": 0.48623853211009177 }, "accuracy": 0.7425742574257426, "ece_hard": 0.27960700182147996, "calibration": 55.23405503899164, "p50_s": 1.3177975565195084, "p95_s": 2.017643444240093, "cost_usd_per_1000": 0.015843906255794116, "axes": { "intelligence": 75.49354865736669, "calibration": 55.23405503899164, "speed": 75.75306881431277, "cost": 64.00413207597434 }, "jevbench_score": 67.05501332389426 } }, { "threshold": 0.45, "public": { "n": 231, "escalation": 0.05627705627705628, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.44144144144144143 }, "accuracy": 0.658008658008658, "ece_hard": 0.3218349479775235, "calibration": 49.436783390450174, "p50_s": 1.3016724586486816, "p95_s": 2.2221655368804933, "cost_usd_per_1000": 0.016144052726299915, "axes": { "intelligence": 67.54462796129464, "calibration": 49.436783390450174, "speed": 75.3872187612086, "cost": 63.75962298111439 }, "jevbench_score": 63.29525010694326 }, "heldout": { "n": 303, "escalation": 0.0462046204620462, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8835616438356164, "hard": 0.48623853211009177 }, "accuracy": 0.7425742574257426, "ece_hard": 0.27960700182147996, "calibration": 55.23405503899164, "p50_s": 1.3177975565195084, "p95_s": 2.0295264974236487, "cost_usd_per_1000": 0.01591798086550228, "axes": { "intelligence": 75.49354865736669, "calibration": 55.23405503899164, "speed": 75.72756576240067, "cost": 63.94336065055826 }, "jevbench_score": 67.03344753718655 } }, { "threshold": 0.46, "public": { "n": 231, "escalation": 0.06060606060606061, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.44144144144144143 }, "accuracy": 0.658008658008658, "ece_hard": 0.3218349479775235, "calibration": 49.436783390450174, "p50_s": 1.3016724586486816, "p95_s": 2.2221655368804933, "cost_usd_per_1000": 0.016241215526046987, "axes": { "intelligence": 67.54462796129464, "calibration": 49.436783390450174, "speed": 75.3872187612086, "cost": 63.68144411152775 }, "jevbench_score": 63.275838809542655 }, "heldout": { "n": 303, "escalation": 0.04950495049504951, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8835616438356164, "hard": 0.48623853211009177 }, "accuracy": 0.7425742574257426, "ece_hard": 0.2827333069315693, "calibration": 54.92142452798271, "p50_s": 1.3177975565195084, "p95_s": 2.031159219145775, "cost_usd_per_1000": 0.015992055475210442, "axes": { "intelligence": 75.49354865736669, "calibration": 54.92142452798271, "speed": 75.72407333712027, "cost": 63.882871370595154 }, "jevbench_score": 66.92178361060901 } }, { "threshold": 0.47, "public": { "n": 231, "escalation": 0.06493506493506493, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.44144144144144143 }, "accuracy": 0.658008658008658, "ece_hard": 0.32410679062054354, "calibration": 49.20959912614818, "p50_s": 1.3016724586486816, "p95_s": 2.266099748015404, "cost_usd_per_1000": 0.016338378325794056, "axes": { "intelligence": 67.54462796129464, "calibration": 49.20959912614818, "speed": 75.30219261545537, "cost": 63.603731553649055 }, "jevbench_score": 63.165903827456624 }, "heldout": { "n": 303, "escalation": 0.04950495049504951, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8835616438356164, "hard": 0.48623853211009177 }, "accuracy": 0.7425742574257426, "ece_hard": 0.2827333069315693, "calibration": 54.92142452798271, "p50_s": 1.3177975565195084, "p95_s": 2.031159219145775, "cost_usd_per_1000": 0.015992055475210442, "axes": { "intelligence": 75.49354865736669, "calibration": 54.92142452798271, "speed": 75.72407333712027, "cost": 63.882871370595154 }, "jevbench_score": 66.92178361060901 } }, { "threshold": 0.48, "public": { "n": 231, "escalation": 0.08225108225108226, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.45045045045045046 }, "accuracy": 0.6623376623376623, "ece_hard": 0.32546376381592856, "calibration": 49.07390180660967, "p50_s": 1.302245020866394, "p95_s": 2.392199069261551, "cost_usd_per_1000": 0.016727029524782333, "axes": { "intelligence": 67.92000333667002, "calibration": 49.07390180660967, "speed": 75.06509977342857, "cost": 63.297435295701106 }, "jevbench_score": 63.08383933124672 }, "heldout": { "n": 303, "escalation": 0.056105610561056105, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8835616438356164, "hard": 0.47706422018348627 }, "accuracy": 0.7392739273927392, "ece_hard": 0.2966592030544505, "calibration": 52.09426753948565, "p50_s": 1.3178558647632599, "p95_s": 2.0585774815082543, "cost_usd_per_1000": 0.016140204694626764, "axes": { "intelligence": 75.21831929956852, "calibration": 52.09426753948565, "speed": 75.66564868594944, "cost": 63.76272885201852 }, "jevbench_score": 65.93937160635697 } }, { "threshold": 0.49, "public": { "n": 231, "escalation": 0.08225108225108226, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.45045045045045046 }, "accuracy": 0.6623376623376623, "ece_hard": 0.32546376381592856, "calibration": 49.07390180660967, "p50_s": 1.302245020866394, "p95_s": 2.392199069261551, "cost_usd_per_1000": 0.016727029524782333, "axes": { "intelligence": 67.92000333667002, "calibration": 49.07390180660967, "speed": 75.06509977342857, "cost": 63.297435295701106 }, "jevbench_score": 63.08383933124672 }, "heldout": { "n": 303, "escalation": 0.066006600660066, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8972602739726028, "hard": 0.47706422018348627 }, "accuracy": 0.7458745874587459, "ece_hard": 0.29116770428846195, "calibration": 53.835908194968866, "p50_s": 1.3178558647632599, "p95_s": 2.101194783151149, "cost_usd_per_1000": 0.01636242852375125, "axes": { "intelligence": 75.60188094340413, "calibration": 53.835908194968866, "speed": 75.57665772508459, "cost": 63.58456712716973 }, "jevbench_score": 66.50219979747693 } }, { "threshold": 0.5, "public": { "n": 231, "escalation": 0.08225108225108226, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.45045045045045046 }, "accuracy": 0.6623376623376623, "ece_hard": 0.32546376381592856, "calibration": 49.07390180660967, "p50_s": 1.302245020866394, "p95_s": 2.392199069261551, "cost_usd_per_1000": 0.016727029524782333, "axes": { "intelligence": 67.92000333667002, "calibration": 49.07390180660967, "speed": 75.06509977342857, "cost": 63.297435295701106 }, "jevbench_score": 63.08383933124672 }, "heldout": { "n": 303, "escalation": 0.06930693069306931, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8972602739726028, "hard": 0.47706422018348627 }, "accuracy": 0.7458745874587459, "ece_hard": 0.28992327508656623, "calibration": 54.42624323298269, "p50_s": 1.3183358311653137, "p95_s": 2.101194783151149, "cost_usd_per_1000": 0.016436503133459412, "axes": { "intelligence": 75.60188094340413, "calibration": 54.42624323298269, "speed": 75.57507630168101, "cost": 63.525717193987816 }, "jevbench_score": 66.66797786024422 } }, { "threshold": 0.51, "public": { "n": 231, "escalation": 0.09090909090909091, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.45045045045045046 }, "accuracy": 0.6623376623376623, "ece_hard": 0.3269175929594109, "calibration": 48.92851889226144, "p50_s": 1.3028961420059204, "p95_s": 2.654650327563286, "cost_usd_per_1000": 0.01692135512427647, "axes": { "intelligence": 67.92000333667002, "calibration": 48.92851889226144, "speed": 74.6108287829623, "cost": 63.146945800205266 }, "jevbench_score": 62.90403373740786 }, "heldout": { "n": 303, "escalation": 0.07920792079207921, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8904109589041096, "hard": 0.48623853211009177 }, "accuracy": 0.7458745874587459, "ece_hard": 0.28357748391858517, "calibration": 55.06082234978079, "p50_s": 1.3189159780740738, "p95_s": 2.101194783151149, "cost_usd_per_1000": 0.016658726962583897, "axes": { "intelligence": 75.6853294792845, "calibration": 55.06082234978079, "speed": 75.57316556563109, "cost": 63.350745695769085 }, "jevbench_score": 66.83338164374287 } }, { "threshold": 0.52, "public": { "n": 231, "escalation": 0.1038961038961039, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.4594594594594595 }, "accuracy": 0.6666666666666666, "ece_hard": 0.31935238363635715, "calibration": 49.685039824566815, "p50_s": 1.304066225886345, "p95_s": 2.654650327563286, "cost_usd_per_1000": 0.01721284352351768, "axes": { "intelligence": 68.29537871204539, "calibration": 49.685039824566815, "speed": 74.60693029165292, "cost": 62.92442137785992 }, "jevbench_score": 63.17624954444758 }, "heldout": { "n": 303, "escalation": 0.08250825082508251, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8904109589041096, "hard": 0.48623853211009177 }, "accuracy": 0.7458745874587459, "ece_hard": 0.28357748391858517, "calibration": 55.06082234978079, "p50_s": 1.319311037659645, "p95_s": 2.101194783151149, "cost_usd_per_1000": 0.01673280157229206, "axes": { "intelligence": 75.6853294792845, "calibration": 55.06082234978079, "speed": 75.57186490273668, "cost": 63.29294017126306 }, "jevbench_score": 66.81784309704518 } }, { "threshold": 0.53, "public": { "n": 231, "escalation": 0.11255411255411256, "tiers": { "easy": 1.0, "standard": 0.7777777777777778, "hard": 0.4594594594594595 }, "accuracy": 0.670995670995671, "ece_hard": 0.3193572315132022, "calibration": 49.6845550368823, "p50_s": 1.3058469742536545, "p95_s": 2.654650327563286, "cost_usd_per_1000": 0.01740716912301182, "axes": { "intelligence": 68.8355021688355, "calibration": 49.6845550368823, "speed": 74.60100389295468, "cost": 62.778155535438955 }, "jevbench_score": 63.26256234272619 }, "heldout": { "n": 303, "escalation": 0.09570957095709572, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8972602739726028, "hard": 0.47706422018348627 }, "accuracy": 0.7458745874587459, "ece_hard": 0.29565738128231167, "calibration": 53.85283261340815, "p50_s": 1.322104200720787, "p95_s": 2.1640617772936803, "cost_usd_per_1000": 0.017029100011124707, "axes": { "intelligence": 75.60188094340413, "calibration": 53.85283261340815, "speed": 75.4346466050429, "cost": 63.06424911754702 }, "jevbench_score": 66.33974692411225 } }, { "threshold": 0.54, "public": { "n": 231, "escalation": 0.11688311688311688, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.4594594594594595 }, "accuracy": 0.6666666666666666, "ece_hard": 0.3193572315132022, "calibration": 49.6845550368823, "p50_s": 1.307793453335762, "p95_s": 2.654650327563286, "cost_usd_per_1000": 0.017504331922758888, "axes": { "intelligence": 68.29537871204539, "calibration": 49.6845550368823, "speed": 74.59453517390777, "cost": 62.70563380111768 }, "jevbench_score": 63.118486145896014 }, "heldout": { "n": 303, "escalation": 0.11221122112211221, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9041095890410958, "hard": 0.45871559633027525 }, "accuracy": 0.7425742574257426, "ece_hard": 0.31663032840420097, "calibration": 51.75553790121921, "p50_s": 1.323771134018898, "p95_s": 2.195301335453987, "cost_usd_per_1000": 0.017399473059665514, "axes": { "intelligence": 75.24320304972562, "calibration": 51.75553790121921, "speed": 75.3669295172276, "cost": 62.78391712177171 }, "jevbench_score": 65.5184075160408 } }, { "threshold": 0.55, "public": { "n": 231, "escalation": 0.12987012987012986, "tiers": { "easy": 1.0, "standard": 0.7777777777777778, "hard": 0.46846846846846846 }, "accuracy": 0.6753246753246753, "ece_hard": 0.31414596930891053, "calibration": 50.20568125731147, "p50_s": 1.3082244992256165, "p95_s": 2.9144957825541495, "cost_usd_per_1000": 0.017795820322000097, "axes": { "intelligence": 69.21087754421087, "calibration": 50.20568125731147, "speed": 74.18754290853735, "cost": 62.49045963451314 }, "jevbench_score": 63.35311619805061 }, "heldout": { "n": 303, "escalation": 0.13861386138613863, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9178082191780822, "hard": 0.46788990825688076 }, "accuracy": 0.7524752475247525, "ece_hard": 0.30766115416919065, "calibration": 52.65245532472025, "p50_s": 1.3345372080802917, "p95_s": 2.227775090932845, "cost_usd_per_1000": 0.01799206993733081, "axes": { "intelligence": 75.9019940513594, "calibration": 52.65245532472025, "speed": 75.267979694634, "cost": 62.347566082434255 }, "jevbench_score": 65.80750027954805 } }, { "threshold": 0.56, "public": { "n": 231, "escalation": 0.13852813852813853, "tiers": { "easy": 1.0, "standard": 0.7777777777777778, "hard": 0.4774774774774775 }, "accuracy": 0.6796536796536796, "ece_hard": 0.3122963694936664, "calibration": 50.39064123883588, "p50_s": 1.309425413608551, "p95_s": 2.9144957825541495, "cost_usd_per_1000": 0.017990145921494234, "axes": { "intelligence": 69.58625291958626, "calibration": 50.39064123883588, "speed": 74.1835580324561, "cost": 62.34895941985127 }, "jevbench_score": 63.46036290064948 }, "heldout": { "n": 303, "escalation": 0.1551155115511551, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9178082191780822, "hard": 0.46788990825688076 }, "accuracy": 0.7524752475247525, "ece_hard": 0.31401223874589945, "calibration": 52.01734686704937, "p50_s": 1.336438462138176, "p95_s": 2.2960563117265695, "cost_usd_per_1000": 0.01836244298587162, "axes": { "intelligence": 75.9019940513594, "calibration": 52.01734686704937, "speed": 75.13068498933522, "cost": 62.08208618962685 }, "jevbench_score": 65.50828932060483 } }, { "threshold": 0.57, "public": { "n": 231, "escalation": 0.14285714285714285, "tiers": { "easy": 1.0, "standard": 0.7777777777777778, "hard": 0.4864864864864865 }, "accuracy": 0.683982683982684, "ece_hard": 0.31621941444676954, "calibration": 49.99833674352557, "p50_s": 1.309425413608551, "p95_s": 2.981084243953228, "cost_usd_per_1000": 0.018087308721241303, "axes": { "intelligence": 69.96162829496163, "calibration": 49.99833674352557, "speed": 74.08544985300612, "cost": 62.27878145932248 }, "jevbench_score": 63.38290260498533 }, "heldout": { "n": 303, "escalation": 0.16501650165016502, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9178082191780822, "hard": 0.47706422018348627 }, "accuracy": 0.7557755775577558, "ece_hard": 0.3067026837648065, "calibration": 52.748302365158665, "p50_s": 1.3393714874982834, "p95_s": 2.2960563117265695, "cost_usd_per_1000": 0.018584666814996105, "axes": { "intelligence": 76.17722340915756, "calibration": 52.748302365158665, "speed": 75.12116415442314, "cost": 61.92535661526514 }, "jevbench_score": 65.75308183723872 } }, { "threshold": 0.58, "public": { "n": 231, "escalation": 0.1471861471861472, "tiers": { "easy": 1.0, "standard": 0.7777777777777778, "hard": 0.4864864864864865 }, "accuracy": 0.683982683982684, "ece_hard": 0.32479703335356686, "calibration": 49.140574852845845, "p50_s": 1.309425413608551, "p95_s": 2.981084243953228, "cost_usd_per_1000": 0.018184471520988375, "axes": { "intelligence": 69.96162829496163, "calibration": 49.140574852845845, "speed": 74.08544985300612, "cost": 62.20897947812458 }, "jevbench_score": 63.09159911493603 }, "heldout": { "n": 303, "escalation": 0.16501650165016502, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9178082191780822, "hard": 0.47706422018348627 }, "accuracy": 0.7557755775577558, "ece_hard": 0.3067026837648065, "calibration": 52.748302365158665, "p50_s": 1.3393714874982834, "p95_s": 2.2960563117265695, "cost_usd_per_1000": 0.018584666814996105, "axes": { "intelligence": 76.17722340915756, "calibration": 52.748302365158665, "speed": 75.12116415442314, "cost": 61.92535661526514 }, "jevbench_score": 65.75308183723872 } }, { "threshold": 0.59, "public": { "n": 231, "escalation": 0.15584415584415584, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "hard": 0.4864864864864865 }, "accuracy": 0.6883116883116883, "ece_hard": 0.32479703335356686, "calibration": 49.140574852845845, "p50_s": 1.3110789507627487, "p95_s": 2.981084243953228, "cost_usd_per_1000": 0.01837879712048251, "axes": { "intelligence": 70.50175175175175, "calibration": 49.140574852845845, "speed": 74.07996905956222, "cost": 62.070487488767014 }, "jevbench_score": 63.17663966793885 }, "heldout": { "n": 303, "escalation": 0.16831683168316833, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9178082191780822, "hard": 0.47706422018348627 }, "accuracy": 0.7557755775577558, "ece_hard": 0.30608234837337184, "calibration": 52.810335904302136, "p50_s": 1.3393714874982834, "p95_s": 2.2960563117265695, "cost_usd_per_1000": 0.018658741424704268, "axes": { "intelligence": 76.17722340915756, "calibration": 52.810335904302136, "speed": 75.12116415442314, "cost": 61.87352961308756 }, "jevbench_score": 65.75863920568746 } }, { "threshold": 0.6, "public": { "n": 231, "escalation": 0.17316017316017315, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "hard": 0.46846846846846846 }, "accuracy": 0.6796536796536796, "ece_hard": 0.3188171835131318, "calibration": 49.760944906544346, "p50_s": 1.3143753111362457, "p95_s": 2.9984570384025573, "cost_usd_per_1000": 0.01876744831947079, "axes": { "intelligence": 69.751001001001, "calibration": 49.760944906544346, "speed": 74.0438277647452, "cost": 61.797843141720584 }, "jevbench_score": 63.12847681913774 }, "heldout": { "n": 303, "escalation": 0.17491749174917492, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9178082191780822, "hard": 0.47706422018348627 }, "accuracy": 0.7557755775577558, "ece_hard": 0.30722705757164526, "calibration": 52.69586498447479, "p50_s": 1.3393714874982834, "p95_s": 2.2960563117265695, "cost_usd_per_1000": 0.01880689064412059, "axes": { "intelligence": 76.17722340915756, "calibration": 52.69586498447479, "speed": 75.12116415442314, "cost": 61.77049002140545 }, "jevbench_score": 65.69559618713718 } }, { "threshold": 0.61, "public": { "n": 231, "escalation": 0.17316017316017315, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "hard": 0.46846846846846846 }, "accuracy": 0.6796536796536796, "ece_hard": 0.3188171835131318, "calibration": 49.760944906544346, "p50_s": 1.3143753111362457, "p95_s": 2.9984570384025573, "cost_usd_per_1000": 0.01876744831947079, "axes": { "intelligence": 69.751001001001, "calibration": 49.760944906544346, "speed": 74.0438277647452, "cost": 61.797843141720584 }, "jevbench_score": 63.12847681913774 }, "heldout": { "n": 303, "escalation": 0.1848184818481848, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.47706422018348627 }, "accuracy": 0.7623762376237624, "ece_hard": 0.30722705757164526, "calibration": 52.69586498447479, "p50_s": 1.3393714874982834, "p95_s": 2.325209452807903, "cost_usd_per_1000": 0.019029114473245075, "axes": { "intelligence": 76.56078505299317, "calibration": 52.69586498447479, "speed": 75.06636870699569, "cost": 61.617442638827185 }, "jevbench_score": 65.72536372962193 } }, { "threshold": 0.62, "public": { "n": 231, "escalation": 0.1774891774891775, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "hard": 0.4774774774774775 }, "accuracy": 0.683982683982684, "ece_hard": 0.3101621336360815, "calibration": 51.65765512025798, "p50_s": 1.3143753111362457, "p95_s": 2.9984570384025573, "cost_usd_per_1000": 0.01886461111921786, "axes": { "intelligence": 70.12637637637638, "calibration": 51.65765512025798, "speed": 74.0438277647452, "cost": 61.730564291443585 }, "jevbench_score": 63.78980857828201 }, "heldout": { "n": 303, "escalation": 0.19471947194719472, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.4954128440366973 }, "accuracy": 0.768976897689769, "ece_hard": 0.29327442618052224, "calibration": 54.09112812358709, "p50_s": 1.3400312662124634, "p95_s": 2.3536960193514815, "cost_usd_per_1000": 0.01925133830236956, "axes": { "intelligence": 77.1112437685895, "calibration": 54.09112812358709, "speed": 75.01134695514986, "cost": 61.46617222290119 }, "jevbench_score": 66.22191263126443 } }, { "threshold": 0.63, "public": { "n": 231, "escalation": 0.19047619047619047, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "hard": 0.4864864864864865 }, "accuracy": 0.6883116883116883, "ece_hard": 0.29853690247834036, "calibration": 53.79119029600767, "p50_s": 1.3146315664052963, "p95_s": 2.9984570384025573, "cost_usd_per_1000": 0.019156099518459067, "axes": { "intelligence": 70.50175175175175, "calibration": 53.79119029600767, "speed": 74.04298113135971, "cost": 61.53078746219671 }, "jevbench_score": 64.4721061686793 }, "heldout": { "n": 303, "escalation": 0.20462046204620463, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9383561643835616, "hard": 0.4954128440366973 }, "accuracy": 0.7722772277227723, "ece_hard": 0.29112360561834233, "calibration": 54.30621017980508, "p50_s": 1.3414461016654968, "p95_s": 2.3566206446290017, "cost_usd_per_1000": 0.019473562131494048, "axes": { "intelligence": 77.3030245905073, "calibration": 54.30621017980508, "speed": 75.00137094381684, "cost": 61.31663798359465 }, "jevbench_score": 66.28623920706795 } }, { "threshold": 0.64, "public": { "n": 231, "escalation": 0.19913419913419914, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "hard": 0.4864864864864865 }, "accuracy": 0.6883116883116883, "ece_hard": 0.30297932131530897, "calibration": 53.34694841231081, "p50_s": 1.3177027255296707, "p95_s": 2.9984570384025573, "cost_usd_per_1000": 0.019350425117953207, "axes": { "intelligence": 70.50175175175175, "calibration": 53.34694841231081, "speed": 74.03284725079573, "cost": 61.39928468004488 }, "jevbench_score": 64.30197516886007 }, "heldout": { "n": 303, "escalation": 0.22112211221122113, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9452054794520548, "hard": 0.5137614678899083 }, "accuracy": 0.7821782178217822, "ece_hard": 0.271759533983223, "calibration": 56.242617343317015, "p50_s": 1.3426922708749771, "p95_s": 2.4181795209646206, "cost_usd_per_1000": 0.019843935180034855, "axes": { "intelligence": 78.04526412802146, "calibration": 56.242617343317015, "speed": 74.88534972162361, "cost": 61.071166007530245 }, "jevbench_score": 66.93623612629719 } }, { "threshold": 0.65, "public": { "n": 231, "escalation": 0.20346320346320346, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "hard": 0.4774774774774775 }, "accuracy": 0.683982683982684, "ece_hard": 0.31510943115307566, "calibration": 52.13393742853414, "p50_s": 1.319142371416092, "p95_s": 2.9984570384025573, "cost_usd_per_1000": 0.019447587917700276, "axes": { "intelligence": 70.12637637637638, "calibration": 52.13393742853414, "speed": 74.02810499022672, "cost": 61.33402769492042 }, "jevbench_score": 63.83002440194463 }, "heldout": { "n": 303, "escalation": 0.23102310231023102, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5229357798165137 }, "accuracy": 0.7887788778877888, "ece_hard": 0.2664959485057827, "calibration": 56.76897589106105, "p50_s": 1.3426922708749771, "p95_s": 2.446256107985973, "cost_usd_per_1000": 0.02006615900915934, "axes": { "intelligence": 78.51227430773741, "calibration": 56.76897589106105, "speed": 74.83521588020507, "cost": 60.9260725186038 }, "jevbench_score": 67.14125489810263 } }, { "threshold": 0.66, "public": { "n": 231, "escalation": 0.2077922077922078, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "hard": 0.4774774774774775 }, "accuracy": 0.683982683982684, "ece_hard": 0.31341529447620553, "calibration": 52.30335109622115, "p50_s": 1.319142371416092, "p95_s": 3.0257777139544486, "cost_usd_per_1000": 0.019544750717447344, "axes": { "intelligence": 70.12637637637638, "calibration": 52.30335109622115, "speed": 73.98871309907929, "cost": 61.269095931514514 }, "jevbench_score": 63.85640498304301 }, "heldout": { "n": 303, "escalation": 0.24092409240924093, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.958904109589041, "hard": 0.5229357798165137 }, "accuracy": 0.7920792079207921, "ece_hard": 0.2593972378461735, "calibration": 57.76637020479299, "p50_s": 1.3426922708749771, "p95_s": 2.447237626612186, "cost_usd_per_1000": 0.020288382838283825, "axes": { "intelligence": 78.70405512965523, "calibration": 57.76637020479299, "speed": 74.83347369700392, "cost": 60.78257705977287 }, "jevbench_score": 67.43522343297572 } }, { "threshold": 0.67, "public": { "n": 231, "escalation": 0.2077922077922078, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "hard": 0.4774774774774775 }, "accuracy": 0.683982683982684, "ece_hard": 0.31341529447620553, "calibration": 52.30335109622115, "p50_s": 1.319142371416092, "p95_s": 3.0257777139544486, "cost_usd_per_1000": 0.019544750717447344, "axes": { "intelligence": 70.12637637637638, "calibration": 52.30335109622115, "speed": 73.98871309907929, "cost": 61.269095931514514 }, "jevbench_score": 63.85640498304301 }, "heldout": { "n": 303, "escalation": 0.24422442244224424, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.958904109589041, "hard": 0.5229357798165137 }, "accuracy": 0.7920792079207921, "ece_hard": 0.2593972378461735, "calibration": 57.76637020479299, "p50_s": 1.3426922708749771, "p95_s": 2.447237626612186, "cost_usd_per_1000": 0.020362457447991988, "axes": { "intelligence": 78.70405512965523, "calibration": 57.76637020479299, "speed": 74.83347369700392, "cost": 60.73509430720149 }, "jevbench_score": 67.42204963947876 } }, { "threshold": 0.68, "public": { "n": 231, "escalation": 0.2077922077922078, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "hard": 0.4774774774774775 }, "accuracy": 0.683982683982684, "ece_hard": 0.31341529447620553, "calibration": 52.30335109622115, "p50_s": 1.319142371416092, "p95_s": 3.0257777139544486, "cost_usd_per_1000": 0.019544750717447344, "axes": { "intelligence": 70.12637637637638, "calibration": 52.30335109622115, "speed": 73.98871309907929, "cost": 61.269095931514514 }, "jevbench_score": 63.85640498304301 }, "heldout": { "n": 303, "escalation": 0.25412541254125415, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9657534246575342, "hard": 0.5137614678899083 }, "accuracy": 0.7920792079207921, "ece_hard": 0.27000575446797237, "calibration": 56.70551854261311, "p50_s": 1.3468152582645416, "p95_s": 2.4500220602750775, "cost_usd_per_1000": 0.020584681277116476, "axes": { "intelligence": 78.62060659377488, "calibration": 56.70551854261311, "speed": 74.81521977252524, "cost": 60.59367559037305 }, "jevbench_score": 67.04937632007214 } }, { "threshold": 0.69, "public": { "n": 231, "escalation": 0.2077922077922078, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "hard": 0.4774774774774775 }, "accuracy": 0.683982683982684, "ece_hard": 0.31341529447620553, "calibration": 52.30335109622115, "p50_s": 1.319142371416092, "p95_s": 3.0257777139544486, "cost_usd_per_1000": 0.019544750717447344, "axes": { "intelligence": 70.12637637637638, "calibration": 52.30335109622115, "speed": 73.98871309907929, "cost": 61.269095931514514 }, "jevbench_score": 63.85640498304301 }, "heldout": { "n": 303, "escalation": 0.26732673267326734, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9657534246575342, "hard": 0.5229357798165137 }, "accuracy": 0.7953795379537953, "ece_hard": 0.2593683266931834, "calibration": 59.007757798138556, "p50_s": 1.3563375920057297, "p95_s": 2.4500220602750775, "cost_usd_per_1000": 0.02088097971594912, "axes": { "intelligence": 78.89583595157303, "calibration": 59.007757798138556, "speed": 74.78462206433798, "cost": 60.407473855080774 }, "jevbench_score": 67.71993664058249 } }, { "threshold": 0.7, "public": { "n": 231, "escalation": 0.21212121212121213, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "hard": 0.4774774774774775 }, "accuracy": 0.683982683982684, "ece_hard": 0.31341529447620553, "calibration": 52.30335109622115, "p50_s": 1.3191694766283035, "p95_s": 3.0257777139544486, "cost_usd_per_1000": 0.019641913517194416, "axes": { "intelligence": 70.12637637637638, "calibration": 52.30335109622115, "speed": 73.98862386289514, "cost": 61.20448616427594 }, "jevbench_score": 63.8395445358421 }, "heldout": { "n": 303, "escalation": 0.2706270627062706, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9657534246575342, "hard": 0.5229357798165137 }, "accuracy": 0.7953795379537953, "ece_hard": 0.2593683266931834, "calibration": 59.007757798138556, "p50_s": 1.3563375920057297, "p95_s": 2.596573760509487, "cost_usd_per_1000": 0.020955054325657283, "axes": { "intelligence": 78.89583595157303, "calibration": 59.007757798138556, "speed": 74.53231537203236, "cost": 60.36133626780661 }, "jevbench_score": 67.64982278689617 } }, { "threshold": 0.71, "public": { "n": 231, "escalation": 0.22077922077922077, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "hard": 0.4864864864864865 }, "accuracy": 0.6883116883116883, "ece_hard": 0.3040165086572414, "calibration": 53.24322967811756, "p50_s": 1.3192902952432632, "p95_s": 3.074162782728672, "cost_usd_per_1000": 0.019836239116688553, "axes": { "intelligence": 70.50175175175175, "calibration": 53.24322967811756, "speed": 73.91932771753181, "cost": 61.07621995390748 }, "jevbench_score": 64.16136792698306 }, "heldout": { "n": 303, "escalation": 0.2871287128712871, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9657534246575342, "hard": 0.5321100917431193 }, "accuracy": 0.7986798679867987, "ece_hard": 0.25106625917785413, "calibration": 59.837964549671476, "p50_s": 1.3565552085638046, "p95_s": 2.9017923897504736, "cost_usd_per_1000": 0.02132542737419809, "axes": { "intelligence": 79.17106530937122, "calibration": 59.837964549671476, "speed": 74.04896189404352, "cost": 60.13306769663579 }, "jevbench_score": 67.77100232233555 } }, { "threshold": 0.72, "public": { "n": 231, "escalation": 0.23376623376623376, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "hard": 0.4864864864864865 }, "accuracy": 0.6883116883116883, "ece_hard": 0.3003850717900295, "calibration": 53.9706794781697, "p50_s": 1.3224025070667267, "p95_s": 3.074162782728672, "cost_usd_per_1000": 0.020127727515929762, "axes": { "intelligence": 70.50175175175175, "calibration": 53.9706794781697, "speed": 73.9090947559582, "cost": 60.88615766690732 }, "jevbench_score": 64.32703901298467 }, "heldout": { "n": 303, "escalation": 0.297029702970297, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.958904109589041, "hard": 0.5321100917431193 }, "accuracy": 0.7953795379537953, "ece_hard": 0.25106625917785413, "calibration": 59.837964549671476, "p50_s": 1.3586038798093796, "p95_s": 2.9017923897504736, "cost_usd_per_1000": 0.02154765120332258, "axes": { "intelligence": 78.9792844874534, "calibration": 59.837964549671476, "speed": 74.04240812126575, "cost": 59.99800189274989 }, "jevbench_score": 67.69036133331711 } }, { "threshold": 0.73, "public": { "n": 231, "escalation": 0.24242424242424243, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "hard": 0.4954954954954955 }, "accuracy": 0.6926406926406926, "ece_hard": 0.2895626228485461, "calibration": 55.05292437231804, "p50_s": 1.3224025070667267, "p95_s": 3.107661770284176, "cost_usd_per_1000": 0.0203220531154239, "axes": { "intelligence": 70.87712712712712, "calibration": 55.05292437231804, "speed": 73.86202590847704, "cost": 60.76097253585188 }, "jevbench_score": 64.68939773062807 }, "heldout": { "n": 303, "escalation": 0.3069306930693069, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.958904109589041, "hard": 0.5321100917431193 }, "accuracy": 0.7953795379537953, "ece_hard": 0.25111425686493755, "calibration": 59.83316478096314, "p50_s": 1.3768931925296783, "p95_s": 2.9017923897504736, "cost_usd_per_1000": 0.021769875032447063, "axes": { "intelligence": 78.9792844874534, "calibration": 59.83316478096314, "speed": 73.98433409859793, "cost": 59.86432191913156 }, "jevbench_score": 67.63799910649782 } }, { "threshold": 0.74, "public": { "n": 231, "escalation": 0.2510822510822511, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.4954954954954955 }, "accuracy": 0.696969696969697, "ece_hard": 0.29128340353498505, "calibration": 54.88084630367415, "p50_s": 1.3224025070667267, "p95_s": 3.107661770284176, "cost_usd_per_1000": 0.02051637871491804, "axes": { "intelligence": 71.41725058391727, "calibration": 54.88084630367415, "speed": 73.86202590847704, "cost": 60.63697878475546 }, "jevbench_score": 64.72851956187773 }, "heldout": { "n": 303, "escalation": 0.3201320132013201, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5229357798165137 }, "accuracy": 0.7887788778877888, "ece_hard": 0.2600011387137997, "calibration": 58.944476596076925, "p50_s": 1.3797449171543121, "p95_s": 2.9017923897504736, "cost_usd_per_1000": 0.02206617347127971, "axes": { "intelligence": 78.51227430773741, "calibration": 58.944476596076925, "speed": 73.97534859770221, "cost": 59.68818915928383 }, "jevbench_score": 67.23401153829303 } }, { "threshold": 0.75, "public": { "n": 231, "escalation": 0.26406926406926406, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.4864864864864865 }, "accuracy": 0.6926406926406926, "ece_hard": 0.30041549276135404, "calibration": 53.96763738103724, "p50_s": 1.3238774687051773, "p95_s": 3.1353327870368957, "cost_usd_per_1000": 0.020807867114159245, "axes": { "intelligence": 71.0418752085419, "calibration": 53.96763738103724, "speed": 73.81868565082098, "cost": 60.453173030495336 }, "jevbench_score": 64.31441265215228 }, "heldout": { "n": 303, "escalation": 0.3234323432343234, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5229357798165137 }, "accuracy": 0.7887788778877888, "ece_hard": 0.25943745197693635, "calibration": 59.00084526976326, "p50_s": 1.3797449171543121, "p95_s": 2.962685215473174, "cost_usd_per_1000": 0.02214024808098787, "axes": { "intelligence": 78.51227430773741, "calibration": 59.00084526976326, "speed": 73.88515686901762, "cost": 59.64452551510034 }, "jevbench_score": 67.21727387175038 } }, { "threshold": 0.76, "public": { "n": 231, "escalation": 0.2683982683982684, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.4864864864864865 }, "accuracy": 0.6926406926406926, "ece_hard": 0.30041549276135404, "calibration": 53.96763738103724, "p50_s": 1.3238774687051773, "p95_s": 3.1353327870368957, "cost_usd_per_1000": 0.020905029913906317, "axes": { "intelligence": 71.0418752085419, "calibration": 53.96763738103724, "speed": 73.81868565082098, "cost": 60.39247619987413 }, "jevbench_score": 64.2982631621525 }, "heldout": { "n": 303, "escalation": 0.33993399339933994, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5229357798165137 }, "accuracy": 0.7887788778877888, "ece_hard": 0.26157038489468054, "calibration": 58.78755197798884, "p50_s": 1.384657546877861, "p95_s": 2.9739170536398887, "cost_usd_per_1000": 0.02251062112952868, "axes": { "intelligence": 78.51227430773741, "calibration": 58.78755197798884, "speed": 73.85328771585951, "cost": 59.42837564389128 }, "jevbench_score": 67.08827986597052 } }, { "threshold": 0.77, "public": { "n": 231, "escalation": 0.27705627705627706, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.4774774774774775 }, "accuracy": 0.6883116883116883, "ece_hard": 0.30628388503201637, "calibration": 53.380798153971014, "p50_s": 1.325474590063095, "p95_s": 3.1353327870368957, "cost_usd_per_1000": 0.021099355513400454, "axes": { "intelligence": 70.66649983316651, "calibration": 53.380798153971014, "speed": 73.81344949449814, "cost": 60.27192430492975 }, "jevbench_score": 64.00476402389863 }, "heldout": { "n": 303, "escalation": 0.34983498349834985, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5229357798165137 }, "accuracy": 0.7887788778877888, "ece_hard": 0.2614914720614379, "calibration": 58.795443261313096, "p50_s": 1.4045170545578003, "p95_s": 2.9739170536398887, "cost_usd_per_1000": 0.022732844958653166, "axes": { "intelligence": 78.51227430773741, "calibration": 58.795443261313096, "speed": 73.79144131620015, "cost": 59.300386300189146 }, "jevbench_score": 67.04033651984736 } }, { "threshold": 0.78, "public": { "n": 231, "escalation": 0.29004329004329005, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.46846846846846846 }, "accuracy": 0.683982683982684, "ece_hard": 0.3110209578769899, "calibration": 53.99640728961536, "p50_s": 1.3266585320234299, "p95_s": 3.1353327870368957, "cost_usd_per_1000": 0.021390843912641663, "axes": { "intelligence": 70.29112445779114, "calibration": 53.99640728961536, "speed": 73.80957201572349, "cost": 60.09316243848716 }, "jevbench_score": 64.05466666735353 }, "heldout": { "n": 303, "escalation": 0.3564356435643564, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5229357798165137 }, "accuracy": 0.7887788778877888, "ece_hard": 0.2673131957569529, "calibration": 58.2132708917616, "p50_s": 1.438198447227478, "p95_s": 2.9739170536398887, "cost_usd_per_1000": 0.02288099417806949, "axes": { "intelligence": 78.51227430773741, "calibration": 58.2132708917616, "speed": 73.68852333150144, "cost": 59.21575328185637 }, "jevbench_score": 66.8265695854812 } }, { "threshold": 0.79, "public": { "n": 231, "escalation": 0.2987012987012987, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.46846846846846846 }, "accuracy": 0.683982683982684, "ece_hard": 0.3092562178833471, "calibration": 54.17288128897964, "p50_s": 1.3277880102396011, "p95_s": 3.1353327870368957, "cost_usd_per_1000": 0.0215851695121358, "axes": { "intelligence": 70.29112445779114, "calibration": 54.17288128897964, "speed": 73.80587613277007, "cost": 59.9753360920992 }, "jevbench_score": 64.07469005216879 }, "heldout": { "n": 303, "escalation": 0.3696369636963696, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5229357798165137 }, "accuracy": 0.7887788778877888, "ece_hard": 0.2673131957569529, "calibration": 58.2132708917616, "p50_s": 1.4431772530078888, "p95_s": 2.9739170536398887, "cost_usd_per_1000": 0.023177292616902136, "axes": { "intelligence": 78.51227430773741, "calibration": 58.2132708917616, "speed": 73.67351473662289, "cost": 59.04811888492279 }, "jevbench_score": 66.77582371994602 } }, { "threshold": 0.8, "public": { "n": 231, "escalation": 0.30735930735930733, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.46846846846846846 }, "accuracy": 0.683982683982684, "ece_hard": 0.3092562178833471, "calibration": 54.17288128897964, "p50_s": 1.3289764374494553, "p95_s": 3.1353327870368957, "cost_usd_per_1000": 0.02177949511162994, "axes": { "intelligence": 70.29112445779114, "calibration": 54.17288128897964, "speed": 73.80199074944782, "cost": 59.85856576602249 }, "jevbench_score": 64.04263646038514 }, "heldout": { "n": 303, "escalation": 0.3795379537953795, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5321100917431193 }, "accuracy": 0.7920792079207921, "ece_hard": 0.25804926641655757, "calibration": 59.13966382580114, "p50_s": 1.4465409815311432, "p95_s": 2.9739170536398887, "cost_usd_per_1000": 0.02339951644602662, "axes": { "intelligence": 78.7875036655356, "calibration": 59.13966382580114, "speed": 73.66340406646403, "cost": 58.923793517429324 }, "jevbench_score": 67.06094495157164 } }, { "threshold": 0.81, "public": { "n": 231, "escalation": 0.31601731601731603, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.46846846846846846 }, "accuracy": 0.683982683982684, "ece_hard": 0.31069727333822195, "calibration": 54.02877574349216, "p50_s": 1.3289764374494553, "p95_s": 3.1660484462976455, "cost_usd_per_1000": 0.02197382071112408, "axes": { "intelligence": 70.29112445779114, "calibration": 54.02877574349216, "speed": 73.75965162891887, "cost": 59.74283269908562 }, "jevbench_score": 63.959869847395176 }, "heldout": { "n": 303, "escalation": 0.3927392739273927, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5321100917431193 }, "accuracy": 0.7920792079207921, "ece_hard": 0.26101126947240505, "calibration": 58.84346352021639, "p50_s": 1.4572032541036606, "p95_s": 2.9739170536398887, "cost_usd_per_1000": 0.02369581488485927, "axes": { "intelligence": 78.7875036655356, "calibration": 58.84346352021639, "speed": 73.63151016274298, "cost": 58.75985054743182 }, "jevbench_score": 66.92293657157087 } }, { "threshold": 0.82, "public": { "n": 231, "escalation": 0.3203463203463203, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.46846846846846846 }, "accuracy": 0.683982683982684, "ece_hard": 0.31069727333822195, "calibration": 54.02877574349216, "p50_s": 1.3397867381572723, "p95_s": 3.1660484462976455, "cost_usd_per_1000": 0.02207098351087115, "axes": { "intelligence": 70.29112445779114, "calibration": 54.02877574349216, "speed": 73.72446769303428, "cost": 59.68534941091298 }, "jevbench_score": 63.93685221533738 }, "heldout": { "n": 303, "escalation": 0.39603960396039606, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5321100917431193 }, "accuracy": 0.7920792079207921, "ece_hard": 0.2618606550357388, "calibration": 58.75852496388301, "p50_s": 1.4642701894044876, "p95_s": 2.9739170536398887, "cost_usd_per_1000": 0.02376988949456743, "axes": { "intelligence": 78.7875036655356, "calibration": 58.75852496388301, "speed": 73.61049927867575, "cost": 58.71918511863563 }, "jevbench_score": 66.88242361008075 } }, { "threshold": 0.83, "public": { "n": 231, "escalation": 0.33766233766233766, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.46846846846846846 }, "accuracy": 0.683982683982684, "ece_hard": 0.3123658957673682, "calibration": 53.861913500577536, "p50_s": 1.349913328886032, "p95_s": 3.1660484462976455, "cost_usd_per_1000": 0.022459634709859427, "axes": { "intelligence": 70.29112445779114, "calibration": 53.861913500577536, "speed": 73.69176558391783, "cost": 59.45791934533004 }, "jevbench_score": 63.81940261051558 }, "heldout": { "n": 303, "escalation": 0.40264026402640263, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5321100917431193 }, "accuracy": 0.7920792079207921, "ece_hard": 0.26153700164947724, "calibration": 58.79089030250917, "p50_s": 1.4727533608675003, "p95_s": 2.9739170536398887, "cost_usd_per_1000": 0.023918038713983757, "axes": { "intelligence": 78.7875036655356, "calibration": 58.79089030250917, "speed": 73.58541126192902, "cost": 58.638233064860295 }, "jevbench_score": 66.86286682140408 } }, { "threshold": 0.84, "public": { "n": 231, "escalation": 0.3593073593073593, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.4774774774774775 }, "accuracy": 0.6926406926406926, "ece_hard": 0.29766268864392564, "calibration": 55.33223421292179, "p50_s": 1.3694269061088562, "p95_s": 3.1660484462976455, "cost_usd_per_1000": 0.022945448708594773, "axes": { "intelligence": 71.20662328995662, "calibration": 55.33223421292179, "speed": 73.62943587418515, "cost": 59.17910335185637 }, "jevbench_score": 64.36942252024147 }, "heldout": { "n": 303, "escalation": 0.41914191419141916, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.958904109589041, "hard": 0.5504587155963303 }, "accuracy": 0.801980198019802, "ece_hard": 0.2408040179614242, "calibration": 60.864188671314466, "p50_s": 1.4812378883361816, "p95_s": 2.9739170536398887, "cost_usd_per_1000": 0.024288411762524567, "axes": { "intelligence": 79.52974320304973, "calibration": 60.864188671314466, "speed": 73.56046336409754, "cost": 58.438026493256245 }, "jevbench_score": 67.53950193435648 } }, { "threshold": 0.85, "public": { "n": 231, "escalation": 0.36796536796536794, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.46846846846846846 }, "accuracy": 0.6883116883116883, "ece_hard": 0.3059041663848725, "calibration": 53.75206629260161, "p50_s": 1.377013847231865, "p95_s": 3.1660484462976455, "cost_usd_per_1000": 0.02313977430808891, "axes": { "intelligence": 70.83124791458125, "calibration": 53.75206629260161, "speed": 73.60544136389544, "cost": 59.06922643658805 }, "jevbench_score": 63.78562501826135 }, "heldout": { "n": 303, "escalation": 0.43234323432343236, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.958904109589041, "hard": 0.5596330275229358 }, "accuracy": 0.8052805280528053, "ece_hard": 0.2326212675141856, "calibration": 61.68246371603833, "p50_s": 1.5059185326099396, "p95_s": 2.9739170536398887, "cost_usd_per_1000": 0.02458471020135721, "axes": { "intelligence": 79.80497256084789, "calibration": 61.68246371603833, "speed": 73.48869670698218, "cost": 58.28004720107208 }, "jevbench_score": 67.76150234438238 } }, { "threshold": 0.86, "public": { "n": 231, "escalation": 0.38095238095238093, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.46846846846846846 }, "accuracy": 0.6883116883116883, "ece_hard": 0.3027280381750075, "calibration": 54.06967911358811, "p50_s": 1.384503349661827, "p95_s": 3.1660484462976455, "cost_usd_per_1000": 0.02343126270733012, "axes": { "intelligence": 70.83124791458125, "calibration": 54.06967911358811, "speed": 73.5818843311444, "cost": 58.90612920118302 }, "jevbench_score": 63.83039310992831 }, "heldout": { "n": 303, "escalation": 0.44554455445544555, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.958904109589041, "hard": 0.5504587155963303 }, "accuracy": 0.801980198019802, "ece_hard": 0.23047635160845834, "calibration": 60.07680867035242, "p50_s": 1.5078626871109009, "p95_s": 2.982789239287376, "cost_usd_per_1000": 0.02488100864018986, "axes": { "intelligence": 79.52974320304973, "calibration": 60.07680867035242, "speed": 73.47015637851801, "cost": 58.12396053858086 }, "jevbench_score": 67.2087239090187 } }, { "threshold": 0.87, "public": { "n": 231, "escalation": 0.39826839826839827, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.4954954954954955 }, "accuracy": 0.7012987012987013, "ece_hard": 0.2705795477762054, "calibration": 57.284528153468315, "p50_s": 1.4003681987524033, "p95_s": 3.1660484462976455, "cost_usd_per_1000": 0.0238199139063184, "axes": { "intelligence": 71.95737404070738, "calibration": 57.284528153468315, "speed": 73.53240204204786, "cost": 58.6917943763761 }, "jevbench_score": 64.94447295113375 }, "heldout": { "n": 303, "escalation": 0.46864686468646866, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.958904109589041, "hard": 0.5596330275229358 }, "accuracy": 0.8052805280528053, "ece_hard": 0.22618283571999814, "calibration": 60.50616025919844, "p50_s": 1.7801171630620956, "p95_s": 2.982789239287376, "cost_usd_per_1000": 0.02539953090814699, "axes": { "intelligence": 79.80497256084789, "calibration": 60.50616025919844, "speed": 72.74928844864009, "cost": 57.85522912252528 }, "jevbench_score": 67.1429206074867 } }, { "threshold": 0.88, "public": { "n": 231, "escalation": 0.4025974025974026, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.4954954954954955 }, "accuracy": 0.7012987012987013, "ece_hard": 0.2714899242670765, "calibration": 57.19349050438121, "p50_s": 1.409024566411972, "p95_s": 3.1660484462976455, "cost_usd_per_1000": 0.02391707670606547, "axes": { "intelligence": 71.95737404070738, "calibration": 57.19349050438121, "speed": 73.50563878806804, "cost": 58.638757108413046 }, "jevbench_score": 64.89807728053425 }, "heldout": { "n": 303, "escalation": 0.48184818481848185, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7986798679867987, "ece_hard": 0.23576459730350177, "calibration": 59.54798410084808, "p50_s": 1.79626307785511, "p95_s": 2.982789239287376, "cost_usd_per_1000": 0.02569582934697964, "axes": { "intelligence": 79.33796238113192, "calibration": 59.54798410084808, "speed": 72.71007489265767, "cost": 57.70412081980294 }, "jevbench_score": 66.7248155083516 } }, { "threshold": 0.89, "public": { "n": 231, "escalation": 0.42857142857142855, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.5135135135135135 }, "accuracy": 0.70995670995671, "ece_hard": 0.24983300665288094, "calibration": 59.35918226580077, "p50_s": 1.4595188051462173, "p95_s": 3.1660484462976455, "cost_usd_per_1000": 0.024500053504547883, "axes": { "intelligence": 72.70812479145812, "calibration": 59.35918226580077, "speed": 73.3527274890738, "cost": 58.32498901595644 }, "jevbench_score": 65.55192451444722 }, "heldout": { "n": 303, "escalation": 0.49174917491749176, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7986798679867987, "ece_hard": 0.23576459730350177, "calibration": 59.54798410084808, "p50_s": 1.8074035435914992, "p95_s": 2.982789239287376, "cost_usd_per_1000": 0.025918053176104128, "axes": { "intelligence": 79.33796238113192, "calibration": 59.54798410084808, "speed": 72.68322302917329, "cost": 57.591928702826806 }, "jevbench_score": 66.68620091730165 } }, { "threshold": 0.9, "public": { "n": 231, "escalation": 0.45454545454545453, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.5405405405405406 }, "accuracy": 0.7229437229437229, "ece_hard": 0.2234243934324515, "calibration": 62.49486064043897, "p50_s": 1.5103851854801178, "p95_s": 3.1876641005277633, "cost_usd_per_1000": 0.0250830303030303, "axes": { "intelligence": 73.83425091758426, "calibration": 62.49486064043897, "speed": 73.17439731335541, "cost": 58.018599915161715 }, "jevbench_score": 66.5284099644264 }, "heldout": { "n": 303, "escalation": 0.5016501650165016, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7986798679867987, "ece_hard": 0.23618353663766378, "calibration": 59.27776823031364, "p50_s": 1.81396327316761, "p95_s": 2.982789239287376, "cost_usd_per_1000": 0.02614027700522861, "axes": { "intelligence": 79.33796238113192, "calibration": 59.27776823031364, "speed": 72.66748942571533, "cost": 57.480694437012176 }, "jevbench_score": 66.57463012897138 } }, { "threshold": 0.91, "public": { "n": 231, "escalation": 0.4588744588744589, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.5315315315315315 }, "accuracy": 0.7186147186147186, "ece_hard": 0.22958055368579958, "calibration": 61.58757567449595, "p50_s": 1.5125302523374557, "p95_s": 3.1876641005277633, "cost_usd_per_1000": 0.02518019310277737, "axes": { "intelligence": 73.45887554220889, "calibration": 61.58757567449595, "speed": 73.16823378761359, "cost": 57.96822831046373 }, "jevbench_score": 66.18544542603377 }, "heldout": { "n": 303, "escalation": 0.5181518151815182, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5412844036697247 }, "accuracy": 0.7953795379537953, "ece_hard": 0.2371764445700075, "calibration": 59.70655467846506, "p50_s": 1.8164630472660064, "p95_s": 2.993203791975975, "cost_usd_per_1000": 0.026510650053769423, "axes": { "intelligence": 79.06273302333375, "calibration": 59.70655467846506, "speed": 72.64637145923454, "cost": 57.29738869086228 }, "jevbench_score": 66.57875162013823 } }, { "threshold": 0.92, "public": { "n": 231, "escalation": 0.47619047619047616, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.5315315315315315 }, "accuracy": 0.7186147186147186, "ece_hard": 0.23073821422698754, "calibration": 61.47180962037716, "p50_s": 1.7855939507484435, "p95_s": 3.1876641005277633, "cost_usd_per_1000": 0.025568844301765647, "axes": { "intelligence": 73.45887554220889, "calibration": 61.47180962037716, "speed": 72.44744742606652, "cost": 57.76866704232554 }, "jevbench_score": 65.93392463626328 }, "heldout": { "n": 303, "escalation": 0.5412541254125413, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5412844036697247 }, "accuracy": 0.7953795379537953, "ece_hard": 0.22516311051190058, "calibration": 60.907888084275754, "p50_s": 1.8254972636699676, "p95_s": 2.993203791975975, "cost_usd_per_1000": 0.027029172321726552, "axes": { "intelligence": 79.06273302333375, "calibration": 60.907888084275754, "speed": 72.62482526957197, "cost": 57.04501758754042 }, "jevbench_score": 66.83239822552551 } }, { "threshold": 0.93, "public": { "n": 231, "escalation": 0.49783549783549785, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.5405405405405406 }, "accuracy": 0.7229437229437229, "ece_hard": 0.2213723792598709, "calibration": 62.40839311708882, "p50_s": 1.796794021129608, "p95_s": 3.1876641005277633, "cost_usd_per_1000": 0.026054658300500994, "axes": { "intelligence": 73.83425091758426, "calibration": 62.40839311708882, "speed": 72.42029154764157, "cost": 57.52343854328734 }, "jevbench_score": 66.19138909629011 }, "heldout": { "n": 303, "escalation": 0.5676567656765676, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5412844036697247 }, "accuracy": 0.7953795379537953, "ece_hard": 0.22579295490924695, "calibration": 60.84490364454112, "p50_s": 1.8264458268880843, "p95_s": 2.993203791975975, "cost_usd_per_1000": 0.027621769199391848, "axes": { "intelligence": 79.06273302333375, "calibration": 60.84490364454112, "speed": 72.6225691786614, "cost": 56.76245523732061 }, "jevbench_score": 66.73170219273588 } }, { "threshold": 0.94, "public": { "n": 231, "escalation": 0.5238095238095238, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.5585585585585585 }, "accuracy": 0.7316017316017316, "ece_hard": 0.2043728841476717, "calibration": 64.10834262830873, "p50_s": 1.8032960534095763, "p95_s": 3.1876641005277633, "cost_usd_per_1000": 0.02663763509898341, "axes": { "intelligence": 74.58500166833501, "calibration": 64.10834262830873, "speed": 72.40460416342503, "cost": 57.23513003938991 }, "jevbench_score": 66.71887698872474 }, "heldout": { "n": 303, "escalation": 0.594059405940594, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5412844036697247 }, "accuracy": 0.7953795379537953, "ece_hard": 0.22579295490924695, "calibration": 60.84490364454112, "p50_s": 1.8317748099565505, "p95_s": 2.993203791975975, "cost_usd_per_1000": 0.028214366077057143, "axes": { "intelligence": 79.06273302333375, "calibration": 60.84490364454112, "speed": 72.60991630827675, "cost": 56.485891091567105 }, "jevbench_score": 66.64736563519493 } }, { "threshold": 0.95, "public": { "n": 231, "escalation": 0.5454545454545454, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.5675675675675675 }, "accuracy": 0.7359307359307359, "ece_hard": 0.20227579433689236, "calibration": 64.08157880084966, "p50_s": 1.8098465740680694, "p95_s": 3.1876641005277633, "cost_usd_per_1000": 0.027123449097718758, "axes": { "intelligence": 74.96037704371038, "calibration": 64.08157880084966, "speed": 72.38885688530304, "cost": 56.999652553814556 }, "jevbench_score": 66.72325477207794 }, "heldout": { "n": 303, "escalation": 0.6204620462046204, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5321100917431193 }, "accuracy": 0.7920792079207921, "ece_hard": 0.23407843864380407, "calibration": 60.0163552710854, "p50_s": 1.8349565058946609, "p95_s": 2.993203791975975, "cost_usd_per_1000": 0.028806962954722435, "axes": { "intelligence": 78.7875036655356, "calibration": 60.0163552710854, "speed": 72.6023793866057, "cost": 56.215075776263944 }, "jevbench_score": 66.2800234042233 } }, { "threshold": 0.96, "public": { "n": 231, "escalation": 0.5627705627705628, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.5675675675675675 }, "accuracy": 0.7359307359307359, "ece_hard": 0.19885699498597081, "calibration": 64.42345873594181, "p50_s": 1.8148353189229964, "p95_s": 3.1876641005277633, "cost_usd_per_1000": 0.02751210029670704, "axes": { "intelligence": 74.96037704371038, "calibration": 64.42345873594181, "speed": 72.3769022597987, "cost": 56.81428761824156 }, "jevbench_score": 66.75492912207086 }, "heldout": { "n": 303, "escalation": 0.6633663366336634, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5412844036697247 }, "accuracy": 0.7953795379537953, "ece_hard": 0.2143179605163461, "calibration": 61.992403083831206, "p50_s": 1.8411511123180389, "p95_s": 3.009263858199119, "cost_usd_per_1000": 0.02976993288092854, "axes": { "intelligence": 79.06273302333375, "calibration": 61.992403083831206, "speed": 72.56450296143515, "cost": 55.78666433527937 }, "jevbench_score": 66.74077239215816 } }, { "threshold": 0.97, "public": { "n": 231, "escalation": 0.6103896103896104, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.5765765765765766 }, "accuracy": 0.7402597402597403, "ece_hard": 0.18044115655653947, "calibration": 66.26504257888493, "p50_s": 1.8212630093097686, "p95_s": 3.197047398984432, "cost_usd_per_1000": 0.0285808910939248, "axes": { "intelligence": 75.33575241908576, "calibration": 66.26504257888493, "speed": 72.34878257929095, "cost": 56.31772704751859 }, "jevbench_score": 67.15687661587678 }, "heldout": { "n": 303, "escalation": 0.6963696369636964, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5412844036697247 }, "accuracy": 0.7953795379537953, "ece_hard": 0.20941178010978975, "calibration": 62.48302112448684, "p50_s": 1.85029279589653, "p95_s": 3.009263858199119, "cost_usd_per_1000": 0.03051067897801016, "axes": { "intelligence": 79.06273302333375, "calibration": 62.48302112448684, "speed": 72.54299272886317, "cost": 55.46644382665497 }, "jevbench_score": 66.77131177663165 } }, { "threshold": 0.98, "public": { "n": 231, "escalation": 0.6796536796536796, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.5855855855855856 }, "accuracy": 0.7445887445887446, "ece_hard": 0.16484832815082107, "calibration": 67.82432541945678, "p50_s": 1.8241762161254882, "p95_s": 3.197047398984432, "cost_usd_per_1000": 0.03013549588987791, "axes": { "intelligence": 75.71112779446113, "calibration": 67.82432541945678, "speed": 72.34184135857217, "cost": 55.627649730197 }, "jevbench_score": 67.42273523422891 }, "heldout": { "n": 303, "escalation": 0.7458745874587459, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7986798679867987, "ece_hard": 0.19846267184176286, "calibration": 64.35757215405391, "p50_s": 1.8569146931171416, "p95_s": 3.0174763172864916, "cost_usd_per_1000": 0.03162179812363258, "axes": { "intelligence": 79.33796238113192, "calibration": 64.35757215405391, "speed": 72.51564177127098, "cost": 55.00040314694519 }, "jevbench_score": 67.1768392003924 } }, { "threshold": 0.99, "public": { "n": 231, "escalation": 0.7316017316017316, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.5945945945945946 }, "accuracy": 0.7489177489177489, "ece_hard": 0.14632171565234448, "calibration": 70.52700185648575, "p50_s": 1.8318715035915374, "p95_s": 3.197047398984432, "cost_usd_per_1000": 0.03130144948684274, "axes": { "intelligence": 76.0865031698365, "calibration": 70.52700185648575, "speed": 72.32355918024906, "cost": 55.13306652897859 }, "jevbench_score": 68.01250120356407 }, "heldout": { "n": 303, "escalation": 0.8547854785478548, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7986798679867987, "ece_hard": 0.19046608330777925, "calibration": 65.15723100745228, "p50_s": 1.8709805101156234, "p95_s": 3.0801832318305964, "cost_usd_per_1000": 0.034066260244001924, "axes": { "intelligence": 79.33796238113192, "calibration": 65.15723100745228, "speed": 72.39354184212674, "cost": 54.03026620535524 }, "jevbench_score": 67.0571588153839 } }, { "threshold": 1.0, "public": { "n": 231, "escalation": 1.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6036036036036037 }, "accuracy": 0.7532467532467533, "ece_hard": 0.12914387221294593, "calibration": 72.24478620042561, "p50_s": 1.845125177502632, "p95_s": 3.197047398984432, "cost_usd_per_1000": 0.037325543071161045, "axes": { "intelligence": 76.46187854521189, "calibration": 72.24478620042561, "speed": 72.29225089867128, "cost": 52.839815942574866 }, "jevbench_score": 67.77603046638914 }, "heldout": { "n": 303, "escalation": 1.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7986798679867987, "ece_hard": 0.17685813516393573, "calibration": 66.51802582183663, "p50_s": 1.884817326068878, "p95_s": 3.083856791257858, "cost_usd_per_1000": 0.037325543071161045, "axes": { "intelligence": 79.33796238113192, "calibration": 66.51802582183663, "speed": 72.35636532653506, "cost": 52.839815942574866 }, "jevbench_score": 67.02157171315109 } } ] }, { "cheap": "system-one-open", "backend": "djev", "chosen": { "threshold": 0.99, "public": { "n": 231, "escalation": 0.7316017316017316, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6396396396396397 }, "accuracy": 0.7705627705627706, "ece_hard": 0.2232823509934215, "calibration": 60.0650561914047, "p50_s": 1.5319301038980484, "p95_s": 2.4070832021534443, "cost_usd_per_1000": 0.03386691919191919, "axes": { "intelligence": 77.96338004671338, "calibration": 60.0650561914047, "speed": 74.33269947282982, "cost": 54.10672925460625 }, "jevbench_score": 65.87732157779108 }, "heldout": { "n": 303, "escalation": 0.8547854785478548, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6605504587155964 }, "accuracy": 0.8283828382838284, "ece_hard": 0.22162323552224517, "calibration": 61.28056017314768, "p50_s": 1.5429361686110497, "p95_s": 2.2624109119176863, "cost_usd_per_1000": 0.037063691981557706, "axes": { "intelligence": 82.06537220895648, "calibration": 61.28056017314768, "speed": 74.5708055357799, "cost": 52.93153966004737 }, "jevbench_score": 66.74849959620211 } }, "full": { "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9315068493150684, "hard": 0.65 }, "accuracy": 0.8033707865168539, "ece_hard": 0.2224603346917936, "calibration": 60.67205403888016, "p50_s": 1.538209691643715, "p95_s": 2.3479351159185167, "cost_usd_per_1000": 0.035680818359073635, "axes": { "intelligence": 82.33219178082192, "calibration": 60.67205403888016, "speed": 74.42298365438822, "cost": 53.426955802010184 }, "jevbench_score": 66.75848470046476 }, "full_escalation": 0.8014981273408239, "backend_heldout_accuracy": 0.834983498349835, "useful": false, "curve": [ { "threshold": 0.0, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 1.29877570271492, "p95_s": 2.095727913081646, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 68.67075408742076, "calibration": 50.471910537865476, "speed": 75.65130946864079, "cost": 64.82109224519658 }, "jevbench_score": 64.2079181671165 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28023325293453055, "calibration": 55.17142992768658, "p50_s": 1.3129856139421463, "p95_s": 1.9534294441342355, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 74.75130911985254, "calibration": 55.17142992768658, "speed": 75.90942302768094, "cost": 64.82109224519658 }, "jevbench_score": 67.11757697107404 } }, { "threshold": 0.01, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 1.29877570271492, "p95_s": 2.095727913081646, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 68.67075408742076, "calibration": 50.471910537865476, "speed": 75.65130946864079, "cost": 64.82109224519658 }, "jevbench_score": 64.2079181671165 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28023325293453055, "calibration": 55.17142992768658, "p50_s": 1.3129856139421463, "p95_s": 1.9534294441342355, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 74.75130911985254, "calibration": 55.17142992768658, "speed": 75.90942302768094, "cost": 64.82109224519658 }, "jevbench_score": 67.11757697107404 } }, { "threshold": 0.02, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 1.29877570271492, "p95_s": 2.095727913081646, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 68.67075408742076, "calibration": 50.471910537865476, "speed": 75.65130946864079, "cost": 64.82109224519658 }, "jevbench_score": 64.2079181671165 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28023325293453055, "calibration": 55.17142992768658, "p50_s": 1.3129856139421463, "p95_s": 1.9534294441342355, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 74.75130911985254, "calibration": 55.17142992768658, "speed": 75.90942302768094, "cost": 64.82109224519658 }, "jevbench_score": 67.11757697107404 } }, { "threshold": 0.03, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 1.29877570271492, "p95_s": 2.095727913081646, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 68.67075408742076, "calibration": 50.471910537865476, "speed": 75.65130946864079, "cost": 64.82109224519658 }, "jevbench_score": 64.2079181671165 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28023325293453055, "calibration": 55.17142992768658, "p50_s": 1.3129856139421463, "p95_s": 1.9534294441342355, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 74.75130911985254, "calibration": 55.17142992768658, "speed": 75.90942302768094, "cost": 64.82109224519658 }, "jevbench_score": 67.11757697107404 } }, { "threshold": 0.04, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 1.29877570271492, "p95_s": 2.095727913081646, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 68.67075408742076, "calibration": 50.471910537865476, "speed": 75.65130946864079, "cost": 64.82109224519658 }, "jevbench_score": 64.2079181671165 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28023325293453055, "calibration": 55.17142992768658, "p50_s": 1.3129856139421463, "p95_s": 1.9534294441342355, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 74.75130911985254, "calibration": 55.17142992768658, "speed": 75.90942302768094, "cost": 64.82109224519658 }, "jevbench_score": 67.11757697107404 } }, { "threshold": 0.05, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 1.29877570271492, "p95_s": 2.095727913081646, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 68.67075408742076, "calibration": 50.471910537865476, "speed": 75.65130946864079, "cost": 64.82109224519658 }, "jevbench_score": 64.2079181671165 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28023325293453055, "calibration": 55.17142992768658, "p50_s": 1.3129856139421463, "p95_s": 1.9534294441342355, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 74.75130911985254, "calibration": 55.17142992768658, "speed": 75.90942302768094, "cost": 64.82109224519658 }, "jevbench_score": 67.11757697107404 } }, { "threshold": 0.06, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 1.29877570271492, "p95_s": 2.095727913081646, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 68.67075408742076, "calibration": 50.471910537865476, "speed": 75.65130946864079, "cost": 64.82109224519658 }, "jevbench_score": 64.2079181671165 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28023325293453055, "calibration": 55.17142992768658, "p50_s": 1.3129856139421463, "p95_s": 1.9534294441342355, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 74.75130911985254, "calibration": 55.17142992768658, "speed": 75.90942302768094, "cost": 64.82109224519658 }, "jevbench_score": 67.11757697107404 } }, { "threshold": 0.07, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 1.29877570271492, "p95_s": 2.095727913081646, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 68.67075408742076, "calibration": 50.471910537865476, "speed": 75.65130946864079, "cost": 64.82109224519658 }, "jevbench_score": 64.2079181671165 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28023325293453055, "calibration": 55.17142992768658, "p50_s": 1.3129856139421463, "p95_s": 1.9534294441342355, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 74.75130911985254, "calibration": 55.17142992768658, "speed": 75.90942302768094, "cost": 64.82109224519658 }, "jevbench_score": 67.11757697107404 } }, { "threshold": 0.08, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 1.29877570271492, "p95_s": 2.095727913081646, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 68.67075408742076, "calibration": 50.471910537865476, "speed": 75.65130946864079, "cost": 64.82109224519658 }, "jevbench_score": 64.2079181671165 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28023325293453055, "calibration": 55.17142992768658, "p50_s": 1.3129856139421463, "p95_s": 1.9534294441342355, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 74.75130911985254, "calibration": 55.17142992768658, "speed": 75.90942302768094, "cost": 64.82109224519658 }, "jevbench_score": 67.11757697107404 } }, { "threshold": 0.09, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 1.29877570271492, "p95_s": 2.095727913081646, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 68.67075408742076, "calibration": 50.471910537865476, "speed": 75.65130946864079, "cost": 64.82109224519658 }, "jevbench_score": 64.2079181671165 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28023325293453055, "calibration": 55.17142992768658, "p50_s": 1.3129856139421463, "p95_s": 1.9534294441342355, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 74.75130911985254, "calibration": 55.17142992768658, "speed": 75.90942302768094, "cost": 64.82109224519658 }, "jevbench_score": 67.11757697107404 } }, { "threshold": 0.1, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 1.29877570271492, "p95_s": 2.095727913081646, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 68.67075408742076, "calibration": 50.471910537865476, "speed": 75.65130946864079, "cost": 64.82109224519658 }, "jevbench_score": 64.2079181671165 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28023325293453055, "calibration": 55.17142992768658, "p50_s": 1.3129856139421463, "p95_s": 1.9534294441342355, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 74.75130911985254, "calibration": 55.17142992768658, "speed": 75.90942302768094, "cost": 64.82109224519658 }, "jevbench_score": 67.11757697107404 } }, { "threshold": 0.11, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 1.29877570271492, "p95_s": 2.095727913081646, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 68.67075408742076, "calibration": 50.471910537865476, "speed": 75.65130946864079, "cost": 64.82109224519658 }, "jevbench_score": 64.2079181671165 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28023325293453055, "calibration": 55.17142992768658, "p50_s": 1.3129856139421463, "p95_s": 1.9534294441342355, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 74.75130911985254, "calibration": 55.17142992768658, "speed": 75.90942302768094, "cost": 64.82109224519658 }, "jevbench_score": 67.11757697107404 } }, { "threshold": 0.12, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 1.29877570271492, "p95_s": 2.095727913081646, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 68.67075408742076, "calibration": 50.471910537865476, "speed": 75.65130946864079, "cost": 64.82109224519658 }, "jevbench_score": 64.2079181671165 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28023325293453055, "calibration": 55.17142992768658, "p50_s": 1.3129856139421463, "p95_s": 1.9534294441342355, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 74.75130911985254, "calibration": 55.17142992768658, "speed": 75.90942302768094, "cost": 64.82109224519658 }, "jevbench_score": 67.11757697107404 } }, { "threshold": 0.13, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 1.29877570271492, "p95_s": 2.095727913081646, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 68.67075408742076, "calibration": 50.471910537865476, "speed": 75.65130946864079, "cost": 64.82109224519658 }, "jevbench_score": 64.2079181671165 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28023325293453055, "calibration": 55.17142992768658, "p50_s": 1.3129856139421463, "p95_s": 1.9534294441342355, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 74.75130911985254, "calibration": 55.17142992768658, "speed": 75.90942302768094, "cost": 64.82109224519658 }, "jevbench_score": 67.11757697107404 } }, { "threshold": 0.14, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 1.29877570271492, "p95_s": 2.095727913081646, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 68.67075408742076, "calibration": 50.471910537865476, "speed": 75.65130946864079, "cost": 64.82109224519658 }, "jevbench_score": 64.2079181671165 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28023325293453055, "calibration": 55.17142992768658, "p50_s": 1.3129856139421463, "p95_s": 1.9534294441342355, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 74.75130911985254, "calibration": 55.17142992768658, "speed": 75.90942302768094, "cost": 64.82109224519658 }, "jevbench_score": 67.11757697107404 } }, { "threshold": 0.15, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 1.29877570271492, "p95_s": 2.095727913081646, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 68.67075408742076, "calibration": 50.471910537865476, "speed": 75.65130946864079, "cost": 64.82109224519658 }, "jevbench_score": 64.2079181671165 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28023325293453055, "calibration": 55.17142992768658, "p50_s": 1.3129856139421463, "p95_s": 1.9534294441342355, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 74.75130911985254, "calibration": 55.17142992768658, "speed": 75.90942302768094, "cost": 64.82109224519658 }, "jevbench_score": 67.11757697107404 } }, { "threshold": 0.16, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 1.29877570271492, "p95_s": 2.095727913081646, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 68.67075408742076, "calibration": 50.471910537865476, "speed": 75.65130946864079, "cost": 64.82109224519658 }, "jevbench_score": 64.2079181671165 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28023325293453055, "calibration": 55.17142992768658, "p50_s": 1.3129856139421463, "p95_s": 1.9534294441342355, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 74.75130911985254, "calibration": 55.17142992768658, "speed": 75.90942302768094, "cost": 64.82109224519658 }, "jevbench_score": 67.11757697107404 } }, { "threshold": 0.17, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 1.29877570271492, "p95_s": 2.095727913081646, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 68.67075408742076, "calibration": 50.471910537865476, "speed": 75.65130946864079, "cost": 64.82109224519658 }, "jevbench_score": 64.2079181671165 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28023325293453055, "calibration": 55.17142992768658, "p50_s": 1.3129856139421463, "p95_s": 1.9534294441342355, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 74.75130911985254, "calibration": 55.17142992768658, "speed": 75.90942302768094, "cost": 64.82109224519658 }, "jevbench_score": 67.11757697107404 } }, { "threshold": 0.18, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 1.29877570271492, "p95_s": 2.095727913081646, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 68.67075408742076, "calibration": 50.471910537865476, "speed": 75.65130946864079, "cost": 64.82109224519658 }, "jevbench_score": 64.2079181671165 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28023325293453055, "calibration": 55.17142992768658, "p50_s": 1.3129856139421463, "p95_s": 1.9534294441342355, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 74.75130911985254, "calibration": 55.17142992768658, "speed": 75.90942302768094, "cost": 64.82109224519658 }, "jevbench_score": 67.11757697107404 } }, { "threshold": 0.19, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 1.29877570271492, "p95_s": 2.095727913081646, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 68.67075408742076, "calibration": 50.471910537865476, "speed": 75.65130946864079, "cost": 64.82109224519658 }, "jevbench_score": 64.2079181671165 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28023325293453055, "calibration": 55.17142992768658, "p50_s": 1.3129856139421463, "p95_s": 1.9534294441342355, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 74.75130911985254, "calibration": 55.17142992768658, "speed": 75.90942302768094, "cost": 64.82109224519658 }, "jevbench_score": 67.11757697107404 } }, { "threshold": 0.2, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 1.29877570271492, "p95_s": 2.095727913081646, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 68.67075408742076, "calibration": 50.471910537865476, "speed": 75.65130946864079, "cost": 64.82109224519658 }, "jevbench_score": 64.2079181671165 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28023325293453055, "calibration": 55.17142992768658, "p50_s": 1.3129856139421463, "p95_s": 1.9534294441342355, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 74.75130911985254, "calibration": 55.17142992768658, "speed": 75.90942302768094, "cost": 64.82109224519658 }, "jevbench_score": 67.11757697107404 } }, { "threshold": 0.21, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 1.29877570271492, "p95_s": 2.095727913081646, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 68.67075408742076, "calibration": 50.471910537865476, "speed": 75.65130946864079, "cost": 64.82109224519658 }, "jevbench_score": 64.2079181671165 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28023325293453055, "calibration": 55.17142992768658, "p50_s": 1.3129856139421463, "p95_s": 1.9534294441342355, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 74.75130911985254, "calibration": 55.17142992768658, "speed": 75.90942302768094, "cost": 64.82109224519658 }, "jevbench_score": 67.11757697107404 } }, { "threshold": 0.22, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 1.29877570271492, "p95_s": 2.095727913081646, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 68.67075408742076, "calibration": 50.471910537865476, "speed": 75.65130946864079, "cost": 64.82109224519658 }, "jevbench_score": 64.2079181671165 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28023325293453055, "calibration": 55.17142992768658, "p50_s": 1.3129856139421463, "p95_s": 1.9534294441342355, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 74.75130911985254, "calibration": 55.17142992768658, "speed": 75.90942302768094, "cost": 64.82109224519658 }, "jevbench_score": 67.11757697107404 } }, { "threshold": 0.23, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 1.29877570271492, "p95_s": 2.095727913081646, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 68.67075408742076, "calibration": 50.471910537865476, "speed": 75.65130946864079, "cost": 64.82109224519658 }, "jevbench_score": 64.2079181671165 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28023325293453055, "calibration": 55.17142992768658, "p50_s": 1.3129856139421463, "p95_s": 1.9534294441342355, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 74.75130911985254, "calibration": 55.17142992768658, "speed": 75.90942302768094, "cost": 64.82109224519658 }, "jevbench_score": 67.11757697107404 } }, { "threshold": 0.24, "public": { "n": 231, "escalation": 0.004329004329004329, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3137236882136536, "calibration": 50.24790936683716, "p50_s": 1.29877570271492, "p95_s": 2.095727913081646, "cost_usd_per_1000": 0.014993279423447962, "axes": { "intelligence": 68.67075408742076, "calibration": 50.24790936683716, "speed": 75.65130946864079, "cost": 64.72310095503776 }, "jevbench_score": 64.11230556497485 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28023325293453055, "calibration": 55.17142992768658, "p50_s": 1.3129856139421463, "p95_s": 1.9534294441342355, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 74.75130911985254, "calibration": 55.17142992768658, "speed": 75.90942302768094, "cost": 64.82109224519658 }, "jevbench_score": 67.11757697107404 } }, { "threshold": 0.25, "public": { "n": 231, "escalation": 0.004329004329004329, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3137236882136536, "calibration": 50.24790936683716, "p50_s": 1.29877570271492, "p95_s": 2.095727913081646, "cost_usd_per_1000": 0.014993279423447962, "axes": { "intelligence": 68.67075408742076, "calibration": 50.24790936683716, "speed": 75.65130946864079, "cost": 64.72310095503776 }, "jevbench_score": 64.11230556497485 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28023325293453055, "calibration": 55.17142992768658, "p50_s": 1.3129856139421463, "p95_s": 1.9534294441342355, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 74.75130911985254, "calibration": 55.17142992768658, "speed": 75.90942302768094, "cost": 64.82109224519658 }, "jevbench_score": 67.11757697107404 } }, { "threshold": 0.26, "public": { "n": 231, "escalation": 0.004329004329004329, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3137236882136536, "calibration": 50.24790936683716, "p50_s": 1.29877570271492, "p95_s": 2.095727913081646, "cost_usd_per_1000": 0.014993279423447962, "axes": { "intelligence": 68.67075408742076, "calibration": 50.24790936683716, "speed": 75.65130946864079, "cost": 64.72310095503776 }, "jevbench_score": 64.11230556497485 }, "heldout": { "n": 303, "escalation": 0.0033003300330033004, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.47706422018348627 }, "accuracy": 0.735973597359736, "ece_hard": 0.27221028216389914, "calibration": 55.97372700474972, "p50_s": 1.3129856139421463, "p95_s": 1.9534294441342355, "cost_usd_per_1000": 0.014966584034807974, "axes": { "intelligence": 75.0265384776507, "calibration": 55.97372700474972, "speed": 75.90942302768094, "cost": 64.74631934466032 }, "jevbench_score": 67.40272853836133 } }, { "threshold": 0.27, "public": { "n": 231, "escalation": 0.004329004329004329, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3137236882136536, "calibration": 50.24790936683716, "p50_s": 1.29877570271492, "p95_s": 2.095727913081646, "cost_usd_per_1000": 0.014993279423447962, "axes": { "intelligence": 68.67075408742076, "calibration": 50.24790936683716, "speed": 75.65130946864079, "cost": 64.72310095503776 }, "jevbench_score": 64.11230556497485 }, "heldout": { "n": 303, "escalation": 0.0033003300330033004, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.47706422018348627 }, "accuracy": 0.735973597359736, "ece_hard": 0.27221028216389914, "calibration": 55.97372700474972, "p50_s": 1.3129856139421463, "p95_s": 1.9534294441342355, "cost_usd_per_1000": 0.014966584034807974, "axes": { "intelligence": 75.0265384776507, "calibration": 55.97372700474972, "speed": 75.90942302768094, "cost": 64.74631934466032 }, "jevbench_score": 67.40272853836133 } }, { "threshold": 0.28, "public": { "n": 231, "escalation": 0.004329004329004329, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3137236882136536, "calibration": 50.24790936683716, "p50_s": 1.29877570271492, "p95_s": 2.095727913081646, "cost_usd_per_1000": 0.014993279423447962, "axes": { "intelligence": 68.67075408742076, "calibration": 50.24790936683716, "speed": 75.65130946864079, "cost": 64.72310095503776 }, "jevbench_score": 64.11230556497485 }, "heldout": { "n": 303, "escalation": 0.0033003300330033004, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.47706422018348627 }, "accuracy": 0.735973597359736, "ece_hard": 0.27221028216389914, "calibration": 55.97372700474972, "p50_s": 1.3129856139421463, "p95_s": 1.9534294441342355, "cost_usd_per_1000": 0.014966584034807974, "axes": { "intelligence": 75.0265384776507, "calibration": 55.97372700474972, "speed": 75.90942302768094, "cost": 64.74631934466032 }, "jevbench_score": 67.40272853836133 } }, { "threshold": 0.29, "public": { "n": 231, "escalation": 0.004329004329004329, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3137236882136536, "calibration": 50.24790936683716, "p50_s": 1.29877570271492, "p95_s": 2.095727913081646, "cost_usd_per_1000": 0.014993279423447962, "axes": { "intelligence": 68.67075408742076, "calibration": 50.24790936683716, "speed": 75.65130946864079, "cost": 64.72310095503776 }, "jevbench_score": 64.11230556497485 }, "heldout": { "n": 303, "escalation": 0.0033003300330033004, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.47706422018348627 }, "accuracy": 0.735973597359736, "ece_hard": 0.27221028216389914, "calibration": 55.97372700474972, "p50_s": 1.3129856139421463, "p95_s": 1.9534294441342355, "cost_usd_per_1000": 0.014966584034807974, "axes": { "intelligence": 75.0265384776507, "calibration": 55.97372700474972, "speed": 75.90942302768094, "cost": 64.74631934466032 }, "jevbench_score": 67.40272853836133 } }, { "threshold": 0.3, "public": { "n": 231, "escalation": 0.004329004329004329, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3137236882136536, "calibration": 50.24790936683716, "p50_s": 1.29877570271492, "p95_s": 2.095727913081646, "cost_usd_per_1000": 0.014993279423447962, "axes": { "intelligence": 68.67075408742076, "calibration": 50.24790936683716, "speed": 75.65130946864079, "cost": 64.72310095503776 }, "jevbench_score": 64.11230556497485 }, "heldout": { "n": 303, "escalation": 0.0033003300330033004, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.47706422018348627 }, "accuracy": 0.735973597359736, "ece_hard": 0.27221028216389914, "calibration": 55.97372700474972, "p50_s": 1.3129856139421463, "p95_s": 1.9534294441342355, "cost_usd_per_1000": 0.014966584034807974, "axes": { "intelligence": 75.0265384776507, "calibration": 55.97372700474972, "speed": 75.90942302768094, "cost": 64.74631934466032 }, "jevbench_score": 67.40272853836133 } }, { "threshold": 0.31, "public": { "n": 231, "escalation": 0.004329004329004329, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3137236882136536, "calibration": 50.24790936683716, "p50_s": 1.29877570271492, "p95_s": 2.095727913081646, "cost_usd_per_1000": 0.014993279423447962, "axes": { "intelligence": 68.67075408742076, "calibration": 50.24790936683716, "speed": 75.65130946864079, "cost": 64.72310095503776 }, "jevbench_score": 64.11230556497485 }, "heldout": { "n": 303, "escalation": 0.006600660066006601, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.47706422018348627 }, "accuracy": 0.735973597359736, "ece_hard": 0.27369515468327243, "calibration": 55.825239752812394, "p50_s": 1.3150212317705154, "p95_s": 1.9534294441342355, "cost_usd_per_1000": 0.015052231740027934, "axes": { "intelligence": 75.0265384776507, "calibration": 55.825239752812394, "speed": 75.90269505611289, "cost": 64.67197312066227 }, "jevbench_score": 67.33714572285932 } }, { "threshold": 0.32, "public": { "n": 231, "escalation": 0.004329004329004329, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3137236882136536, "calibration": 50.24790936683716, "p50_s": 1.29877570271492, "p95_s": 2.095727913081646, "cost_usd_per_1000": 0.014993279423447962, "axes": { "intelligence": 68.67075408742076, "calibration": 50.24790936683716, "speed": 75.65130946864079, "cost": 64.72310095503776 }, "jevbench_score": 64.11230556497485 }, "heldout": { "n": 303, "escalation": 0.0165016501650165, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.47706422018348627 }, "accuracy": 0.735973597359736, "ece_hard": 0.27508525847724086, "calibration": 55.68622937341555, "p50_s": 1.315921351313591, "p95_s": 1.9534294441342355, "cost_usd_per_1000": 0.015309174855687816, "axes": { "intelligence": 75.0265384776507, "calibration": 55.68622937341555, "speed": 75.89972336856351, "cost": 64.45144649710343 }, "jevbench_score": 67.23708795904888 } }, { "threshold": 0.33, "public": { "n": 231, "escalation": 0.012987012987012988, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.31205958096794917, "calibration": 50.41432009140761, "p50_s": 1.29877570271492, "p95_s": 2.1450010761618614, "cost_usd_per_1000": 0.015217965611167858, "axes": { "intelligence": 68.67075408742076, "calibration": 50.41432009140761, "speed": 75.5503833027139, "cost": 64.52930204896883 }, "jevbench_score": 64.09583988288226 }, "heldout": { "n": 303, "escalation": 0.0165016501650165, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.47706422018348627 }, "accuracy": 0.735973597359736, "ece_hard": 0.27508525847724086, "calibration": 55.68622937341555, "p50_s": 1.315921351313591, "p95_s": 1.9534294441342355, "cost_usd_per_1000": 0.015309174855687816, "axes": { "intelligence": 75.0265384776507, "calibration": 55.68622937341555, "speed": 75.89972336856351, "cost": 64.45144649710343 }, "jevbench_score": 67.23708795904888 } }, { "threshold": 0.34, "public": { "n": 231, "escalation": 0.017316017316017316, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.4594594594594595 }, "accuracy": 0.6666666666666666, "ece_hard": 0.3147516652372091, "calibration": 50.14511166448162, "p50_s": 1.29877570271492, "p95_s": 2.182773359119892, "cost_usd_per_1000": 0.015330308705027804, "axes": { "intelligence": 68.29537871204539, "calibration": 50.14511166448162, "speed": 75.47457200127542, "cost": 64.4334729912763 }, "jevbench_score": 63.882665680586896 }, "heldout": { "n": 303, "escalation": 0.019801980198019802, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.47706422018348627 }, "accuracy": 0.735973597359736, "ece_hard": 0.27780488746821846, "calibration": 55.41426647431779, "p50_s": 1.31616772711277, "p95_s": 1.9534294441342355, "cost_usd_per_1000": 0.015394822560907776, "axes": { "intelligence": 75.0265384776507, "calibration": 55.41426647431779, "speed": 75.89891032882828, "cost": 64.37875937092234 }, "jevbench_score": 67.13572150890808 } }, { "threshold": 0.35, "public": { "n": 231, "escalation": 0.017316017316017316, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.4594594594594595 }, "accuracy": 0.6666666666666666, "ece_hard": 0.3147516652372091, "calibration": 50.14511166448162, "p50_s": 1.29877570271492, "p95_s": 2.182773359119892, "cost_usd_per_1000": 0.015330308705027804, "axes": { "intelligence": 68.29537871204539, "calibration": 50.14511166448162, "speed": 75.47457200127542, "cost": 64.4334729912763 }, "jevbench_score": 63.882665680586896 }, "heldout": { "n": 303, "escalation": 0.019801980198019802, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.47706422018348627 }, "accuracy": 0.735973597359736, "ece_hard": 0.27780488746821846, "calibration": 55.41426647431779, "p50_s": 1.31616772711277, "p95_s": 1.9534294441342355, "cost_usd_per_1000": 0.015394822560907776, "axes": { "intelligence": 75.0265384776507, "calibration": 55.41426647431779, "speed": 75.89891032882828, "cost": 64.37875937092234 }, "jevbench_score": 67.13572150890808 } }, { "threshold": 0.36, "public": { "n": 231, "escalation": 0.021645021645021644, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.31794820319197226, "calibration": 49.825457869005305, "p50_s": 1.29877570271492, "p95_s": 2.182773359119892, "cost_usd_per_1000": 0.015442651798887753, "axes": { "intelligence": 68.67075408742076, "calibration": 49.825457869005305, "speed": 75.47457200127542, "cost": 64.33834362764338 }, "jevbench_score": 63.84448857577704 }, "heldout": { "n": 303, "escalation": 0.026402640264026403, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.48623853211009177 }, "accuracy": 0.7392739273927392, "ece_hard": 0.2705991466401015, "calibration": 56.13484055712949, "p50_s": 1.3169443011283875, "p95_s": 1.9534294441342355, "cost_usd_per_1000": 0.015566117971347696, "axes": { "intelligence": 75.30176783544887, "calibration": 56.13484055712949, "speed": 75.89634863136966, "cost": 64.23459047427286 }, "jevbench_score": 67.37625642169525 } }, { "threshold": 0.37, "public": { "n": 231, "escalation": 0.030303030303030304, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.4594594594594595 }, "accuracy": 0.6666666666666666, "ece_hard": 0.3270182854964311, "calibration": 48.91844963855942, "p50_s": 1.2999516874551773, "p95_s": 2.182773359119892, "cost_usd_per_1000": 0.01566733798660765, "axes": { "intelligence": 68.29537871204539, "calibration": 48.91844963855942, "speed": 75.47064143341139, "cost": 64.1501436271999 }, "jevbench_score": 63.417617041660876 }, "heldout": { "n": 303, "escalation": 0.033003300330033, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.48623853211009177 }, "accuracy": 0.7392739273927392, "ece_hard": 0.2712500241685222, "calibration": 56.06975280428742, "p50_s": 1.3169443011283875, "p95_s": 1.9534294441342355, "cost_usd_per_1000": 0.015737413381787618, "axes": { "intelligence": 75.30176783544887, "calibration": 56.06975280428742, "speed": 75.89634863136966, "cost": 64.091999416607 }, "jevbench_score": 67.31930582602433 } }, { "threshold": 0.38, "public": { "n": 231, "escalation": 0.03463203463203463, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.45045045045045046 }, "accuracy": 0.6623376623376623, "ece_hard": 0.3273856569311282, "calibration": 48.88171249508971, "p50_s": 1.3007893413305283, "p95_s": 2.182773359119892, "cost_usd_per_1000": 0.015779681080467595, "axes": { "intelligence": 67.92000333667002, "calibration": 48.88171249508971, "speed": 75.46784385791761, "cost": 64.05705335393623 }, "jevbench_score": 63.29483215719252 }, "heldout": { "n": 303, "escalation": 0.036303630363036306, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.48623853211009177 }, "accuracy": 0.7392739273927392, "ece_hard": 0.2775051050016622, "calibration": 55.444244720973415, "p50_s": 1.3169443011283875, "p95_s": 1.9534294441342355, "cost_usd_per_1000": 0.015823061087007578, "axes": { "intelligence": 75.30176783544887, "calibration": 55.444244720973415, "speed": 75.89634863136966, "cost": 64.02128485790242 }, "jevbench_score": 67.11223900908644 } }, { "threshold": 0.39, "public": { "n": 231, "escalation": 0.03896103896103896, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.45045045045045046 }, "accuracy": 0.6623376623376623, "ece_hard": 0.32885003610551833, "calibration": 48.73527457765069, "p50_s": 1.3007893413305283, "p95_s": 2.194945380091667, "cost_usd_per_1000": 0.015892024174327543, "axes": { "intelligence": 67.92000333667002, "calibration": 48.73527457765069, "speed": 75.44369312791835, "cost": 63.96462348953019 }, "jevbench_score": 63.219488236771376 }, "heldout": { "n": 303, "escalation": 0.0429042904290429, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.48623853211009177 }, "accuracy": 0.7392739273927392, "ece_hard": 0.28313089827038374, "calibration": 54.88166539410126, "p50_s": 1.3177975565195084, "p95_s": 1.9534294441342355, "cost_usd_per_1000": 0.015994356497447498, "axes": { "intelligence": 75.30176783544887, "calibration": 54.88166539410126, "speed": 75.89353572489918, "cost": 63.8809968472578 }, "jevbench_score": 66.90402270029472 } }, { "threshold": 0.4, "public": { "n": 231, "escalation": 0.04329004329004329, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.4594594594594595 }, "accuracy": 0.6666666666666666, "ece_hard": 0.32026052115097897, "calibration": 49.59422607310463, "p50_s": 1.3007893413305283, "p95_s": 2.194945380091667, "cost_usd_per_1000": 0.01600436726818749, "axes": { "intelligence": 68.29537871204539, "calibration": 49.59422607310463, "speed": 75.44369312791835, "cost": 63.872844729694 }, "jevbench_score": 63.56095421323766 }, "heldout": { "n": 303, "escalation": 0.0429042904290429, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.48623853211009177 }, "accuracy": 0.7392739273927392, "ece_hard": 0.28313089827038374, "calibration": 54.88166539410126, "p50_s": 1.3177975565195084, "p95_s": 1.9534294441342355, "cost_usd_per_1000": 0.015994356497447498, "axes": { "intelligence": 75.30176783544887, "calibration": 54.88166539410126, "speed": 75.89353572489918, "cost": 63.8809968472578 }, "jevbench_score": 66.90402270029472 } }, { "threshold": 0.41, "public": { "n": 231, "escalation": 0.04329004329004329, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.4594594594594595 }, "accuracy": 0.6666666666666666, "ece_hard": 0.32026052115097897, "calibration": 49.59422607310463, "p50_s": 1.3007893413305283, "p95_s": 2.194945380091667, "cost_usd_per_1000": 0.01600436726818749, "axes": { "intelligence": 68.29537871204539, "calibration": 49.59422607310463, "speed": 75.44369312791835, "cost": 63.872844729694 }, "jevbench_score": 63.56095421323766 }, "heldout": { "n": 303, "escalation": 0.0429042904290429, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.48623853211009177 }, "accuracy": 0.7392739273927392, "ece_hard": 0.28313089827038374, "calibration": 54.88166539410126, "p50_s": 1.3177975565195084, "p95_s": 1.9534294441342355, "cost_usd_per_1000": 0.015994356497447498, "axes": { "intelligence": 75.30176783544887, "calibration": 54.88166539410126, "speed": 75.89353572489918, "cost": 63.8809968472578 }, "jevbench_score": 66.90402270029472 } }, { "threshold": 0.42, "public": { "n": 231, "escalation": 0.047619047619047616, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.4594594594594595 }, "accuracy": 0.6666666666666666, "ece_hard": 0.3236153872635375, "calibration": 49.258739461848776, "p50_s": 1.3007893413305283, "p95_s": 2.194945380091667, "cost_usd_per_1000": 0.01611671036204744, "axes": { "intelligence": 68.29537871204539, "calibration": 49.258739461848776, "speed": 75.44369312791835, "cost": 63.78170796539293 }, "jevbench_score": 63.430542246997625 }, "heldout": { "n": 303, "escalation": 0.0429042904290429, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.48623853211009177 }, "accuracy": 0.7392739273927392, "ece_hard": 0.28313089827038374, "calibration": 54.88166539410126, "p50_s": 1.3177975565195084, "p95_s": 1.9534294441342355, "cost_usd_per_1000": 0.015994356497447498, "axes": { "intelligence": 75.30176783544887, "calibration": 54.88166539410126, "speed": 75.89353572489918, "cost": 63.8809968472578 }, "jevbench_score": 66.90402270029472 } }, { "threshold": 0.43, "public": { "n": 231, "escalation": 0.05194805194805195, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.4594594594594595 }, "accuracy": 0.6666666666666666, "ece_hard": 0.3262603935144741, "calibration": 48.994238836755116, "p50_s": 1.3012879490852356, "p95_s": 2.194945380091667, "cost_usd_per_1000": 0.01622905345590739, "axes": { "intelligence": 68.29537871204539, "calibration": 48.994238836755116, "speed": 75.44202874538865, "cost": 63.69120427741999 }, "jevbench_score": 63.32238850435525 }, "heldout": { "n": 303, "escalation": 0.0429042904290429, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.48623853211009177 }, "accuracy": 0.7392739273927392, "ece_hard": 0.28313089827038374, "calibration": 54.88166539410126, "p50_s": 1.3177975565195084, "p95_s": 1.9534294441342355, "cost_usd_per_1000": 0.015994356497447498, "axes": { "intelligence": 75.30176783544887, "calibration": 54.88166539410126, "speed": 75.89353572489918, "cost": 63.8809968472578 }, "jevbench_score": 66.90402270029472 } }, { "threshold": 0.44, "public": { "n": 231, "escalation": 0.05194805194805195, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.4594594594594595 }, "accuracy": 0.6666666666666666, "ece_hard": 0.3262603935144741, "calibration": 48.994238836755116, "p50_s": 1.3012879490852356, "p95_s": 2.194945380091667, "cost_usd_per_1000": 0.01622905345590739, "axes": { "intelligence": 68.29537871204539, "calibration": 48.994238836755116, "speed": 75.44202874538865, "cost": 63.69120427741999 }, "jevbench_score": 63.32238850435525 }, "heldout": { "n": 303, "escalation": 0.0429042904290429, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.48623853211009177 }, "accuracy": 0.7392739273927392, "ece_hard": 0.28313089827038374, "calibration": 54.88166539410126, "p50_s": 1.3177975565195084, "p95_s": 1.9534294441342355, "cost_usd_per_1000": 0.015994356497447498, "axes": { "intelligence": 75.30176783544887, "calibration": 54.88166539410126, "speed": 75.89353572489918, "cost": 63.8809968472578 }, "jevbench_score": 66.90402270029472 } }, { "threshold": 0.45, "public": { "n": 231, "escalation": 0.05627705627705628, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.4594594594594595 }, "accuracy": 0.6666666666666666, "ece_hard": 0.3291992177734602, "calibration": 48.70035641085651, "p50_s": 1.3016724586486816, "p95_s": 2.194945380091667, "cost_usd_per_1000": 0.016341396549767337, "axes": { "intelligence": 68.29537871204539, "calibration": 48.70035641085651, "speed": 75.44074566492104, "cost": 63.60132493115769 }, "jevbench_score": 63.204630699247566 }, "heldout": { "n": 303, "escalation": 0.0462046204620462, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8698630136986302, "hard": 0.48623853211009177 }, "accuracy": 0.735973597359736, "ece_hard": 0.28313089827038374, "calibration": 54.88166539410126, "p50_s": 1.3177975565195084, "p95_s": 1.9764527156949037, "cost_usd_per_1000": 0.016080004202667458, "axes": { "intelligence": 75.10998701353107, "calibration": 54.88166539410126, "speed": 75.8426487324432, "cost": 63.81141526241238 }, "jevbench_score": 66.83196174536938 } }, { "threshold": 0.46, "public": { "n": 231, "escalation": 0.06060606060606061, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.4594594594594595 }, "accuracy": 0.6666666666666666, "ece_hard": 0.3291992177734602, "calibration": 48.70035641085651, "p50_s": 1.3016724586486816, "p95_s": 2.194945380091667, "cost_usd_per_1000": 0.016453739643627282, "axes": { "intelligence": 68.29537871204539, "calibration": 48.70035641085651, "speed": 75.44074566492104, "cost": 63.51206137151936 }, "jevbench_score": 63.182442316701106 }, "heldout": { "n": 303, "escalation": 0.04950495049504951, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8698630136986302, "hard": 0.48623853211009177 }, "accuracy": 0.735973597359736, "ece_hard": 0.28633470539458994, "calibration": 54.56128468168065, "p50_s": 1.3177975565195084, "p95_s": 1.9910572722554205, "cost_usd_per_1000": 0.016165651907887418, "axes": { "intelligence": 75.10998701353107, "calibration": 54.56128468168065, "speed": 75.8106754948598, "cost": 63.74220331068289 }, "jevbench_score": 66.70907642869633 } }, { "threshold": 0.47, "public": { "n": 231, "escalation": 0.06493506493506493, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3248757328126922, "calibration": 49.132704906933306, "p50_s": 1.3016724586486816, "p95_s": 2.2172104194760323, "cost_usd_per_1000": 0.01656608273748723, "axes": { "intelligence": 68.67075408742076, "calibration": 49.132704906933306, "speed": 75.39691373212693, "cost": 63.42340521806249 }, "jevbench_score": 63.37768969312123 }, "heldout": { "n": 303, "escalation": 0.04950495049504951, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8698630136986302, "hard": 0.48623853211009177 }, "accuracy": 0.735973597359736, "ece_hard": 0.28633470539458994, "calibration": 54.56128468168065, "p50_s": 1.3177975565195084, "p95_s": 1.9910572722554205, "cost_usd_per_1000": 0.016165651907887418, "axes": { "intelligence": 75.10998701353107, "calibration": 54.56128468168065, "speed": 75.8106754948598, "cost": 63.74220331068289 }, "jevbench_score": 66.70907642869633 } }, { "threshold": 0.48, "public": { "n": 231, "escalation": 0.08225108225108226, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.33660379985411315, "calibration": 47.95989820279121, "p50_s": 1.302245020866394, "p95_s": 2.248997263610363, "cost_usd_per_1000": 0.01701545511292702, "axes": { "intelligence": 68.67075408742076, "calibration": 47.95989820279121, "speed": 75.33318367393802, "cost": 63.074692909728945 }, "jevbench_score": 62.89597934850328 }, "heldout": { "n": 303, "escalation": 0.056105610561056105, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8698630136986302, "hard": 0.48623853211009177 }, "accuracy": 0.735973597359736, "ece_hard": 0.29333743380202687, "calibration": 53.818945440641826, "p50_s": 1.3178558647632599, "p95_s": 1.9910572722554205, "cost_usd_per_1000": 0.016336947318327338, "axes": { "intelligence": 75.10998701353107, "calibration": 53.818945440641826, "speed": 75.81048333796761, "cost": 63.60487274254527 }, "jevbench_score": 66.44512626451868 } }, { "threshold": 0.49, "public": { "n": 231, "escalation": 0.08225108225108226, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.33660379985411315, "calibration": 47.95989820279121, "p50_s": 1.302245020866394, "p95_s": 2.248997263610363, "cost_usd_per_1000": 0.01701545511292702, "axes": { "intelligence": 68.67075408742076, "calibration": 47.95989820279121, "speed": 75.33318367393802, "cost": 63.074692909728945 }, "jevbench_score": 62.89597934850328 }, "heldout": { "n": 303, "escalation": 0.066006600660066, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8835616438356164, "hard": 0.48623853211009177 }, "accuracy": 0.7425742574257426, "ece_hard": 0.29653336655395157, "calibration": 55.863099622700986, "p50_s": 1.3178558647632599, "p95_s": 1.9910572722554205, "cost_usd_per_1000": 0.016593890433987218, "axes": { "intelligence": 75.49354865736669, "calibration": 55.863099622700986, "speed": 75.81048333796761, "cost": 63.40155345374494 }, "jevbench_score": 67.09899394063247 } }, { "threshold": 0.5, "public": { "n": 231, "escalation": 0.08225108225108226, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.33660379985411315, "calibration": 47.95989820279121, "p50_s": 1.302245020866394, "p95_s": 2.248997263610363, "cost_usd_per_1000": 0.01701545511292702, "axes": { "intelligence": 68.67075408742076, "calibration": 47.95989820279121, "speed": 75.33318367393802, "cost": 63.074692909728945 }, "jevbench_score": 62.89597934850328 }, "heldout": { "n": 303, "escalation": 0.06930693069306931, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8835616438356164, "hard": 0.48623853211009177 }, "accuracy": 0.7425742574257426, "ece_hard": 0.2997419430006619, "calibration": 54.64508335398057, "p50_s": 1.3183358311653137, "p95_s": 1.9910572722554205, "cost_usd_per_1000": 0.016679538139207178, "axes": { "intelligence": 75.49354865736669, "calibration": 54.64508335398057, "speed": 75.80890191456402, "cost": 63.33447937780238 }, "jevbench_score": 66.71221123963154 } }, { "threshold": 0.51, "public": { "n": 231, "escalation": 0.09090909090909091, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.34088879222866203, "calibration": 47.53139896533632, "p50_s": 1.3028961420059204, "p95_s": 2.248997263610363, "cost_usd_per_1000": 0.017240141300646918, "axes": { "intelligence": 68.67075408742076, "calibration": 47.53139896533632, "speed": 75.33101274878535, "cost": 62.90377537023518 }, "jevbench_score": 62.71201174993864 }, "heldout": { "n": 303, "escalation": 0.07920792079207921, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8835616438356164, "hard": 0.4954128440366973 }, "accuracy": 0.7458745874587459, "ece_hard": 0.2950659812054216, "calibration": 55.11267953350459, "p50_s": 1.3189159780740738, "p95_s": 1.9910572722554205, "cost_usd_per_1000": 0.016936481254867058, "axes": { "intelligence": 75.76877801516484, "calibration": 55.11267953350459, "speed": 75.8069911785141, "cost": 63.13530442641245 }, "jevbench_score": 66.86222690240555 } }, { "threshold": 0.52, "public": { "n": 231, "escalation": 0.1038961038961039, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3473584898990505, "calibration": 46.884429198297475, "p50_s": 1.304066225886345, "p95_s": 2.248997263610363, "cost_usd_per_1000": 0.017577170582226763, "axes": { "intelligence": 68.67075408742076, "calibration": 46.884429198297475, "speed": 75.32711425747598, "cost": 62.65153097585604 }, "jevbench_score": 62.433957859578605 }, "heldout": { "n": 303, "escalation": 0.08250825082508251, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.4954128440366973 }, "accuracy": 0.7425742574257426, "ece_hard": 0.2950659812054216, "calibration": 55.11267953350459, "p50_s": 1.319311037659645, "p95_s": 1.9910572722554205, "cost_usd_per_1000": 0.017022128960087018, "axes": { "intelligence": 75.57699719324704, "calibration": 55.11267953350459, "speed": 75.80569051561969, "cost": 63.06958370745611 }, "jevbench_score": 66.80219501411457 } }, { "threshold": 0.53, "public": { "n": 231, "escalation": 0.11255411255411256, "tiers": { "easy": 1.0, "standard": 0.7777777777777778, "hard": 0.4594594594594595 }, "accuracy": 0.670995670995671, "ece_hard": 0.3590681224689467, "calibration": 45.71346594130786, "p50_s": 1.3058469742536545, "p95_s": 2.248997263610363, "cost_usd_per_1000": 0.017801856769946656, "axes": { "intelligence": 68.8355021688355, "calibration": 45.71346594130786, "speed": 75.32118785877775, "cost": 62.48604092586789 }, "jevbench_score": 62.035344904871366 }, "heldout": { "n": 303, "escalation": 0.09570957095709572, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.5045871559633027 }, "accuracy": 0.7458745874587459, "ece_hard": 0.2980414335221253, "calibration": 54.81513430183422, "p50_s": 1.322104200720787, "p95_s": 1.9910572722554205, "cost_usd_per_1000": 0.01736471978096686, "axes": { "intelligence": 75.85222655104519, "calibration": 54.81513430183422, "speed": 75.79650562307398, "cost": 62.809966618615704 }, "jevbench_score": 66.70165975311951 } }, { "threshold": 0.54, "public": { "n": 231, "escalation": 0.11688311688311688, "tiers": { "easy": 1.0, "standard": 0.7777777777777778, "hard": 0.4594594594594595 }, "accuracy": 0.670995670995671, "ece_hard": 0.3590681224689467, "calibration": 45.71346594130786, "p50_s": 1.307793453335762, "p95_s": 2.248997263610363, "cost_usd_per_1000": 0.017914199863806605, "axes": { "intelligence": 68.8355021688355, "calibration": 45.71346594130786, "speed": 75.31471913973084, "cost": 62.40407754370898 }, "jevbench_score": 62.013660307776654 }, "heldout": { "n": 303, "escalation": 0.11221122112211221, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8835616438356164, "hard": 0.48623853211009177 }, "accuracy": 0.7425742574257426, "ece_hard": 0.32449275083228557, "calibration": 52.1700025708182, "p50_s": 1.323771134018898, "p95_s": 2.0295264974236487, "cost_usd_per_1000": 0.01779295830706666, "axes": { "intelligence": 75.49354865736669, "calibration": 52.1700025708182, "speed": 75.70792367490351, "cost": 62.492555166885886 }, "jevbench_score": 65.70146887213569 } }, { "threshold": 0.55, "public": { "n": 231, "escalation": 0.12987012987012986, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "hard": 0.46846846846846846 }, "accuracy": 0.6796536796536796, "ece_hard": 0.35383433082457016, "calibration": 46.23684510574551, "p50_s": 1.3082244992256165, "p95_s": 2.248997263610363, "cost_usd_per_1000": 0.018251229145386447, "axes": { "intelligence": 69.751001001001, "calibration": 46.23684510574551, "speed": 75.31328795036976, "cost": 62.1612364681429 }, "jevbench_score": 62.33507301986222 }, "heldout": { "n": 303, "escalation": 0.13861386138613863, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8972602739726028, "hard": 0.4954128440366973 }, "accuracy": 0.7524752475247525, "ece_hard": 0.3176220937314532, "calibration": 52.85706828090143, "p50_s": 1.3345372080802917, "p95_s": 2.031159219145775, "cost_usd_per_1000": 0.01847813994882634, "axes": { "intelligence": 76.15233965900046, "calibration": 52.85706828090143, "speed": 75.66925344841344, "cost": 62.00025243928482 }, "jevbench_score": 65.92115489392727 } }, { "threshold": 0.56, "public": { "n": 231, "escalation": 0.13852813852813853, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "hard": 0.4774774774774775 }, "accuracy": 0.683982683982684, "ece_hard": 0.3517845783415379, "calibration": 46.44182035404873, "p50_s": 1.309425413608551, "p95_s": 2.248997263610363, "cost_usd_per_1000": 0.018475915333106344, "axes": { "intelligence": 70.12637637637638, "calibration": 46.44182035404873, "speed": 75.30930307428851, "cost": 62.00182109797522 }, "jevbench_score": 62.44690614770926 }, "heldout": { "n": 303, "escalation": 0.1551155115511551, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8972602739726028, "hard": 0.5045871559633027 }, "accuracy": 0.7557755775577558, "ece_hard": 0.32041957886902966, "calibration": 52.57731976714379, "p50_s": 1.336438462138176, "p95_s": 2.0953346848487846, "cost_usd_per_1000": 0.01890637847492614, "axes": { "intelligence": 76.42756901679863, "calibration": 52.57731976714379, "speed": 75.52797631954061, "cost": 61.70174957459926 }, "jevbench_score": 65.78296648548358 } }, { "threshold": 0.57, "public": { "n": 231, "escalation": 0.14285714285714285, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "hard": 0.4864864864864865 }, "accuracy": 0.6883116883116883, "ece_hard": 0.35644323332475714, "calibration": 45.97595485572681, "p50_s": 1.309425413608551, "p95_s": 2.248997263610363, "cost_usd_per_1000": 0.018588258426966292, "axes": { "intelligence": 70.50175175175175, "calibration": 45.97595485572681, "speed": 75.30930307428851, "cost": 61.92283894856903 }, "jevbench_score": 62.35302652225532 }, "heldout": { "n": 303, "escalation": 0.16501650165016502, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8972602739726028, "hard": 0.5045871559633027 }, "accuracy": 0.7557755775577558, "ece_hard": 0.3259532605582373, "calibration": 52.02395159822302, "p50_s": 1.3393714874982834, "p95_s": 2.0953346848487846, "cost_usd_per_1000": 0.019163321590586025, "axes": { "intelligence": 76.42756901679863, "calibration": 52.02395159822302, "speed": 75.51845548462853, "cost": 61.525876365913405 }, "jevbench_score": 65.56032119402256 } }, { "threshold": 0.58, "public": { "n": 231, "escalation": 0.1471861471861472, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "hard": 0.4954954954954955 }, "accuracy": 0.6926406926406926, "ece_hard": 0.356558152231641, "calibration": 45.96446296503842, "p50_s": 1.309425413608551, "p95_s": 2.248997263610363, "cost_usd_per_1000": 0.01870060152082624, "axes": { "intelligence": 70.87712712712712, "calibration": 45.96446296503842, "speed": 75.30930307428851, "cost": 61.84433271357228 }, "jevbench_score": 62.41215928234267 }, "heldout": { "n": 303, "escalation": 0.16501650165016502, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8972602739726028, "hard": 0.5045871559633027 }, "accuracy": 0.7557755775577558, "ece_hard": 0.3259532605582373, "calibration": 52.02395159822302, "p50_s": 1.3393714874982834, "p95_s": 2.0953346848487846, "cost_usd_per_1000": 0.019163321590586025, "axes": { "intelligence": 76.42756901679863, "calibration": 52.02395159822302, "speed": 75.51845548462853, "cost": 61.525876365913405 }, "jevbench_score": 65.56032119402256 } }, { "threshold": 0.59, "public": { "n": 231, "escalation": 0.15584415584415584, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.4954954954954955 }, "accuracy": 0.696969696969697, "ece_hard": 0.356558152231641, "calibration": 45.96446296503842, "p50_s": 1.3110789507627487, "p95_s": 2.248997263610363, "cost_usd_per_1000": 0.018925287708546134, "axes": { "intelligence": 71.41725058391727, "calibration": 45.96446296503842, "speed": 75.3038222808446, "cost": 61.68872528463715 }, "jevbench_score": 62.49021710449933 }, "heldout": { "n": 303, "escalation": 0.16831683168316833, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8972602739726028, "hard": 0.5045871559633027 }, "accuracy": 0.7557755775577558, "ece_hard": 0.3256308000243119, "calibration": 52.05619765161556, "p50_s": 1.3393714874982834, "p95_s": 2.0953346848487846, "cost_usd_per_1000": 0.019248969295805985, "axes": { "intelligence": 76.42756901679863, "calibration": 52.05619765161556, "speed": 75.51845548462853, "cost": 61.46777560720169 }, "jevbench_score": 65.55499237730398 } }, { "threshold": 0.6, "public": { "n": 231, "escalation": 0.17316017316017315, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.4954954954954955 }, "accuracy": 0.696969696969697, "ece_hard": 0.34903236231877766, "calibration": 46.00942839427438, "p50_s": 1.3143753111362457, "p95_s": 2.276942200958729, "cost_usd_per_1000": 0.019374660083985924, "axes": { "intelligence": 71.41725058391727, "calibration": 46.00942839427438, "speed": 75.23928600757874, "cost": 61.382977245208316 }, "jevbench_score": 62.41452142162182 }, "heldout": { "n": 303, "escalation": 0.17491749174917492, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8972602739726028, "hard": 0.5045871559633027 }, "accuracy": 0.7557755775577558, "ece_hard": 0.3328694544116618, "calibration": 51.33233221288057, "p50_s": 1.3393714874982834, "p95_s": 2.0953346848487846, "cost_usd_per_1000": 0.019420264706245905, "axes": { "intelligence": 76.42756901679863, "calibration": 51.33233221288057, "speed": 75.51845548462853, "cost": 61.352345643223885 }, "jevbench_score": 65.29521078425478 } }, { "threshold": 0.61, "public": { "n": 231, "escalation": 0.17316017316017315, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.4954954954954955 }, "accuracy": 0.696969696969697, "ece_hard": 0.34903236231877766, "calibration": 46.00942839427438, "p50_s": 1.3143753111362457, "p95_s": 2.276942200958729, "cost_usd_per_1000": 0.019374660083985924, "axes": { "intelligence": 71.41725058391727, "calibration": 46.00942839427438, "speed": 75.23928600757874, "cost": 61.382977245208316 }, "jevbench_score": 62.41452142162182 }, "heldout": { "n": 303, "escalation": 0.1848184818481848, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.910958904109589, "hard": 0.5045871559633027 }, "accuracy": 0.7623762376237624, "ece_hard": 0.3328694544116618, "calibration": 51.33233221288057, "p50_s": 1.3393714874982834, "p95_s": 2.0953346848487846, "cost_usd_per_1000": 0.019677207821905785, "axes": { "intelligence": 76.81113066063425, "calibration": 51.33233221288057, "speed": 75.51845548462853, "cost": 61.18109582587525 }, "jevbench_score": 65.33131144567757 } }, { "threshold": 0.62, "public": { "n": 231, "escalation": 0.1774891774891775, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.5045045045045045 }, "accuracy": 0.7012987012987013, "ece_hard": 0.3398243741240551, "calibration": 47.95553821815212, "p50_s": 1.3143753111362457, "p95_s": 2.276942200958729, "cost_usd_per_1000": 0.019487003177845873, "axes": { "intelligence": 71.79262595929265, "calibration": 47.95553821815212, "speed": 75.23928600757874, "cost": 61.30764832066727 }, "jevbench_score": 63.127629701853635 }, "heldout": { "n": 303, "escalation": 0.19471947194719472, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.910958904109589, "hard": 0.5229357798165137 }, "accuracy": 0.768976897689769, "ece_hard": 0.32139224584852266, "calibration": 52.48005306919449, "p50_s": 1.3400312662124634, "p95_s": 2.1863119244575477, "cost_usd_per_1000": 0.019934150937565665, "axes": { "intelligence": 77.36158937623057, "calibration": 52.48005306919449, "speed": 75.33172944872274, "cost": 61.012067730044464 }, "jevbench_score": 65.72465766644507 } }, { "threshold": 0.63, "public": { "n": 231, "escalation": 0.19047619047619047, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "hard": 0.5225225225225225 }, "accuracy": 0.7056277056277056, "ece_hard": 0.32387781334960386, "calibration": 50.420827805420906, "p50_s": 1.3146315664052963, "p95_s": 2.276942200958729, "cost_usd_per_1000": 0.019824032459425718, "axes": { "intelligence": 72.00325325325325, "calibration": 50.420827805420906, "speed": 75.23843937419326, "cost": 61.08423999590183 }, "jevbench_score": 63.91205193570623 }, "heldout": { "n": 303, "escalation": 0.20462046204620463, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9178082191780822, "hard": 0.5229357798165137 }, "accuracy": 0.7722772277227723, "ece_hard": 0.3247510575663571, "calibration": 52.14417189741104, "p50_s": 1.3414461016654968, "p95_s": 2.197334446012974, "cost_usd_per_1000": 0.02019109405322555, "axes": { "intelligence": 77.55337019814839, "calibration": 52.14417189741104, "speed": 75.3053060839359, "cost": 60.84520444594672 }, "jevbench_score": 65.60917787020048 } }, { "threshold": 0.64, "public": { "n": 231, "escalation": 0.19913419913419914, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "hard": 0.5225225225225225 }, "accuracy": 0.7056277056277056, "ece_hard": 0.324347078691158, "calibration": 50.373901271265495, "p50_s": 1.3177027255296707, "p95_s": 2.276942200958729, "cost_usd_per_1000": 0.020048718647145615, "axes": { "intelligence": 72.00325325325325, "calibration": 50.373901271265495, "speed": 75.22830549362926, "cost": 60.93740136301969 }, "jevbench_score": 63.856590845585856 }, "heldout": { "n": 303, "escalation": 0.22112211221122113, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9246575342465754, "hard": 0.5412844036697247 }, "accuracy": 0.7821782178217822, "ece_hard": 0.31306257837112866, "calibration": 53.31301981693389, "p50_s": 1.3426922708749771, "p95_s": 2.202181353420019, "cost_usd_per_1000": 0.02061933257932535, "axes": { "intelligence": 78.29560973566252, "calibration": 53.31301981693389, "speed": 75.2917043005962, "cost": 60.57176189098419 }, "jevbench_score": 66.05367830323306 } }, { "threshold": 0.65, "public": { "n": 231, "escalation": 0.20346320346320346, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "hard": 0.5135135135135135 }, "accuracy": 0.7012987012987013, "ece_hard": 0.3365413941814197, "calibration": 49.15446972223933, "p50_s": 1.319142371416092, "p95_s": 2.276942200958729, "cost_usd_per_1000": 0.02016106174100556, "axes": { "intelligence": 71.62787787787788, "calibration": 49.15446972223933, "speed": 75.22356323306026, "cost": 60.864598011857794 }, "jevbench_score": 63.3637599237806 }, "heldout": { "n": 303, "escalation": 0.23102310231023102, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.5412844036697247 }, "accuracy": 0.7854785478547854, "ece_hard": 0.31449053341049393, "calibration": 53.17022431299736, "p50_s": 1.3426922708749771, "p95_s": 2.2078663267195227, "cost_usd_per_1000": 0.02087627569498523, "axes": { "intelligence": 78.48739055758033, "calibration": 53.17022431299736, "speed": 75.28050735180821, "cost": 60.410409292770694 }, "jevbench_score": 66.00330378524163 } }, { "threshold": 0.66, "public": { "n": 231, "escalation": 0.2077922077922078, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "hard": 0.5135135135135135 }, "accuracy": 0.7012987012987013, "ece_hard": 0.3355801916966704, "calibration": 49.25058997071425, "p50_s": 1.319142371416092, "p95_s": 2.2920106947422028, "cost_usd_per_1000": 0.02027340483486551, "axes": { "intelligence": 71.62787787787788, "calibration": 49.25058997071425, "speed": 75.19491689952562, "cost": 60.792199216344336 }, "jevbench_score": 63.36981876137649 }, "heldout": { "n": 303, "escalation": 0.24092409240924093, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9383561643835616, "hard": 0.5412844036697247 }, "accuracy": 0.7887788778877888, "ece_hard": 0.3144061322573718, "calibration": 52.547829652003706, "p50_s": 1.3426922708749771, "p95_s": 2.2078663267195227, "cost_usd_per_1000": 0.02113321881064511, "axes": { "intelligence": 78.67917137949814, "calibration": 52.547829652003706, "speed": 75.28050735180821, "cost": 60.25103050929512 }, "jevbench_score": 65.80598503474204 } }, { "threshold": 0.67, "public": { "n": 231, "escalation": 0.2077922077922078, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "hard": 0.5135135135135135 }, "accuracy": 0.7012987012987013, "ece_hard": 0.3355801916966704, "calibration": 49.25058997071425, "p50_s": 1.319142371416092, "p95_s": 2.2920106947422028, "cost_usd_per_1000": 0.02027340483486551, "axes": { "intelligence": 71.62787787787788, "calibration": 49.25058997071425, "speed": 75.19491689952562, "cost": 60.792199216344336 }, "jevbench_score": 63.36981876137649 }, "heldout": { "n": 303, "escalation": 0.24422442244224424, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9383561643835616, "hard": 0.5412844036697247 }, "accuracy": 0.7887788778877888, "ece_hard": 0.3144061322573718, "calibration": 52.547829652003706, "p50_s": 1.3426922708749771, "p95_s": 2.2078663267195227, "cost_usd_per_1000": 0.02121886651586507, "axes": { "intelligence": 78.67917137949814, "calibration": 52.547829652003706, "speed": 75.28050735180821, "cost": 60.19833457777125 }, "jevbench_score": 65.79159173075132 } }, { "threshold": 0.68, "public": { "n": 231, "escalation": 0.2077922077922078, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "hard": 0.5135135135135135 }, "accuracy": 0.7012987012987013, "ece_hard": 0.3355801916966704, "calibration": 49.25058997071425, "p50_s": 1.319142371416092, "p95_s": 2.2920106947422028, "cost_usd_per_1000": 0.02027340483486551, "axes": { "intelligence": 71.62787787787788, "calibration": 49.25058997071425, "speed": 75.19491689952562, "cost": 60.792199216344336 }, "jevbench_score": 63.36981876137649 }, "heldout": { "n": 303, "escalation": 0.25412541254125415, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9452054794520548, "hard": 0.5321100917431193 }, "accuracy": 0.7887788778877888, "ece_hard": 0.32305386053867047, "calibration": 51.68305682387384, "p50_s": 1.3468152582645416, "p95_s": 2.2078663267195227, "cost_usd_per_1000": 0.02147580963152495, "axes": { "intelligence": 78.59572284361778, "calibration": 51.68305682387384, "speed": 75.26719196186778, "cost": 60.041513632525884 }, "jevbench_score": 65.4562487700284 } }, { "threshold": 0.69, "public": { "n": 231, "escalation": 0.2077922077922078, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "hard": 0.5135135135135135 }, "accuracy": 0.7012987012987013, "ece_hard": 0.3355801916966704, "calibration": 49.25058997071425, "p50_s": 1.319142371416092, "p95_s": 2.2920106947422028, "cost_usd_per_1000": 0.02027340483486551, "axes": { "intelligence": 71.62787787787788, "calibration": 49.25058997071425, "speed": 75.19491689952562, "cost": 60.792199216344336 }, "jevbench_score": 63.36981876137649 }, "heldout": { "n": 303, "escalation": 0.26732673267326734, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9452054794520548, "hard": 0.5321100917431193 }, "accuracy": 0.7887788778877888, "ece_hard": 0.3268502746038982, "calibration": 50.795837363201, "p50_s": 1.3563375920057297, "p95_s": 2.2078663267195227, "cost_usd_per_1000": 0.021818400452404792, "axes": { "intelligence": 78.59572284361778, "calibration": 50.795837363201, "speed": 75.23659425368052, "cost": 59.83531274555239 }, "jevbench_score": 65.11086012321053 } }, { "threshold": 0.7, "public": { "n": 231, "escalation": 0.21212121212121213, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "hard": 0.5135135135135135 }, "accuracy": 0.7012987012987013, "ece_hard": 0.3355801916966704, "calibration": 49.25058997071425, "p50_s": 1.3191694766283035, "p95_s": 2.2920106947422028, "cost_usd_per_1000": 0.020385747928725457, "axes": { "intelligence": 71.62787787787788, "calibration": 49.25058997071425, "speed": 75.19482766334148, "cost": 60.72020050522214 }, "jevbench_score": 63.351028754227634 }, "heldout": { "n": 303, "escalation": 0.2706270627062706, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9452054794520548, "hard": 0.5321100917431193 }, "accuracy": 0.7887788778877888, "ece_hard": 0.3268502746038982, "calibration": 50.795837363201, "p50_s": 1.3563375920057297, "p95_s": 2.2290131688117976, "cost_usd_per_1000": 0.02190404815762475, "axes": { "intelligence": 78.59572284361778, "calibration": 50.795837363201, "speed": 75.19519566991417, "cost": 59.78426843144175 }, "jevbench_score": 65.08801281007027 } }, { "threshold": 0.71, "public": { "n": 231, "escalation": 0.22077922077922077, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "hard": 0.5315315315315315 }, "accuracy": 0.70995670995671, "ece_hard": 0.32282934453689316, "calibration": 50.52567468669198, "p50_s": 1.3192902952432632, "p95_s": 2.2920106947422028, "cost_usd_per_1000": 0.02061043411644535, "axes": { "intelligence": 72.37862862862865, "calibration": 50.52567468669198, "speed": 75.19442992479429, "cost": 60.57738581810664 }, "jevbench_score": 63.88584855579207 }, "heldout": { "n": 303, "escalation": 0.2871287128712871, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9452054794520548, "hard": 0.5412844036697247 }, "accuracy": 0.7920792079207921, "ece_hard": 0.32145534809373666, "calibration": 51.335330014217156, "p50_s": 1.3565552085638046, "p95_s": 2.238805812597275, "cost_usd_per_1000": 0.022332286683724552, "axes": { "intelligence": 78.87095220141595, "calibration": 51.335330014217156, "speed": 75.17546101000235, "cost": 59.532004169999574 }, "jevbench_score": 65.24391473347592 } }, { "threshold": 0.72, "public": { "n": 231, "escalation": 0.23376623376623376, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "hard": 0.5315315315315315 }, "accuracy": 0.70995670995671, "ece_hard": 0.32491676041704365, "calibration": 51.19707318904378, "p50_s": 1.3224025070667267, "p95_s": 2.2920106947422028, "cost_usd_per_1000": 0.020947463398025196, "axes": { "intelligence": 72.37862862862865, "calibration": 51.19707318904378, "speed": 75.18419696322064, "cost": 60.366056792774984 }, "jevbench_score": 64.03887803224104 }, "heldout": { "n": 303, "escalation": 0.297029702970297, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9452054794520548, "hard": 0.5412844036697247 }, "accuracy": 0.7920792079207921, "ece_hard": 0.32145534809373666, "calibration": 51.335330014217156, "p50_s": 1.3586038798093796, "p95_s": 2.238805812597275, "cost_usd_per_1000": 0.022589229799384433, "axes": { "intelligence": 78.87095220141595, "calibration": 51.335330014217156, "speed": 75.1689072372246, "cost": 59.38295729493848 }, "jevbench_score": 65.20161833077793 } }, { "threshold": 0.73, "public": { "n": 231, "escalation": 0.24242424242424243, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "hard": 0.5495495495495496 }, "accuracy": 0.7186147186147186, "ece_hard": 0.3135893553444104, "calibration": 52.3298136963071, "p50_s": 1.3224025070667267, "p95_s": 2.3059075847268105, "cost_usd_per_1000": 0.021172149585745093, "axes": { "intelligence": 73.12937937937939, "calibration": 52.3298136963071, "speed": 75.15794438260922, "cost": 60.22705138860819 }, "jevbench_score": 64.51369144881795 }, "heldout": { "n": 303, "escalation": 0.3069306930693069, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9452054794520548, "hard": 0.5412844036697247 }, "accuracy": 0.7920792079207921, "ece_hard": 0.31970343632127673, "calibration": 51.51052119146315, "p50_s": 1.3768931925296783, "p95_s": 2.238805812597275, "cost_usd_per_1000": 0.022846172915044313, "axes": { "intelligence": 78.87095220141595, "calibration": 51.51052119146315, "speed": 75.11083321455678, "cost": 59.23559621497804 }, "jevbench_score": 65.20405318517275 } }, { "threshold": 0.74, "public": { "n": 231, "escalation": 0.2510822510822511, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.5495495495495496 }, "accuracy": 0.7229437229437229, "ece_hard": 0.3159748374696105, "calibration": 52.09126548378709, "p50_s": 1.3224025070667267, "p95_s": 2.3059075847268105, "cost_usd_per_1000": 0.021396835773464986, "axes": { "intelligence": 73.66950283616951, "calibration": 52.09126548378709, "speed": 75.15794438260922, "cost": 60.089513399145 }, "jevbench_score": 64.52181222274736 }, "heldout": { "n": 303, "escalation": 0.3201320132013201, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9452054794520548, "hard": 0.5412844036697247 }, "accuracy": 0.7920792079207921, "ece_hard": 0.32204872048973565, "calibration": 51.27599277461725, "p50_s": 1.3797449171543121, "p95_s": 2.238805812597275, "cost_usd_per_1000": 0.023188763735924156, "axes": { "intelligence": 78.87095220141595, "calibration": 51.27599277461725, "speed": 75.10184771366106, "cost": 59.04167212944773 }, "jevbench_score": 65.07439034957754 } }, { "threshold": 0.75, "public": { "n": 231, "escalation": 0.26406926406926406, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.5405405405405406 }, "accuracy": 0.7186147186147186, "ece_hard": 0.32681307417580896, "calibration": 51.00744181316725, "p50_s": 1.3238774687051773, "p95_s": 2.325567226856947, "cost_usd_per_1000": 0.021733865055044828, "axes": { "intelligence": 73.29412746079414, "calibration": 51.00744181316725, "speed": 75.11623310607298, "cost": 59.885891013677735 }, "jevbench_score": 64.03836860680198 }, "heldout": { "n": 303, "escalation": 0.3234323432343234, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9452054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7953795379537953, "ece_hard": 0.3151405951328064, "calibration": 51.96680531031018, "p50_s": 1.3797449171543121, "p95_s": 2.238805812597275, "cost_usd_per_1000": 0.023274411441144116, "axes": { "intelligence": 79.14618155921411, "calibration": 51.96680531031018, "speed": 75.10184771366106, "cost": 58.993638776901584 }, "jevbench_score": 65.3360610201011 } }, { "threshold": 0.76, "public": { "n": 231, "escalation": 0.2683982683982684, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.5405405405405406 }, "accuracy": 0.7186147186147186, "ece_hard": 0.32681307417580896, "calibration": 51.00744181316725, "p50_s": 1.3238774687051773, "p95_s": 2.325567226856947, "cost_usd_per_1000": 0.021846208148904776, "axes": { "intelligence": 73.29412746079414, "calibration": 51.00744181316725, "speed": 75.11623310607298, "cost": 59.818717982199786 }, "jevbench_score": 64.02040334874253 }, "heldout": { "n": 303, "escalation": 0.33993399339933994, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9452054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7953795379537953, "ece_hard": 0.319522910214323, "calibration": 51.52857380215852, "p50_s": 1.384657546877861, "p95_s": 2.2529926203191275, "cost_usd_per_1000": 0.023702649967243916, "axes": { "intelligence": 79.14618155921411, "calibration": 51.52857380215852, "speed": 75.05897853716904, "cost": 58.756092908538434 }, "jevbench_score": 65.12285202437495 } }, { "threshold": 0.77, "public": { "n": 231, "escalation": 0.27705627705627706, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.5315315315315315 }, "accuracy": 0.7142857142857143, "ece_hard": 0.3363344019046055, "calibration": 50.0553090402876, "p50_s": 1.325474590063095, "p95_s": 2.325567226856947, "cost_usd_per_1000": 0.022070894336624673, "axes": { "intelligence": 72.91875208541877, "calibration": 50.0553090402876, "speed": 75.11099694975015, "cost": 59.68540205192201 }, "jevbench_score": 63.60119214916753 }, "heldout": { "n": 303, "escalation": 0.34983498349834985, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9452054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7953795379537953, "ece_hard": 0.32167946256856206, "calibration": 51.312918566734616, "p50_s": 1.4045170545578003, "p95_s": 2.2529926203191275, "cost_usd_per_1000": 0.023959593082903796, "axes": { "intelligence": 79.14618155921411, "calibration": 51.312918566734616, "speed": 74.99713213750968, "cost": 58.61561686145775 }, "jevbench_score": 65.00229197184659 } }, { "threshold": 0.78, "public": { "n": 231, "escalation": 0.29004329004329005, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.5315315315315315 }, "accuracy": 0.7142857142857143, "ece_hard": 0.3376125281159751, "calibration": 50.34072219076827, "p50_s": 1.3266585320234299, "p95_s": 2.325567226856947, "cost_usd_per_1000": 0.02240792361820452, "axes": { "intelligence": 72.91875208541877, "calibration": 50.34072219076827, "speed": 75.10711947097549, "cost": 59.48795153679246 }, "jevbench_score": 63.63809892975694 }, "heldout": { "n": 303, "escalation": 0.3564356435643564, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9452054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7953795379537953, "ece_hard": 0.32363307363110694, "calibration": 51.11755746048013, "p50_s": 1.438198447227478, "p95_s": 2.2529926203191275, "cost_usd_per_1000": 0.024130888493343716, "axes": { "intelligence": 79.14618155921411, "calibration": 51.11755746048013, "speed": 74.89421415281097, "cost": 58.522800615443614 }, "jevbench_score": 64.89232842939326 } }, { "threshold": 0.79, "public": { "n": 231, "escalation": 0.2987012987012987, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.5315315315315315 }, "accuracy": 0.7142857142857143, "ece_hard": 0.3367757276792861, "calibration": 50.42440223443717, "p50_s": 1.3277880102396011, "p95_s": 2.325567226856947, "cost_usd_per_1000": 0.022632609805924412, "axes": { "intelligence": 72.91875208541877, "calibration": 50.42440223443717, "speed": 75.10342358802205, "cost": 59.35796091698246 }, "jevbench_score": 63.62893783902645 }, "heldout": { "n": 303, "escalation": 0.3696369636963696, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9452054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7953795379537953, "ece_hard": 0.32363307363110694, "calibration": 51.11755746048013, "p50_s": 1.4431772530078888, "p95_s": 2.2529926203191275, "cost_usd_per_1000": 0.024473479314223556, "axes": { "intelligence": 79.14618155921411, "calibration": 51.11755746048013, "speed": 74.87920555793242, "cost": 58.33912852113895 }, "jevbench_score": 64.83810397637531 } }, { "threshold": 0.8, "public": { "n": 231, "escalation": 0.30735930735930733, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.5315315315315315 }, "accuracy": 0.7142857142857143, "ece_hard": 0.3367757276792861, "calibration": 50.42440223443717, "p50_s": 1.3289764374494553, "p95_s": 2.325567226856947, "cost_usd_per_1000": 0.022857295993644305, "axes": { "intelligence": 72.91875208541877, "calibration": 50.42440223443717, "speed": 75.09953820469983, "cost": 59.229254431474324 }, "jevbench_score": 63.59359542458724 }, "heldout": { "n": 303, "escalation": 0.3795379537953795, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9452054794520548, "hard": 0.5596330275229358 }, "accuracy": 0.7986798679867987, "ece_hard": 0.31613547301545264, "calibration": 51.86731752204555, "p50_s": 1.4465409815311432, "p95_s": 2.2529926203191275, "cost_usd_per_1000": 0.024730422429883436, "axes": { "intelligence": 79.42141091701227, "calibration": 51.86731752204555, "speed": 74.86909488777356, "cost": 58.203053956714065 }, "jevbench_score": 65.09084887657042 } }, { "threshold": 0.81, "public": { "n": 231, "escalation": 0.31601731601731603, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.5405405405405406 }, "accuracy": 0.7186147186147186, "ece_hard": 0.3289213818753172, "calibration": 51.209836814834055, "p50_s": 1.3289764374494553, "p95_s": 2.325567226856947, "cost_usd_per_1000": 0.023081982181364202, "axes": { "intelligence": 73.29412746079414, "calibration": 51.209836814834055, "speed": 75.09953820469983, "cost": 59.10180695717314 }, "jevbench_score": 63.887390431206384 }, "heldout": { "n": 303, "escalation": 0.3927392739273927, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.5596330275229358 }, "accuracy": 0.801980198019802, "ece_hard": 0.3210831043858974, "calibration": 51.37255438500108, "p50_s": 1.4572032541036606, "p95_s": 2.2529926203191275, "cost_usd_per_1000": 0.025073013250763276, "axes": { "intelligence": 80.58807758367894, "calibration": 51.37255438500108, "speed": 74.83720098405252, "cost": 58.02380409428842 }, "jevbench_score": 65.11505663446664 } }, { "threshold": 0.82, "public": { "n": 231, "escalation": 0.3203463203463203, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.5405405405405406 }, "accuracy": 0.7186147186147186, "ece_hard": 0.3289213818753172, "calibration": 51.209836814834055, "p50_s": 1.3397867381572723, "p95_s": 2.325567226856947, "cost_usd_per_1000": 0.02319432527522415, "axes": { "intelligence": 73.29412746079414, "calibration": 51.209836814834055, "speed": 75.06435426881524, "cost": 59.03854769867961 }, "jevbench_score": 63.862806115469894 }, "heldout": { "n": 303, "escalation": 0.39603960396039606, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.5596330275229358 }, "accuracy": 0.801980198019802, "ece_hard": 0.32279292321333164, "calibration": 51.20157250225766, "p50_s": 1.4642701894044876, "p95_s": 2.2529926203191275, "cost_usd_per_1000": 0.02515866095598324, "axes": { "intelligence": 80.58807758367894, "calibration": 51.20157250225766, "speed": 74.81619009998528, "cost": 57.979374324747994 }, "jevbench_score": 65.04378446704938 } }, { "threshold": 0.83, "public": { "n": 231, "escalation": 0.33766233766233766, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.5405405405405406 }, "accuracy": 0.7186147186147186, "ece_hard": 0.3320861724431001, "calibration": 50.893357758055764, "p50_s": 1.349913328886032, "p95_s": 2.325567226856947, "cost_usd_per_1000": 0.023643697650663945, "axes": { "intelligence": 73.29412746079414, "calibration": 50.893357758055764, "speed": 75.03165215969878, "cost": 58.78853808790947 }, "jevbench_score": 63.68935688568036 }, "heldout": { "n": 303, "escalation": 0.40264026402640263, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.5688073394495413 }, "accuracy": 0.8052805280528053, "ece_hard": 0.3141924146364148, "calibration": 52.061623359949344, "p50_s": 1.4727533608675003, "p95_s": 2.2529926203191275, "cost_usd_per_1000": 0.02532995636642316, "axes": { "intelligence": 80.86330694147712, "calibration": 52.061623359949344, "speed": 74.79110208323856, "cost": 57.890966749834256 }, "jevbench_score": 65.34050470268456 } }, { "threshold": 0.84, "public": { "n": 231, "escalation": 0.3593073593073593, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.5495495495495496 }, "accuracy": 0.7272727272727273, "ece_hard": 0.31481709211706915, "calibration": 52.62026579065886, "p50_s": 1.3694269061088562, "p95_s": 2.352534409612417, "cost_usd_per_1000": 0.02420541311996368, "axes": { "intelligence": 74.20962629295963, "calibration": 52.62026579065886, "speed": 74.9192515908745, "cost": 58.482625022352096 }, "jevbench_score": 64.31442520035921 }, "heldout": { "n": 303, "escalation": 0.41914191419141916, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.952054794520548, "hard": 0.5871559633027523 }, "accuracy": 0.8151815181518152, "ece_hard": 0.29516760398508357, "calibration": 53.96410442508247, "p50_s": 1.480877622961998, "p95_s": 2.2529926203191275, "cost_usd_per_1000": 0.025758194892522963, "axes": { "intelligence": 81.60554647899126, "calibration": 53.96410442508247, "speed": 74.76721060109514, "cost": 57.67253725452788 }, "jevbench_score": 66.01250631113899 } }, { "threshold": 0.85, "public": { "n": 231, "escalation": 0.36796536796536794, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.5405405405405406 }, "accuracy": 0.7229437229437229, "ece_hard": 0.3243483480188111, "calibration": 50.19529317501791, "p50_s": 1.377013847231865, "p95_s": 2.352534409612417, "cost_usd_per_1000": 0.024430099307683577, "axes": { "intelligence": 73.83425091758426, "calibration": 50.19529317501791, "speed": 74.89525708058478, "cost": 58.36224302883456 }, "jevbench_score": 63.44198891466033 }, "heldout": { "n": 303, "escalation": 0.43234323432343236, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.952054794520548, "hard": 0.5963302752293578 }, "accuracy": 0.8184818481848185, "ece_hard": 0.2873960025862724, "calibration": 54.74126456496358, "p50_s": 1.4825191721320152, "p95_s": 2.2529926203191275, "cost_usd_per_1000": 0.026100785713402803, "axes": { "intelligence": 81.88077583678941, "calibration": 54.74126456496358, "speed": 74.76239912380387, "cost": 57.50039256622043 }, "jevbench_score": 66.25409154937286 } }, { "threshold": 0.86, "public": { "n": 231, "escalation": 0.38095238095238093, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.5405405405405406 }, "accuracy": 0.7229437229437229, "ece_hard": 0.32548167346164436, "calibration": 50.081960630734585, "p50_s": 1.384503349661827, "p95_s": 2.352534409612417, "cost_usd_per_1000": 0.024767128589263422, "axes": { "intelligence": 73.83425091758426, "calibration": 50.081960630734585, "speed": 74.87170004783374, "cost": 58.18373023051197 }, "jevbench_score": 63.352624747008036 }, "heldout": { "n": 303, "escalation": 0.44554455445544555, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.952054794520548, "hard": 0.6055045871559633 }, "accuracy": 0.8217821782178217, "ece_hard": 0.27232037027224454, "calibration": 56.131590956090065, "p50_s": 1.4871234074234962, "p95_s": 2.2529926203191275, "cost_usd_per_1000": 0.026443376534282644, "axes": { "intelligence": 82.15600519458759, "calibration": 56.131590956090065, "speed": 74.74893221302827, "cost": 57.330492727580726 }, "jevbench_score": 66.67443332233417 } }, { "threshold": 0.87, "public": { "n": 231, "escalation": 0.39826839826839827, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.5585585585585585 }, "accuracy": 0.7316017316017316, "ece_hard": 0.3101857267084698, "calibration": 51.61155530605204, "p50_s": 1.4003681987524033, "p95_s": 2.352534409612417, "cost_usd_per_1000": 0.025216500964703212, "axes": { "intelligence": 74.58500166833501, "calibration": 51.61155530605204, "speed": 74.8222177587372, "cost": 57.94945528524375 }, "jevbench_score": 63.91747294621732 }, "heldout": { "n": 303, "escalation": 0.46864686468646866, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.952054794520548, "hard": 0.6238532110091743 }, "accuracy": 0.8283828382838284, "ece_hard": 0.2592636112717428, "calibration": 57.43726685614023, "p50_s": 1.4923389479517937, "p95_s": 2.2529926203191275, "cost_usd_per_1000": 0.027042910470822364, "axes": { "intelligence": 82.70646391018391, "calibration": 57.43726685614023, "speed": 74.73372757195546, "cost": 57.03839708894212 }, "jevbench_score": 67.08173612888206 } }, { "threshold": 0.88, "public": { "n": 231, "escalation": 0.4025974025974026, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.5585585585585585 }, "accuracy": 0.7316017316017316, "ece_hard": 0.31127895922953897, "calibration": 51.50223205394512, "p50_s": 1.409024566411972, "p95_s": 2.352534409612417, "cost_usd_per_1000": 0.02532884405856316, "axes": { "intelligence": 74.58500166833501, "calibration": 51.50223205394512, "speed": 74.79545450475737, "cost": 57.89153889425355 }, "jevbench_score": 63.86191887344474 }, "heldout": { "n": 303, "escalation": 0.48184818481848185, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.952054794520548, "hard": 0.6238532110091743 }, "accuracy": 0.8283828382838284, "ece_hard": 0.2603541133893722, "calibration": 57.32821664437729, "p50_s": 1.4997273087501526, "p95_s": 2.2529926203191275, "cost_usd_per_1000": 0.027385501291702204, "axes": { "intelligence": 82.70646391018391, "calibration": 57.32821664437729, "speed": 74.71227931131193, "cost": 56.87437914701023 }, "jevbench_score": 66.9968116022626 } }, { "threshold": 0.89, "public": { "n": 231, "escalation": 0.42857142857142855, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.5765765765765766 }, "accuracy": 0.7402597402597403, "ece_hard": 0.29492572111178783, "calibration": 53.13755586572023, "p50_s": 1.4595188051462173, "p95_s": 2.352534409612417, "cost_usd_per_1000": 0.026002902621722848, "axes": { "intelligence": 75.33575241908576, "calibration": 53.13755586572023, "speed": 74.64254320576313, "cost": 57.54934511214083 }, "jevbench_score": 64.39577580437128 }, "heldout": { "n": 303, "escalation": 0.49174917491749176, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.6238532110091743 }, "accuracy": 0.8250825082508251, "ece_hard": 0.2603541133893722, "calibration": 57.32821664437729, "p50_s": 1.502712868154049, "p95_s": 2.2529926203191275, "cost_usd_per_1000": 0.027642444407362084, "axes": { "intelligence": 81.53979724351724, "calibration": 57.32821664437729, "speed": 74.7036422539488, "cost": 56.75270665475705 }, "jevbench_score": 66.72162326957086 } }, { "threshold": 0.9, "public": { "n": 231, "escalation": 0.45454545454545453, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.5945945945945946 }, "accuracy": 0.7489177489177489, "ece_hard": 0.274714603478593, "calibration": 54.630566669340155, "p50_s": 1.483648955821991, "p95_s": 2.378658790141344, "cost_usd_per_1000": 0.026676961184882532, "axes": { "intelligence": 76.0865031698365, "calibration": 54.630566669340155, "speed": 74.52336696175482, "cost": 57.21590929341548 }, "jevbench_score": 64.8840799372722 }, "heldout": { "n": 303, "escalation": 0.5016501650165016, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.6238532110091743 }, "accuracy": 0.8250825082508251, "ece_hard": 0.26076454222725237, "calibration": 57.06349004394461, "p50_s": 1.5047701001167297, "p95_s": 2.2529926203191275, "cost_usd_per_1000": 0.027899387523021964, "axes": { "intelligence": 81.53979724351724, "calibration": 57.06349004394461, "speed": 74.69770077638657, "cost": 56.63215992146039 }, "jevbench_score": 66.6077217445303 } }, { "threshold": 0.91, "public": { "n": 231, "escalation": 0.4588744588744589, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.5945945945945946 }, "accuracy": 0.7489177489177489, "ece_hard": 0.2753828763648298, "calibration": 54.192847928855, "p50_s": 1.4956383556127548, "p95_s": 2.378658790141344, "cost_usd_per_1000": 0.02678930427874248, "axes": { "intelligence": 76.0865031698365, "calibration": 54.192847928855, "speed": 74.48841257344327, "cost": 57.16115694766242 }, "jevbench_score": 64.73062975861806 }, "heldout": { "n": 303, "escalation": 0.5181518151815182, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.6238532110091743 }, "accuracy": 0.8250825082508251, "ece_hard": 0.2612418352600219, "calibration": 56.98252548099002, "p50_s": 1.5054559037089348, "p95_s": 2.2623383209109305, "cost_usd_per_1000": 0.028327626049121767, "axes": { "intelligence": 81.53979724351724, "calibration": 56.98252548099002, "speed": 74.67774408955884, "cost": 56.43369457903486 }, "jevbench_score": 66.52122657493969 } }, { "threshold": 0.92, "public": { "n": 231, "escalation": 0.47619047619047616, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.5945945945945946 }, "accuracy": 0.7489177489177489, "ece_hard": 0.2768439868969, "calibration": 54.04673687564798, "p50_s": 1.4978486150503159, "p95_s": 2.378658790141344, "cost_usd_per_1000": 0.027238676654182274, "axes": { "intelligence": 76.0865031698365, "calibration": 54.04673687564798, "speed": 74.48199929246928, "cost": 56.94441987166195 }, "jevbench_score": 64.62415841609092 }, "heldout": { "n": 303, "escalation": 0.5412541254125413, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.6330275229357798 }, "accuracy": 0.8283828382838284, "ece_hard": 0.2509902219591433, "calibration": 58.00768681107789, "p50_s": 1.5091186985373497, "p95_s": 2.2623383209109305, "cost_usd_per_1000": 0.028927159985661487, "axes": { "intelligence": 81.81502660131541, "calibration": 58.00768681107789, "speed": 74.66719047868496, "cost": 56.16082607431475 }, "jevbench_score": 66.79138977197242 } }, { "threshold": 0.93, "public": { "n": 231, "escalation": 0.49783549783549785, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.5945945945945946 }, "accuracy": 0.7489177489177489, "ece_hard": 0.2741391031198916, "calibration": 54.317225253348816, "p50_s": 1.5025348737835884, "p95_s": 2.378658790141344, "cost_usd_per_1000": 0.02780039212348201, "axes": { "intelligence": 76.0865031698365, "calibration": 54.317225253348816, "speed": 74.46843290689776, "cost": 56.678472349943206 }, "jevbench_score": 64.6262398467473 }, "heldout": { "n": 303, "escalation": 0.5676567656765676, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.6330275229357798 }, "accuracy": 0.8283828382838284, "ece_hard": 0.25232453166063606, "calibration": 57.874255840928605, "p50_s": 1.5146773904561996, "p95_s": 2.2623383209109305, "cost_usd_per_1000": 0.02961234162742117, "axes": { "intelligence": 81.81502660131541, "calibration": 57.874255840928605, "speed": 74.65122305313554, "cost": 55.855817468636936 }, "jevbench_score": 66.65856480790966 } }, { "threshold": 0.94, "public": { "n": 231, "escalation": 0.5238095238095238, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6126126126126126 }, "accuracy": 0.7575757575757576, "ece_hard": 0.25460248540131386, "calibration": 56.27088702520659, "p50_s": 1.5059989392757416, "p95_s": 2.378658790141344, "cost_usd_per_1000": 0.0284744506866417, "axes": { "intelligence": 76.83725392058726, "calibration": 56.27088702520659, "speed": 74.45843185464769, "cost": 56.366339360111816 }, "jevbench_score": 65.2675526584936 }, "heldout": { "n": 303, "escalation": 0.594059405940594, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.6330275229357798 }, "accuracy": 0.8283828382838284, "ece_hard": 0.25232453166063606, "calibration": 57.874255840928605, "p50_s": 1.5168466866016388, "p95_s": 2.2623383209109305, "cost_usd_per_1000": 0.03029752326918085, "axes": { "intelligence": 81.81502660131541, "calibration": 57.874255840928605, "speed": 74.64500760847844, "cost": 55.55778616917985 }, "jevbench_score": 66.56808270532134 } }, { "threshold": 0.95, "public": { "n": 231, "escalation": 0.5454545454545454, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6216216216216216 }, "accuracy": 0.7619047619047619, "ece_hard": 0.24324824143016402, "calibration": 58.11994995969944, "p50_s": 1.5125302523374557, "p95_s": 2.378658790141344, "cost_usd_per_1000": 0.029036166155941436, "axes": { "intelligence": 77.21262929596264, "calibration": 58.11994995969944, "speed": 74.43963781643322, "cost": 56.11182181192289 }, "jevbench_score": 65.7988114011007 }, "heldout": { "n": 303, "escalation": 0.6204620462046204, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.6330275229357798 }, "accuracy": 0.8283828382838284, "ece_hard": 0.25169165229039847, "calibration": 57.937543777952364, "p50_s": 1.5221686363220215, "p95_s": 2.2623383209109305, "cost_usd_per_1000": 0.03098270491094053, "axes": { "intelligence": 81.81502660131541, "calibration": 57.937543777952364, "speed": 74.62979678843548, "cost": 55.26642007960117 }, "jevbench_score": 66.4954129014222 } }, { "threshold": 0.96, "public": { "n": 231, "escalation": 0.5627705627705628, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6216216216216216 }, "accuracy": 0.7619047619047619, "ece_hard": 0.24349911780775105, "calibration": 58.09486232194074, "p50_s": 1.5147403478622437, "p95_s": 2.378658790141344, "cost_usd_per_1000": 0.02948553853138123, "axes": { "intelligence": 77.21262929596264, "calibration": 58.09486232194074, "speed": 74.43329657649676, "cost": 55.91172807246312 }, "jevbench_score": 65.73157815100484 }, "heldout": { "n": 303, "escalation": 0.6633663366336634, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.6513761467889908 }, "accuracy": 0.834983498349835, "ece_hard": 0.23498163420577456, "calibration": 59.608545586414756, "p50_s": 1.528146542608738, "p95_s": 2.2624109119176863, "cost_usd_per_1000": 0.03209612507880002, "axes": { "intelligence": 82.36548531691173, "calibration": 59.608545586414756, "speed": 74.61263510027285, "cost": 54.80642188624204 }, "jevbench_score": 66.93825763495565 } }, { "threshold": 0.97, "public": { "n": 231, "escalation": 0.6103896103896104, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6216216216216216 }, "accuracy": 0.7619047619047619, "ece_hard": 0.2413490759263409, "calibration": 58.30986651008175, "p50_s": 1.518095463514328, "p95_s": 2.400771364569664, "cost_usd_per_1000": 0.030721312563840655, "axes": { "intelligence": 77.21262929596264, "calibration": 58.30986651008175, "speed": 74.38350115163612, "cost": 55.37680699216934 }, "jevbench_score": 65.62340018233552 }, "heldout": { "n": 303, "escalation": 0.6963696369636964, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.6513761467889908 }, "accuracy": 0.834983498349835, "ece_hard": 0.2352858263640096, "calibration": 59.57812637059125, "p50_s": 1.5309525728225708, "p95_s": 2.2624109119176863, "cost_usd_per_1000": 0.03295260213099962, "axes": { "intelligence": 82.36548531691173, "calibration": 59.57812637059125, "speed": 74.60466776261086, "cost": 54.463308558186704 }, "jevbench_score": 66.82293257224701 } }, { "threshold": 0.98, "public": { "n": 231, "escalation": 0.6796536796536796, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6216216216216216 }, "accuracy": 0.7619047619047619, "ece_hard": 0.24127136259492354, "calibration": 58.31763784322349, "p50_s": 1.5273330509662628, "p95_s": 2.4070832021534443, "cost_usd_per_1000": 0.03251880206559982, "axes": { "intelligence": 77.21262929596264, "calibration": 58.31763784322349, "speed": 74.34575147994991, "cost": 54.63596384241493 }, "jevbench_score": 65.396688533231 }, "heldout": { "n": 303, "escalation": 0.7458745874587459, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9383561643835616, "hard": 0.6605504587155964 }, "accuracy": 0.8316831683168316, "ece_hard": 0.22161031624000246, "calibration": 61.28185210137195, "p50_s": 1.5344456285238266, "p95_s": 2.2624109119176863, "cost_usd_per_1000": 0.034237317709299026, "axes": { "intelligence": 82.25715303087429, "calibration": 61.28185210137195, "speed": 74.59477008946695, "cost": 53.96500801173972 }, "jevbench_score": 67.11684691377232 } }, { "threshold": 0.99, "public": { "n": 231, "escalation": 0.7316017316017316, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6396396396396397 }, "accuracy": 0.7705627705627706, "ece_hard": 0.2232823509934215, "calibration": 60.0650561914047, "p50_s": 1.5319301038980484, "p95_s": 2.4070832021534443, "cost_usd_per_1000": 0.03386691919191919, "axes": { "intelligence": 77.96338004671338, "calibration": 60.0650561914047, "speed": 74.33269947282982, "cost": 54.10672925460625 }, "jevbench_score": 65.87732157779108 }, "heldout": { "n": 303, "escalation": 0.8547854785478548, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6605504587155964 }, "accuracy": 0.8283828382838284, "ece_hard": 0.22162323552224517, "calibration": 61.28056017314768, "p50_s": 1.5429361686110497, "p95_s": 2.2624109119176863, "cost_usd_per_1000": 0.037063691981557706, "axes": { "intelligence": 82.06537220895648, "calibration": 61.28056017314768, "speed": 74.5708055357799, "cost": 52.93153966004737 }, "jevbench_score": 66.74849959620211 } }, { "threshold": 1.0, "public": { "n": 231, "escalation": 1.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6486486486486487 }, "accuracy": 0.7748917748917749, "ece_hard": 0.21041641822205612, "calibration": 61.35164946854124, "p50_s": 1.5394722446799278, "p95_s": 2.4070832021534443, "cost_usd_per_1000": 0.04083219101123596, "axes": { "intelligence": 78.33875542208877, "calibration": 61.35164946854124, "speed": 74.31137034472908, "cost": 51.66991947037791 }, "jevbench_score": 65.54264921032468 }, "heldout": { "n": 303, "escalation": 1.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6788990825688074 }, "accuracy": 0.834983498349835, "ece_hard": 0.1935286337386598, "calibration": 64.09002035150621, "p50_s": 1.550565131008625, "p95_s": 2.284891256690025, "cost_usd_per_1000": 0.04083219101123596, "axes": { "intelligence": 82.6158309245528, "calibration": 64.09002035150621, "speed": 74.50644450828901, "cost": 51.66991947037791 }, "jevbench_score": 67.19258155743307 } } ] }, { "cheap": "open-alternative-jev", "backend": "semif-qwen3.5-4b", "chosen": { "threshold": 0.99, "public": { "n": 231, "escalation": 0.6320346320346321, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.6036036036036037 }, "accuracy": 0.7489177489177489, "ece_hard": 0.12839466947184652, "calibration": 72.31970647453554, "p50_s": 1.0721421420574189, "p95_s": 2.441754038631916, "cost_usd_per_1000": 0.036357173257454155, "axes": { "intelligence": 75.92175508842178, "calibration": 72.31970647453554, "speed": 75.82045718386495, "cost": 53.182296706086404 }, "jevbench_score": 68.59525822825742 }, "heldout": { "n": 303, "escalation": 0.7326732673267327, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7986798679867987, "ece_hard": 0.18252396785705838, "calibration": 65.95144255252437, "p50_s": 1.1211026340723038, "p95_s": 2.2278682902455325, "cost_usd_per_1000": 0.03861596784959395, "axes": { "intelligence": 79.33796238113192, "calibration": 65.95144255252437, "speed": 76.02465114913076, "cost": 52.39699227306265 }, "jevbench_score": 67.56807309945889 } }, "full": { "tiers": { "easy": 1.0, "standard": 0.8020833333333334, "judge": 0.952054794520548, "hard": 0.5772727272727273 }, "accuracy": 0.7771535580524345, "ece_hard": 0.14436113847874998, "calibration": 70.2453925321002, "p50_s": 1.1042310401797295, "p95_s": 2.2586303554475307, "cost_usd_per_1000": 0.0376388488406346, "axes": { "intelligence": 80.43404939809051, "calibration": 70.2453925321002, "speed": 76.03094888253057, "cost": 52.73091002955807 }, "jevbench_score": 68.98876389696247 }, "full_escalation": 0.6891385767790262, "backend_heldout_accuracy": 0.7986798679867987, "useful": false, "curve": [ { "threshold": 0.0, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 63.4379581919484, "calibration": 59.97952523584675, "speed": 81.86491236780878, "cost": 59.62620384142349 }, "jevbench_score": 65.64804472134733 } }, { "threshold": 0.01, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 63.4379581919484, "calibration": 59.97952523584675, "speed": 81.86491236780878, "cost": 59.62620384142349 }, "jevbench_score": 65.64804472134733 } }, { "threshold": 0.02, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 63.4379581919484, "calibration": 59.97952523584675, "speed": 81.86491236780878, "cost": 59.62620384142349 }, "jevbench_score": 65.64804472134733 } }, { "threshold": 0.03, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 63.4379581919484, "calibration": 59.97952523584675, "speed": 81.86491236780878, "cost": 59.62620384142349 }, "jevbench_score": 65.64804472134733 } }, { "threshold": 0.04, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 63.4379581919484, "calibration": 59.97952523584675, "speed": 81.86491236780878, "cost": 59.62620384142349 }, "jevbench_score": 65.64804472134733 } }, { "threshold": 0.05, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 63.4379581919484, "calibration": 59.97952523584675, "speed": 81.86491236780878, "cost": 59.62620384142349 }, "jevbench_score": 65.64804472134733 } }, { "threshold": 0.06, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 63.4379581919484, "calibration": 59.97952523584675, "speed": 81.86491236780878, "cost": 59.62620384142349 }, "jevbench_score": 65.64804472134733 } }, { "threshold": 0.07, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 63.4379581919484, "calibration": 59.97952523584675, "speed": 81.86491236780878, "cost": 59.62620384142349 }, "jevbench_score": 65.64804472134733 } }, { "threshold": 0.08, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 63.4379581919484, "calibration": 59.97952523584675, "speed": 81.86491236780878, "cost": 59.62620384142349 }, "jevbench_score": 65.64804472134733 } }, { "threshold": 0.09, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 63.4379581919484, "calibration": 59.97952523584675, "speed": 81.86491236780878, "cost": 59.62620384142349 }, "jevbench_score": 65.64804472134733 } }, { "threshold": 0.1, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 63.4379581919484, "calibration": 59.97952523584675, "speed": 81.86491236780878, "cost": 59.62620384142349 }, "jevbench_score": 65.64804472134733 } }, { "threshold": 0.11, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 63.4379581919484, "calibration": 59.97952523584675, "speed": 81.86491236780878, "cost": 59.62620384142349 }, "jevbench_score": 65.64804472134733 } }, { "threshold": 0.12, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 63.4379581919484, "calibration": 59.97952523584675, "speed": 81.86491236780878, "cost": 59.62620384142349 }, "jevbench_score": 65.64804472134733 } }, { "threshold": 0.13, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 63.4379581919484, "calibration": 59.97952523584675, "speed": 81.86491236780878, "cost": 59.62620384142349 }, "jevbench_score": 65.64804472134733 } }, { "threshold": 0.14, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 63.4379581919484, "calibration": 59.97952523584675, "speed": 81.86491236780878, "cost": 59.62620384142349 }, "jevbench_score": 65.64804472134733 } }, { "threshold": 0.15, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 63.4379581919484, "calibration": 59.97952523584675, "speed": 81.86491236780878, "cost": 59.62620384142349 }, "jevbench_score": 65.64804472134733 } }, { "threshold": 0.16, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 63.4379581919484, "calibration": 59.97952523584675, "speed": 81.86491236780878, "cost": 59.62620384142349 }, "jevbench_score": 65.64804472134733 } }, { "threshold": 0.17, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 63.4379581919484, "calibration": 59.97952523584675, "speed": 81.86491236780878, "cost": 59.62620384142349 }, "jevbench_score": 65.64804472134733 } }, { "threshold": 0.18, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 63.4379581919484, "calibration": 59.97952523584675, "speed": 81.86491236780878, "cost": 59.62620384142349 }, "jevbench_score": 65.64804472134733 } }, { "threshold": 0.19, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 63.4379581919484, "calibration": 59.97952523584675, "speed": 81.86491236780878, "cost": 59.62620384142349 }, "jevbench_score": 65.64804472134733 } }, { "threshold": 0.2, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 63.4379581919484, "calibration": 59.97952523584675, "speed": 81.86491236780878, "cost": 59.62620384142349 }, "jevbench_score": 65.64804472134733 } }, { "threshold": 0.21, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 63.4379581919484, "calibration": 59.97952523584675, "speed": 81.86491236780878, "cost": 59.62620384142349 }, "jevbench_score": 65.64804472134733 } }, { "threshold": 0.22, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 63.4379581919484, "calibration": 59.97952523584675, "speed": 81.86491236780878, "cost": 59.62620384142349 }, "jevbench_score": 65.64804472134733 } }, { "threshold": 0.23, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 63.4379581919484, "calibration": 59.97952523584675, "speed": 81.86491236780878, "cost": 59.62620384142349 }, "jevbench_score": 65.64804472134733 } }, { "threshold": 0.24, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 63.4379581919484, "calibration": 59.97952523584675, "speed": 81.86491236780878, "cost": 59.62620384142349 }, "jevbench_score": 65.64804472134733 } }, { "threshold": 0.25, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 63.4379581919484, "calibration": 59.97952523584675, "speed": 81.86491236780878, "cost": 59.62620384142349 }, "jevbench_score": 65.64804472134733 } }, { "threshold": 0.26, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 63.4379581919484, "calibration": 59.97952523584675, "speed": 81.86491236780878, "cost": 59.62620384142349 }, "jevbench_score": 65.64804472134733 } }, { "threshold": 0.27, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 63.4379581919484, "calibration": 59.97952523584675, "speed": 81.86491236780878, "cost": 59.62620384142349 }, "jevbench_score": 65.64804472134733 } }, { "threshold": 0.28, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 63.4379581919484, "calibration": 59.97952523584675, "speed": 81.86491236780878, "cost": 59.62620384142349 }, "jevbench_score": 65.64804472134733 } }, { "threshold": 0.29, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 63.4379581919484, "calibration": 59.97952523584675, "speed": 81.86491236780878, "cost": 59.62620384142349 }, "jevbench_score": 65.64804472134733 } }, { "threshold": 0.3, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 63.4379581919484, "calibration": 59.97952523584675, "speed": 81.86491236780878, "cost": 59.62620384142349 }, "jevbench_score": 65.64804472134733 } }, { "threshold": 0.31, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.0033003300330033004, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5229357798165137 }, "accuracy": 0.5676567656765676, "ece_hard": 0.2819868215442428, "calibration": 59.01428965858395, "p50_s": 0.5870145052671433, "p95_s": 1.1089038744568824, "cost_usd_per_1000": 0.022245479104090186, "axes": { "intelligence": 63.71318754974655, "calibration": 59.01428965858395, "speed": 81.86457266219443, "cost": 59.58274709057052 }, "jevbench_score": 65.44112546186749 } }, { "threshold": 0.32, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.006600660066006601, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5229357798165137 }, "accuracy": 0.5676567656765676, "ece_hard": 0.277431004333902, "calibration": 59.46987137961803, "p50_s": 0.5870145052671433, "p95_s": 1.1089038744568824, "cost_usd_per_1000": 0.022319553713798346, "axes": { "intelligence": 63.71318754974655, "calibration": 59.46987137961803, "speed": 81.86457266219443, "cost": 59.53943480489997 }, "jevbench_score": 65.55514160021767 } }, { "threshold": 0.33, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.006600660066006601, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5229357798165137 }, "accuracy": 0.5676567656765676, "ece_hard": 0.277431004333902, "calibration": 59.46987137961803, "p50_s": 0.5870145052671433, "p95_s": 1.1089038744568824, "cost_usd_per_1000": 0.022319553713798346, "axes": { "intelligence": 63.71318754974655, "calibration": 59.46987137961803, "speed": 81.86457266219443, "cost": 59.53943480489997 }, "jevbench_score": 65.55514160021767 } }, { "threshold": 0.34, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.013201320132013201, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5229357798165137 }, "accuracy": 0.5676567656765676, "ece_hard": 0.2773528356845132, "calibration": 59.477688244556916, "p50_s": 0.5870145052671433, "p95_s": 1.1165145859122274, "cost_usd_per_1000": 0.02246770293321467, "axes": { "intelligence": 63.71318754974655, "calibration": 59.477688244556916, "speed": 81.83486766124035, "cost": 59.45323980930129 }, "jevbench_score": 65.52761041713606 } }, { "threshold": 0.35, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.013201320132013201, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5229357798165137 }, "accuracy": 0.5676567656765676, "ece_hard": 0.2773528356845132, "calibration": 59.477688244556916, "p50_s": 0.5870145052671433, "p95_s": 1.1165145859122274, "cost_usd_per_1000": 0.02246770293321467, "axes": { "intelligence": 63.71318754974655, "calibration": 59.477688244556916, "speed": 81.83486766124035, "cost": 59.45323980930129 }, "jevbench_score": 65.52761041713606 } }, { "threshold": 0.36, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.0165016501650165, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5229357798165137 }, "accuracy": 0.5676567656765676, "ece_hard": 0.2782143758655505, "calibration": 59.39153422645318, "p50_s": 0.5870145052671433, "p95_s": 1.120798410475254, "cost_usd_per_1000": 0.022541777542922834, "axes": { "intelligence": 63.71318754974655, "calibration": 59.39153422645318, "speed": 81.81823660742067, "cost": 59.410355213056796 }, "jevbench_score": 65.48872502745635 } }, { "threshold": 0.37, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.019801980198019802, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5229357798165137 }, "accuracy": 0.5676567656765676, "ece_hard": 0.28696561771421053, "calibration": 57.5009831444821, "p50_s": 0.5870145052671433, "p95_s": 1.1283639609813687, "cost_usd_per_1000": 0.022615852152630993, "axes": { "intelligence": 63.71318754974655, "calibration": 57.5009831444821, "speed": 81.78901960404914, "cost": 59.367611309114295 }, "jevbench_score": 64.94373999586213 } }, { "threshold": 0.38, "public": { "n": 231, "escalation": 0.004329004329004329, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5495495495495496 }, "accuracy": 0.6666666666666666, "ece_hard": 0.22725384582267993, "calibration": 58.5345276116532, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022268567294129092, "axes": { "intelligence": 66.6478978978979, "calibration": 58.5345276116532, "speed": 81.73312596625158, "cost": 59.56923170577649 }, "jevbench_score": 66.01683060804717 }, "heldout": { "n": 303, "escalation": 0.019801980198019802, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5229357798165137 }, "accuracy": 0.5676567656765676, "ece_hard": 0.28696561771421053, "calibration": 57.5009831444821, "p50_s": 0.5870145052671433, "p95_s": 1.1283639609813687, "cost_usd_per_1000": 0.022615852152630993, "axes": { "intelligence": 63.71318754974655, "calibration": 57.5009831444821, "speed": 81.78901960404914, "cost": 59.367611309114295 }, "jevbench_score": 64.94373999586213 } }, { "threshold": 0.39, "public": { "n": 231, "escalation": 0.008658008658008658, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5495495495495496 }, "accuracy": 0.6666666666666666, "ece_hard": 0.23082635440535254, "calibration": 58.17727675338594, "p50_s": 0.5530259400606156, "p95_s": 1.2218917965888978, "cost_usd_per_1000": 0.022365730093876164, "axes": { "intelligence": 66.6478978978979, "calibration": 58.17727675338594, "speed": 81.70221748243355, "cost": 59.51250761210872 }, "jevbench_score": 65.89394128555004 }, "heldout": { "n": 303, "escalation": 0.019801980198019802, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5229357798165137 }, "accuracy": 0.5676567656765676, "ece_hard": 0.28696561771421053, "calibration": 57.5009831444821, "p50_s": 0.5870145052671433, "p95_s": 1.1283639609813687, "cost_usd_per_1000": 0.022615852152630993, "axes": { "intelligence": 63.71318754974655, "calibration": 57.5009831444821, "speed": 81.78901960404914, "cost": 59.367611309114295 }, "jevbench_score": 64.94373999586213 } }, { "threshold": 0.4, "public": { "n": 231, "escalation": 0.017316017316017316, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5585585585585585 }, "accuracy": 0.670995670995671, "ece_hard": 0.223201586861134, "calibration": 58.939753507807794, "p50_s": 0.5530259400606156, "p95_s": 1.2218917965888978, "cost_usd_per_1000": 0.0225600556933703, "axes": { "intelligence": 67.02327327327328, "calibration": 58.939753507807794, "speed": 81.70221748243355, "cost": 59.39979497670668 }, "jevbench_score": 66.17031236512273 }, "heldout": { "n": 303, "escalation": 0.019801980198019802, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5229357798165137 }, "accuracy": 0.5676567656765676, "ece_hard": 0.28696561771421053, "calibration": 57.5009831444821, "p50_s": 0.5870145052671433, "p95_s": 1.1283639609813687, "cost_usd_per_1000": 0.022615852152630993, "axes": { "intelligence": 63.71318754974655, "calibration": 57.5009831444821, "speed": 81.78901960404914, "cost": 59.367611309114295 }, "jevbench_score": 64.94373999586213 } }, { "threshold": 0.41, "public": { "n": 231, "escalation": 0.017316017316017316, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5585585585585585 }, "accuracy": 0.670995670995671, "ece_hard": 0.223201586861134, "calibration": 58.939753507807794, "p50_s": 0.5530259400606156, "p95_s": 1.2218917965888978, "cost_usd_per_1000": 0.0225600556933703, "axes": { "intelligence": 67.02327327327328, "calibration": 58.939753507807794, "speed": 81.70221748243355, "cost": 59.39979497670668 }, "jevbench_score": 66.17031236512273 }, "heldout": { "n": 303, "escalation": 0.0231023102310231, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5229357798165137 }, "accuracy": 0.5676567656765676, "ece_hard": 0.28778001991628044, "calibration": 57.419542924275106, "p50_s": 0.5870145052671433, "p95_s": 1.1592439234256735, "cost_usd_per_1000": 0.022689926762339156, "axes": { "intelligence": 63.71318754974655, "calibration": 57.419542924275106, "speed": 81.6717633903008, "cost": 59.32500717734787 }, "jevbench_score": 64.88580530622325 } }, { "threshold": 0.42, "public": { "n": 231, "escalation": 0.017316017316017316, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5585585585585585 }, "accuracy": 0.670995670995671, "ece_hard": 0.223201586861134, "calibration": 58.939753507807794, "p50_s": 0.5530259400606156, "p95_s": 1.2218917965888978, "cost_usd_per_1000": 0.0225600556933703, "axes": { "intelligence": 67.02327327327328, "calibration": 58.939753507807794, "speed": 81.70221748243355, "cost": 59.39979497670668 }, "jevbench_score": 66.17031236512273 }, "heldout": { "n": 303, "escalation": 0.0231023102310231, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5229357798165137 }, "accuracy": 0.5676567656765676, "ece_hard": 0.28778001991628044, "calibration": 57.419542924275106, "p50_s": 0.5870145052671433, "p95_s": 1.1592439234256735, "cost_usd_per_1000": 0.022689926762339156, "axes": { "intelligence": 63.71318754974655, "calibration": 57.419542924275106, "speed": 81.6717633903008, "cost": 59.32500717734787 }, "jevbench_score": 64.88580530622325 } }, { "threshold": 0.43, "public": { "n": 231, "escalation": 0.030303030303030304, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5585585585585585 }, "accuracy": 0.670995670995671, "ece_hard": 0.22475728178209348, "calibration": 58.784184015711844, "p50_s": 0.5530259400606156, "p95_s": 1.3188491851091384, "cost_usd_per_1000": 0.02285154409261151, "axes": { "intelligence": 67.02327327327328, "calibration": 58.784184015711844, "speed": 81.37059362003006, "cost": 59.23253347187982 }, "jevbench_score": 66.01284956120963 }, "heldout": { "n": 303, "escalation": 0.0231023102310231, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5229357798165137 }, "accuracy": 0.5676567656765676, "ece_hard": 0.28778001991628044, "calibration": 57.419542924275106, "p50_s": 0.5870145052671433, "p95_s": 1.1592439234256735, "cost_usd_per_1000": 0.022689926762339156, "axes": { "intelligence": 63.71318754974655, "calibration": 57.419542924275106, "speed": 81.6717633903008, "cost": 59.32500717734787 }, "jevbench_score": 64.88580530622325 } }, { "threshold": 0.44, "public": { "n": 231, "escalation": 0.030303030303030304, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5585585585585585 }, "accuracy": 0.670995670995671, "ece_hard": 0.22475728178209348, "calibration": 58.784184015711844, "p50_s": 0.5530259400606156, "p95_s": 1.3188491851091384, "cost_usd_per_1000": 0.02285154409261151, "axes": { "intelligence": 67.02327327327328, "calibration": 58.784184015711844, "speed": 81.37059362003006, "cost": 59.23253347187982 }, "jevbench_score": 66.01284956120963 }, "heldout": { "n": 303, "escalation": 0.026402640264026403, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5229357798165137 }, "accuracy": 0.5676567656765676, "ece_hard": 0.2897562627080228, "calibration": 57.22191864510087, "p50_s": 0.5878239214420319, "p95_s": 1.1592439234256735, "cost_usd_per_1000": 0.02276400137204732, "axes": { "intelligence": 63.71318754974655, "calibration": 57.22191864510087, "speed": 81.66577916258157, "cost": 59.28254190662861 }, "jevbench_score": 64.81711084956397 } }, { "threshold": 0.45, "public": { "n": 231, "escalation": 0.03463203463203463, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5495495495495496 }, "accuracy": 0.6666666666666666, "ece_hard": 0.23349052087682726, "calibration": 57.91086010623847, "p50_s": 0.5530259400606156, "p95_s": 1.3188491851091384, "cost_usd_per_1000": 0.02294870689235858, "axes": { "intelligence": 66.6478978978979, "calibration": 57.91086010623847, "speed": 81.37059362003006, "cost": 59.177253428422546 }, "jevbench_score": 65.65868644273655 }, "heldout": { "n": 303, "escalation": 0.0297029702970297, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5229357798165137 }, "accuracy": 0.5676567656765676, "ece_hard": 0.28297572404223104, "calibration": 57.89997251168005, "p50_s": 0.5878239214420319, "p95_s": 1.205813025534152, "cost_usd_per_1000": 0.02283807598175548, "axes": { "intelligence": 63.71318754974655, "calibration": 57.89997251168005, "speed": 81.49472773401806, "cost": 59.24021459470771 }, "jevbench_score": 64.96260873376468 } }, { "threshold": 0.46, "public": { "n": 231, "escalation": 0.04329004329004329, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.2406888055383759, "calibration": 57.1910316400836, "p50_s": 0.5530259400606156, "p95_s": 1.4152807354927064, "cost_usd_per_1000": 0.02314303249185272, "axes": { "intelligence": 66.27252252252254, "calibration": 57.1910316400836, "speed": 81.06411902153799, "cost": 59.06739204753974 }, "jevbench_score": 65.26937798345838 }, "heldout": { "n": 303, "escalation": 0.039603960396039604, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5136986301369864, "hard": 0.5229357798165137 }, "accuracy": 0.570957095709571, "ece_hard": 0.2835027170973235, "calibration": 57.8472732061708, "p50_s": 0.5895052790641785, "p95_s": 1.2240840876102446, "cost_usd_per_1000": 0.023060299810879967, "axes": { "intelligence": 63.904968371664374, "calibration": 57.8472732061708, "speed": 81.41701049018167, "cost": 59.114051520045706 }, "jevbench_score": 64.94651446921276 } }, { "threshold": 0.47, "public": { "n": 231, "escalation": 0.047619047619047616, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.24195821713687382, "calibration": 57.06409048023381, "p50_s": 0.5530259400606156, "p95_s": 1.4152807354927064, "cost_usd_per_1000": 0.023240195291599788, "axes": { "intelligence": 66.27252252252254, "calibration": 57.06409048023381, "speed": 81.06411902153799, "cost": 59.01280680431894 }, "jevbench_score": 65.21805383220494 }, "heldout": { "n": 303, "escalation": 0.0462046204620462, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5205479452054794, "hard": 0.5229357798165137 }, "accuracy": 0.5742574257425742, "ece_hard": 0.2835027170973235, "calibration": 57.8472732061708, "p50_s": 0.5900527179241181, "p95_s": 1.2240840876102446, "cost_usd_per_1000": 0.02320844903029629, "axes": { "intelligence": 64.09674919358217, "calibration": 57.8472732061708, "speed": 81.41297932437669, "cost": 59.03061644563777 }, "jevbench_score": 64.97143599869442 } }, { "threshold": 0.48, "public": { "n": 231, "escalation": 0.05194805194805195, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.24501305624006303, "calibration": 56.75860656991489, "p50_s": 0.5530259400606156, "p95_s": 1.4313000380992889, "cost_usd_per_1000": 0.023337358091346856, "axes": { "intelligence": 66.27252252252254, "calibration": 56.75860656991489, "speed": 81.01523814498383, "cost": 58.958449296137424 }, "jevbench_score": 65.10577274021477 }, "heldout": { "n": 303, "escalation": 0.0462046204620462, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5205479452054794, "hard": 0.5229357798165137 }, "accuracy": 0.5742574257425742, "ece_hard": 0.2835027170973235, "calibration": 57.8472732061708, "p50_s": 0.5900527179241181, "p95_s": 1.2240840876102446, "cost_usd_per_1000": 0.02320844903029629, "axes": { "intelligence": 64.09674919358217, "calibration": 57.8472732061708, "speed": 81.41297932437669, "cost": 59.03061644563777 }, "jevbench_score": 64.97143599869442 } }, { "threshold": 0.49, "public": { "n": 231, "escalation": 0.05627705627705628, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.24456146331384865, "calibration": 56.80376586253632, "p50_s": 0.5530259400606156, "p95_s": 1.4596813708543777, "cost_usd_per_1000": 0.023434520891093925, "axes": { "intelligence": 66.27252252252254, "calibration": 56.80376586253632, "speed": 80.92996431951336, "cost": 58.90431763062138 }, "jevbench_score": 65.08662872321877 }, "heldout": { "n": 303, "escalation": 0.0627062706270627, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5273972602739726, "hard": 0.5137614678899083 }, "accuracy": 0.5742574257425742, "ece_hard": 0.2902747991409586, "calibration": 56.60426664087775, "p50_s": 0.5905896663665772, "p95_s": 1.2347744405269623, "cost_usd_per_1000": 0.0235788220788371, "axes": { "intelligence": 64.01330065770182, "calibration": 56.60426664087775, "speed": 81.37126525085077, "cost": 58.82433683910958 }, "jevbench_score": 64.53374585752454 } }, { "threshold": 0.5, "public": { "n": 231, "escalation": 0.06060606060606061, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5495495495495496 }, "accuracy": 0.6666666666666666, "ece_hard": 0.2352967437259098, "calibration": 58.5606194380997, "p50_s": 0.5530259400606156, "p95_s": 1.4893719941377639, "cost_usd_per_1000": 0.023531683690840997, "axes": { "intelligence": 66.6478978978979, "calibration": 58.5606194380997, "speed": 80.8425131414242, "cost": 58.85040993888652 }, "jevbench_score": 65.64404795757174 }, "heldout": { "n": 303, "escalation": 0.06930693069306931, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5273972602739726, "hard": 0.5137614678899083 }, "accuracy": 0.5742574257425742, "ece_hard": 0.2923513586025238, "calibration": 55.92843837570963, "p50_s": 0.5923556029796601, "p95_s": 1.2456217154860494, "cost_usd_per_1000": 0.02372697129825342, "axes": { "intelligence": 64.01330065770182, "calibration": 55.92843837570963, "speed": 81.32031328401001, "cost": 58.74273085272526 }, "jevbench_score": 64.30785477943854 } }, { "threshold": 0.51, "public": { "n": 231, "escalation": 0.06060606060606061, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5495495495495496 }, "accuracy": 0.6666666666666666, "ece_hard": 0.2352967437259098, "calibration": 58.5606194380997, "p50_s": 0.5530259400606156, "p95_s": 1.4893719941377639, "cost_usd_per_1000": 0.023531683690840997, "axes": { "intelligence": 66.6478978978979, "calibration": 58.5606194380997, "speed": 80.8425131414242, "cost": 58.85040993888652 }, "jevbench_score": 65.64404795757174 }, "heldout": { "n": 303, "escalation": 0.0924092409240924, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5616438356164384, "hard": 0.5137614678899083 }, "accuracy": 0.5907590759075908, "ece_hard": 0.2923854145957692, "calibration": 55.92503277638509, "p50_s": 0.5930232793092728, "p95_s": 1.2973083615303036, "cost_usd_per_1000": 0.024245493566210554, "axes": { "intelligence": 64.97220476729086, "calibration": 55.92503277638509, "speed": 81.13885040687786, "cost": 58.4610691153959 }, "jevbench_score": 64.43285330518007 } }, { "threshold": 0.52, "public": { "n": 231, "escalation": 0.06060606060606061, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5495495495495496 }, "accuracy": 0.6666666666666666, "ece_hard": 0.2352967437259098, "calibration": 58.5606194380997, "p50_s": 0.5530259400606156, "p95_s": 1.4893719941377639, "cost_usd_per_1000": 0.023531683690840997, "axes": { "intelligence": 66.6478978978979, "calibration": 58.5606194380997, "speed": 80.8425131414242, "cost": 58.85040993888652 }, "jevbench_score": 65.64404795757174 }, "heldout": { "n": 303, "escalation": 0.0924092409240924, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5616438356164384, "hard": 0.5137614678899083 }, "accuracy": 0.5907590759075908, "ece_hard": 0.2923854145957692, "calibration": 55.92503277638509, "p50_s": 0.5930232793092728, "p95_s": 1.2973083615303036, "cost_usd_per_1000": 0.024245493566210554, "axes": { "intelligence": 64.97220476729086, "calibration": 55.92503277638509, "speed": 81.13885040687786, "cost": 58.4610691153959 }, "jevbench_score": 64.43285330518007 } }, { "threshold": 0.53, "public": { "n": 231, "escalation": 0.06926406926406926, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.24323344670207736, "calibration": 57.76694914048295, "p50_s": 0.5530259400606156, "p95_s": 1.5442360624670983, "cost_usd_per_1000": 0.023726009290335134, "axes": { "intelligence": 66.27252252252254, "calibration": 57.76694914048295, "speed": 80.68540807043262, "cost": 58.743259116356334 }, "jevbench_score": 65.26667567161648 }, "heldout": { "n": 303, "escalation": 0.10561056105610561, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5753424657534246, "hard": 0.5137614678899083 }, "accuracy": 0.5973597359735974, "ece_hard": 0.2913872048610434, "calibration": 55.991903982105654, "p50_s": 0.5963243931531906, "p95_s": 1.3021944677829742, "cost_usd_per_1000": 0.024541792005043202, "axes": { "intelligence": 65.35576641112648, "calibration": 55.991903982105654, "speed": 81.09841579141369, "cost": 58.30281186747887 }, "jevbench_score": 64.49525359551258 } }, { "threshold": 0.54, "public": { "n": 231, "escalation": 0.08225108225108226, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.24428583049673314, "calibration": 58.9029124371901, "p50_s": 0.5530259400606156, "p95_s": 1.5958027049899102, "cost_usd_per_1000": 0.024017497689576343, "axes": { "intelligence": 66.27252252252254, "calibration": 58.9029124371901, "speed": 80.5427530044208, "cost": 58.58416727213047 }, "jevbench_score": 65.51175747642127 }, "heldout": { "n": 303, "escalation": 0.12871287128712872, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5958904109589042, "hard": 0.5137614678899083 }, "accuracy": 0.6072607260726073, "ece_hard": 0.2949073111144308, "calibration": 55.63989335676692, "p50_s": 0.5986908316612244, "p95_s": 1.4577348420023917, "cost_usd_per_1000": 0.025060314273000334, "axes": { "intelligence": 65.9311088768799, "calibration": 55.63989335676692, "speed": 80.59118858397633, "cost": 58.030404608986274 }, "jevbench_score": 64.35835828140166 } }, { "threshold": 0.55, "public": { "n": 231, "escalation": 0.08225108225108226, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.24428583049673314, "calibration": 58.9029124371901, "p50_s": 0.5530259400606156, "p95_s": 1.5958027049899102, "cost_usd_per_1000": 0.024017497689576343, "axes": { "intelligence": 66.27252252252254, "calibration": 58.9029124371901, "speed": 80.5427530044208, "cost": 58.58416727213047 }, "jevbench_score": 65.51175747642127 }, "heldout": { "n": 303, "escalation": 0.12871287128712872, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5958904109589042, "hard": 0.5137614678899083 }, "accuracy": 0.6072607260726073, "ece_hard": 0.2949073111144308, "calibration": 55.63989335676692, "p50_s": 0.5986908316612244, "p95_s": 1.4577348420023917, "cost_usd_per_1000": 0.025060314273000334, "axes": { "intelligence": 65.9311088768799, "calibration": 55.63989335676692, "speed": 80.59118858397633, "cost": 58.030404608986274 }, "jevbench_score": 64.35835828140166 } }, { "threshold": 0.56, "public": { "n": 231, "escalation": 0.09090909090909091, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.24957859186439785, "calibration": 58.373636300423634, "p50_s": 0.5530259400606156, "p95_s": 1.7163118824362755, "cost_usd_per_1000": 0.02421182328907048, "axes": { "intelligence": 66.27252252252254, "calibration": 58.373636300423634, "speed": 80.2265828811735, "cost": 58.47917513389019 }, "jevbench_score": 65.27057603267316 }, "heldout": { "n": 303, "escalation": 0.12871287128712872, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5958904109589042, "hard": 0.5137614678899083 }, "accuracy": 0.6072607260726073, "ece_hard": 0.2949073111144308, "calibration": 55.63989335676692, "p50_s": 0.5986908316612244, "p95_s": 1.4577348420023917, "cost_usd_per_1000": 0.025060314273000334, "axes": { "intelligence": 65.9311088768799, "calibration": 55.63989335676692, "speed": 80.59118858397633, "cost": 58.030404608986274 }, "jevbench_score": 64.35835828140166 } }, { "threshold": 0.57, "public": { "n": 231, "escalation": 0.11255411255411256, "tiers": { "easy": 1.0, "standard": 0.6388888888888888, "hard": 0.5315315315315315 }, "accuracy": 0.6623376623376623, "ece_hard": 0.2610313234561977, "calibration": 56.2646707113345, "p50_s": 0.5537255644798279, "p95_s": 1.9435866132378576, "cost_usd_per_1000": 0.02469763728780583, "axes": { "intelligence": 66.43727060393728, "calibration": 56.2646707113345, "speed": 79.6810152632751, "cost": 58.220337752754766 }, "jevbench_score": 64.53113234675673 }, "heldout": { "n": 303, "escalation": 0.13861386138613863, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.6095890410958904, "hard": 0.5229357798165137 }, "accuracy": 0.6171617161716172, "ece_hard": 0.2888749628044775, "calibration": 56.243128187762245, "p50_s": 0.6008343935012818, "p95_s": 1.4577348420023917, "cost_usd_per_1000": 0.02528253810212482, "axes": { "intelligence": 66.58989987851368, "calibration": 56.243128187762245, "speed": 80.57566680835744, "cost": 57.91537988746103 }, "jevbench_score": 64.65750049480488 } }, { "threshold": 0.58, "public": { "n": 231, "escalation": 0.12121212121212122, "tiers": { "easy": 1.0, "standard": 0.6388888888888888, "hard": 0.5315315315315315 }, "accuracy": 0.6623376623376623, "ece_hard": 0.25467101696644084, "calibration": 56.90070136031018, "p50_s": 0.5537255644798279, "p95_s": 2.1041721060872076, "cost_usd_per_1000": 0.024891962887299966, "axes": { "intelligence": 66.43727060393728, "calibration": 56.90070136031018, "speed": 79.33624166368674, "cost": 58.118225655967905 }, "jevbench_score": 64.61425524928798 }, "heldout": { "n": 303, "escalation": 0.1419141914191419, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.6095890410958904, "hard": 0.5229357798165137 }, "accuracy": 0.6171617161716172, "ece_hard": 0.28665283754096466, "calibration": 56.46534071411353, "p50_s": 0.6008917182683945, "p95_s": 1.4577348420023917, "cost_usd_per_1000": 0.025356612711832982, "axes": { "intelligence": 66.58989987851368, "calibration": 56.46534071411353, "speed": 80.57525247384629, "cost": 57.877262877194624 }, "jevbench_score": 64.71053547192186 } }, { "threshold": 0.59, "public": { "n": 231, "escalation": 0.12121212121212122, "tiers": { "easy": 1.0, "standard": 0.6388888888888888, "hard": 0.5315315315315315 }, "accuracy": 0.6623376623376623, "ece_hard": 0.25467101696644084, "calibration": 56.90070136031018, "p50_s": 0.5537255644798279, "p95_s": 2.1041721060872076, "cost_usd_per_1000": 0.024891962887299966, "axes": { "intelligence": 66.43727060393728, "calibration": 56.90070136031018, "speed": 79.33624166368674, "cost": 58.118225655967905 }, "jevbench_score": 64.61425524928798 }, "heldout": { "n": 303, "escalation": 0.1419141914191419, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.6095890410958904, "hard": 0.5229357798165137 }, "accuracy": 0.6171617161716172, "ece_hard": 0.28665283754096466, "calibration": 56.46534071411353, "p50_s": 0.6008917182683945, "p95_s": 1.4577348420023917, "cost_usd_per_1000": 0.025356612711832982, "axes": { "intelligence": 66.58989987851368, "calibration": 56.46534071411353, "speed": 80.57525247384629, "cost": 57.877262877194624 }, "jevbench_score": 64.71053547192186 } }, { "threshold": 0.6, "public": { "n": 231, "escalation": 0.12987012987012986, "tiers": { "easy": 1.0, "standard": 0.6527777777777778, "hard": 0.5225225225225225 }, "accuracy": 0.6623376623376623, "ece_hard": 0.25929098458746425, "calibration": 56.43870459820784, "p50_s": 0.5537255644798279, "p95_s": 2.1041721060872076, "cost_usd_per_1000": 0.025086288486794107, "axes": { "intelligence": 66.60201868535202, "calibration": 56.43870459820784, "speed": 79.33624166368674, "cost": 58.01690763238918 }, "jevbench_score": 64.4944966155557 }, "heldout": { "n": 303, "escalation": 0.1617161716171617, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.6301369863013698, "hard": 0.5229357798165137 }, "accuracy": 0.6270627062706271, "ece_hard": 0.2884364868728114, "calibration": 56.943319904848934, "p50_s": 0.6067677825689316, "p95_s": 1.4577348420023917, "cost_usd_per_1000": 0.025801060370081952, "axes": { "intelligence": 67.16524234426711, "calibration": 56.943319904848934, "speed": 80.53298952740565, "cost": 57.650873352019175 }, "jevbench_score": 64.91450809226812 } }, { "threshold": 0.61, "public": { "n": 231, "escalation": 0.13852813852813853, "tiers": { "easy": 1.0, "standard": 0.6527777777777778, "hard": 0.5315315315315315 }, "accuracy": 0.6666666666666666, "ece_hard": 0.24921977322485295, "calibration": 57.44582573446897, "p50_s": 0.5537255644798279, "p95_s": 2.1041721060872076, "cost_usd_per_1000": 0.025280614086288244, "axes": { "intelligence": 66.9773940607274, "calibration": 57.44582573446897, "speed": 79.33624166368674, "cost": 57.91637142706071 }, "jevbench_score": 64.84327219295653 }, "heldout": { "n": 303, "escalation": 0.1716171617161716, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.636986301369863, "hard": 0.5137614678899083 }, "accuracy": 0.6270627062706271, "ece_hard": 0.29396748498481823, "calibration": 54.668143642346934, "p50_s": 0.6076879739761353, "p95_s": 1.4577348420023917, "cost_usd_per_1000": 0.026023284199206437, "axes": { "intelligence": 67.08179380838675, "calibration": 54.668143642346934, "speed": 80.52640823986565, "cost": 57.53913686015007 }, "jevbench_score": 64.20371620251956 } }, { "threshold": 0.62, "public": { "n": 231, "escalation": 0.14285714285714285, "tiers": { "easy": 1.0, "standard": 0.6527777777777778, "hard": 0.5315315315315315 }, "accuracy": 0.6666666666666666, "ece_hard": 0.24921977322485295, "calibration": 57.44582573446897, "p50_s": 0.5538872867822647, "p95_s": 2.1041721060872076, "cost_usd_per_1000": 0.025377776886035312, "axes": { "intelligence": 66.9773940607274, "calibration": 57.44582573446897, "speed": 79.33497343889198, "cost": 57.86639275378192 }, "jevbench_score": 64.82901952823677 }, "heldout": { "n": 303, "escalation": 0.1716171617161716, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.636986301369863, "hard": 0.5137614678899083 }, "accuracy": 0.6270627062706271, "ece_hard": 0.29396748498481823, "calibration": 54.668143642346934, "p50_s": 0.6076879739761353, "p95_s": 1.4577348420023917, "cost_usd_per_1000": 0.026023284199206437, "axes": { "intelligence": 67.08179380838675, "calibration": 54.668143642346934, "speed": 80.52640823986565, "cost": 57.53913686015007 }, "jevbench_score": 64.20371620251956 } }, { "threshold": 0.63, "public": { "n": 231, "escalation": 0.16017316017316016, "tiers": { "easy": 1.0, "standard": 0.6666666666666666, "hard": 0.5315315315315315 }, "accuracy": 0.670995670995671, "ece_hard": 0.252209336700876, "calibration": 56.99790573062774, "p50_s": 0.5563413441181183, "p95_s": 2.1041721060872076, "cost_usd_per_1000": 0.02576642808502359, "axes": { "intelligence": 67.51751751751752, "calibration": 56.99790573062774, "speed": 79.31577405481292, "cost": 57.66837346266069 }, "jevbench_score": 64.77287267588792 }, "heldout": { "n": 303, "escalation": 0.18151815181518152, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.6575342465753424, "hard": 0.5137614678899083 }, "accuracy": 0.636963696369637, "ece_hard": 0.29396748498481823, "calibration": 54.668143642346934, "p50_s": 0.6084316313266754, "p95_s": 1.4577348420023917, "cost_usd_per_1000": 0.02624550802833092, "axes": { "intelligence": 67.65713627414017, "calibration": 54.668143642346934, "speed": 80.52109681617651, "cost": 57.428350488248384 }, "jevbench_score": 64.3088866097052 } }, { "threshold": 0.64, "public": { "n": 231, "escalation": 0.1645021645021645, "tiers": { "easy": 1.0, "standard": 0.6666666666666666, "hard": 0.5405405405405406 }, "accuracy": 0.6753246753246753, "ece_hard": 0.24283447526188007, "calibration": 57.93539187452733, "p50_s": 0.5563413441181183, "p95_s": 2.1041721060872076, "cost_usd_per_1000": 0.025863590884770662, "axes": { "intelligence": 67.89289289289289, "calibration": 57.93539187452733, "speed": 79.31577405481292, "cost": 57.61933534286139 }, "jevbench_score": 65.11394653139754 }, "heldout": { "n": 303, "escalation": 0.1848184818481848, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.6643835616438356, "hard": 0.5137614678899083 }, "accuracy": 0.6402640264026402, "ece_hard": 0.29396748498481823, "calibration": 54.668143642346934, "p50_s": 0.6108522057533264, "p95_s": 1.4577348420023917, "cost_usd_per_1000": 0.026319582638039084, "axes": { "intelligence": 67.84891709605799, "calibration": 54.668143642346934, "speed": 80.5038531938859, "cost": 57.391630054440476 }, "jevbench_score": 64.34067588161305 } }, { "threshold": 0.65, "public": { "n": 231, "escalation": 0.16883116883116883, "tiers": { "easy": 1.0, "standard": 0.6666666666666666, "hard": 0.5405405405405406 }, "accuracy": 0.6753246753246753, "ece_hard": 0.24162404465566106, "calibration": 58.056434935149234, "p50_s": 0.5563413441181183, "p95_s": 2.1041721060872076, "cost_usd_per_1000": 0.02596075368451773, "axes": { "intelligence": 67.89289289289289, "calibration": 58.056434935149234, "speed": 79.31577405481292, "cost": 57.57048110159408 }, "jevbench_score": 65.13411639112752 }, "heldout": { "n": 303, "escalation": 0.19141914191419143, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.6643835616438356, "hard": 0.5045871559633027 }, "accuracy": 0.636963696369637, "ece_hard": 0.30001253394765626, "calibration": 54.06363874606313, "p50_s": 0.6108522057533264, "p95_s": 1.4646646618843078, "cost_usd_per_1000": 0.02646773185745541, "axes": { "intelligence": 67.5736877382598, "calibration": 54.06363874606313, "speed": 80.48325650152927, "cost": 57.31849821429897 }, "jevbench_score": 64.07237303051436 } }, { "threshold": 0.66, "public": { "n": 231, "escalation": 0.1774891774891775, "tiers": { "easy": 1.0, "standard": 0.6666666666666666, "hard": 0.5405405405405406 }, "accuracy": 0.6753246753246753, "ece_hard": 0.24420389391728747, "calibration": 56.57834268588597, "p50_s": 0.5563752442598343, "p95_s": 2.1041721060872076, "cost_usd_per_1000": 0.026155079284011867, "axes": { "intelligence": 67.89289289289289, "calibration": 56.57834268588597, "speed": 79.31550942957696, "cost": 57.473318774670815 }, "jevbench_score": 64.6881502173252 }, "heldout": { "n": 303, "escalation": 0.21782178217821782, "tiers": { "easy": 1.0, "standard": 0.75, "judge": 0.684931506849315, "hard": 0.5045871559633027 }, "accuracy": 0.6501650165016502, "ece_hard": 0.29876209432189565, "calibration": 54.18868270863919, "p50_s": 0.6203177213668823, "p95_s": 1.4646646618843078, "cost_usd_per_1000": 0.027060328735120702, "axes": { "intelligence": 69.31569687067991, "calibration": 54.18868270863919, "speed": 80.41647608223919, "cost": 57.03000795390162 }, "jevbench_score": 64.42392261711254 } }, { "threshold": 0.67, "public": { "n": 231, "escalation": 0.18614718614718614, "tiers": { "easy": 1.0, "standard": 0.6666666666666666, "hard": 0.5405405405405406 }, "accuracy": 0.6753246753246753, "ece_hard": 0.24655401443244032, "calibration": 56.34333063437069, "p50_s": 0.5563752442598343, "p95_s": 2.1041721060872076, "cost_usd_per_1000": 0.026349404883506008, "axes": { "intelligence": 67.89289289289289, "calibration": 56.34333063437069, "speed": 79.31550942957696, "cost": 57.37687567395453 }, "jevbench_score": 64.59374445751737 }, "heldout": { "n": 303, "escalation": 0.23102310231023102, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.6917808219178082, "hard": 0.5045871559633027 }, "accuracy": 0.6567656765676567, "ece_hard": 0.2945137228110856, "calibration": 55.56553450037815, "p50_s": 0.6225624173879624, "p95_s": 1.4759698948264122, "cost_usd_per_1000": 0.02735662717395335, "axes": { "intelligence": 70.67414435926437, "calibration": 55.56553450037815, "speed": 80.36739600859806, "cost": 56.8881234476312 }, "jevbench_score": 65.09413947621499 } }, { "threshold": 0.68, "public": { "n": 231, "escalation": 0.19047619047619047, "tiers": { "easy": 1.0, "standard": 0.6805555555555556, "hard": 0.5405405405405406 }, "accuracy": 0.6796536796536796, "ece_hard": 0.24655401443244032, "calibration": 56.34333063437069, "p50_s": 0.5565016955137253, "p95_s": 2.1041721060872076, "cost_usd_per_1000": 0.026446567683253076, "axes": { "intelligence": 68.43301634968303, "calibration": 56.34333063437069, "speed": 79.31452249058904, "cost": 57.328920521282186 }, "jevbench_score": 64.70810317403661 }, "heldout": { "n": 303, "escalation": 0.264026402640264, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.7328767123287672, "hard": 0.4954128440366973 }, "accuracy": 0.6732673267326733, "ece_hard": 0.3036880347376911, "calibration": 54.6481033077176, "p50_s": 0.635247477889061, "p95_s": 1.5004552006721497, "cost_usd_per_1000": 0.028097373271034967, "axes": { "intelligence": 71.54959993297308, "calibration": 54.6481033077176, "speed": 80.20834017389743, "cost": 56.54002836784782 }, "jevbench_score": 64.89174659503935 } }, { "threshold": 0.69, "public": { "n": 231, "escalation": 0.19047619047619047, "tiers": { "easy": 1.0, "standard": 0.6805555555555556, "hard": 0.5405405405405406 }, "accuracy": 0.6796536796536796, "ece_hard": 0.24655401443244032, "calibration": 56.34333063437069, "p50_s": 0.5565016955137253, "p95_s": 2.1041721060872076, "cost_usd_per_1000": 0.026446567683253076, "axes": { "intelligence": 68.43301634968303, "calibration": 56.34333063437069, "speed": 79.31452249058904, "cost": 57.328920521282186 }, "jevbench_score": 64.70810317403661 }, "heldout": { "n": 303, "escalation": 0.26732673267326734, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.7328767123287672, "hard": 0.4954128440366973 }, "accuracy": 0.6732673267326733, "ece_hard": 0.3036880347376911, "calibration": 54.6481033077176, "p50_s": 0.635247477889061, "p95_s": 1.5108010032773014, "cost_usd_per_1000": 0.028171447880743133, "axes": { "intelligence": 71.54959993297308, "calibration": 54.6481033077176, "speed": 80.17849785987838, "cost": 56.50572495202631 }, "jevbench_score": 64.87586590054869 } }, { "threshold": 0.7, "public": { "n": 231, "escalation": 0.19913419913419914, "tiers": { "easy": 1.0, "standard": 0.6805555555555556, "hard": 0.5585585585585585 }, "accuracy": 0.6883116883116883, "ece_hard": 0.23454697136129485, "calibration": 57.54403494148522, "p50_s": 0.5565016955137253, "p95_s": 2.1702152371406553, "cost_usd_per_1000": 0.026640893282747217, "axes": { "intelligence": 69.18376710043377, "calibration": 57.54403494148522, "speed": 79.18030699797217, "cost": 57.233536514259555 }, "jevbench_score": 65.17305364763585 }, "heldout": { "n": 303, "escalation": 0.2739273927392739, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.7397260273972602, "hard": 0.4954128440366973 }, "accuracy": 0.6765676567656765, "ece_hard": 0.3006460990797747, "calibration": 54.95229687350924, "p50_s": 0.6424911558628082, "p95_s": 1.5108010032773014, "cost_usd_per_1000": 0.028319597100159452, "axes": { "intelligence": 71.74138075489087, "calibration": 54.95229687350924, "speed": 80.12925581628215, "cost": 56.43738788796614 }, "jevbench_score": 64.97980425058329 } }, { "threshold": 0.71, "public": { "n": 231, "escalation": 0.22077922077922077, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "hard": 0.5495495495495496 }, "accuracy": 0.6926406926406926, "ece_hard": 0.248338016747443, "calibration": 56.16493040287041, "p50_s": 0.5587625443935395, "p95_s": 2.1702152371406553, "cost_usd_per_1000": 0.027126707281482563, "axes": { "intelligence": 69.88863863863864, "calibration": 56.16493040287041, "speed": 79.16269905354449, "cost": 56.99808756894544 }, "jevbench_score": 64.87287883052925 }, "heldout": { "n": 303, "escalation": 0.297029702970297, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.7808219178082192, "hard": 0.4954128440366973 }, "accuracy": 0.6963696369636964, "ece_hard": 0.3006460990797747, "calibration": 54.95229687350924, "p50_s": 0.6549014985561371, "p95_s": 1.5108010032773014, "cost_usd_per_1000": 0.028838119368116588, "axes": { "intelligence": 72.89206568639773, "calibration": 54.95229687350924, "speed": 80.04616751253019, "cost": 56.20099194549955 }, "jevbench_score": 65.15348608969627 } }, { "threshold": 0.72, "public": { "n": 231, "escalation": 0.22077922077922077, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "hard": 0.5495495495495496 }, "accuracy": 0.6926406926406926, "ece_hard": 0.248338016747443, "calibration": 56.16493040287041, "p50_s": 0.5587625443935395, "p95_s": 2.1702152371406553, "cost_usd_per_1000": 0.027126707281482563, "axes": { "intelligence": 69.88863863863864, "calibration": 56.16493040287041, "speed": 79.16269905354449, "cost": 56.99808756894544 }, "jevbench_score": 64.87287883052925 }, "heldout": { "n": 303, "escalation": 0.297029702970297, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.7808219178082192, "hard": 0.4954128440366973 }, "accuracy": 0.6963696369636964, "ece_hard": 0.3006460990797747, "calibration": 54.95229687350924, "p50_s": 0.6549014985561371, "p95_s": 1.5108010032773014, "cost_usd_per_1000": 0.028838119368116588, "axes": { "intelligence": 72.89206568639773, "calibration": 54.95229687350924, "speed": 80.04616751253019, "cost": 56.20099194549955 }, "jevbench_score": 65.15348608969627 } }, { "threshold": 0.73, "public": { "n": 231, "escalation": 0.22077922077922077, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "hard": 0.5495495495495496 }, "accuracy": 0.6926406926406926, "ece_hard": 0.248338016747443, "calibration": 56.16493040287041, "p50_s": 0.5587625443935395, "p95_s": 2.1702152371406553, "cost_usd_per_1000": 0.027126707281482563, "axes": { "intelligence": 69.88863863863864, "calibration": 56.16493040287041, "speed": 79.16269905354449, "cost": 56.99808756894544 }, "jevbench_score": 64.87287883052925 }, "heldout": { "n": 303, "escalation": 0.30363036303630364, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.7876712328767124, "hard": 0.4954128440366973 }, "accuracy": 0.6996699669966997, "ece_hard": 0.3006460990797747, "calibration": 54.95229687350924, "p50_s": 0.655505919456482, "p95_s": 1.5437380111217491, "cost_usd_per_1000": 0.02898626858753291, "axes": { "intelligence": 73.08384650831555, "calibration": 54.95229687350924, "speed": 79.94849784292181, "cost": 56.13423063781041 }, "jevbench_score": 65.15703785069677 } }, { "threshold": 0.74, "public": { "n": 231, "escalation": 0.22510822510822512, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "hard": 0.5495495495495496 }, "accuracy": 0.6926406926406926, "ece_hard": 0.25042463417109956, "calibration": 55.95626866050476, "p50_s": 0.5587625443935395, "p95_s": 2.2142491221427916, "cost_usd_per_1000": 0.02722387008122963, "axes": { "intelligence": 69.88863863863864, "calibration": 55.95626866050476, "speed": 79.07546232533612, "cost": 56.95150409390983 }, "jevbench_score": 64.78143535850222 }, "heldout": { "n": 303, "escalation": 0.31683168316831684, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8082191780821918, "hard": 0.4954128440366973 }, "accuracy": 0.7095709570957096, "ece_hard": 0.3023704730191378, "calibration": 54.779859479572934, "p50_s": 0.6719379365444184, "p95_s": 1.5437380111217491, "cost_usd_per_1000": 0.029282567026365558, "axes": { "intelligence": 73.65918897406895, "calibration": 54.779859479572934, "speed": 79.84097240817019, "cost": 56.001725618862096 }, "jevbench_score": 65.17315844458936 } }, { "threshold": 0.75, "public": { "n": 231, "escalation": 0.23376623376623376, "tiers": { "easy": 1.0, "standard": 0.7222222222222222, "hard": 0.5495495495495496 }, "accuracy": 0.696969696969697, "ece_hard": 0.24711291992924483, "calibration": 56.28744008469023, "p50_s": 0.5630453020334244, "p95_s": 2.2142491221427916, "cost_usd_per_1000": 0.027418195680723772, "axes": { "intelligence": 70.42876209542877, "calibration": 56.28744008469023, "speed": 79.04230179694869, "cost": 56.85883385148076 }, "jevbench_score": 64.9687887375871 }, "heldout": { "n": 303, "escalation": 0.31683168316831684, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8082191780821918, "hard": 0.4954128440366973 }, "accuracy": 0.7095709570957096, "ece_hard": 0.3023704730191378, "calibration": 54.779859479572934, "p50_s": 0.6719379365444184, "p95_s": 1.5437380111217491, "cost_usd_per_1000": 0.029282567026365558, "axes": { "intelligence": 73.65918897406895, "calibration": 54.779859479572934, "speed": 79.84097240817019, "cost": 56.001725618862096 }, "jevbench_score": 65.17315844458936 } }, { "threshold": 0.76, "public": { "n": 231, "escalation": 0.24675324675324675, "tiers": { "easy": 1.0, "standard": 0.7222222222222222, "hard": 0.5585585585585585 }, "accuracy": 0.7012987012987013, "ece_hard": 0.22941837042387542, "calibration": 59.05200989076844, "p50_s": 0.5630453020334244, "p95_s": 2.2142491221427916, "cost_usd_per_1000": 0.027709684079964977, "axes": { "intelligence": 70.80413747080415, "calibration": 59.05200989076844, "speed": 79.04230179694869, "cost": 56.721052768333905 }, "jevbench_score": 65.79975788066827 }, "heldout": { "n": 303, "escalation": 0.3333333333333333, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8287671232876712, "hard": 0.5045871559633027 }, "accuracy": 0.7227722772277227, "ece_hard": 0.29408641627694215, "calibration": 55.608265153792495, "p50_s": 0.6774831205606461, "p95_s": 1.5540282458066939, "cost_usd_per_1000": 0.02965294007490637, "axes": { "intelligence": 74.50976079762054, "calibration": 55.608265153792495, "speed": 79.77642612517812, "cost": 55.837967202206215 }, "jevbench_score": 65.54494039247575 } }, { "threshold": 0.77, "public": { "n": 231, "escalation": 0.2510822510822511, "tiers": { "easy": 1.0, "standard": 0.7222222222222222, "hard": 0.5585585585585585 }, "accuracy": 0.7012987012987013, "ece_hard": 0.229769758795002, "calibration": 59.01687105365578, "p50_s": 0.5630453020334244, "p95_s": 2.2142491221427916, "cost_usd_per_1000": 0.02780684687971205, "axes": { "intelligence": 70.80413747080415, "calibration": 59.01687105365578, "speed": 79.04230179694869, "cost": 56.67544763761196 }, "jevbench_score": 65.77673899658566 }, "heldout": { "n": 303, "escalation": 0.3333333333333333, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8287671232876712, "hard": 0.5045871559633027 }, "accuracy": 0.7227722772277227, "ece_hard": 0.29408641627694215, "calibration": 55.608265153792495, "p50_s": 0.6774831205606461, "p95_s": 1.5540282458066939, "cost_usd_per_1000": 0.02965294007490637, "axes": { "intelligence": 74.50976079762054, "calibration": 55.608265153792495, "speed": 79.77642612517812, "cost": 55.837967202206215 }, "jevbench_score": 65.54494039247575 } }, { "threshold": 0.78, "public": { "n": 231, "escalation": 0.2727272727272727, "tiers": { "easy": 1.0, "standard": 0.7222222222222222, "hard": 0.5585585585585585 }, "accuracy": 0.7012987012987013, "ece_hard": 0.22453657056380089, "calibration": 59.54018987677589, "p50_s": 0.5651318520307541, "p95_s": 2.2475591257214544, "cost_usd_per_1000": 0.028292660878447395, "axes": { "intelligence": 70.80413747080415, "calibration": 59.54018987677589, "speed": 78.961390888707, "cost": 56.44978617811038 }, "jevbench_score": 65.83949418397965 }, "heldout": { "n": 303, "escalation": 0.35313531353135313, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8493150684931506, "hard": 0.5137614678899083 }, "accuracy": 0.735973597359736, "ece_hard": 0.30022451840695236, "calibration": 54.18138749753497, "p50_s": 0.6951174080371857, "p95_s": 1.5727281272411342, "cost_usd_per_1000": 0.03009738773315534, "axes": { "intelligence": 75.36033262117215, "calibration": 54.18138749753497, "speed": 79.6128818124823, "cost": 55.64413590524911 }, "jevbench_score": 65.21521330670018 } }, { "threshold": 0.79, "public": { "n": 231, "escalation": 0.27705627705627706, "tiers": { "easy": 1.0, "standard": 0.7222222222222222, "hard": 0.5585585585585585 }, "accuracy": 0.7012987012987013, "ece_hard": 0.2235313490677503, "calibration": 59.64071202638095, "p50_s": 0.5651318520307541, "p95_s": 2.278913828730583, "cost_usd_per_1000": 0.028389823678194464, "axes": { "intelligence": 70.80413747080415, "calibration": 59.64071202638095, "speed": 78.90122310492231, "cost": 56.40511914267266 }, "jevbench_score": 65.84168371419557 }, "heldout": { "n": 303, "escalation": 0.35973597359735976, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8493150684931506, "hard": 0.5137614678899083 }, "accuracy": 0.735973597359736, "ece_hard": 0.286275601150194, "calibration": 55.576279223210804, "p50_s": 0.6960485965013504, "p95_s": 1.5753223180770874, "cost_usd_per_1000": 0.030245536952571664, "axes": { "intelligence": 75.36033262117215, "calibration": 55.576279223210804, "speed": 79.59991012061599, "cost": 55.58016103021659 }, "jevbench_score": 65.60941465367875 } }, { "threshold": 0.8, "public": { "n": 231, "escalation": 0.2857142857142857, "tiers": { "easy": 1.0, "standard": 0.7222222222222222, "hard": 0.5675675675675675 }, "accuracy": 0.7056277056277056, "ece_hard": 0.2156612376362702, "calibration": 60.42772316952896, "p50_s": 0.5651318520307541, "p95_s": 2.278913828730583, "cost_usd_per_1000": 0.028584149277688604, "axes": { "intelligence": 71.17951284617952, "calibration": 60.42772316952896, "speed": 78.90122310492231, "cost": 56.31624186228442 }, "jevbench_score": 66.11913462390677 }, "heldout": { "n": 303, "escalation": 0.38613861386138615, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.863013698630137, "hard": 0.5137614678899083 }, "accuracy": 0.7425742574257426, "ece_hard": 0.28054157060144713, "calibration": 56.14968227808549, "p50_s": 0.7260566294193268, "p95_s": 1.5829851537942887, "cost_usd_per_1000": 0.030838133830236956, "axes": { "intelligence": 75.74389426500775, "calibration": 56.14968227808549, "speed": 79.3955266300097, "cost": 55.327357334734174 }, "jevbench_score": 65.74424214231335 } }, { "threshold": 0.81, "public": { "n": 231, "escalation": 0.2857142857142857, "tiers": { "easy": 1.0, "standard": 0.7222222222222222, "hard": 0.5675675675675675 }, "accuracy": 0.7056277056277056, "ece_hard": 0.2156612376362702, "calibration": 60.42772316952896, "p50_s": 0.5651318520307541, "p95_s": 2.278913828730583, "cost_usd_per_1000": 0.028584149277688604, "axes": { "intelligence": 71.17951284617952, "calibration": 60.42772316952896, "speed": 78.90122310492231, "cost": 56.31624186228442 }, "jevbench_score": 66.11913462390677 }, "heldout": { "n": 303, "escalation": 0.38943894389438943, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.863013698630137, "hard": 0.5137614678899083 }, "accuracy": 0.7425742574257426, "ece_hard": 0.2812809624435205, "calibration": 56.07574309387815, "p50_s": 0.7260566294193268, "p95_s": 1.6031569147109979, "cost_usd_per_1000": 0.03091220843994512, "axes": { "intelligence": 75.74389426500775, "calibration": 56.07574309387815, "speed": 79.34053472243677, "cost": 55.296099004664754 }, "jevbench_score": 65.70192152060612 } }, { "threshold": 0.82, "public": { "n": 231, "escalation": 0.2857142857142857, "tiers": { "easy": 1.0, "standard": 0.7222222222222222, "hard": 0.5675675675675675 }, "accuracy": 0.7056277056277056, "ece_hard": 0.2156612376362702, "calibration": 60.42772316952896, "p50_s": 0.5651318520307541, "p95_s": 2.278913828730583, "cost_usd_per_1000": 0.028584149277688604, "axes": { "intelligence": 71.17951284617952, "calibration": 60.42772316952896, "speed": 78.90122310492231, "cost": 56.31624186228442 }, "jevbench_score": 66.11913462390677 }, "heldout": { "n": 303, "escalation": 0.429042904290429, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8904109589041096, "hard": 0.5229357798165137 }, "accuracy": 0.759075907590759, "ece_hard": 0.27484722912670784, "calibration": 56.71911642555942, "p50_s": 0.8348341524600983, "p95_s": 1.668169368803501, "cost_usd_per_1000": 0.03180110375644306, "axes": { "intelligence": 76.78624691047715, "calibration": 56.71911642555942, "speed": 78.56159650224245, "cost": 54.92673418630312 }, "jevbench_score": 65.8418073925914 } }, { "threshold": 0.83, "public": { "n": 231, "escalation": 0.29004329004329005, "tiers": { "easy": 1.0, "standard": 0.7222222222222222, "hard": 0.5675675675675675 }, "accuracy": 0.7056277056277056, "ece_hard": 0.21538782447232896, "calibration": 60.45506448592308, "p50_s": 0.5651318520307541, "p95_s": 2.278913828730583, "cost_usd_per_1000": 0.028681312077435673, "axes": { "intelligence": 71.17951284617952, "calibration": 60.45506448592308, "speed": 78.90122310492231, "cost": 56.27202954937812 }, "jevbench_score": 66.11363010971661 }, "heldout": { "n": 303, "escalation": 0.43564356435643564, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8904109589041096, "hard": 0.5229357798165137 }, "accuracy": 0.759075907590759, "ece_hard": 0.27484722912670784, "calibration": 56.71911642555942, "p50_s": 0.9954077243804932, "p95_s": 1.668169368803501, "cost_usd_per_1000": 0.031949252975859384, "axes": { "intelligence": 76.78624691047715, "calibration": 56.71911642555942, "speed": 77.79758851100084, "cost": 54.8661787563927 }, "jevbench_score": 65.66303265910913 } }, { "threshold": 0.84, "public": { "n": 231, "escalation": 0.30735930735930733, "tiers": { "easy": 1.0, "standard": 0.7222222222222222, "hard": 0.5675675675675675 }, "accuracy": 0.7056277056277056, "ece_hard": 0.21178517721836176, "calibration": 60.815329211319806, "p50_s": 0.5724019199609757, "p95_s": 2.278913828730583, "cost_usd_per_1000": 0.02906996327642395, "axes": { "intelligence": 71.17951284617952, "calibration": 60.815329211319806, "speed": 78.8457101443015, "cost": 56.09666550597541 }, "jevbench_score": 66.14862136976876 }, "heldout": { "n": 303, "escalation": 0.44224422442244227, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9041095890410958, "hard": 0.5229357798165137 }, "accuracy": 0.7656765676567657, "ece_hard": 0.27484722912670784, "calibration": 56.71911642555942, "p50_s": 1.012372413277626, "p95_s": 1.668169368803501, "cost_usd_per_1000": 0.03209740219527571, "axes": { "intelligence": 77.16980855431277, "calibration": 56.71911642555942, "speed": 77.72419555348043, "cost": 54.80590347450014 }, "jevbench_score": 65.71130833269706 } }, { "threshold": 0.85, "public": { "n": 231, "escalation": 0.31601731601731603, "tiers": { "easy": 1.0, "standard": 0.7222222222222222, "hard": 0.5675675675675675 }, "accuracy": 0.7056277056277056, "ece_hard": 0.20935362617817202, "calibration": 61.05848431533878, "p50_s": 0.5724860668182373, "p95_s": 2.342836084961891, "cost_usd_per_1000": 0.02926428887591809, "axes": { "intelligence": 71.17951284617952, "calibration": 61.05848431533878, "speed": 78.72493174208226, "cost": 56.00986074415376 }, "jevbench_score": 66.16364977152917 }, "heldout": { "n": 303, "escalation": 0.4521452145214521, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9041095890410958, "hard": 0.5229357798165137 }, "accuracy": 0.7656765676567657, "ece_hard": 0.26732201460625155, "calibration": 57.47163787760505, "p50_s": 1.0215922087430953, "p95_s": 1.668169368803501, "cost_usd_per_1000": 0.032319626024400194, "axes": { "intelligence": 77.16980855431277, "calibration": 57.47163787760505, "speed": 77.6848228556812, "cost": 54.716010194793924 }, "jevbench_score": 65.89279046013976 } }, { "threshold": 0.86, "public": { "n": 231, "escalation": 0.3203463203463203, "tiers": { "easy": 1.0, "standard": 0.7222222222222222, "hard": 0.5675675675675675 }, "accuracy": 0.7056277056277056, "ece_hard": 0.2092052544226326, "calibration": 61.07332149089272, "p50_s": 0.5724860668182373, "p95_s": 2.342836084961891, "cost_usd_per_1000": 0.02936145167566516, "axes": { "intelligence": 71.17951284617952, "calibration": 61.07332149089272, "speed": 78.72493174208226, "cost": 55.96667428099103 }, "jevbench_score": 66.15491048465488 }, "heldout": { "n": 303, "escalation": 0.47194719471947194, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9246575342465754, "hard": 0.5412844036697247 }, "accuracy": 0.7821782178217822, "ece_hard": 0.24698194853132377, "calibration": 59.505644485097825, "p50_s": 1.0425333261489869, "p95_s": 1.668169368803501, "cost_usd_per_1000": 0.032764073682649164, "axes": { "intelligence": 78.29560973566252, "calibration": 59.505644485097825, "speed": 77.59669910908227, "cost": 54.538063181793014 }, "jevbench_score": 66.63610946184546 } }, { "threshold": 0.87, "public": { "n": 231, "escalation": 0.3333333333333333, "tiers": { "easy": 1.0, "standard": 0.7222222222222222, "hard": 0.5675675675675675 }, "accuracy": 0.7056277056277056, "ece_hard": 0.20645877461257517, "calibration": 61.347969471898466, "p50_s": 0.5749294698238373, "p95_s": 2.342836084961891, "cost_usd_per_1000": 0.02965294007490637, "axes": { "intelligence": 71.17951284617952, "calibration": 61.347969471898466, "speed": 78.70643525127272, "cost": 55.837967202206215 }, "jevbench_score": 66.18716229737319 }, "heldout": { "n": 303, "escalation": 0.5016501650165016, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.5412844036697247 }, "accuracy": 0.7854785478547854, "ece_hard": 0.23256952478243548, "calibration": 60.946886859986655, "p50_s": 1.0793453931808472, "p95_s": 2.0541040048003194, "cost_usd_per_1000": 0.03343074517002262, "axes": { "intelligence": 78.48739055758033, "calibration": 60.946886859986655, "speed": 76.54217128460095, "cost": 54.27561828676787 }, "jevbench_score": 66.76736602970277 } }, { "threshold": 0.88, "public": { "n": 231, "escalation": 0.3463203463203463, "tiers": { "easy": 1.0, "standard": 0.7222222222222222, "hard": 0.5675675675675675 }, "accuracy": 0.7056277056277056, "ece_hard": 0.1995747691485335, "calibration": 62.03637001830263, "p50_s": 0.5749294698238373, "p95_s": 2.4169830560684202, "cost_usd_per_1000": 0.029944428474147574, "axes": { "intelligence": 71.17951284617952, "calibration": 62.03637001830263, "speed": 78.57111823793818, "cost": 55.710519146178605 }, "jevbench_score": 66.30562638632732 }, "heldout": { "n": 303, "escalation": 0.5016501650165016, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.5412844036697247 }, "accuracy": 0.7854785478547854, "ece_hard": 0.23256952478243548, "calibration": 60.946886859986655, "p50_s": 1.0793453931808472, "p95_s": 2.0541040048003194, "cost_usd_per_1000": 0.03343074517002262, "axes": { "intelligence": 78.48739055758033, "calibration": 60.946886859986655, "speed": 76.54217128460095, "cost": 54.27561828676787 }, "jevbench_score": 66.76736602970277 } }, { "threshold": 0.89, "public": { "n": 231, "escalation": 0.3593073593073593, "tiers": { "easy": 1.0, "standard": 0.7222222222222222, "hard": 0.5675675675675675 }, "accuracy": 0.7056277056277056, "ece_hard": 0.1995747691485335, "calibration": 62.03637001830263, "p50_s": 0.5810501962900162, "p95_s": 2.4169830560684202, "cost_usd_per_1000": 0.030235916873388783, "axes": { "intelligence": 71.17951284617952, "calibration": 62.03637001830263, "speed": 78.52512741908583, "cost": 55.584305719613624 }, "jevbench_score": 66.25834075631887 }, "heldout": { "n": 303, "escalation": 0.5247524752475248, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9383561643835616, "hard": 0.5504587155963303 }, "accuracy": 0.7920792079207921, "ece_hard": 0.21184827081524943, "calibration": 63.01901225670526, "p50_s": 1.0899055927991868, "p95_s": 2.0541040048003194, "cost_usd_per_1000": 0.033949267437979755, "axes": { "intelligence": 78.95440073729631, "calibration": 63.01901225670526, "speed": 76.49988689536717, "cost": 54.07508777647541 }, "jevbench_score": 67.35603127172101 } }, { "threshold": 0.9, "public": { "n": 231, "escalation": 0.3722943722943723, "tiers": { "easy": 1.0, "standard": 0.7222222222222222, "hard": 0.5675675675675675 }, "accuracy": 0.7056277056277056, "ece_hard": 0.20098782472908017, "calibration": 61.895064460247966, "p50_s": 0.5851256638765335, "p95_s": 2.4169830560684202, "cost_usd_per_1000": 0.030527405272629992, "axes": { "intelligence": 71.17951284617952, "calibration": 61.895064460247966, "speed": 78.49477247263114, "cost": 55.459303231342616 }, "jevbench_score": 66.17691898028767 }, "heldout": { "n": 303, "escalation": 0.5346534653465347, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9383561643835616, "hard": 0.5504587155963303 }, "accuracy": 0.7920792079207921, "ece_hard": 0.21115345494496776, "calibration": 63.08849384373343, "p50_s": 1.090947213768959, "p95_s": 2.0541040048003194, "cost_usd_per_1000": 0.03417149126710424, "axes": { "intelligence": 78.95440073729631, "calibration": 63.08849384373343, "speed": 76.49573833226889, "cost": 53.99008203643897 }, "jevbench_score": 67.34718261322551 } }, { "threshold": 0.91, "public": { "n": 231, "escalation": 0.3939393939393939, "tiers": { "easy": 1.0, "standard": 0.7361111111111112, "hard": 0.5765765765765766 }, "accuracy": 0.7142857142857143, "ece_hard": 0.1930200175408916, "calibration": 62.69184517906682, "p50_s": 0.5868345886468888, "p95_s": 2.441754038631916, "cost_usd_per_1000": 0.031013219271365338, "axes": { "intelligence": 72.09501167834502, "calibration": 62.69184517906682, "speed": 78.43782381592251, "cost": 55.25359450798859 }, "jevbench_score": 66.52740527859257 }, "heldout": { "n": 303, "escalation": 0.5445544554455446, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9452054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7953795379537953, "ece_hard": 0.21073559409390288, "calibration": 63.13027992883992, "p50_s": 1.0932887375354767, "p95_s": 2.0541040048003194, "cost_usd_per_1000": 0.034393715096228725, "axes": { "intelligence": 79.14618155921411, "calibration": 63.13027992883992, "speed": 76.48642696423666, "cost": 53.90562731738742 }, "jevbench_score": 67.37077448806997 } }, { "threshold": 0.92, "public": { "n": 231, "escalation": 0.4155844155844156, "tiers": { "easy": 1.0, "standard": 0.7361111111111112, "hard": 0.5765765765765766 }, "accuracy": 0.7142857142857143, "ece_hard": 0.18789177266343093, "calibration": 64.19816556287442, "p50_s": 0.6046394348144531, "p95_s": 2.441754038631916, "cost_usd_per_1000": 0.03149903327010069, "axes": { "intelligence": 72.09501167834502, "calibration": 64.19816556287442, "speed": 78.30801616673523, "cost": 55.05108324526759 }, "jevbench_score": 66.83438890316854 }, "heldout": { "n": 303, "escalation": 0.570957095709571, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5596330275229358 }, "accuracy": 0.801980198019802, "ece_hard": 0.19825622046000793, "calibration": 64.37821729222941, "p50_s": 1.0996689289808272, "p95_s": 2.0541040048003194, "cost_usd_per_1000": 0.03498631197389402, "axes": { "intelligence": 79.61319173893008, "calibration": 64.37821729222941, "speed": 76.46115616399555, "cost": 53.68305506687068 }, "jevbench_score": 67.725234339397 } }, { "threshold": 0.93, "public": { "n": 231, "escalation": 0.43722943722943725, "tiers": { "easy": 1.0, "standard": 0.75, "hard": 0.5855855855855856 }, "accuracy": 0.7229437229437229, "ece_hard": 0.17420301182141057, "calibration": 66.21390455707581, "p50_s": 0.635255092382431, "p95_s": 2.441754038631916, "cost_usd_per_1000": 0.031984847268836034, "axes": { "intelligence": 73.01051051051051, "calibration": 66.21390455707581, "speed": 78.0934993012837, "cost": 54.85167156236169 }, "jevbench_score": 67.45821318664692 }, "heldout": { "n": 303, "escalation": 0.5874587458745875, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5596330275229358 }, "accuracy": 0.801980198019802, "ece_hard": 0.19356497128627032, "calibration": 64.84734220960317, "p50_s": 1.1005460798740387, "p95_s": 2.0572781085968015, "cost_usd_per_1000": 0.03535668502243483, "axes": { "intelligence": 79.61319173893008, "calibration": 64.84734220960317, "speed": 76.45098764012627, "cost": 53.545853811991314 }, "jevbench_score": 67.80263017300842 } }, { "threshold": 0.94, "public": { "n": 231, "escalation": 0.4588744588744589, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.5855855855855856 }, "accuracy": 0.7272727272727273, "ece_hard": 0.17002974664185122, "calibration": 66.63123107503175, "p50_s": 0.7952923119068146, "p95_s": 2.441754038631916, "cost_usd_per_1000": 0.03247066126757138, "axes": { "intelligence": 73.55063396730064, "calibration": 66.63123107503175, "speed": 77.11771301143744, "cost": 54.65526600527937 }, "jevbench_score": 67.41594133148536 }, "heldout": { "n": 303, "escalation": 0.6204620462046204, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5596330275229358 }, "accuracy": 0.801980198019802, "ece_hard": 0.18326411189342057, "calibration": 65.87742814888814, "p50_s": 1.1066845685243607, "p95_s": 2.074221782386302, "cost_usd_per_1000": 0.036097431119516445, "axes": { "intelligence": 79.61319173893008, "calibration": 65.87742814888814, "speed": 76.39120955078744, "cost": 53.27571110929742 }, "jevbench_score": 67.97098731028649 } }, { "threshold": 0.95, "public": { "n": 231, "escalation": 0.47186147186147187, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.5945945945945946 }, "accuracy": 0.7316017316017316, "ece_hard": 0.15407890827547954, "calibration": 68.22631491166892, "p50_s": 1.006430798768997, "p95_s": 2.441754038631916, "cost_usd_per_1000": 0.03276214966681259, "axes": { "intelligence": 73.92600934267601, "calibration": 68.22631491166892, "speed": 76.09514166717787, "cost": 54.53882830093321 }, "jevbench_score": 67.63990324087649 }, "heldout": { "n": 303, "escalation": 0.636963696369637, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5596330275229358 }, "accuracy": 0.801980198019802, "ece_hard": 0.18067606357520324, "calibration": 66.13623298070988, "p50_s": 1.1089836835861206, "p95_s": 2.106213764846324, "cost_usd_per_1000": 0.036467804168057255, "axes": { "intelligence": 79.61319173893008, "calibration": 66.13623298070988, "speed": 76.31572396845932, "cost": 53.14271158076698 }, "jevbench_score": 67.97834032514812 } }, { "threshold": 0.96, "public": { "n": 231, "escalation": 0.5064935064935064, "tiers": { "easy": 1.0, "standard": 0.7777777777777778, "hard": 0.5945945945945946 }, "accuracy": 0.7359307359307359, "ece_hard": 0.14686611612638323, "calibration": 70.47256180908187, "p50_s": 1.0422662526369095, "p95_s": 2.441754038631916, "cost_usd_per_1000": 0.033539452064789144, "axes": { "intelligence": 74.46613279946614, "calibration": 70.47256180908187, "speed": 75.94319409244372, "cost": 54.23332110268719 }, "jevbench_score": 68.18415768268827 }, "heldout": { "n": 303, "escalation": 0.6501650165016502, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5688073394495413 }, "accuracy": 0.8052805280528053, "ece_hard": 0.17045614913608453, "calibration": 67.15822442462175, "p50_s": 1.1113397508859635, "p95_s": 2.106213764846324, "cost_usd_per_1000": 0.0367641026068899, "axes": { "intelligence": 79.88842109672825, "calibration": 67.15822442462175, "speed": 76.30650704631651, "cost": 53.037280914131046 }, "jevbench_score": 68.2623864382706 } }, { "threshold": 0.97, "public": { "n": 231, "escalation": 0.5194805194805194, "tiers": { "easy": 1.0, "standard": 0.7777777777777778, "hard": 0.5945945945945946 }, "accuracy": 0.7359307359307359, "ece_hard": 0.14686611612638323, "calibration": 70.47256180908187, "p50_s": 1.0466676831245423, "p95_s": 2.441754038631916, "cost_usd_per_1000": 0.03383094046403035, "axes": { "intelligence": 74.46613279946614, "calibration": 70.47256180908187, "speed": 75.92489270177457, "cost": 54.120577873636655 }, "jevbench_score": 68.14458765034412 }, "heldout": { "n": 303, "escalation": 0.66996699669967, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5688073394495413 }, "accuracy": 0.8052805280528053, "ece_hard": 0.1678272170596774, "calibration": 67.42111763226247, "p50_s": 1.112243089079857, "p95_s": 2.106213764846324, "cost_usd_per_1000": 0.037208550265138876, "axes": { "intelligence": 79.88842109672825, "calibration": 67.42111763226247, "speed": 76.30297837351483, "cost": 52.88071752427941 }, "jevbench_score": 68.27782116694536 } }, { "threshold": 0.98, "public": { "n": 231, "escalation": 0.5670995670995671, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "hard": 0.5945945945945946 }, "accuracy": 0.7402597402597403, "ece_hard": 0.13860595269225418, "calibration": 71.29857815249477, "p50_s": 1.0590197175741196, "p95_s": 2.441754038631916, "cost_usd_per_1000": 0.03489973126124812, "axes": { "intelligence": 75.00625625625626, "calibration": 71.29857815249477, "speed": 75.87394039107771, "cost": 53.71533751690343 }, "jevbench_score": 68.32699588229077 }, "heldout": { "n": 303, "escalation": 0.6864686468646864, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7986798679867987, "ece_hard": 0.18302724489094155, "calibration": 65.90111484913605, "p50_s": 1.1126901239156723, "p95_s": 2.2278682902455325, "cost_usd_per_1000": 0.03757892331367968, "axes": { "intelligence": 79.33796238113192, "calibration": 65.90111484913605, "speed": 76.05736254547242, "cost": 52.75167001521774 }, "jevbench_score": 67.67648897008748 } }, { "threshold": 0.99, "public": { "n": 231, "escalation": 0.6320346320346321, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.6036036036036037 }, "accuracy": 0.7489177489177489, "ece_hard": 0.12839466947184652, "calibration": 72.31970647453554, "p50_s": 1.0721421420574189, "p95_s": 2.441754038631916, "cost_usd_per_1000": 0.036357173257454155, "axes": { "intelligence": 75.92175508842178, "calibration": 72.31970647453554, "speed": 75.82045718386495, "cost": 53.182296706086404 }, "jevbench_score": 68.59525822825742 }, "heldout": { "n": 303, "escalation": 0.7326732673267327, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7986798679867987, "ece_hard": 0.18252396785705838, "calibration": 65.95144255252437, "p50_s": 1.1211026340723038, "p95_s": 2.2278682902455325, "cost_usd_per_1000": 0.03861596784959395, "axes": { "intelligence": 79.33796238113192, "calibration": 65.95144255252437, "speed": 76.02465114913076, "cost": 52.39699227306265 }, "jevbench_score": 67.56807309945889 } }, { "threshold": 1.0, "public": { "n": 231, "escalation": 1.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6036036036036037 }, "accuracy": 0.7532467532467533, "ece_hard": 0.12914387221294593, "calibration": 72.24478620042561, "p50_s": 1.0902809530496598, "p95_s": 2.441754038631916, "cost_usd_per_1000": 0.04461601123595506, "axes": { "intelligence": 76.46187854521189, "calibration": 72.24478620042561, "speed": 75.7475966016049, "cost": 50.51527777435402 }, "jevbench_score": 67.80485151468261 }, "heldout": { "n": 303, "escalation": 1.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.952054794520548, "hard": 0.5504587155963303 }, "accuracy": 0.7986798679867987, "ece_hard": 0.17685813516393573, "calibration": 66.51802582183663, "p50_s": 1.1560284227132798, "p95_s": 2.2278682902455325, "cost_usd_per_1000": 0.04461601123595506, "axes": { "intelligence": 79.33796238113192, "calibration": 66.51802582183663, "speed": 75.89141975869856, "cost": 50.51527777435402 }, "jevbench_score": 67.06701143047529 } } ] }, { "cheap": "openjev-verdict-1.4", "backend": "jev-1.13.0", "chosen": { "threshold": 0.71, "public": { "n": 231, "escalation": 0.8051948051948052, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.7027027027027027 }, "accuracy": 0.8008658008658008, "ece_hard": 0.08887587268226801, "calibration": 78.53341273177321, "p50_s": 1.3159398227930068, "p95_s": 2.337883038073778, "cost_usd_per_1000": 0.036011656792645556, "axes": { "intelligence": 80.59100767434103, "calibration": 78.53341273177321, "speed": 75.11941190261479, "cost": 53.30670692591103 }, "jevbench_score": 70.95267483938372 }, "heldout": { "n": 303, "escalation": 0.9174917491749175, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6972477064220184 }, "accuracy": 0.8514851485148515, "ece_hard": 0.09415119034291637, "calibration": 80.59188096570836, "p50_s": 1.8783646464347838, "p95_s": 2.6674607589840886, "cost_usd_per_1000": 0.04049390351169948, "axes": { "intelligence": 84.71651795065142, "calibration": 80.59188096570836, "speed": 73.00122055174546, "cost": 51.77831068913449 }, "jevbench_score": 71.27452749270998 } }, "full": { "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9452054794520548, "hard": 0.7 }, "accuracy": 0.8295880149812734, "ece_hard": 0.09148955279595289, "calibration": 79.56504472040471, "p50_s": 1.6719033174216746, "p95_s": 2.5153479777276515, "cost_usd_per_1000": 0.03855495408828851, "axes": { "intelligence": 84.5074200913242, "calibration": 79.56504472040471, "speed": 73.76190765866312, "cost": 52.417594291059316 }, "jevbench_score": 71.40548855809061 }, "full_escalation": 0.8689138576779026, "backend_heldout_accuracy": 0.8514851485148515, "useful": false, "curve": [ { "threshold": 0.0, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.01, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.02, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.03, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.04, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.05, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.06, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.07, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.08, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.09, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.1, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.11, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.12, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.13, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.14, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.15, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.16, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.17, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.18, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.19, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.2, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.21, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.22, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.23, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.12665915118090862, "calibration": 73.72562045082995, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 53.09371871871872, "calibration": 73.72562045082995, "speed": 79.65074585196557, "cost": 82.35883967864214 }, "jevbench_score": 71.18531509987325 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.24, "public": { "n": 231, "escalation": 0.004329004329004329, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36036036036036034 }, "accuracy": 0.5238095238095238, "ece_hard": 0.1269620600392055, "calibration": 73.69532956500026, "p50_s": 0.6819422334432602, "p95_s": 1.5891973316669463, "cost_usd_per_1000": 0.004045710248552945, "axes": { "intelligence": 53.09371871871872, "calibration": 73.69532956500026, "speed": 79.65074585196557, "cost": 81.79015673119277 }, "jevbench_score": 71.05481275710818 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.25, "public": { "n": 231, "escalation": 0.008658008658008658, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36936936936936937 }, "accuracy": 0.5281385281385281, "ece_hard": 0.12575800961928585, "calibration": 73.81573460699222, "p50_s": 0.6819422334432602, "p95_s": 1.6338174432516097, "cost_usd_per_1000": 0.004218499148791284, "axes": { "intelligence": 53.46909409409409, "calibration": 73.81573460699222, "speed": 79.53048883940582, "cost": 81.24526102566108 }, "jevbench_score": 71.06338090715253 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5616438356164384, "hard": 0.3577981651376147 }, "accuracy": 0.5016501650165016, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9867176026105877, "cost_usd_per_1000": 0.003872921348314607, "axes": { "intelligence": 50.959972351388714, "calibration": 70.2289302105776, "speed": 76.13981363278096, "cost": 82.35883967864214 }, "jevbench_score": 68.82826408417213 } }, { "threshold": 0.26, "public": { "n": 231, "escalation": 0.008658008658008658, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36936936936936937 }, "accuracy": 0.5281385281385281, "ece_hard": 0.12575800961928585, "calibration": 73.81573460699222, "p50_s": 0.6819422334432602, "p95_s": 1.6338174432516097, "cost_usd_per_1000": 0.004218499148791284, "axes": { "intelligence": 53.46909409409409, "calibration": 73.81573460699222, "speed": 79.53048883940582, "cost": 81.24526102566108 }, "jevbench_score": 71.06338090715253 }, "heldout": { "n": 303, "escalation": 0.0033003300330033004, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5684931506849316, "hard": 0.3577981651376147 }, "accuracy": 0.504950495049505, "ece_hard": 0.14782087231475824, "calibration": 70.2289302105776, "p50_s": 1.2242849111557006, "p95_s": 1.9941574066877361, "cost_usd_per_1000": 0.004004651499981459, "axes": { "intelligence": 51.151753173306524, "calibration": 70.2289302105776, "speed": 76.12358067087503, "cost": 81.92305815683352 }, "jevbench_score": 68.79794802483471 } }, { "threshold": 0.27, "public": { "n": 231, "escalation": 0.008658008658008658, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36936936936936937 }, "accuracy": 0.5281385281385281, "ece_hard": 0.12575800961928585, "calibration": 73.81573460699222, "p50_s": 0.6819422334432602, "p95_s": 1.6338174432516097, "cost_usd_per_1000": 0.004218499148791284, "axes": { "intelligence": 53.46909409409409, "calibration": 73.81573460699222, "speed": 79.53048883940582, "cost": 81.24526102566108 }, "jevbench_score": 71.06338090715253 }, "heldout": { "n": 303, "escalation": 0.006600660066006601, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5684931506849316, "hard": 0.3669724770642202 }, "accuracy": 0.5082508250825083, "ece_hard": 0.14535413126842384, "calibration": 70.47560431521106, "p50_s": 1.2242849111557006, "p95_s": 1.9994657054543494, "cost_usd_per_1000": 0.004136381651648311, "axes": { "intelligence": 51.42698253110469, "calibration": 70.47560431521106, "speed": 76.11203543410345, "cost": 81.50138191010686 }, "jevbench_score": 68.85921066571657 } }, { "threshold": 0.28, "public": { "n": 231, "escalation": 0.008658008658008658, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.36936936936936937 }, "accuracy": 0.5281385281385281, "ece_hard": 0.12575800961928585, "calibration": 73.81573460699222, "p50_s": 0.6819422334432602, "p95_s": 1.6338174432516097, "cost_usd_per_1000": 0.004218499148791284, "axes": { "intelligence": 53.46909409409409, "calibration": 73.81573460699222, "speed": 79.53048883940582, "cost": 81.24526102566108 }, "jevbench_score": 71.06338090715253 }, "heldout": { "n": 303, "escalation": 0.013201320132013201, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5684931506849316, "hard": 0.3853211009174312 }, "accuracy": 0.5148514851485149, "ece_hard": 0.1402327780284442, "calibration": 70.98773963920902, "p50_s": 1.234055870771408, "p95_s": 2.002006083726883, "cost_usd_per_1000": 0.004399841954982015, "axes": { "intelligence": 51.97744124670103, "calibration": 70.98773963920902, "speed": 76.07199784678924, "cost": 80.69688770072312 }, "jevbench_score": 68.9874294324266 } }, { "threshold": 0.29, "public": { "n": 231, "escalation": 0.017316017316017316, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.38738738738738737 }, "accuracy": 0.5367965367965368, "ece_hard": 0.12286488717698521, "calibration": 74.10504685122228, "p50_s": 0.6819422334432602, "p95_s": 1.6590973153710364, "cost_usd_per_1000": 0.0045640769492679605, "axes": { "intelligence": 54.21984484484484, "calibration": 74.10504685122228, "speed": 79.46380551799933, "cost": 80.21941126266677 }, "jevbench_score": 71.13997719614932 }, "heldout": { "n": 303, "escalation": 0.019801980198019802, "tiers": { "easy": 0.8333333333333334, "standard": 0.4583333333333333, "judge": 0.5684931506849316, "hard": 0.4036697247706422 }, "accuracy": 0.5214521452145214, "ece_hard": 0.12468978750868424, "calibration": 73.71703959581147, "p50_s": 1.234055870771408, "p95_s": 2.06843184530735, "cost_usd_per_1000": 0.004663302258315719, "axes": { "intelligence": 52.527899962297354, "calibration": 73.71703959581147, "speed": 75.93023962072186, "cost": 79.9391930272659 }, "jevbench_score": 69.627870618252 } }, { "threshold": 0.3, "public": { "n": 231, "escalation": 0.030303030303030304, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.38738738738738737 }, "accuracy": 0.5367965367965368, "ece_hard": 0.12347518324220808, "calibration": 74.0440172447, "p50_s": 0.6859634757041931, "p95_s": 1.669744887948036, "cost_usd_per_1000": 0.005082443649982976, "axes": { "intelligence": 54.21984484484484, "calibration": 74.0440172447, "speed": 79.41048885282348, "cost": 78.81782283305648 }, "jevbench_score": 70.80071391541993 }, "heldout": { "n": 303, "escalation": 0.033003300330033, "tiers": { "easy": 0.875, "standard": 0.4583333333333333, "judge": 0.5821917808219178, "hard": 0.41284403669724773 }, "accuracy": 0.5346534653465347, "ece_hard": 0.12196142160188074, "calibration": 73.98987618649181, "p50_s": 1.2708158284425735, "p95_s": 2.0718681916594504, "cost_usd_per_1000": 0.005190222864983127, "axes": { "intelligence": 53.77002429726447, "calibration": 73.98987618649181, "speed": 75.79555261568528, "cost": 78.54441980238255 }, "jevbench_score": 69.76183541435546 } }, { "threshold": 0.31, "public": { "n": 231, "escalation": 0.030303030303030304, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "hard": 0.38738738738738737 }, "accuracy": 0.5367965367965368, "ece_hard": 0.12347518324220808, "calibration": 74.0440172447, "p50_s": 0.6859634757041931, "p95_s": 1.669744887948036, "cost_usd_per_1000": 0.005082443649982976, "axes": { "intelligence": 54.21984484484484, "calibration": 74.0440172447, "speed": 79.41048885282348, "cost": 78.81782283305648 }, "jevbench_score": 70.80071391541993 }, "heldout": { "n": 303, "escalation": 0.0462046204620462, "tiers": { "easy": 0.875, "standard": 0.4583333333333333, "judge": 0.5958904109589042, "hard": 0.41284403669724773 }, "accuracy": 0.5412541254125413, "ece_hard": 0.12156486031497392, "calibration": 74.02953231518251, "p50_s": 1.2708158284425735, "p95_s": 2.129326386749744, "cost_usd_per_1000": 0.0057171434716505355, "axes": { "intelligence": 54.15358594110009, "calibration": 74.02953231518251, "speed": 75.67675148592534, "cost": 77.2846272710164 }, "jevbench_score": 69.58601332768515 } }, { "threshold": 0.32, "public": { "n": 231, "escalation": 0.05194805194805195, "tiers": { "easy": 0.875, "standard": 0.5555555555555556, "hard": 0.40540540540540543 }, "accuracy": 0.5497835497835498, "ece_hard": 0.12410571892511613, "calibration": 73.9809636764092, "p50_s": 0.6935107052326203, "p95_s": 1.683034674823284, "cost_usd_per_1000": 0.005946388151174668, "axes": { "intelligence": 55.510719052385724, "calibration": 73.9809636764092, "speed": 79.32853766997002, "cost": 76.77240236749748 }, "jevbench_score": 70.71846967816448 }, "heldout": { "n": 303, "escalation": 0.0627062706270627, "tiers": { "easy": 0.875, "standard": 0.4583333333333333, "judge": 0.6164383561643836, "hard": 0.43119266055045874 }, "accuracy": 0.5577557755775577, "ece_hard": 0.11447138690478628, "calibration": 74.73887965620126, "p50_s": 1.2787838667631148, "p95_s": 2.1685877382755274, "cost_usd_per_1000": 0.006375794229984796, "axes": { "intelligence": 55.27938712244984, "calibration": 74.73887965620126, "speed": 75.5702585362144, "cost": 75.86397122933789 }, "jevbench_score": 69.76282695550769 } }, { "threshold": 0.33, "public": { "n": 231, "escalation": 0.05627705627705628, "tiers": { "easy": 0.875, "standard": 0.5555555555555556, "hard": 0.4144144144144144 }, "accuracy": 0.5541125541125541, "ece_hard": 0.11514955734457474, "calibration": 74.87657983446333, "p50_s": 0.6935107052326203, "p95_s": 1.6896898761391639, "cost_usd_per_1000": 0.006119177051413006, "axes": { "intelligence": 55.8860944277611, "calibration": 74.87657983446333, "speed": 79.3113982865718, "cost": 76.39920942417274 }, "jevbench_score": 70.96080891139748 }, "heldout": { "n": 303, "escalation": 0.10561056105610561, "tiers": { "easy": 0.875, "standard": 0.4583333333333333, "judge": 0.6712328767123288, "hard": 0.44036697247706424 }, "accuracy": 0.5874587458745875, "ece_hard": 0.12007683905953936, "calibration": 74.17833444072596, "p50_s": 1.3324144572019576, "p95_s": 2.2292815722525114, "cost_usd_per_1000": 0.008088286201653873, "axes": { "intelligence": 57.088863055590465, "calibration": 74.17833444072596, "speed": 75.27195737632299, "cost": 72.76430469283852 }, "jevbench_score": 69.39769026573775 } }, { "threshold": 0.34, "public": { "n": 231, "escalation": 0.06926406926406926, "tiers": { "easy": 0.875, "standard": 0.5694444444444444, "hard": 0.42342342342342343 }, "accuracy": 0.5627705627705628, "ece_hard": 0.11160922151924879, "calibration": 75.23061341699594, "p50_s": 0.6951739430427551, "p95_s": 1.7372671008110045, "cost_usd_per_1000": 0.006637543752128021, "axes": { "intelligence": 56.801593259926605, "calibration": 75.23061341699594, "speed": 79.18039919403824, "cost": 75.33977809598025 }, "jevbench_score": 71.05575953040456 }, "heldout": { "n": 303, "escalation": 0.1353135313531353, "tiers": { "easy": 0.875, "standard": 0.4583333333333333, "judge": 0.7123287671232876, "hard": 0.46788990825688076 }, "accuracy": 0.6171617161716172, "ece_hard": 0.10631300236478948, "calibration": 75.55471811020095, "p50_s": 1.366723546385765, "p95_s": 2.2310479655861855, "cost_usd_per_1000": 0.009273857566655542, "axes": { "intelligence": 59.06523606049181, "calibration": 75.55471811020095, "speed": 75.15810416003889, "cost": 70.98218735615195 }, "jevbench_score": 69.85213458870918 } }, { "threshold": 0.35, "public": { "n": 231, "escalation": 0.08225108225108226, "tiers": { "easy": 0.875, "standard": 0.5694444444444444, "hard": 0.44144144144144143 }, "accuracy": 0.5714285714285714, "ece_hard": 0.10003086103888648, "calibration": 76.38844946503215, "p50_s": 0.6951739430427551, "p95_s": 1.897957244515419, "cost_usd_per_1000": 0.007155910452843037, "axes": { "intelligence": 57.552344010677345, "calibration": 76.38844946503215, "speed": 78.79620089956148, "cost": 74.36005308148314 }, "jevbench_score": 71.24164193766039 }, "heldout": { "n": 303, "escalation": 0.16501650165016502, "tiers": { "easy": 0.875, "standard": 0.4583333333333333, "judge": 0.7602739726027398, "hard": 0.46788990825688076 }, "accuracy": 0.6402640264026402, "ece_hard": 0.11010938908827053, "calibration": 76.94371270836044, "p50_s": 1.366723546385765, "p95_s": 2.313778032362461, "cost_usd_per_1000": 0.01045942893165721, "axes": { "intelligence": 60.40770181391647, "calibration": 76.94371270836044, "speed": 74.9999762989361, "cost": 69.4147607984805 }, "jevbench_score": 70.13658279636418 } }, { "threshold": 0.36, "public": { "n": 231, "escalation": 0.1038961038961039, "tiers": { "easy": 0.875, "standard": 0.5833333333333334, "hard": 0.4594594594594595 }, "accuracy": 0.5844155844155844, "ece_hard": 0.09847105380245193, "calibration": 76.54443018867562, "p50_s": 0.7169283866882324, "p95_s": 1.9970385447144507, "cost_usd_per_1000": 0.00801985495403473, "axes": { "intelligence": 58.84321821821823, "calibration": 76.54443018867562, "speed": 78.441377761969, "cost": 72.87500458700617 }, "jevbench_score": 71.23336407670443 }, "heldout": { "n": 303, "escalation": 0.1782178217821782, "tiers": { "easy": 0.875, "standard": 0.4583333333333333, "judge": 0.773972602739726, "hard": 0.47706422018348627 }, "accuracy": 0.6501650165016502, "ece_hard": 0.1077036602758031, "calibration": 77.18428558960719, "p50_s": 1.3695352911949157, "p95_s": 2.313778032362461, "cost_usd_per_1000": 0.010986349538324617, "axes": { "intelligence": 61.066492815550255, "calibration": 77.18428558960719, "speed": 74.99105078598262, "cost": 68.77439763144767 }, "jevbench_score": 70.21696036957563 } }, { "threshold": 0.37, "public": { "n": 231, "escalation": 0.12554112554112554, "tiers": { "easy": 0.875, "standard": 0.5833333333333334, "hard": 0.4594594594594595 }, "accuracy": 0.5844155844155844, "ece_hard": 0.1167499777665573, "calibration": 74.99520678309995, "p50_s": 0.7518308401107788, "p95_s": 2.05018679946661, "cost_usd_per_1000": 0.00888379945522642, "axes": { "intelligence": 58.84321821821823, "calibration": 74.99520678309995, "speed": 78.12086430591104, "cost": 71.54203761606031 }, "jevbench_score": 70.47167062745172 }, "heldout": { "n": 303, "escalation": 0.2079207920792079, "tiers": { "easy": 0.875, "standard": 0.5, "judge": 0.815068493150685, "hard": 0.47706422018348627 }, "accuracy": 0.6732673267326733, "ece_hard": 0.10818323588812019, "calibration": 77.13632802837547, "p50_s": 1.3695352911949157, "p95_s": 2.313778032362461, "cost_usd_per_1000": 0.012171920903326285, "axes": { "intelligence": 63.383844413723764, "calibration": 77.13632802837547, "speed": 74.99105078598262, "cost": 67.43922635428765 }, "jevbench_score": 70.5163604634153 } }, { "threshold": 0.38, "public": { "n": 231, "escalation": 0.12987012987012986, "tiers": { "easy": 0.875, "standard": 0.5833333333333334, "hard": 0.45045045045045046 }, "accuracy": 0.5800865800865801, "ece_hard": 0.12657471109806545, "calibration": 74.01273344994912, "p50_s": 0.7518308401107788, "p95_s": 2.0756869927048682, "cost_usd_per_1000": 0.00905658835546476, "axes": { "intelligence": 58.46784284284285, "calibration": 74.01273344994912, "speed": 78.06717999666785, "cost": 71.29106114728984 }, "jevbench_score": 70.05381182171558 }, "heldout": { "n": 303, "escalation": 0.22772277227722773, "tiers": { "easy": 0.875, "standard": 0.5, "judge": 0.8561643835616438, "hard": 0.47706422018348627 }, "accuracy": 0.693069306930693, "ece_hard": 0.10818323588812019, "calibration": 77.13632802837547, "p50_s": 1.3695352911949157, "p95_s": 2.313778032362461, "cost_usd_per_1000": 0.012962301813327397, "axes": { "intelligence": 64.53452934523062, "calibration": 77.13632802837547, "speed": 74.99105078598262, "cost": 66.61953612048544 }, "jevbench_score": 70.6180202090003 } }, { "threshold": 0.39, "public": { "n": 231, "escalation": 0.1645021645021645, "tiers": { "easy": 0.875, "standard": 0.5833333333333334, "hard": 0.44144144144144143 }, "accuracy": 0.5757575757575758, "ece_hard": 0.13936793703650008, "calibration": 72.31584837593715, "p50_s": 0.7781706541776657, "p95_s": 2.084478536248207, "cost_usd_per_1000": 0.010438899557371466, "axes": { "intelligence": 58.09246746746747, "calibration": 72.31584837593715, "speed": 77.89927723449536, "cost": 69.44035843469862 }, "jevbench_score": 69.04379791900303 }, "heldout": { "n": 303, "escalation": 0.24422442244224424, "tiers": { "easy": 0.875, "standard": 0.5, "judge": 0.8904109589041096, "hard": 0.47706422018348627 }, "accuracy": 0.7095709570957096, "ece_hard": 0.10818323588812019, "calibration": 77.13632802837547, "p50_s": 1.3831777215003966, "p95_s": 2.313778032362461, "cost_usd_per_1000": 0.013620952571661658, "axes": { "intelligence": 65.49343345481967, "calibration": 77.13632802837547, "speed": 74.94800321303805, "cost": 65.97377557853369 }, "jevbench_score": 70.69635638579517 } }, { "threshold": 0.4, "public": { "n": 231, "escalation": 0.18614718614718614, "tiers": { "easy": 0.875, "standard": 0.5833333333333334, "hard": 0.46846846846846846 }, "accuracy": 0.5887445887445888, "ece_hard": 0.12868502702049311, "calibration": 73.43859744996794, "p50_s": 0.8036727398633957, "p95_s": 2.093044375628233, "cost_usd_per_1000": 0.011302844058563157, "axes": { "intelligence": 59.2185935935936, "calibration": 73.43859744996794, "speed": 77.74142326331935, "cost": 68.40436792506435 }, "jevbench_score": 69.34732225964737 }, "heldout": { "n": 303, "escalation": 0.2739273927392739, "tiers": { "easy": 0.875, "standard": 0.5, "judge": 0.9383561643835616, "hard": 0.4954128440366973 }, "accuracy": 0.7392739273927392, "ece_hard": 0.11884060672817376, "calibration": 76.07059094437011, "p50_s": 1.3944507777690887, "p95_s": 2.3205819219350814, "cost_usd_per_1000": 0.014806523936663325, "axes": { "intelligence": 67.38635792384065, "calibration": 76.07059094437011, "speed": 74.89999906506361, "cost": 64.88640660827443 }, "jevbench_score": 70.64900953108574 } }, { "threshold": 0.41, "public": { "n": 231, "escalation": 0.21645021645021645, "tiers": { "easy": 0.875, "standard": 0.5833333333333334, "hard": 0.4774774774774775 }, "accuracy": 0.5930735930735931, "ece_hard": 0.14476149874498234, "calibration": 72.83238380871973, "p50_s": 0.8354266524314881, "p95_s": 2.093044375628233, "cost_usd_per_1000": 0.012512366360231526, "axes": { "intelligence": 59.59396896896897, "calibration": 72.83238380871973, "speed": 77.57313237529257, "cost": 67.07981644060102 }, "jevbench_score": 68.93781389758355 }, "heldout": { "n": 303, "escalation": 0.29042904290429045, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "judge": 0.9452054794520548, "hard": 0.5137614678899083 }, "accuracy": 0.7524752475247525, "ece_hard": 0.11529743544343218, "calibration": 76.29266154215988, "p50_s": 1.3992734491825103, "p95_s": 2.3233821764588356, "cost_usd_per_1000": 0.015465174694997589, "axes": { "intelligence": 69.29526412802144, "calibration": 76.29266154215988, "speed": 74.87976753007163, "cost": 64.31935509495509 }, "jevbench_score": 71.03512062528264 } }, { "threshold": 0.42, "public": { "n": 231, "escalation": 0.22943722943722944, "tiers": { "easy": 0.875, "standard": 0.5972222222222222, "hard": 0.4864864864864865 }, "accuracy": 0.6017316017316018, "ece_hard": 0.14041069668674439, "calibration": 73.26746401454352, "p50_s": 0.8431772083044052, "p95_s": 2.0982027254998683, "cost_usd_per_1000": 0.013030733060946542, "axes": { "intelligence": 60.509467801134484, "calibration": 73.26746401454352, "speed": 77.52233694441585, "cost": 66.55093455386694 }, "jevbench_score": 69.15584255894613 }, "heldout": { "n": 303, "escalation": 0.3102310231023102, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "judge": 0.952054794520548, "hard": 0.5137614678899083 }, "accuracy": 0.7557755775577558, "ece_hard": 0.12577164559352327, "calibration": 75.24524052715077, "p50_s": 1.4169122666120528, "p95_s": 2.342274042963981, "cost_usd_per_1000": 0.0162555556049987, "axes": { "intelligence": 69.48704494993927, "calibration": 75.24524052715077, "speed": 74.79019334287837, "cost": 63.66994545979172 }, "jevbench_score": 70.63834294630331 } }, { "threshold": 0.43, "public": { "n": 231, "escalation": 0.24242424242424243, "tiers": { "easy": 0.875, "standard": 0.6111111111111112, "hard": 0.4864864864864865 }, "accuracy": 0.6060606060606061, "ece_hard": 0.13983752983737438, "calibration": 73.32478069948051, "p50_s": 0.8988664388656616, "p95_s": 2.1083709128201007, "cost_usd_per_1000": 0.013549099761661559, "axes": { "intelligence": 61.049591257924604, "calibration": 73.32478069948051, "speed": 77.22357818561146, "cost": 66.0426867854533 }, "jevbench_score": 69.12371117242004 }, "heldout": { "n": 303, "escalation": 0.3234323432343234, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "judge": 0.9452054794520548, "hard": 0.5229357798165137 }, "accuracy": 0.7557755775577558, "ece_hard": 0.12601595966455229, "calibration": 75.68955705123345, "p50_s": 1.4169122666120528, "p95_s": 2.3799273982644076, "cost_usd_per_1000": 0.016782476211666106, "axes": { "intelligence": 69.57049348581961, "calibration": 75.68955705123345, "speed": 74.72093331145373, "cost": 63.254318792009656 }, "jevbench_score": 70.63149189720548 } }, { "threshold": 0.44, "public": { "n": 231, "escalation": 0.2597402597402597, "tiers": { "easy": 0.875, "standard": 0.6111111111111112, "hard": 0.5045045045045045 }, "accuracy": 0.6147186147186147, "ece_hard": 0.137765361895848, "calibration": 72.83876040422787, "p50_s": 0.935780817270279, "p95_s": 2.140915212780237, "cost_usd_per_1000": 0.01424025536261491, "axes": { "intelligence": 61.80034200867535, "calibration": 72.83876040422787, "speed": 76.98226393570712, "cost": 65.39446668000545 }, "jevbench_score": 68.99558105720222 }, "heldout": { "n": 303, "escalation": 0.33003300330033003, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "judge": 0.9452054794520548, "hard": 0.5321100917431193 }, "accuracy": 0.759075907590759, "ece_hard": 0.11318490062205647, "calibration": 76.97266295548303, "p50_s": 1.4169122666120528, "p95_s": 2.3799273982644076, "cost_usd_per_1000": 0.01704593651499981, "axes": { "intelligence": 69.84572284361778, "calibration": 76.97266295548303, "speed": 74.72093331145373, "cost": 63.05137400062628 }, "jevbench_score": 70.9419779110635 } }, { "threshold": 0.45, "public": { "n": 231, "escalation": 0.27705627705627706, "tiers": { "easy": 0.875, "standard": 0.6111111111111112, "hard": 0.5135135135135135 }, "accuracy": 0.6190476190476191, "ece_hard": 0.13986956326078082, "calibration": 72.62834026773459, "p50_s": 0.9514630824327469, "p95_s": 2.1748134322464465, "cost_usd_per_1000": 0.014931410963568266, "axes": { "intelligence": 62.17571738405072, "calibration": 72.62834026773459, "speed": 76.84186051380722, "cost": 64.77697453300775 }, "jevbench_score": 68.85513973951512 }, "heldout": { "n": 303, "escalation": 0.35313531353135313, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "judge": 0.9452054794520548, "hard": 0.5412844036697247 }, "accuracy": 0.7623762376237624, "ece_hard": 0.12419895130641571, "calibration": 76.90218395931856, "p50_s": 1.4429965257644652, "p95_s": 2.410200057923793, "cost_usd_per_1000": 0.017968047576667777, "axes": { "intelligence": 70.12095220141595, "calibration": 76.90218395931856, "speed": 74.58681621983229, "cost": 62.36497333405311 }, "jevbench_score": 70.76969472758492 } }, { "threshold": 0.46, "public": { "n": 231, "escalation": 0.30735930735930733, "tiers": { "easy": 0.875, "standard": 0.6111111111111112, "hard": 0.5405405405405406 }, "accuracy": 0.6320346320346321, "ece_hard": 0.1353703532112359, "calibration": 73.07826127268908, "p50_s": 0.993661054968834, "p95_s": 2.209390891343355, "cost_usd_per_1000": 0.016140933265236634, "axes": { "intelligence": 63.30184351017685, "calibration": 73.07826127268908, "speed": 76.58489172146759, "cost": 63.76214074227594 }, "jevbench_score": 68.94100958329923 }, "heldout": { "n": 303, "escalation": 0.3795379537953795, "tiers": { "easy": 0.875, "standard": 0.5416666666666666, "judge": 0.9383561643835616, "hard": 0.5321100917431193 }, "accuracy": 0.7557755775577558, "ece_hard": 0.14302121592128636, "calibration": 75.01995749783148, "p50_s": 1.4721267461776733, "p95_s": 2.411423055827617, "cost_usd_per_1000": 0.019021888790002592, "axes": { "intelligence": 69.65394202169996, "calibration": 75.01995749783148, "speed": 74.49781387871587, "cost": 61.62239085169222 }, "jevbench_score": 69.98438438346126 } }, { "threshold": 0.47, "public": { "n": 231, "escalation": 0.3203463203463203, "tiers": { "easy": 0.875, "standard": 0.6111111111111112, "hard": 0.5585585585585585 }, "accuracy": 0.6406926406926406, "ece_hard": 0.13123599369290917, "calibration": 73.49169722452174, "p50_s": 1.0137497425079345, "p95_s": 2.212811952084303, "cost_usd_per_1000": 0.016659299965951647, "axes": { "intelligence": 64.05259426092759, "calibration": 73.49169722452174, "speed": 76.49124734560525, "cost": 63.35029755602549 }, "jevbench_score": 69.10887939002042 }, "heldout": { "n": 303, "escalation": 0.38943894389438943, "tiers": { "easy": 0.875, "standard": 0.5833333333333334, "judge": 0.9383561643835616, "hard": 0.5412844036697247 }, "accuracy": 0.7623762376237624, "ece_hard": 0.13712652149220106, "calibration": 75.60942694074002, "p50_s": 1.4721267461776733, "p95_s": 2.411423055827617, "cost_usd_per_1000": 0.01941707924500315, "axes": { "intelligence": 71.0958380461648, "calibration": 75.60942694074002, "speed": 74.49781387871587, "cost": 61.3544829081896 }, "jevbench_score": 70.40483553344802 } }, { "threshold": 0.48, "public": { "n": 231, "escalation": 0.354978354978355, "tiers": { "easy": 0.875, "standard": 0.6111111111111112, "hard": 0.5945945945945946 }, "accuracy": 0.658008658008658, "ece_hard": 0.11133859697046719, "calibration": 75.48143689676594, "p50_s": 1.099877917766571, "p95_s": 2.2162962310016154, "cost_usd_per_1000": 0.018041611167858358, "axes": { "intelligence": 65.5540957624291, "calibration": 75.48143689676594, "speed": 76.13027709287087, "cost": 62.31174043941132 }, "jevbench_score": 69.60522162587806 }, "heldout": { "n": 303, "escalation": 0.41254125412541254, "tiers": { "easy": 0.875, "standard": 0.625, "judge": 0.9315068493150684, "hard": 0.5596330275229358 }, "accuracy": 0.768976897689769, "ece_hard": 0.13342913520446537, "calibration": 75.97916556951358, "p50_s": 1.4737490206956863, "p95_s": 2.4158167533576487, "cost_usd_per_1000": 0.020339190306671116, "axes": { "intelligence": 72.62118260651, "calibration": 75.97916556951358, "speed": 74.48512481770811, "cost": 60.749990203672695 }, "jevbench_score": 70.68762653437477 } }, { "threshold": 0.49, "public": { "n": 231, "escalation": 0.36363636363636365, "tiers": { "easy": 0.875, "standard": 0.6111111111111112, "hard": 0.6036036036036037 }, "accuracy": 0.6623376623376623, "ece_hard": 0.1021309721627022, "calibration": 76.40219937754244, "p50_s": 1.1170677185058593, "p95_s": 2.220161111652851, "cost_usd_per_1000": 0.018387188968335032, "axes": { "intelligence": 65.92947113780448, "calibration": 76.40219937754244, "speed": 76.05536007150619, "cost": 62.06453981756384 }, "jevbench_score": 69.82962419407043 }, "heldout": { "n": 303, "escalation": 0.42244224422442245, "tiers": { "easy": 0.875, "standard": 0.625, "judge": 0.9246575342465754, "hard": 0.5688073394495413 }, "accuracy": 0.768976897689769, "ece_hard": 0.1207794204145117, "calibration": 77.24413704850895, "p50_s": 1.4737490206956863, "p95_s": 2.4385307766497135, "cost_usd_per_1000": 0.02073438076167167, "axes": { "intelligence": 72.70463114239037, "calibration": 77.24413704850895, "speed": 74.44448229300457, "cost": 60.49926791312931 }, "jevbench_score": 70.91735978060184 } }, { "threshold": 0.5, "public": { "n": 231, "escalation": 0.36363636363636365, "tiers": { "easy": 0.875, "standard": 0.6111111111111112, "hard": 0.6036036036036037 }, "accuracy": 0.6623376623376623, "ece_hard": 0.1021309721627022, "calibration": 76.40219937754244, "p50_s": 1.1170677185058593, "p95_s": 2.220161111652851, "cost_usd_per_1000": 0.018387188968335032, "axes": { "intelligence": 65.92947113780448, "calibration": 76.40219937754244, "speed": 76.05536007150619, "cost": 62.06453981756384 }, "jevbench_score": 69.82962419407043 }, "heldout": { "n": 303, "escalation": 0.43234323432343236, "tiers": { "easy": 0.875, "standard": 0.625, "judge": 0.9246575342465754, "hard": 0.5688073394495413 }, "accuracy": 0.768976897689769, "ece_hard": 0.12057722516646169, "calibration": 77.26435657331396, "p50_s": 1.4789225697517394, "p95_s": 2.4385307766497135, "cost_usd_per_1000": 0.02112957121667223, "axes": { "intelligence": 72.70463114239037, "calibration": 77.26435657331396, "speed": 74.42926322080238, "cost": 60.253279480464634 }, "jevbench_score": 70.84617692465 } }, { "threshold": 0.51, "public": { "n": 231, "escalation": 0.38095238095238093, "tiers": { "easy": 0.875, "standard": 0.6111111111111112, "hard": 0.6036036036036037 }, "accuracy": 0.6623376623376623, "ece_hard": 0.10912023247264396, "calibration": 75.70327334654827, "p50_s": 1.119548463821411, "p95_s": 2.2212680965662, "cost_usd_per_1000": 0.019078344569288384, "axes": { "intelligence": 65.92947113780448, "calibration": 75.70327334654827, "speed": 76.04356122620854, "cost": 61.583779353759255 }, "jevbench_score": 69.53136557257945 }, "heldout": { "n": 303, "escalation": 0.45544554455445546, "tiers": { "easy": 0.875, "standard": 0.625, "judge": 0.9246575342465754, "hard": 0.5779816513761468 }, "accuracy": 0.7722772277227723, "ece_hard": 0.11810119241746043, "calibration": 77.54301225667113, "p50_s": 1.4937828123569488, "p95_s": 2.4385307766497135, "cost_usd_per_1000": 0.02205168227834019, "axes": { "intelligence": 72.97986050018852, "calibration": 77.54301225667113, "speed": 74.38584300728186, "cost": 59.696748204481146 }, "jevbench_score": 70.8021857632831 } }, { "threshold": 0.52, "public": { "n": 231, "escalation": 0.4025974025974026, "tiers": { "easy": 0.8958333333333334, "standard": 0.6111111111111112, "hard": 0.6036036036036037 }, "accuracy": 0.6666666666666666, "ece_hard": 0.1160476126237192, "calibration": 74.46368475899726, "p50_s": 1.1292717546224593, "p95_s": 2.225298771262169, "cost_usd_per_1000": 0.019942289070480082, "axes": { "intelligence": 66.33456373039708, "calibration": 74.46368475899726, "speed": 75.99813206183876, "cost": 61.00674978350849 }, "jevbench_score": 69.17772913945407 }, "heldout": { "n": 303, "escalation": 0.46864686468646866, "tiers": { "easy": 0.875, "standard": 0.625, "judge": 0.9246575342465754, "hard": 0.5963302752293578 }, "accuracy": 0.7788778877887789, "ece_hard": 0.10511144336133871, "calibration": 79.17207686775743, "p50_s": 1.4949380844831466, "p95_s": 2.4750396698713297, "cost_usd_per_1000": 0.022578602885007598, "axes": { "intelligence": 73.53031921578486, "calibration": 79.17207686775743, "speed": 74.31794629858655, "cost": 59.38908804341728 }, "jevbench_score": 71.19667491478972 } }, { "threshold": 0.53, "public": { "n": 231, "escalation": 0.42424242424242425, "tiers": { "easy": 0.8958333333333334, "standard": 0.6111111111111112, "hard": 0.6306306306306306 }, "accuracy": 0.6796536796536796, "ece_hard": 0.09717548664362775, "calibration": 76.35089735700642, "p50_s": 1.1295514345169066, "p95_s": 2.225298771262169, "cost_usd_per_1000": 0.020806233571671773, "axes": { "intelligence": 67.46068985652319, "calibration": 76.35089735700642, "speed": 75.99705660417513, "cost": 60.45419591236989 }, "jevbench_score": 69.7464385567765 }, "heldout": { "n": 303, "escalation": 0.4884488448844885, "tiers": { "easy": 0.875, "standard": 0.625, "judge": 0.9246575342465754, "hard": 0.6146788990825688 }, "accuracy": 0.7854785478547854, "ece_hard": 0.10281998418439209, "calibration": 79.40122278545209, "p50_s": 1.501078763604164, "p95_s": 2.4980192244052883, "cost_usd_per_1000": 0.02336898379500871, "axes": { "intelligence": 74.08077793138118, "calibration": 79.40122278545209, "speed": 74.2600074282798, "cost": 58.94080517741904 }, "jevbench_score": 71.23213221245514 } }, { "threshold": 0.54, "public": { "n": 231, "escalation": 0.4458874458874459, "tiers": { "easy": 0.8958333333333334, "standard": 0.6111111111111112, "hard": 0.6216216216216216 }, "accuracy": 0.6753246753246753, "ece_hard": 0.112849529396887, "calibration": 74.78349308168049, "p50_s": 1.1450107634067535, "p95_s": 2.2469268083572387, "cost_usd_per_1000": 0.021670178072863464, "axes": { "intelligence": 67.08531448114782, "calibration": 74.78349308168049, "speed": 75.89601504927901, "cost": 59.92412559661558 }, "jevbench_score": 69.11359305808898 }, "heldout": { "n": 303, "escalation": 0.5082508250825083, "tiers": { "easy": 0.875, "standard": 0.625, "judge": 0.9246575342465754, "hard": 0.6422018348623854 }, "accuracy": 0.7953795379537953, "ece_hard": 0.09685187199139139, "calibration": 79.99803400475216, "p50_s": 1.5230290234088897, "p95_s": 2.4980192244052883, "cost_usd_per_1000": 0.024159364705009827, "axes": { "intelligence": 74.90646600477568, "calibration": 79.99803400475216, "speed": 74.19696044187194, "cost": 58.50743470343468 }, "jevbench_score": 71.41656346144978 } }, { "threshold": 0.55, "public": { "n": 231, "escalation": 0.4675324675324675, "tiers": { "easy": 0.8958333333333334, "standard": 0.6111111111111112, "hard": 0.6216216216216216 }, "accuracy": 0.6753246753246753, "ece_hard": 0.11765353031484617, "calibration": 74.30309298988458, "p50_s": 1.154820704460144, "p95_s": 2.2469268083572387, "cost_usd_per_1000": 0.022534122574055154, "axes": { "intelligence": 67.08531448114782, "calibration": 74.30309298988458, "speed": 75.85896512483224, "cost": 59.41478043020615 }, "jevbench_score": 68.8468287878965 }, "heldout": { "n": 303, "escalation": 0.528052805280528, "tiers": { "easy": 0.875, "standard": 0.7083333333333334, "judge": 0.9246575342465754, "hard": 0.6330275229357798 }, "accuracy": 0.7986798679867987, "ece_hard": 0.10574355595339485, "calibration": 79.26742625009321, "p50_s": 1.5264378130435943, "p95_s": 2.4980192244052883, "cost_usd_per_1000": 0.024949745615010933, "axes": { "intelligence": 76.96456998031084, "calibration": 79.26742625009321, "speed": 74.18725107807599, "cost": 58.08801634116993 }, "jevbench_score": 71.60615551246639 } }, { "threshold": 0.56, "public": { "n": 231, "escalation": 0.48484848484848486, "tiers": { "easy": 0.8958333333333334, "standard": 0.6111111111111112, "hard": 0.6216216216216216 }, "accuracy": 0.6753246753246753, "ece_hard": 0.11660963901007718, "calibration": 74.40748212036148, "p50_s": 1.1590392157435416, "p95_s": 2.313111515343189, "cost_usd_per_1000": 0.02322527817500851, "axes": { "intelligence": 67.08531448114782, "calibration": 74.40748212036148, "speed": 75.71705298907725, "cost": 59.02117226946207 }, "jevbench_score": 68.72447019823016 }, "heldout": { "n": 303, "escalation": 0.5445544554455446, "tiers": { "easy": 0.875, "standard": 0.7083333333333334, "judge": 0.9246575342465754, "hard": 0.6605504587155964 }, "accuracy": 0.8085808580858086, "ece_hard": 0.09397854959984454, "calibration": 80.44392688544823, "p50_s": 1.547201508283615, "p95_s": 2.4980192244052883, "cost_usd_per_1000": 0.025608396373345197, "axes": { "intelligence": 77.79025805370534, "calibration": 80.44392688544823, "speed": 74.12857343570167, "cost": 57.74852850070644 }, "jevbench_score": 71.94262166194797 } }, { "threshold": 0.57, "public": { "n": 231, "escalation": 0.5108225108225108, "tiers": { "easy": 0.8958333333333334, "standard": 0.6111111111111112, "hard": 0.6306306306306306 }, "accuracy": 0.6796536796536796, "ece_hard": 0.11622774534562194, "calibration": 74.44567148680699, "p50_s": 1.162949138879776, "p95_s": 2.313111515343189, "cost_usd_per_1000": 0.02426201157643854, "axes": { "intelligence": 67.46068985652319, "calibration": 74.44567148680699, "speed": 75.70242707896661, "cost": 58.45219583162232 }, "jevbench_score": 68.65943352017122 }, "heldout": { "n": 303, "escalation": 0.5742574257425742, "tiers": { "easy": 0.875, "standard": 0.75, "judge": 0.9315068493150684, "hard": 0.6605504587155964 }, "accuracy": 0.8151815181518152, "ece_hard": 0.09813731462593095, "calibration": 80.0280503828396, "p50_s": 1.5591047048568725, "p95_s": 2.5291963659226884, "cost_usd_per_1000": 0.026793967738346865, "axes": { "intelligence": 79.1487055422898, "calibration": 80.0280503828396, "speed": 74.04142168774572, "cost": 57.15888909648241 }, "jevbench_score": 71.95502783309968 } }, { "threshold": 0.58, "public": { "n": 231, "escalation": 0.5238095238095238, "tiers": { "easy": 0.8958333333333334, "standard": 0.6111111111111112, "hard": 0.6306306306306306 }, "accuracy": 0.6796536796536796, "ece_hard": 0.1176465395050862, "calibration": 74.31620787528934, "p50_s": 1.162949138879776, "p95_s": 2.313111515343189, "cost_usd_per_1000": 0.024780378277153556, "axes": { "intelligence": 67.46068985652319, "calibration": 74.31620787528934, "speed": 75.70242707896661, "cost": 58.17676204966246 }, "jevbench_score": 68.54857280166019 }, "heldout": { "n": 303, "escalation": 0.5841584158415841, "tiers": { "easy": 0.875, "standard": 0.75, "judge": 0.9315068493150684, "hard": 0.6605504587155964 }, "accuracy": 0.8151815181518152, "ece_hard": 0.10072827405076733, "calibration": 79.76895444035596, "p50_s": 1.5591047048568725, "p95_s": 2.544618652015924, "cost_usd_per_1000": 0.02718915819334742, "axes": { "intelligence": 79.1487055422898, "calibration": 79.76895444035596, "speed": 74.01502011672964, "cost": 56.96812715337193 }, "jevbench_score": 71.83025032558727 } }, { "threshold": 0.59, "public": { "n": 231, "escalation": 0.5584415584415584, "tiers": { "easy": 0.9166666666666666, "standard": 0.6111111111111112, "hard": 0.6306306306306306 }, "accuracy": 0.683982683982684, "ece_hard": 0.12618431533955146, "calibration": 73.21218804740104, "p50_s": 1.1785981595516204, "p95_s": 2.3275452978909015, "cost_usd_per_1000": 0.02616268947906026, "axes": { "intelligence": 67.86578244911578, "calibration": 73.21218804740104, "speed": 75.61736099589623, "cost": 57.469528400023144 }, "jevbench_score": 68.166869403084 }, "heldout": { "n": 303, "escalation": 0.6171617161716172, "tiers": { "easy": 0.875, "standard": 0.75, "judge": 0.9383561643835616, "hard": 0.6697247706422018 }, "accuracy": 0.8217821782178217, "ece_hard": 0.09654905351382166, "calibration": 80.18687649405052, "p50_s": 1.5772460341453551, "p95_s": 2.586803795397281, "cost_usd_per_1000": 0.028506459710015943, "axes": { "intelligence": 79.61571572200579, "calibration": 80.18687649405052, "speed": 73.89337066439228, "cost": 56.351701464450244 }, "jevbench_score": 71.80482909438457 } }, { "threshold": 0.6, "public": { "n": 231, "escalation": 0.5670995670995671, "tiers": { "easy": 0.9166666666666666, "standard": 0.625, "hard": 0.6396396396396397 }, "accuracy": 0.6926406926406926, "ece_hard": 0.11723942217933289, "calibration": 74.1066773634229, "p50_s": 1.1785981595516204, "p95_s": 2.3275452978909015, "cost_usd_per_1000": 0.02650826727953694, "axes": { "intelligence": 68.7812812812813, "calibration": 74.1066773634229, "speed": 75.61736099589623, "cost": 57.29855977366087 }, "jevbench_score": 68.55248564724302 }, "heldout": { "n": 303, "escalation": 0.6303630363036303, "tiers": { "easy": 0.875, "standard": 0.75, "judge": 0.9383561643835616, "hard": 0.6697247706422018 }, "accuracy": 0.8217821782178217, "ece_hard": 0.10223743980653034, "calibration": 80.15398149098134, "p50_s": 1.5808078438043593, "p95_s": 2.586803795397281, "cost_usd_per_1000": 0.029033380316683347, "axes": { "intelligence": 79.61571572200579, "calibration": 80.15398149098134, "speed": 73.88357428348064, "cost": 56.11307190739461 }, "jevbench_score": 71.71895639877927 } }, { "threshold": 0.61, "public": { "n": 231, "escalation": 0.5887445887445888, "tiers": { "easy": 0.9166666666666666, "standard": 0.625, "hard": 0.6576576576576577 }, "accuracy": 0.7012987012987013, "ece_hard": 0.11423628406108075, "calibration": 74.40699117524812, "p50_s": 1.1857077792286872, "p95_s": 2.3275452978909015, "cost_usd_per_1000": 0.02737221178072863, "axes": { "intelligence": 69.53203203203203, "calibration": 74.40699117524812, "speed": 75.59124188964805, "cost": 56.88070325487284 }, "jevbench_score": 68.67659831856164 }, "heldout": { "n": 303, "escalation": 0.66996699669967, "tiers": { "easy": 0.875, "standard": 0.75, "judge": 0.9383561643835616, "hard": 0.6697247706422018 }, "accuracy": 0.8217821782178217, "ece_hard": 0.10758683480523762, "calibration": 79.6190419911106, "p50_s": 1.6551591843366622, "p95_s": 2.635781662911176, "cost_usd_per_1000": 0.030614142136685576, "axes": { "intelligence": 79.61571572200579, "calibration": 79.6190419911106, "speed": 73.60250799484653, "cost": 55.42233717310282 }, "jevbench_score": 71.30964922453664 } }, { "threshold": 0.62, "public": { "n": 231, "escalation": 0.6017316017316018, "tiers": { "easy": 0.9375, "standard": 0.625, "hard": 0.6666666666666666 }, "accuracy": 0.70995670995671, "ece_hard": 0.10626158634275225, "calibration": 75.20446094708097, "p50_s": 1.1857077792286872, "p95_s": 2.3275452978909015, "cost_usd_per_1000": 0.027890578481443648, "axes": { "intelligence": 70.3125, "calibration": 75.20446094708097, "speed": 75.59124188964805, "cost": 56.63627433677996 }, "jevbench_score": 68.97799633925646 }, "heldout": { "n": 303, "escalation": 0.7227722772277227, "tiers": { "easy": 0.9166666666666666, "standard": 0.75, "judge": 0.9383561643835616, "hard": 0.6697247706422018 }, "accuracy": 0.8250825082508251, "ece_hard": 0.10758683480523762, "calibration": 79.6190419911106, "p50_s": 1.7229696750640868, "p95_s": 2.6609175033867354, "cost_usd_per_1000": 0.032721824563355206, "axes": { "intelligence": 80.19904905533912, "calibration": 79.6190419911106, "speed": 73.38690955938762, "cost": 54.55487464417976 }, "jevbench_score": 71.10654578133537 } }, { "threshold": 0.63, "public": { "n": 231, "escalation": 0.6190476190476191, "tiers": { "easy": 0.9375, "standard": 0.6527777777777778, "hard": 0.6756756756756757 }, "accuracy": 0.7229437229437229, "ece_hard": 0.10043320699316122, "calibration": 77.3776793006839, "p50_s": 1.1920731678605079, "p95_s": 2.337883038073778, "cost_usd_per_1000": 0.028581734082397, "axes": { "intelligence": 71.76812228895562, "calibration": 77.3776793006839, "speed": 75.54874307306666, "cost": 56.3173427699519 }, "jevbench_score": 69.71948502974693 }, "heldout": { "n": 303, "escalation": 0.768976897689769, "tiers": { "easy": 0.9583333333333334, "standard": 0.75, "judge": 0.9383561643835616, "hard": 0.6697247706422018 }, "accuracy": 0.8283828382838284, "ece_hard": 0.11287859054780455, "calibration": 79.08986641685391, "p50_s": 1.7949005156755446, "p95_s": 2.6637706011533737, "cost_usd_per_1000": 0.03456604668669113, "axes": { "intelligence": 80.78238238867246, "calibration": 79.08986641685391, "speed": 73.20462796169662, "cost": 53.84050863426507 }, "jevbench_score": 70.83881664192374 } }, { "threshold": 0.64, "public": { "n": 231, "escalation": 0.6536796536796536, "tiers": { "easy": 0.9375, "standard": 0.6944444444444444, "hard": 0.6846846846846847 }, "accuracy": 0.7402597402597403, "ece_hard": 0.09209432984834855, "calibration": 78.21156701516514, "p50_s": 1.235545241832733, "p95_s": 2.337883038073778, "cost_usd_per_1000": 0.029964045284303707, "axes": { "intelligence": 73.76386803470137, "calibration": 78.21156701516514, "speed": 75.393185673993, "cost": 55.701986657694064 }, "jevbench_score": 70.1583492861244 }, "heldout": { "n": 303, "escalation": 0.8085808580858086, "tiers": { "easy": 0.9583333333333334, "standard": 0.75, "judge": 0.9383561643835616, "hard": 0.6697247706422018 }, "accuracy": 0.8283828382838284, "ece_hard": 0.10591769230181856, "calibration": 79.78595624145251, "p50_s": 1.8254774525761603, "p95_s": 2.6637706011533737, "cost_usd_per_1000": 0.03614680850669336, "axes": { "intelligence": 80.78238238867246, "calibration": 79.78595624145251, "speed": 73.13126705383063, "cost": 53.2579012491077 }, "jevbench_score": 70.78358657337637 } }, { "threshold": 0.65, "public": { "n": 231, "escalation": 0.6926406926406926, "tiers": { "easy": 0.9583333333333334, "standard": 0.75, "hard": 0.6936936936936937 }, "accuracy": 0.7662337662337663, "ece_hard": 0.09327060227619592, "calibration": 78.09393977238042, "p50_s": 1.2900475278496741, "p95_s": 2.337883038073778, "cost_usd_per_1000": 0.03151914538644875, "axes": { "intelligence": 76.70482982982985, "calibration": 78.09393977238042, "speed": 75.2057150910091, "cost": 55.042766995926094 }, "jevbench_score": 70.56637149446382 }, "heldout": { "n": 303, "escalation": 0.8283828382838284, "tiers": { "easy": 0.9583333333333334, "standard": 0.75, "judge": 0.9383561643835616, "hard": 0.6697247706422018 }, "accuracy": 0.8283828382838284, "ece_hard": 0.10591769230181856, "calibration": 79.78595624145251, "p50_s": 1.840062716603279, "p95_s": 2.6637706011533737, "cost_usd_per_1000": 0.03693718941669447, "axes": { "intelligence": 80.78238238867246, "calibration": 79.78595624145251, "speed": 73.09670552771688, "cost": 52.97608460465762 }, "jevbench_score": 70.6814081324882 } }, { "threshold": 0.66, "public": { "n": 231, "escalation": 0.7056277056277056, "tiers": { "easy": 0.9583333333333334, "standard": 0.75, "hard": 0.7027027027027027 }, "accuracy": 0.7705627705627706, "ece_hard": 0.08715595125544924, "calibration": 78.70540487445508, "p50_s": 1.2999952971935271, "p95_s": 2.337883038073778, "cost_usd_per_1000": 0.032037512087163766, "axes": { "intelligence": 77.08020520520522, "calibration": 78.70540487445508, "speed": 75.17235438719982, "cost": 54.83023650869057 }, "jevbench_score": 70.71416585638556 }, "heldout": { "n": 303, "escalation": 0.8646864686468647, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9452054794520548, "hard": 0.6697247706422018 }, "accuracy": 0.8382838283828383, "ece_hard": 0.10925714520721262, "calibration": 79.08128547927873, "p50_s": 1.8709524363279342, "p95_s": 2.6674607589840886, "cost_usd_per_1000": 0.03838622108502985, "axes": { "intelligence": 82.72416321059026, "calibration": 79.08128547927873, "speed": 73.01839213830807, "cost": 52.474739191334486 }, "jevbench_score": 70.7574485630423 } }, { "threshold": 0.67, "public": { "n": 231, "escalation": 0.7272727272727273, "tiers": { "easy": 0.9791666666666666, "standard": 0.75, "hard": 0.7027027027027027 }, "accuracy": 0.7748917748917749, "ece_hard": 0.08618303437918026, "calibration": 78.80269656208198, "p50_s": 1.2999952971935271, "p95_s": 2.337883038073778, "cost_usd_per_1000": 0.03290145658835546, "axes": { "intelligence": 77.4852977977978, "calibration": 78.80269656208198, "speed": 75.17235438719982, "cost": 54.48354624604584 }, "jevbench_score": 70.71653557745687 }, "heldout": { "n": 303, "escalation": 0.8811881188118812, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9452054794520548, "hard": 0.6788990825688074 }, "accuracy": 0.8415841584158416, "ece_hard": 0.10653994871644337, "calibration": 79.35300512835566, "p50_s": 1.8709524363279342, "p95_s": 2.6674607589840886, "cost_usd_per_1000": 0.03904487184336411, "axes": { "intelligence": 82.99939256838842, "calibration": 79.35300512835566, "speed": 73.01839213830807, "cost": 52.253079949409866 }, "jevbench_score": 70.80201422863415 } }, { "threshold": 0.68, "public": { "n": 231, "escalation": 0.7402597402597403, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.7027027027027027 }, "accuracy": 0.7835497835497836, "ece_hard": 0.08618303437918026, "calibration": 78.80269656208198, "p50_s": 1.3072779864072799, "p95_s": 2.337883038073778, "cost_usd_per_1000": 0.03341982328907048, "axes": { "intelligence": 78.43051384718053, "calibration": 78.80269656208198, "speed": 75.14809271900847, "cost": 54.27987552446776 }, "jevbench_score": 70.85911701380947 }, "heldout": { "n": 303, "escalation": 0.900990099009901, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6880733944954128 }, "accuracy": 0.8481848184818482, "ece_hard": 0.09734076304128228, "calibration": 80.27292369587178, "p50_s": 1.8759940281510352, "p95_s": 2.6674607589840886, "cost_usd_per_1000": 0.03983525275336522, "axes": { "intelligence": 84.44128859285325, "calibration": 80.27292369587178, "speed": 73.00670509206458, "cost": 51.99197268716513 }, "jevbench_score": 71.22061907259572 } }, { "threshold": 0.69, "public": { "n": 231, "escalation": 0.7619047619047619, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.7027027027027027 }, "accuracy": 0.7835497835497836, "ece_hard": 0.08843099021765283, "calibration": 78.57790097823472, "p50_s": 1.3148568361997603, "p95_s": 2.337883038073778, "cost_usd_per_1000": 0.03428376779026216, "axes": { "intelligence": 78.43051384718053, "calibration": 78.57790097823472, "speed": 75.12298751250758, "cost": 53.947343653281926 }, "jevbench_score": 70.69392583002812 }, "heldout": { "n": 303, "escalation": 0.9075907590759076, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6972477064220184 }, "accuracy": 0.8514851485148515, "ece_hard": 0.09247605725152011, "calibration": 80.75939427484799, "p50_s": 1.8783646464347838, "p95_s": 2.6674607589840886, "cost_usd_per_1000": 0.040098713056698924, "axes": { "intelligence": 84.71651795065142, "calibration": 80.75939427484799, "speed": 73.00122055174546, "cost": 51.90608696703771 }, "jevbench_score": 71.35548959436896 } }, { "threshold": 0.7, "public": { "n": 231, "escalation": 0.7965367965367965, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.7027027027027027 }, "accuracy": 0.7965367965367965, "ece_hard": 0.08887587268226801, "calibration": 78.53341273177321, "p50_s": 1.3159398227930068, "p95_s": 2.337883038073778, "cost_usd_per_1000": 0.03566607899216887, "axes": { "intelligence": 80.0508842175509, "calibration": 78.53341273177321, "speed": 75.11941190261479, "cost": 53.43233898701938 }, "jevbench_score": 70.87519098889629 }, "heldout": { "n": 303, "escalation": 0.9141914191419142, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6972477064220184 }, "accuracy": 0.8514851485148515, "ece_hard": 0.09247605725152011, "calibration": 80.75939427484799, "p50_s": 1.8783646464347838, "p95_s": 2.6674607589840886, "cost_usd_per_1000": 0.04036217336003262, "axes": { "intelligence": 84.71651795065142, "calibration": 80.75939427484799, "speed": 73.00122055174546, "cost": 51.8207636975905 }, "jevbench_score": 71.3261479477718 } }, { "threshold": 0.71, "public": { "n": 231, "escalation": 0.8051948051948052, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.7027027027027027 }, "accuracy": 0.8008658008658008, "ece_hard": 0.08887587268226801, "calibration": 78.53341273177321, "p50_s": 1.3159398227930068, "p95_s": 2.337883038073778, "cost_usd_per_1000": 0.036011656792645556, "axes": { "intelligence": 80.59100767434103, "calibration": 78.53341273177321, "speed": 75.11941190261479, "cost": 53.30670692591103 }, "jevbench_score": 70.95267483938372 }, "heldout": { "n": 303, "escalation": 0.9174917491749175, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6972477064220184 }, "accuracy": 0.8514851485148515, "ece_hard": 0.09415119034291637, "calibration": 80.59188096570836, "p50_s": 1.8783646464347838, "p95_s": 2.6674607589840886, "cost_usd_per_1000": 0.04049390351169948, "axes": { "intelligence": 84.71651795065142, "calibration": 80.59188096570836, "speed": 73.00122055174546, "cost": 51.77831068913449 }, "jevbench_score": 71.27452749270998 } }, { "threshold": 0.72, "public": { "n": 231, "escalation": 0.8225108225108225, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.7027027027027027 }, "accuracy": 0.7965367965367965, "ece_hard": 0.09027889123004468, "calibration": 78.39311087699554, "p50_s": 1.318672539293766, "p95_s": 2.337883038073778, "cost_usd_per_1000": 0.036702812393598905, "axes": { "intelligence": 80.0508842175509, "calibration": 78.39311087699554, "speed": 75.11040257631257, "cost": 53.059019685333304 }, "jevbench_score": 70.71732687636384 }, "heldout": { "n": 303, "escalation": 0.9207920792079208, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6972477064220184 }, "accuracy": 0.8514851485148515, "ece_hard": 0.0917804434951582, "calibration": 80.82895565048418, "p50_s": 1.8784535765647887, "p95_s": 2.6674607589840886, "cost_usd_per_1000": 0.040625633663366335, "axes": { "intelligence": 84.71651795065142, "calibration": 80.82895565048418, "speed": 73.00101494230444, "cost": 51.7359955599437 }, "jevbench_score": 71.31225888068501 } }, { "threshold": 0.73, "public": { "n": 231, "escalation": 0.8225108225108225, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.7027027027027027 }, "accuracy": 0.7965367965367965, "ece_hard": 0.09027889123004468, "calibration": 78.39311087699554, "p50_s": 1.318672539293766, "p95_s": 2.337883038073778, "cost_usd_per_1000": 0.036702812393598905, "axes": { "intelligence": 80.0508842175509, "calibration": 78.39311087699554, "speed": 75.11040257631257, "cost": 53.059019685333304 }, "jevbench_score": 70.71732687636384 }, "heldout": { "n": 303, "escalation": 0.9273927392739274, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6972477064220184 }, "accuracy": 0.8514851485148515, "ece_hard": 0.09396684788836207, "calibration": 80.6103152111638, "p50_s": 1.8784535765647887, "p95_s": 2.6674607589840886, "cost_usd_per_1000": 0.040889093966700034, "axes": { "intelligence": 84.71651795065142, "calibration": 80.6103152111638, "speed": 73.00101494230444, "cost": 51.65177537714985 }, "jevbench_score": 71.23496529571261 } }, { "threshold": 0.74, "public": { "n": 231, "escalation": 0.8268398268398268, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.7027027027027027 }, "accuracy": 0.7965367965367965, "ece_hard": 0.09027889123004468, "calibration": 78.39311087699554, "p50_s": 1.318672539293766, "p95_s": 2.337883038073778, "cost_usd_per_1000": 0.03687560129383724, "axes": { "intelligence": 80.0508842175509, "calibration": 78.39311087699554, "speed": 75.11040257631257, "cost": 52.997826680406824 }, "jevbench_score": 70.69692846640578 }, "heldout": { "n": 303, "escalation": 0.9273927392739274, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6972477064220184 }, "accuracy": 0.8514851485148515, "ece_hard": 0.09396684788836207, "calibration": 80.6103152111638, "p50_s": 1.8784535765647887, "p95_s": 2.6674607589840886, "cost_usd_per_1000": 0.040889093966700034, "axes": { "intelligence": 84.71651795065142, "calibration": 80.6103152111638, "speed": 73.00101494230444, "cost": 51.65177537714985 }, "jevbench_score": 71.23496529571261 } }, { "threshold": 0.75, "public": { "n": 231, "escalation": 0.8311688311688312, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.7117117117117117 }, "accuracy": 0.8008658008658008, "ece_hard": 0.09203203203203206, "calibration": 78.2177967967968, "p50_s": 1.318672539293766, "p95_s": 2.3501844495534896, "cost_usd_per_1000": 0.03704839019407558, "axes": { "intelligence": 80.42625959292629, "calibration": 78.2177967967968, "speed": 75.08761089338216, "cost": 52.936919739866546 }, "jevbench_score": 70.71435768361991 }, "heldout": { "n": 303, "escalation": 0.933993399339934, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6972477064220184 }, "accuracy": 0.8514851485148515, "ece_hard": 0.08432949509502849, "calibration": 81.57405049049714, "p50_s": 1.8784535765647887, "p95_s": 2.6674607589840886, "cost_usd_per_1000": 0.04115255427003374, "axes": { "intelligence": 84.71651795065142, "calibration": 81.57405049049714, "speed": 73.00101494230444, "cost": 51.56809611054731 }, "jevbench_score": 71.41797468271136 } }, { "threshold": 0.76, "public": { "n": 231, "escalation": 0.8311688311688312, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.7117117117117117 }, "accuracy": 0.8008658008658008, "ece_hard": 0.09203203203203206, "calibration": 78.2177967967968, "p50_s": 1.318672539293766, "p95_s": 2.3501844495534896, "cost_usd_per_1000": 0.03704839019407558, "axes": { "intelligence": 80.42625959292629, "calibration": 78.2177967967968, "speed": 75.08761089338216, "cost": 52.936919739866546 }, "jevbench_score": 70.71435768361991 }, "heldout": { "n": 303, "escalation": 0.933993399339934, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6972477064220184 }, "accuracy": 0.8514851485148515, "ece_hard": 0.08432949509502849, "calibration": 81.57405049049714, "p50_s": 1.8784535765647887, "p95_s": 2.6674607589840886, "cost_usd_per_1000": 0.04115255427003374, "axes": { "intelligence": 84.71651795065142, "calibration": 81.57405049049714, "speed": 73.00101494230444, "cost": 51.56809611054731 }, "jevbench_score": 71.41797468271136 } }, { "threshold": 0.77, "public": { "n": 231, "escalation": 0.8441558441558441, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.7117117117117117 }, "accuracy": 0.8008658008658008, "ece_hard": 0.09203203203203206, "calibration": 78.2177967967968, "p50_s": 1.318672539293766, "p95_s": 2.3501844495534896, "cost_usd_per_1000": 0.037566756894790596, "axes": { "intelligence": 80.42625959292629, "calibration": 78.2177967967968, "speed": 75.08761089338216, "cost": 52.75588886741489 }, "jevbench_score": 70.65382369599216 }, "heldout": { "n": 303, "escalation": 0.9372937293729373, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6880733944954128 }, "accuracy": 0.8481848184818482, "ece_hard": 0.09354813597584669, "calibration": 80.65218640241534, "p50_s": 1.8849402159452437, "p95_s": 2.6674607589840886, "cost_usd_per_1000": 0.04128428442170059, "axes": { "intelligence": 84.44128859285325, "calibration": 80.65218640241534, "speed": 72.98604380383695, "cost": 51.526457158177536 }, "jevbench_score": 71.13941274685207 } }, { "threshold": 0.78, "public": { "n": 231, "escalation": 0.8441558441558441, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.7117117117117117 }, "accuracy": 0.8008658008658008, "ece_hard": 0.09203203203203206, "calibration": 78.2177967967968, "p50_s": 1.318672539293766, "p95_s": 2.3501844495534896, "cost_usd_per_1000": 0.037566756894790596, "axes": { "intelligence": 80.42625959292629, "calibration": 78.2177967967968, "speed": 75.08761089338216, "cost": 52.75588886741489 }, "jevbench_score": 70.65382369599216 }, "heldout": { "n": 303, "escalation": 0.9405940594059405, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6880733944954128 }, "accuracy": 0.8481848184818482, "ece_hard": 0.09354813597584669, "calibration": 80.65218640241534, "p50_s": 1.8849402159452437, "p95_s": 2.6674607589840886, "cost_usd_per_1000": 0.04141601457336744, "axes": { "intelligence": 84.44128859285325, "calibration": 80.65218640241534, "speed": 72.98604380383695, "cost": 51.48495085623033 }, "jevbench_score": 71.12508211713448 } }, { "threshold": 0.79, "public": { "n": 231, "escalation": 0.8441558441558441, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.7117117117117117 }, "accuracy": 0.8008658008658008, "ece_hard": 0.09203203203203206, "calibration": 78.2177967967968, "p50_s": 1.318672539293766, "p95_s": 2.3501844495534896, "cost_usd_per_1000": 0.037566756894790596, "axes": { "intelligence": 80.42625959292629, "calibration": 78.2177967967968, "speed": 75.08761089338216, "cost": 52.75588886741489 }, "jevbench_score": 70.65382369599216 }, "heldout": { "n": 303, "escalation": 0.9438943894389439, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6880733944954128 }, "accuracy": 0.8481848184818482, "ece_hard": 0.09354813597584669, "calibration": 80.65218640241534, "p50_s": 1.8849402159452437, "p95_s": 2.6674607589840886, "cost_usd_per_1000": 0.041547744725034294, "axes": { "intelligence": 84.44128859285325, "calibration": 80.65218640241534, "speed": 72.98604380383695, "cost": 51.44357636221229 }, "jevbench_score": 71.11078836974299 } }, { "threshold": 0.8, "public": { "n": 231, "escalation": 0.8484848484848485, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.7117117117117117 }, "accuracy": 0.8008658008658008, "ece_hard": 0.09203203203203206, "calibration": 78.2177967967968, "p50_s": 1.318672539293766, "p95_s": 2.3501844495534896, "cost_usd_per_1000": 0.03773954579502894, "axes": { "intelligence": 80.42625959292629, "calibration": 78.2177967967968, "speed": 75.08761089338216, "cost": 52.696099929504335 }, "jevbench_score": 70.63379695746458 }, "heldout": { "n": 303, "escalation": 0.9438943894389439, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6880733944954128 }, "accuracy": 0.8481848184818482, "ece_hard": 0.09354813597584669, "calibration": 80.65218640241534, "p50_s": 1.8849402159452437, "p95_s": 2.6674607589840886, "cost_usd_per_1000": 0.041547744725034294, "axes": { "intelligence": 84.44128859285325, "calibration": 80.65218640241534, "speed": 72.98604380383695, "cost": 51.44357636221229 }, "jevbench_score": 71.11078836974299 } }, { "threshold": 0.81, "public": { "n": 231, "escalation": 0.8484848484848485, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.7117117117117117 }, "accuracy": 0.8008658008658008, "ece_hard": 0.09203203203203206, "calibration": 78.2177967967968, "p50_s": 1.318672539293766, "p95_s": 2.3501844495534896, "cost_usd_per_1000": 0.03773954579502894, "axes": { "intelligence": 80.42625959292629, "calibration": 78.2177967967968, "speed": 75.08761089338216, "cost": 52.696099929504335 }, "jevbench_score": 70.63379695746458 }, "heldout": { "n": 303, "escalation": 0.9471947194719472, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6880733944954128 }, "accuracy": 0.8481848184818482, "ece_hard": 0.09354813597584669, "calibration": 80.65218640241534, "p50_s": 1.8849402159452437, "p95_s": 2.6674607589840886, "cost_usd_per_1000": 0.04167947487670115, "axes": { "intelligence": 84.44128859285325, "calibration": 80.65218640241534, "speed": 72.98604380383695, "cost": 51.40233284163096 }, "jevbench_score": 71.09653128636647 } }, { "threshold": 0.82, "public": { "n": 231, "escalation": 0.8484848484848485, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.7117117117117117 }, "accuracy": 0.8008658008658008, "ece_hard": 0.09203203203203206, "calibration": 78.2177967967968, "p50_s": 1.318672539293766, "p95_s": 2.3501844495534896, "cost_usd_per_1000": 0.03773954579502894, "axes": { "intelligence": 80.42625959292629, "calibration": 78.2177967967968, "speed": 75.08761089338216, "cost": 52.696099929504335 }, "jevbench_score": 70.63379695746458 }, "heldout": { "n": 303, "escalation": 0.9504950495049505, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6972477064220184 }, "accuracy": 0.8514851485148515, "ece_hard": 0.08560682566419815, "calibration": 81.44631743358019, "p50_s": 1.8944206550717353, "p95_s": 2.6674607589840886, "cost_usd_per_1000": 0.041811205028368, "axes": { "intelligence": 84.71651795065142, "calibration": 81.44631743358019, "speed": 72.96425540534744, "cost": 51.36121946789377 }, "jevbench_score": 71.30931387482124 } }, { "threshold": 0.83, "public": { "n": 231, "escalation": 0.8528138528138528, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.7117117117117117 }, "accuracy": 0.8008658008658008, "ece_hard": 0.09203203203203206, "calibration": 78.2177967967968, "p50_s": 1.318672539293766, "p95_s": 2.3501844495534896, "cost_usd_per_1000": 0.03791233469526727, "axes": { "intelligence": 80.42625959292629, "calibration": 78.2177967967968, "speed": 75.08761089338216, "cost": 52.63658410842239 }, "jevbench_score": 70.61384476909898 }, "heldout": { "n": 303, "escalation": 0.9504950495049505, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6972477064220184 }, "accuracy": 0.8514851485148515, "ece_hard": 0.08560682566419815, "calibration": 81.44631743358019, "p50_s": 1.8944206550717353, "p95_s": 2.6674607589840886, "cost_usd_per_1000": 0.041811205028368, "axes": { "intelligence": 84.71651795065142, "calibration": 81.44631743358019, "speed": 72.96425540534744, "cost": 51.36121946789377 }, "jevbench_score": 71.30931387482124 } }, { "threshold": 0.84, "public": { "n": 231, "escalation": 0.8571428571428571, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.7117117117117117 }, "accuracy": 0.8008658008658008, "ece_hard": 0.09203203203203206, "calibration": 78.2177967967968, "p50_s": 1.318672539293766, "p95_s": 2.3501844495534896, "cost_usd_per_1000": 0.03808512359550561, "axes": { "intelligence": 80.42625959292629, "calibration": 78.2177967967968, "speed": 75.08761089338216, "cost": 52.577338920306964 }, "jevbench_score": 70.59396649650465 }, "heldout": { "n": 303, "escalation": 0.9504950495049505, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6972477064220184 }, "accuracy": 0.8514851485148515, "ece_hard": 0.08560682566419815, "calibration": 81.44631743358019, "p50_s": 1.8944206550717353, "p95_s": 2.6674607589840886, "cost_usd_per_1000": 0.041811205028368, "axes": { "intelligence": 84.71651795065142, "calibration": 81.44631743358019, "speed": 72.96425540534744, "cost": 51.36121946789377 }, "jevbench_score": 71.30931387482124 } }, { "threshold": 0.85, "public": { "n": 231, "escalation": 0.8571428571428571, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.7117117117117117 }, "accuracy": 0.8008658008658008, "ece_hard": 0.09203203203203206, "calibration": 78.2177967967968, "p50_s": 1.318672539293766, "p95_s": 2.3501844495534896, "cost_usd_per_1000": 0.03808512359550561, "axes": { "intelligence": 80.42625959292629, "calibration": 78.2177967967968, "speed": 75.08761089338216, "cost": 52.577338920306964 }, "jevbench_score": 70.59396649650465 }, "heldout": { "n": 303, "escalation": 0.9537953795379538, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6972477064220184 }, "accuracy": 0.8514851485148515, "ece_hard": 0.08560682566419815, "calibration": 81.44631743358019, "p50_s": 1.8944206550717353, "p95_s": 2.6674607589840886, "cost_usd_per_1000": 0.04194293518003485, "axes": { "intelligence": 84.71651795065142, "calibration": 81.44631743358019, "speed": 72.96425540534744, "cost": 51.32023542220867 }, "jevbench_score": 71.29508417418604 } }, { "threshold": 0.86, "public": { "n": 231, "escalation": 0.8658008658008658, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.7117117117117117 }, "accuracy": 0.8008658008658008, "ece_hard": 0.09203203203203206, "calibration": 78.2177967967968, "p50_s": 1.318672539293766, "p95_s": 2.3501844495534896, "cost_usd_per_1000": 0.038430701395982286, "axes": { "intelligence": 80.42625959292629, "calibration": 78.2177967967968, "speed": 75.08761089338216, "cost": 52.45965067557384 }, "jevbench_score": 70.55442920226784 }, "heldout": { "n": 303, "escalation": 0.9537953795379538, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6972477064220184 }, "accuracy": 0.8514851485148515, "ece_hard": 0.08560682566419815, "calibration": 81.44631743358019, "p50_s": 1.8944206550717353, "p95_s": 2.6674607589840886, "cost_usd_per_1000": 0.04194293518003485, "axes": { "intelligence": 84.71651795065142, "calibration": 81.44631743358019, "speed": 72.96425540534744, "cost": 51.32023542220867 }, "jevbench_score": 71.29508417418604 } }, { "threshold": 0.87, "public": { "n": 231, "escalation": 0.8701298701298701, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.7117117117117117 }, "accuracy": 0.8008658008658008, "ece_hard": 0.09203203203203206, "calibration": 78.2177967967968, "p50_s": 1.318672539293766, "p95_s": 2.3501844495534896, "cost_usd_per_1000": 0.03860349029622063, "axes": { "intelligence": 80.42625959292629, "calibration": 78.2177967967968, "speed": 75.08761089338216, "cost": 52.401202817468146 }, "jevbench_score": 70.53476895261343 }, "heldout": { "n": 303, "escalation": 0.9537953795379538, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6972477064220184 }, "accuracy": 0.8514851485148515, "ece_hard": 0.08560682566419815, "calibration": 81.44631743358019, "p50_s": 1.8944206550717353, "p95_s": 2.6674607589840886, "cost_usd_per_1000": 0.04194293518003485, "axes": { "intelligence": 84.71651795065142, "calibration": 81.44631743358019, "speed": 72.96425540534744, "cost": 51.32023542220867 }, "jevbench_score": 71.29508417418604 } }, { "threshold": 0.88, "public": { "n": 231, "escalation": 0.8831168831168831, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.7117117117117117 }, "accuracy": 0.8008658008658008, "ece_hard": 0.09203203203203206, "calibration": 78.2177967967968, "p50_s": 1.318672539293766, "p95_s": 2.3501844495534896, "cost_usd_per_1000": 0.03912185699693564, "axes": { "intelligence": 80.42625959292629, "calibration": 78.2177967967968, "speed": 75.08761089338216, "cost": 52.227416162253114 }, "jevbench_score": 70.47621459018097 }, "heldout": { "n": 303, "escalation": 0.9537953795379538, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6972477064220184 }, "accuracy": 0.8514851485148515, "ece_hard": 0.08560682566419815, "calibration": 81.44631743358019, "p50_s": 1.8944206550717353, "p95_s": 2.6674607589840886, "cost_usd_per_1000": 0.04194293518003485, "axes": { "intelligence": 84.71651795065142, "calibration": 81.44631743358019, "speed": 72.96425540534744, "cost": 51.32023542220867 }, "jevbench_score": 71.29508417418604 } }, { "threshold": 0.89, "public": { "n": 231, "escalation": 0.8961038961038961, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.7117117117117117 }, "accuracy": 0.8051948051948052, "ece_hard": 0.09203203203203206, "calibration": 78.2177967967968, "p50_s": 1.318672539293766, "p95_s": 2.3501844495534896, "cost_usd_per_1000": 0.039640223697650655, "axes": { "intelligence": 80.9663830497164, "calibration": 78.2177967967968, "speed": 75.08761089338216, "cost": 52.055917101096426 }, "jevbench_score": 70.53621905372219 }, "heldout": { "n": 303, "escalation": 0.9570957095709571, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6972477064220184 }, "accuracy": 0.8514851485148515, "ece_hard": 0.08560682566419815, "calibration": 81.44631743358019, "p50_s": 1.8944206550717353, "p95_s": 2.6674607589840886, "cost_usd_per_1000": 0.042074665331701705, "axes": { "intelligence": 84.71651795065142, "calibration": 81.44631743358019, "speed": 72.96425540534744, "cost": 51.27937989348627 }, "jevbench_score": 71.2808906094645 } }, { "threshold": 0.9, "public": { "n": 231, "escalation": 0.8961038961038961, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.7117117117117117 }, "accuracy": 0.8051948051948052, "ece_hard": 0.09203203203203206, "calibration": 78.2177967967968, "p50_s": 1.318672539293766, "p95_s": 2.3501844495534896, "cost_usd_per_1000": 0.039640223697650655, "axes": { "intelligence": 80.9663830497164, "calibration": 78.2177967967968, "speed": 75.08761089338216, "cost": 52.055917101096426 }, "jevbench_score": 70.53621905372219 }, "heldout": { "n": 303, "escalation": 0.9570957095709571, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6972477064220184 }, "accuracy": 0.8514851485148515, "ece_hard": 0.08560682566419815, "calibration": 81.44631743358019, "p50_s": 1.8944206550717353, "p95_s": 2.6674607589840886, "cost_usd_per_1000": 0.042074665331701705, "axes": { "intelligence": 84.71651795065142, "calibration": 81.44631743358019, "speed": 72.96425540534744, "cost": 51.27937989348627 }, "jevbench_score": 71.2808906094645 } }, { "threshold": 0.91, "public": { "n": 231, "escalation": 0.9090909090909091, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.7117117117117117 }, "accuracy": 0.8051948051948052, "ece_hard": 0.09203203203203206, "calibration": 78.2177967967968, "p50_s": 1.318672539293766, "p95_s": 2.3501844495534896, "cost_usd_per_1000": 0.040158590398365675, "axes": { "intelligence": 80.9663830497164, "calibration": 78.2177967967968, "speed": 75.08761089338216, "cost": 51.88664619134585 }, "jevbench_score": 70.47880811213322 }, "heldout": { "n": 303, "escalation": 0.9603960396039604, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6972477064220184 }, "accuracy": 0.8514851485148515, "ece_hard": 0.08560682566419815, "calibration": 81.44631743358019, "p50_s": 1.8944206550717353, "p95_s": 2.6674607589840886, "cost_usd_per_1000": 0.042206395483368554, "axes": { "intelligence": 84.71651795065142, "calibration": 81.44631743358019, "speed": 72.96425540534744, "cost": 51.238652078243554 }, "jevbench_score": 71.26673296945937 } }, { "threshold": 0.92, "public": { "n": 231, "escalation": 0.922077922077922, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.7117117117117117 }, "accuracy": 0.8051948051948052, "ece_hard": 0.09203203203203206, "calibration": 78.2177967967968, "p50_s": 1.318672539293766, "p95_s": 2.3501844495534896, "cost_usd_per_1000": 0.04067695709908069, "axes": { "intelligence": 80.9663830497164, "calibration": 78.2177967967968, "speed": 75.08761089338216, "cost": 51.71954627757169 }, "jevbench_score": 70.42199555411581 }, "heldout": { "n": 303, "escalation": 0.9636963696369637, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6972477064220184 }, "accuracy": 0.8514851485148515, "ece_hard": 0.08560682566419815, "calibration": 81.44631743358019, "p50_s": 1.8944206550717353, "p95_s": 2.6674607589840886, "cost_usd_per_1000": 0.04233812563503541, "axes": { "intelligence": 84.71651795065142, "calibration": 81.44631743358019, "speed": 72.96425540534744, "cost": 51.19805118050903 }, "jevbench_score": 71.25261104486721 } }, { "threshold": 0.93, "public": { "n": 231, "escalation": 0.922077922077922, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.7117117117117117 }, "accuracy": 0.8051948051948052, "ece_hard": 0.09203203203203206, "calibration": 78.2177967967968, "p50_s": 1.318672539293766, "p95_s": 2.3501844495534896, "cost_usd_per_1000": 0.04067695709908069, "axes": { "intelligence": 80.9663830497164, "calibration": 78.2177967967968, "speed": 75.08761089338216, "cost": 51.71954627757169 }, "jevbench_score": 70.42199555411581 }, "heldout": { "n": 303, "escalation": 0.9702970297029703, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6972477064220184 }, "accuracy": 0.8514851485148515, "ece_hard": 0.08560682566419815, "calibration": 81.44631743358019, "p50_s": 1.8944206550717353, "p95_s": 2.6674607589840886, "cost_usd_per_1000": 0.04260158593836911, "axes": { "intelligence": 84.71651795065142, "calibration": 81.44631743358019, "speed": 72.96425540534744, "cost": 51.117226990677736 }, "jevbench_score": 71.22447351403979 } }, { "threshold": 0.94, "public": { "n": 231, "escalation": 0.9264069264069265, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.7117117117117117 }, "accuracy": 0.8051948051948052, "ece_hard": 0.09203203203203206, "calibration": 78.2177967967968, "p50_s": 1.318672539293766, "p95_s": 2.3501844495534896, "cost_usd_per_1000": 0.04084974599931903, "axes": { "intelligence": 80.9663830497164, "calibration": 78.2177967967968, "speed": 75.08761089338216, "cost": 51.664319186018275 }, "jevbench_score": 70.403188542998 }, "heldout": { "n": 303, "escalation": 0.9735973597359736, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6972477064220184 }, "accuracy": 0.8514851485148515, "ece_hard": 0.08560682566419815, "calibration": 81.44631743358019, "p50_s": 1.8944206550717353, "p95_s": 2.6674607589840886, "cost_usd_per_1000": 0.042733316090035965, "axes": { "intelligence": 84.71651795065142, "calibration": 81.44631743358019, "speed": 72.96425540534744, "cost": 51.07700214336283 }, "jevbench_score": 71.21045749846279 } }, { "threshold": 0.95, "public": { "n": 231, "escalation": 0.935064935064935, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.7117117117117117 }, "accuracy": 0.8051948051948052, "ece_hard": 0.09203203203203206, "calibration": 78.2177967967968, "p50_s": 1.318672539293766, "p95_s": 2.3501844495534896, "cost_usd_per_1000": 0.0411953237997957, "axes": { "intelligence": 80.9663830497164, "calibration": 78.2177967967968, "speed": 75.08761089338216, "cost": 51.55456237570816 }, "jevbench_score": 70.36576719915459 }, "heldout": { "n": 303, "escalation": 0.976897689768977, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6972477064220184 }, "accuracy": 0.8514851485148515, "ece_hard": 0.08560682566419815, "calibration": 81.44631743358019, "p50_s": 1.8944206550717353, "p95_s": 2.6674607589840886, "cost_usd_per_1000": 0.042865046241702814, "axes": { "intelligence": 84.71651795065142, "calibration": 81.44631743358019, "speed": 72.96425540534744, "cost": 51.03690110294031 }, "jevbench_score": 71.19647637957114 } }, { "threshold": 0.96, "public": { "n": 231, "escalation": 0.9523809523809523, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.7117117117117117 }, "accuracy": 0.8051948051948052, "ece_hard": 0.09203203203203206, "calibration": 78.2177967967968, "p50_s": 1.318672539293766, "p95_s": 2.3501844495534896, "cost_usd_per_1000": 0.041886479400749056, "axes": { "intelligence": 80.9663830497164, "calibration": 78.2177967967968, "speed": 75.08761089338216, "cost": 51.337784229327085 }, "jevbench_score": 70.29168126309153 }, "heldout": { "n": 303, "escalation": 0.976897689768977, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6972477064220184 }, "accuracy": 0.8514851485148515, "ece_hard": 0.08560682566419815, "calibration": 81.44631743358019, "p50_s": 1.8944206550717353, "p95_s": 2.6674607589840886, "cost_usd_per_1000": 0.042865046241702814, "axes": { "intelligence": 84.71651795065142, "calibration": 81.44631743358019, "speed": 72.96425540534744, "cost": 51.03690110294031 }, "jevbench_score": 71.19647637957114 } }, { "threshold": 0.97, "public": { "n": 231, "escalation": 0.9696969696969697, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.7117117117117117 }, "accuracy": 0.8051948051948052, "ece_hard": 0.09203203203203206, "calibration": 78.2177967967968, "p50_s": 1.318672539293766, "p95_s": 2.3501844495534896, "cost_usd_per_1000": 0.04257763500170241, "axes": { "intelligence": 80.9663830497164, "calibration": 78.2177967967968, "speed": 75.08761089338216, "cost": 51.124553960652406 }, "jevbench_score": 70.21857859546948 }, "heldout": { "n": 303, "escalation": 0.9933993399339934, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6972477064220184 }, "accuracy": 0.8514851485148515, "ece_hard": 0.08064220183486236, "calibration": 81.94277981651376, "p50_s": 1.8944206550717353, "p95_s": 2.6674607589840886, "cost_usd_per_1000": 0.043523697000037075, "axes": { "intelligence": 84.71651795065142, "calibration": 81.94277981651376, "speed": 72.96425540534744, "cost": 50.83822665399112 }, "jevbench_score": 71.2352305827793 } }, { "threshold": 0.98, "public": { "n": 231, "escalation": 0.9826839826839827, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.7117117117117117 }, "accuracy": 0.8051948051948052, "ece_hard": 0.09203203203203206, "calibration": 78.2177967967968, "p50_s": 1.318672539293766, "p95_s": 2.3501844495534896, "cost_usd_per_1000": 0.043096001702417425, "axes": { "intelligence": 80.9663830497164, "calibration": 78.2177967967968, "speed": 75.08761089338216, "cost": 50.96689060918618 }, "jevbench_score": 70.1643789899812 }, "heldout": { "n": 303, "escalation": 0.9966996699669967, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6972477064220184 }, "accuracy": 0.8514851485148515, "ece_hard": 0.08064220183486236, "calibration": 81.94277981651376, "p50_s": 1.8944206550717353, "p95_s": 2.6674607589840886, "cost_usd_per_1000": 0.043655427151703924, "axes": { "intelligence": 84.71651795065142, "calibration": 81.94277981651376, "speed": 72.96425540534744, "cost": 50.7988527405229 }, "jevbench_score": 71.22143375619021 } }, { "threshold": 0.99, "public": { "n": 231, "escalation": 1.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.7117117117117117 }, "accuracy": 0.8051948051948052, "ece_hard": 0.09203203203203206, "calibration": 78.2177967967968, "p50_s": 1.318672539293766, "p95_s": 2.3501844495534896, "cost_usd_per_1000": 0.04378715730337078, "axes": { "intelligence": 80.9663830497164, "calibration": 78.2177967967968, "speed": 75.08761089338216, "cost": 50.759597458942885 }, "jevbench_score": 70.09292656644448 }, "heldout": { "n": 303, "escalation": 1.0, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6972477064220184 }, "accuracy": 0.8514851485148515, "ece_hard": 0.08064220183486236, "calibration": 81.94277981651376, "p50_s": 1.8944206550717353, "p95_s": 2.6674607589840886, "cost_usd_per_1000": 0.04378715730337078, "axes": { "intelligence": 84.71651795065142, "calibration": 81.94277981651376, "speed": 72.96425540534744, "cost": 50.759597458942885 }, "jevbench_score": 71.20767051235549 } }, { "threshold": 1.0, "public": { "n": 231, "escalation": 1.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.7117117117117117 }, "accuracy": 0.8051948051948052, "ece_hard": 0.09203203203203206, "calibration": 78.2177967967968, "p50_s": 1.318672539293766, "p95_s": 2.3501844495534896, "cost_usd_per_1000": 0.04378715730337078, "axes": { "intelligence": 80.9663830497164, "calibration": 78.2177967967968, "speed": 75.08761089338216, "cost": 50.759597458942885 }, "jevbench_score": 70.09292656644448 }, "heldout": { "n": 303, "escalation": 1.0, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6972477064220184 }, "accuracy": 0.8514851485148515, "ece_hard": 0.08064220183486236, "calibration": 81.94277981651376, "p50_s": 1.8944206550717353, "p95_s": 2.6674607589840886, "cost_usd_per_1000": 0.04378715730337078, "axes": { "intelligence": 84.71651795065142, "calibration": 81.94277981651376, "speed": 72.96425540534744, "cost": 50.759597458942885 }, "jevbench_score": 71.20767051235549 } } ] }, { "cheap": "laya", "backend": "jev-1.13.0", "chosen": { "threshold": 0.96, "public": { "n": 231, "escalation": 0.8874458874458875, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.7117117117117117 }, "accuracy": 0.8051948051948052, "ece_hard": 0.09203203203203206, "calibration": 78.2177967967968, "p50_s": 1.7860921561717986, "p95_s": 5.118455225229264, "cost_usd_per_1000": 0.03830485188968335, "axes": { "intelligence": 80.9663830497164, "calibration": 78.2177967967968, "speed": 70.38957228221139, "cost": 52.50238637718715 }, "jevbench_score": 69.55437009803765 }, "heldout": { "n": 303, "escalation": 0.9537953795379538, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6972477064220184 }, "accuracy": 0.8514851485148515, "ece_hard": 0.08064220183486236, "calibration": 81.94277981651376, "p50_s": 3.443461710214615, "p95_s": 5.44839422404766, "cost_usd_per_1000": 0.040953141172544213, "axes": { "intelligence": 84.71651795065142, "calibration": 81.94277981651376, "speed": 67.26736217960183, "cost": 51.63138344598353 }, "jevbench_score": 70.07276220626505 } }, "full": { "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9452054794520548, "hard": 0.7045454545454546 }, "accuracy": 0.8314606741573034, "ece_hard": 0.08502525252525245, "calibration": 80.21147474747477, "p50_s": 2.8905016608536243, "p95_s": 5.293572343885899, "cost_usd_per_1000": 0.03980753288726171, "axes": { "intelligence": 84.64378372768783, "calibration": 80.21147474747477, "speed": 68.15277924064857, "cost": 52.00104212285385 }, "jevbench_score": 70.03768732176295 }, "full_escalation": 0.9250936329588015, "backend_heldout_accuracy": 0.8514851485148515, "useful": false, "curve": [ { "threshold": 0.0, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.01, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.02, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.03, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.04, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.05, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.06, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.07, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.08, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.09, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.1, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.11, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.12, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.13, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.14, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.15, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.16, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.17, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.18, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.19, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.2, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.21, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.2060783783783784, "calibration": 64.07466216216217, "p50_s": 1.1657145261764525, "p95_s": 4.443649256229401, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 53.58796296296297, "calibration": 64.07466216216217, "speed": 72.85668019555155, "cost": 86.20408526325653 }, "jevbench_score": 68.14558119804416 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.22, "public": { "n": 231, "escalation": 0.004329004329004329, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.34234234234234234 }, "accuracy": 0.5324675324675324, "ece_hard": 0.2027765765765766, "calibration": 64.40484234234235, "p50_s": 1.1657145261764525, "p95_s": 4.4528342053294185, "cost_usd_per_1000": 0.0030559162410623087, "axes": { "intelligence": 53.963338338338346, "calibration": 64.40484234234235, "speed": 72.84771266552642, "cost": 85.44575660246133 }, "jevbench_score": 68.19946023428056 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2599752293577982, "calibration": 55.35122706422018, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.0028831273408239703, "axes": { "intelligence": 59.67720246323992, "calibration": 55.35122706422018, "speed": 68.8164811989393, "cost": 86.20408526325653 }, "jevbench_score": 66.5332908249994 } }, { "threshold": 0.23, "public": { "n": 231, "escalation": 0.004329004329004329, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.34234234234234234 }, "accuracy": 0.5324675324675324, "ece_hard": 0.2027765765765766, "calibration": 64.40484234234235, "p50_s": 1.1657145261764525, "p95_s": 4.4528342053294185, "cost_usd_per_1000": 0.0030559162410623087, "axes": { "intelligence": 53.963338338338346, "calibration": 64.40484234234235, "speed": 72.84771266552642, "cost": 85.44575660246133 }, "jevbench_score": 68.19946023428056 }, "heldout": { "n": 303, "escalation": 0.0033003300330033004, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.26593394495412853, "calibration": 54.75535550458714, "p50_s": 2.7680240362882613, "p95_s": 4.744407929480074, "cost_usd_per_1000": 0.003014857492490822, "axes": { "intelligence": 59.67720246323992, "calibration": 54.75535550458714, "speed": 68.8164811989393, "cost": 85.62199634340215 }, "jevbench_score": 66.24120401482321 } }, { "threshold": 0.24, "public": { "n": 231, "escalation": 0.004329004329004329, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.34234234234234234 }, "accuracy": 0.5324675324675324, "ece_hard": 0.2027765765765766, "calibration": 64.40484234234235, "p50_s": 1.1657145261764525, "p95_s": 4.4528342053294185, "cost_usd_per_1000": 0.0030559162410623087, "axes": { "intelligence": 53.963338338338346, "calibration": 64.40484234234235, "speed": 72.84771266552642, "cost": 85.44575660246133 }, "jevbench_score": 68.19946023428056 }, "heldout": { "n": 303, "escalation": 0.006600660066006601, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.30275229357798167 }, "accuracy": 0.5676567656765676, "ece_hard": 0.2681311926605505, "calibration": 54.53563073394494, "p50_s": 2.7680240362882613, "p95_s": 4.776861734688282, "cost_usd_per_1000": 0.0031465876441576744, "axes": { "intelligence": 59.95243182103809, "calibration": 54.53563073394494, "speed": 68.78687472081019, "cost": 85.06480501098979 }, "jevbench_score": 66.13565487379515 } }, { "threshold": 0.25, "public": { "n": 231, "escalation": 0.008658008658008658, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.35135135135135137 }, "accuracy": 0.5367965367965368, "ece_hard": 0.20499549549549553, "calibration": 64.18295045045045, "p50_s": 1.1657145261764525, "p95_s": 4.4528342053294185, "cost_usd_per_1000": 0.003228705141300647, "axes": { "intelligence": 54.33871371371373, "calibration": 64.18295045045045, "speed": 72.84771266552642, "cost": 84.72914844217487 }, "jevbench_score": 68.1152646286431 }, "heldout": { "n": 303, "escalation": 0.009900990099009901, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2654128440366973, "calibration": 54.807465596330275, "p50_s": 2.7680240362882613, "p95_s": 4.808721804618835, "cost_usd_per_1000": 0.0032783177958245263, "axes": { "intelligence": 59.67720246323992, "calibration": 54.807465596330275, "speed": 68.75800490091856, "cost": 84.53046846363226 }, "jevbench_score": 66.03074231569374 } }, { "threshold": 0.26, "public": { "n": 231, "escalation": 0.008658008658008658, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.35135135135135137 }, "accuracy": 0.5367965367965368, "ece_hard": 0.20499549549549553, "calibration": 64.18295045045045, "p50_s": 1.1657145261764525, "p95_s": 4.4528342053294185, "cost_usd_per_1000": 0.003228705141300647, "axes": { "intelligence": 54.33871371371373, "calibration": 64.18295045045045, "speed": 72.84771266552642, "cost": 84.72914844217487 }, "jevbench_score": 68.1152646286431 }, "heldout": { "n": 303, "escalation": 0.013201320132013201, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2756587155963304, "calibration": 53.78287844036696, "p50_s": 2.7680240362882613, "p95_s": 4.8142633035779, "cost_usd_per_1000": 0.0034100479474913786, "axes": { "intelligence": 59.67720246323992, "calibration": 53.78287844036696, "speed": 68.75300303786179, "cost": 84.01718543505324 }, "jevbench_score": 65.61876838218721 } }, { "threshold": 0.27, "public": { "n": 231, "escalation": 0.021645021645021644, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.35135135135135137 }, "accuracy": 0.5367965367965368, "ece_hard": 0.21230590590590592, "calibration": 63.45190940940941, "p50_s": 1.1657145261764525, "p95_s": 4.455602088570595, "cost_usd_per_1000": 0.0037470718420156623, "axes": { "intelligence": 54.33871371371373, "calibration": 63.45190940940941, "speed": 72.84501392824589, "cost": 82.78923940500938 }, "jevbench_score": 67.527697713906 }, "heldout": { "n": 303, "escalation": 0.013201320132013201, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2756587155963304, "calibration": 53.78287844036696, "p50_s": 2.7680240362882613, "p95_s": 4.8142633035779, "cost_usd_per_1000": 0.0034100479474913786, "axes": { "intelligence": 59.67720246323992, "calibration": 53.78287844036696, "speed": 68.75300303786179, "cost": 84.01718543505324 }, "jevbench_score": 65.61876838218721 } }, { "threshold": 0.28, "public": { "n": 231, "escalation": 0.021645021645021644, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.35135135135135137 }, "accuracy": 0.5367965367965368, "ece_hard": 0.21230590590590592, "calibration": 63.45190940940941, "p50_s": 1.1657145261764525, "p95_s": 4.455602088570595, "cost_usd_per_1000": 0.0037470718420156623, "axes": { "intelligence": 54.33871371371373, "calibration": 63.45190940940941, "speed": 72.84501392824589, "cost": 82.78923940500938 }, "jevbench_score": 67.527697713906 }, "heldout": { "n": 303, "escalation": 0.019801980198019802, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2756504587155964, "calibration": 53.78370412844036, "p50_s": 2.7680240362882613, "p95_s": 4.822164891660214, "cost_usd_per_1000": 0.0036735082508250827, "axes": { "intelligence": 59.67720246323992, "calibration": 53.78370412844036, "speed": 68.74588086188128, "cost": 83.0475694113478 }, "jevbench_score": 65.42717866647327 } }, { "threshold": 0.29, "public": { "n": 231, "escalation": 0.03896103896103896, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.3333333333333333 }, "accuracy": 0.5281385281385281, "ece_hard": 0.233477977977978, "calibration": 61.33470220220221, "p50_s": 1.1657145261764525, "p95_s": 4.538626746833325, "cost_usd_per_1000": 0.004438227442969016, "axes": { "intelligence": 53.58796296296297, "calibration": 61.33470220220221, "speed": 72.76483323506824, "cost": 80.58371336576546 }, "jevbench_score": 66.25758173957432 }, "heldout": { "n": 303, "escalation": 0.0231023102310231, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6917808219178082, "hard": 0.29357798165137616 }, "accuracy": 0.5643564356435643, "ece_hard": 0.27133853211009185, "calibration": 54.21489678899081, "p50_s": 2.7680240362882613, "p95_s": 4.822164891660214, "cost_usd_per_1000": 0.0038052384024919346, "axes": { "intelligence": 59.67720246323992, "calibration": 54.21489678899081, "speed": 68.74588086188128, "cost": 82.58854387000882 }, "jevbench_score": 65.46714402212004 } }, { "threshold": 0.3, "public": { "n": 231, "escalation": 0.04329004329004329, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.34234234234234234 }, "accuracy": 0.5324675324675324, "ece_hard": 0.2301716716716717, "calibration": 61.66533283283283, "p50_s": 1.1657145261764525, "p95_s": 4.6030811160802845, "cost_usd_per_1000": 0.004611016343207354, "axes": { "intelligence": 53.963338338338346, "calibration": 61.66533283283283, "speed": 72.70359163520703, "cost": 80.08610015415823 }, "jevbench_score": 66.34576790643531 }, "heldout": { "n": 303, "escalation": 0.033003300330033, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6986301369863014, "hard": 0.29357798165137616 }, "accuracy": 0.5676567656765676, "ece_hard": 0.27714220183486243, "calibration": 53.634529816513755, "p50_s": 2.79284847676754, "p95_s": 4.840475116670132, "cost_usd_per_1000": 0.004200428857492491, "axes": { "intelligence": 59.868983285157725, "calibration": 53.634529816513755, "speed": 68.69064633980814, "cost": 81.30119099567554 }, "jevbench_score": 65.07440678589502 } }, { "threshold": 0.31, "public": { "n": 231, "escalation": 0.05627705627705628, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.34234234234234234 }, "accuracy": 0.5324675324675324, "ece_hard": 0.24274824824824828, "calibration": 60.40767517517518, "p50_s": 1.1657145261764525, "p95_s": 4.6030811160802845, "cost_usd_per_1000": 0.00512938304392237, "axes": { "intelligence": 53.963338338338346, "calibration": 60.40767517517518, "speed": 72.70359163520703, "cost": 78.69804604508548 }, "jevbench_score": 65.71699349405755 }, "heldout": { "n": 303, "escalation": 0.0429042904290429, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6986301369863014, "hard": 0.3211009174311927 }, "accuracy": 0.5775577557755776, "ece_hard": 0.2626073394495413, "calibration": 55.08801605504587, "p50_s": 2.79284847676754, "p95_s": 4.911906263977289, "cost_usd_per_1000": 0.004595619312493047, "axes": { "intelligence": 60.69467135855222, "calibration": 55.08801605504587, "speed": 68.62702555493969, "cost": 80.12967862491175 }, "jevbench_score": 65.48232157747644 } }, { "threshold": 0.32, "public": { "n": 231, "escalation": 0.06060606060606061, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.34234234234234234 }, "accuracy": 0.5324675324675324, "ece_hard": 0.24291041041041048, "calibration": 60.39145895895895, "p50_s": 1.1657145261764525, "p95_s": 4.684279622137547, "cost_usd_per_1000": 0.005302171944160708, "axes": { "intelligence": 53.963338338338346, "calibration": 60.39145895895895, "speed": 72.62764979841634, "cost": 78.2663857790985 }, "jevbench_score": 65.6051450536322 }, "heldout": { "n": 303, "escalation": 0.0462046204620462, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6986301369863014, "hard": 0.3211009174311927 }, "accuracy": 0.5775577557755776, "ece_hard": 0.26556697247706423, "calibration": 54.79205275229357, "p50_s": 2.79284847676754, "p95_s": 4.911906263977289, "cost_usd_per_1000": 0.0047273494641598985, "axes": { "intelligence": 60.69467135855222, "calibration": 54.79205275229357, "speed": 68.62702555493969, "cost": 79.76146875291555 }, "jevbench_score": 65.31893777110234 } }, { "threshold": 0.33, "public": { "n": 231, "escalation": 0.06493506493506493, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.34234234234234234 }, "accuracy": 0.5324675324675324, "ece_hard": 0.24891311311311318, "calibration": 59.79118868868869, "p50_s": 1.1657145261764525, "p95_s": 4.729233426600695, "cost_usd_per_1000": 0.005474960844399046, "axes": { "intelligence": 53.963338338338346, "calibration": 59.79118868868869, "speed": 72.58617041122389, "cost": 77.8485694733873 }, "jevbench_score": 65.34466399794091 }, "heldout": { "n": 303, "escalation": 0.0594059405940594, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.6986301369863014, "hard": 0.3486238532110092 }, "accuracy": 0.5874587458745875, "ece_hard": 0.24522844036697247, "calibration": 57.938155963302755, "p50_s": 2.8226395934820174, "p95_s": 4.911906263977289, "cost_usd_per_1000": 0.005254270070827308, "axes": { "intelligence": 61.520359431946716, "calibration": 57.938155963302755, "speed": 68.58094500479204, "cost": 78.38462824385897 }, "jevbench_score": 66.1613678120411 } }, { "threshold": 0.34, "public": { "n": 231, "escalation": 0.0735930735930736, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.35135135135135137 }, "accuracy": 0.5367965367965368, "ece_hard": 0.24608248248248257, "calibration": 60.074251751751746, "p50_s": 1.1657145261764525, "p95_s": 4.735011649876833, "cost_usd_per_1000": 0.005820538644875723, "axes": { "intelligence": 54.33871371371373, "calibration": 60.074251751751746, "speed": 72.58086739809278, "cost": 77.05110468899464 }, "jevbench_score": 65.36566545990713 }, "heldout": { "n": 303, "escalation": 0.06930693069306931, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7054794520547946, "hard": 0.3577981651376147 }, "accuracy": 0.594059405940594, "ece_hard": 0.24798073394495412, "calibration": 57.66292660550459, "p50_s": 2.8226395934820174, "p95_s": 4.911906263977289, "cost_usd_per_1000": 0.005649460525827864, "axes": { "intelligence": 61.98736961166269, "calibration": 57.66292660550459, "speed": 68.58094500479204, "cost": 77.4397906459943 }, "jevbench_score": 66.00728669474688 } }, { "threshold": 0.35, "public": { "n": 231, "escalation": 0.0735930735930736, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.35135135135135137 }, "accuracy": 0.5367965367965368, "ece_hard": 0.24608248248248257, "calibration": 60.074251751751746, "p50_s": 1.1657145261764525, "p95_s": 4.735011649876833, "cost_usd_per_1000": 0.005820538644875723, "axes": { "intelligence": 54.33871371371373, "calibration": 60.074251751751746, "speed": 72.58086739809278, "cost": 77.05110468899464 }, "jevbench_score": 65.36566545990713 }, "heldout": { "n": 303, "escalation": 0.07920792079207921, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7054794520547946, "hard": 0.3761467889908257 }, "accuracy": 0.6006600660066007, "ece_hard": 0.24359908256880736, "calibration": 58.10109174311926, "p50_s": 2.8226395934820174, "p95_s": 4.911906263977289, "cost_usd_per_1000": 0.00604465098082842, "axes": { "intelligence": 62.53782832725903, "calibration": 58.10109174311926, "speed": 68.58094500479204, "cost": 76.55886310948712 }, "jevbench_score": 66.08935409406172 } }, { "threshold": 0.36, "public": { "n": 231, "escalation": 0.08658008658008658, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.36936936936936937 }, "accuracy": 0.5454545454545454, "ece_hard": 0.24189509509509513, "calibration": 60.4929904904905, "p50_s": 1.1657145261764525, "p95_s": 4.753057958185673, "cost_usd_per_1000": 0.006338905345590739, "axes": { "intelligence": 55.08946446446447, "calibration": 60.4929904904905, "speed": 72.56434681615796, "cost": 75.9395719957974 }, "jevbench_score": 65.4623000090849 }, "heldout": { "n": 303, "escalation": 0.08250825082508251, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7054794520547946, "hard": 0.3761467889908257 }, "accuracy": 0.6006600660066007, "ece_hard": 0.2543926605504587, "calibration": 58.36023394495413, "p50_s": 2.8226395934820174, "p95_s": 4.911906263977289, "cost_usd_per_1000": 0.006176381132495272, "axes": { "intelligence": 62.53782832725903, "calibration": 58.36023394495413, "speed": 68.58094500479204, "cost": 76.27797737126875 }, "jevbench_score": 66.10215428082049 } }, { "threshold": 0.37, "public": { "n": 231, "escalation": 0.11255411255411256, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.40540540540540543 }, "accuracy": 0.5627705627705628, "ece_hard": 0.22461671671671674, "calibration": 62.22082832832833, "p50_s": 1.169974410533905, "p95_s": 4.861264364421368, "cost_usd_per_1000": 0.0073756387470207685, "axes": { "intelligence": 56.59096596596597, "calibration": 62.22082832832833, "speed": 72.45074397516674, "cost": 73.96601088506 }, "jevbench_score": 65.90820525270126 }, "heldout": { "n": 303, "escalation": 0.0891089108910891, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7054794520547946, "hard": 0.3944954128440367 }, "accuracy": 0.6072607260726073, "ece_hard": 0.2584889908256881, "calibration": 57.95060091743119, "p50_s": 2.8226395934820174, "p95_s": 4.911906263977289, "cost_usd_per_1000": 0.006439841435828975, "axes": { "intelligence": 63.08828704285535, "calibration": 57.95060091743119, "speed": 68.58094500479204, "cost": 75.73374477606413 }, "jevbench_score": 66.01230446588 } }, { "threshold": 0.38, "public": { "n": 231, "escalation": 0.12554112554112554, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.40540540540540543 }, "accuracy": 0.5627705627705628, "ece_hard": 0.22675185185185187, "calibration": 62.00731481481482, "p50_s": 1.2405554592609405, "p95_s": 4.878736702352763, "cost_usd_per_1000": 0.007894005447735783, "axes": { "intelligence": 56.59096596596597, "calibration": 62.00731481481482, "speed": 72.18076435179336, "cost": 73.08107734687698 }, "jevbench_score": 65.59248919449297 }, "heldout": { "n": 303, "escalation": 0.09900990099009901, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7123287671232876, "hard": 0.4036697247706422 }, "accuracy": 0.6138613861386139, "ece_hard": 0.25609449541284407, "calibration": 58.1900504587156, "p50_s": 2.8282473921775817, "p95_s": 4.911906263977289, "cost_usd_per_1000": 0.006835031890829532, "axes": { "intelligence": 63.555297222571326, "calibration": 58.1900504587156, "speed": 68.57232534265842, "cost": 74.95778364292583 }, "jevbench_score": 66.0300346873197 } }, { "threshold": 0.39, "public": { "n": 231, "escalation": 0.14285714285714285, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.4144144144144144 }, "accuracy": 0.5670995670995671, "ece_hard": 0.22774194194194192, "calibration": 61.90830580580581, "p50_s": 1.2405554592609405, "p95_s": 4.936991681903601, "cost_usd_per_1000": 0.008585161048689137, "axes": { "intelligence": 56.96634134134135, "calibration": 61.90830580580581, "speed": 72.12921420244184, "cost": 71.98754660542049 }, "jevbench_score": 65.41599554240025 }, "heldout": { "n": 303, "escalation": 0.11221122112211221, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7191780821917808, "hard": 0.41284403669724773 }, "accuracy": 0.6204620462046204, "ece_hard": 0.26134678899082575, "calibration": 57.66482110091742, "p50_s": 2.8427457749843597, "p95_s": 4.911906263977289, "cost_usd_per_1000": 0.007361952497496939, "axes": { "intelligence": 64.0223074022873, "calibration": 57.66482110091742, "speed": 68.55011906879295, "cost": 73.99020967408688 }, "jevbench_score": 65.78186554319022 } }, { "threshold": 0.4, "public": { "n": 231, "escalation": 0.1471861471861472, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.42342342342342343 }, "accuracy": 0.5714285714285714, "ece_hard": 0.2242005005005005, "calibration": 62.26244994994995, "p50_s": 1.2405554592609405, "p95_s": 4.936991681903601, "cost_usd_per_1000": 0.008757949948927476, "axes": { "intelligence": 57.34171671671673, "calibration": 62.26244994994995, "speed": 72.12921420244184, "cost": 71.72792623298335 }, "jevbench_score": 65.55775804297578 }, "heldout": { "n": 303, "escalation": 0.132013201320132, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7191780821917808, "hard": 0.42201834862385323 }, "accuracy": 0.6237623762376238, "ece_hard": 0.2659963302752294, "calibration": 56.983116972477056, "p50_s": 2.8427457749843597, "p95_s": 4.941431261599064, "cost_usd_per_1000": 0.008152333407498052, "axes": { "intelligence": 64.29753676008545, "calibration": 56.983116972477056, "speed": 68.52409218827898, "cost": 72.6615420164828 }, "jevbench_score": 65.35399031615282 } }, { "threshold": 0.41, "public": { "n": 231, "escalation": 0.1645021645021645, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.45045045045045046 }, "accuracy": 0.5844155844155844, "ece_hard": 0.22026716716716715, "calibration": 62.655783283283284, "p50_s": 1.2405554592609405, "p95_s": 4.980368090420962, "cost_usd_per_1000": 0.00944910554988083, "axes": { "intelligence": 58.46784284284285, "calibration": 62.655783283283284, "speed": 72.09122373840252, "cost": 70.738278992773 }, "jevbench_score": 65.74364524439402 }, "heldout": { "n": 303, "escalation": 0.13861386138613863, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7191780821917808, "hard": 0.42201834862385323 }, "accuracy": 0.6237623762376238, "ece_hard": 0.2703770642201835, "calibration": 56.13254357798165, "p50_s": 2.8707323759794234, "p95_s": 4.9580711171031, "cost_usd_per_1000": 0.008415793710831758, "axes": { "intelligence": 64.29753676008545, "calibration": 56.13254357798165, "speed": 68.46694540850847, "cost": 72.24714755649475 }, "jevbench_score": 65.0021436084267 } }, { "threshold": 0.42, "public": { "n": 231, "escalation": 0.16883116883116883, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.4594594594594595 }, "accuracy": 0.5887445887445888, "ece_hard": 0.21444644644644642, "calibration": 63.237855355355364, "p50_s": 1.2405554592609405, "p95_s": 4.980368090420962, "cost_usd_per_1000": 0.009621894450119168, "axes": { "intelligence": 58.84321821821823, "calibration": 63.237855355355364, "speed": 72.09122373840252, "cost": 70.5021823454176 }, "jevbench_score": 65.94617746696389 }, "heldout": { "n": 303, "escalation": 0.15841584158415842, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.726027397260274, "hard": 0.44036697247706424 }, "accuracy": 0.6336633663366337, "ece_hard": 0.2605669724770642, "calibration": 57.113552752293586, "p50_s": 2.8707323759794234, "p95_s": 4.9580711171031, "cost_usd_per_1000": 0.009206174620832869, "axes": { "intelligence": 65.0397762975996, "calibration": 57.113552752293586, "speed": 68.46694540850847, "cost": 71.07762375265659 }, "jevbench_score": 65.20531795931618 } }, { "threshold": 0.43, "public": { "n": 231, "escalation": 0.18614718614718614, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.4774774774774775 }, "accuracy": 0.5974025974025974, "ece_hard": 0.2154905905905906, "calibration": 62.674940940940935, "p50_s": 1.2405554592609405, "p95_s": 4.980368090420962, "cost_usd_per_1000": 0.010313050051072522, "axes": { "intelligence": 59.59396896896897, "calibration": 62.674940940940935, "speed": 72.09122373840252, "cost": 69.59838623609706 }, "jevbench_score": 65.7952365780763 }, "heldout": { "n": 303, "escalation": 0.19141914191419143, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.726027397260274, "hard": 0.44954128440366975 }, "accuracy": 0.636963696369637, "ece_hard": 0.25558348623853205, "calibration": 57.6119013761468, "p50_s": 2.9190098881721496, "p95_s": 5.103181174397469, "cost_usd_per_1000": 0.010523476137501389, "axes": { "intelligence": 65.31500565539777, "calibration": 57.6119013761468, "speed": 68.26923446797237, "cost": 69.33522338160118 }, "jevbench_score": 64.96448984320301 } }, { "threshold": 0.44, "public": { "n": 231, "escalation": 0.19913419913419914, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.4864864864864865 }, "accuracy": 0.6017316017316018, "ece_hard": 0.2152023023023023, "calibration": 63.384769769769775, "p50_s": 1.2405554592609405, "p95_s": 5.010124215483666, "cost_usd_per_1000": 0.010831416751787537, "axes": { "intelligence": 59.96934434434435, "calibration": 63.384769769769775, "speed": 72.06535322324531, "cost": 68.95944201525167 }, "jevbench_score": 65.92628734831554 }, "heldout": { "n": 303, "escalation": 0.20132013201320131, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.726027397260274, "hard": 0.44954128440366975 }, "accuracy": 0.636963696369637, "ece_hard": 0.25558348623853205, "calibration": 57.6119013761468, "p50_s": 2.9190098881721496, "p95_s": 5.103181174397469, "cost_usd_per_1000": 0.010918666592501946, "axes": { "intelligence": 65.31500565539777, "calibration": 57.6119013761468, "speed": 68.26923446797237, "cost": 68.85491185684216 }, "jevbench_score": 64.85168793882677 } }, { "threshold": 0.45, "public": { "n": 231, "escalation": 0.21212121212121213, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.5135135135135135 }, "accuracy": 0.6147186147186147, "ece_hard": 0.19979509509509508, "calibration": 64.92549049049049, "p50_s": 1.2405554592609405, "p95_s": 5.010124215483666, "cost_usd_per_1000": 0.011349783452502553, "axes": { "intelligence": 61.095470470470474, "calibration": 64.92549049049049, "speed": 72.06535322324531, "cost": 68.35037273460252 }, "jevbench_score": 66.4848839312715 }, "heldout": { "n": 303, "escalation": 0.2145214521452145, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7328767123287672, "hard": 0.44954128440366975 }, "accuracy": 0.6402640264026402, "ece_hard": 0.26197798165137615, "calibration": 56.97245183486238, "p50_s": 2.9652757436037063, "p95_s": 5.103181174397469, "cost_usd_per_1000": 0.011445587199169353, "axes": { "intelligence": 65.50678647731559, "calibration": 56.97245183486238, "speed": 68.20093927959584, "cost": 68.24085764717393 }, "jevbench_score": 64.5574710732496 } }, { "threshold": 0.46, "public": { "n": 231, "escalation": 0.22510822510822512, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.5315315315315315 }, "accuracy": 0.6233766233766234, "ece_hard": 0.18507077077077078, "calibration": 66.39792292292293, "p50_s": 1.3233075231313705, "p95_s": 5.056547989696265, "cost_usd_per_1000": 0.011868150153217568, "axes": { "intelligence": 61.846221221221235, "calibration": 66.39792292292293, "speed": 71.74485085175436, "cost": 67.7685090316738 }, "jevbench_score": 66.84540868333009 }, "heldout": { "n": 303, "escalation": 0.21782178217821782, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7328767123287672, "hard": 0.45871559633027525 }, "accuracy": 0.6435643564356436, "ece_hard": 0.2567935779816513, "calibration": 58.22939220183487, "p50_s": 2.9652757436037063, "p95_s": 5.103181174397469, "cost_usd_per_1000": 0.011577317350836206, "axes": { "intelligence": 65.78201583511374, "calibration": 58.22939220183487, "speed": 68.20093927959584, "cost": 68.09176185737603 }, "jevbench_score": 64.94318600808592 } }, { "threshold": 0.47, "public": { "n": 231, "escalation": 0.24242424242424243, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.5225225225225225 }, "accuracy": 0.6190476190476191, "ece_hard": 0.19789509509509512, "calibration": 65.36899049049049, "p50_s": 1.3951993823051452, "p95_s": 5.056547989696265, "cost_usd_per_1000": 0.012559305754170922, "axes": { "intelligence": 61.470845845845844, "calibration": 65.36899049049049, "speed": 71.51509591372808, "cost": 67.0310309982173 }, "jevbench_score": 66.24889639402485 }, "heldout": { "n": 303, "escalation": 0.23432343234323433, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7534246575342466, "hard": 0.46788990825688076 }, "accuracy": 0.6567656765676567, "ece_hard": 0.25255137614678896, "calibration": 58.6536123853211, "p50_s": 3.0245840936899184, "p95_s": 5.103181174397469, "cost_usd_per_1000": 0.012235968109170466, "axes": { "intelligence": 66.63258765866533, "calibration": 58.6536123853211, "speed": 68.11493349094279, "cost": 67.37084990786825 }, "jevbench_score": 65.07646471156544 } }, { "threshold": 0.48, "public": { "n": 231, "escalation": 0.2510822510822511, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.5135135135135135 }, "accuracy": 0.6147186147186147, "ece_hard": 0.2079356356356356, "calibration": 63.62793643643644, "p50_s": 1.416904667019844, "p95_s": 5.056547989696265, "cost_usd_per_1000": 0.012904883554647598, "axes": { "intelligence": 61.095470470470474, "calibration": 63.62793643643644, "speed": 71.44805235902912, "cost": 66.67737729687843 }, "jevbench_score": 65.6003926542719 }, "heldout": { "n": 303, "escalation": 0.264026402640264, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7671232876712328, "hard": 0.48623853211009177 }, "accuracy": 0.66996699669967, "ece_hard": 0.23725688073394494, "calibration": 60.183061926605504, "p50_s": 3.0606545895338058, "p95_s": 5.190079523622989, "cost_usd_per_1000": 0.013421539474172134, "axes": { "intelligence": 67.56660801809727, "calibration": 60.183061926605504, "speed": 67.9901166772053, "cost": 66.16593000908105 }, "jevbench_score": 65.39908182264469 } }, { "threshold": 0.49, "public": { "n": 231, "escalation": 0.2554112554112554, "tiers": { "easy": 0.9583333333333334, "standard": 0.5416666666666666, "hard": 0.5225225225225225 }, "accuracy": 0.6190476190476191, "ece_hard": 0.20244464464464468, "calibration": 64.17703553553554, "p50_s": 1.416904667019844, "p95_s": 5.056547989696265, "cost_usd_per_1000": 0.013077672454885936, "axes": { "intelligence": 61.470845845845844, "calibration": 64.17703553553554, "speed": 71.44805235902912, "cost": 66.50408632702226 }, "jevbench_score": 65.79939368997783 }, "heldout": { "n": 303, "escalation": 0.26732673267326734, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7671232876712328, "hard": 0.4954128440366973 }, "accuracy": 0.6732673267326733, "ece_hard": 0.22927247706422016, "calibration": 60.98150229357798, "p50_s": 3.0606545895338058, "p95_s": 5.190079523622989, "cost_usd_per_1000": 0.013553269625838988, "axes": { "intelligence": 67.84183737589544, "calibration": 60.98150229357798, "speed": 67.9901166772053, "cost": 66.03867765475862 }, "jevbench_score": 65.65003604618202 } }, { "threshold": 0.5, "public": { "n": 231, "escalation": 0.26406926406926406, "tiers": { "easy": 0.9583333333333334, "standard": 0.5555555555555556, "hard": 0.5315315315315315 }, "accuracy": 0.6277056277056277, "ece_hard": 0.19311041041041044, "calibration": 65.11045895895896, "p50_s": 1.448715904355049, "p95_s": 5.056547989696265, "cost_usd_per_1000": 0.013423250255362614, "axes": { "intelligence": 62.386344678011355, "calibration": 65.11045895895896, "speed": 71.35162638750217, "cost": 66.1642693900054 }, "jevbench_score": 66.17469126079018 }, "heldout": { "n": 303, "escalation": 0.27722772277227725, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7671232876712328, "hard": 0.48623853211009177 }, "accuracy": 0.66996699669967, "ece_hard": 0.23291926605504584, "calibration": 60.61682339449541, "p50_s": 3.0606545895338058, "p95_s": 5.190673627704383, "cost_usd_per_1000": 0.013948460080839544, "axes": { "intelligence": 67.56660801809727, "calibration": 60.61682339449541, "speed": 67.98961957237918, "cost": 65.66421208420482 }, "jevbench_score": 65.39193065065288 } }, { "threshold": 0.51, "public": { "n": 231, "escalation": 0.2813852813852814, "tiers": { "easy": 0.9583333333333334, "standard": 0.5972222222222222, "hard": 0.5315315315315315 }, "accuracy": 0.6406926406926406, "ece_hard": 0.1972347347347348, "calibration": 64.69802652652652, "p50_s": 1.5303658336400985, "p95_s": 5.056547989696265, "cost_usd_per_1000": 0.014114405856315968, "axes": { "intelligence": 64.00671504838174, "calibration": 64.69802652652652, "speed": 71.11350605451048, "cost": 65.51012196069632 }, "jevbench_score": 66.27416548229517 }, "heldout": { "n": 303, "escalation": 0.29042904290429045, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7671232876712328, "hard": 0.4954128440366973 }, "accuracy": 0.6732673267326733, "ece_hard": 0.23115229357798164, "calibration": 62.19652064220185, "p50_s": 3.0780361533164977, "p95_s": 5.190673627704383, "cost_usd_per_1000": 0.014475380687506952, "axes": { "intelligence": 67.84183737589544, "calibration": 62.19652064220185, "speed": 67.96502560689913, "cost": 65.18110017866952 }, "jevbench_score": 65.75332451421096 } }, { "threshold": 0.52, "public": { "n": 231, "escalation": 0.2943722943722944, "tiers": { "easy": 0.9583333333333334, "standard": 0.5972222222222222, "hard": 0.5315315315315315 }, "accuracy": 0.6406926406926406, "ece_hard": 0.20055815815815822, "calibration": 64.36568418418418, "p50_s": 1.5417469084262847, "p95_s": 5.056547989696265, "cost_usd_per_1000": 0.014632772557030983, "axes": { "intelligence": 64.00671504838174, "calibration": 64.36568418418418, "speed": 71.0813278045247, "cost": 65.04020133284166 }, "jevbench_score": 66.06239805026384 }, "heldout": { "n": 303, "escalation": 0.32673267326732675, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7808219178082192, "hard": 0.48623853211009177 }, "accuracy": 0.6765676567656765, "ece_hard": 0.24421743119266054, "calibration": 60.89000688073395, "p50_s": 3.0780361533164977, "p95_s": 5.19665765836835, "cost_usd_per_1000": 0.015924412355842323, "axes": { "intelligence": 67.9501696619329, "calibration": 60.89000688073395, "speed": 67.96002175805128, "cost": 63.93809755217807 }, "jevbench_score": 65.11595988216928 } }, { "threshold": 0.53, "public": { "n": 231, "escalation": 0.30735930735930733, "tiers": { "easy": 0.9583333333333334, "standard": 0.6111111111111112, "hard": 0.5405405405405406 }, "accuracy": 0.6493506493506493, "ece_hard": 0.20003113113113116, "calibration": 64.41838688688689, "p50_s": 1.5417469084262847, "p95_s": 5.071261623501778, "cost_usd_per_1000": 0.015151139257745997, "axes": { "intelligence": 64.92221388054722, "calibration": 64.41838688688689, "speed": 71.06870897634727, "cost": 64.58664130247331 }, "jevbench_score": 66.19208683501327 }, "heldout": { "n": 303, "escalation": 0.3465346534653465, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7945205479452054, "hard": 0.4954128440366973 }, "accuracy": 0.6864686468646864, "ece_hard": 0.24285688073394496, "calibration": 61.02606192660551, "p50_s": 3.0780361533164977, "p95_s": 5.281180657446384, "cost_usd_per_1000": 0.016714793265843436, "axes": { "intelligence": 68.60896066356668, "calibration": 61.02606192660551, "speed": 67.88995259443405, "cost": 63.30696971613038 }, "jevbench_score": 65.13108368668195 } }, { "threshold": 0.54, "public": { "n": 231, "escalation": 0.3246753246753247, "tiers": { "easy": 0.9583333333333334, "standard": 0.6111111111111112, "hard": 0.5495495495495496 }, "accuracy": 0.6536796536796536, "ece_hard": 0.20120860860860862, "calibration": 64.30063913913914, "p50_s": 1.5417469084262847, "p95_s": 5.071261623501778, "cost_usd_per_1000": 0.01584229485869935, "axes": { "intelligence": 65.2975892559226, "calibration": 64.30063913913914, "speed": 71.06870897634727, "cost": 64.0054572348935 }, "jevbench_score": 66.10768773675565 }, "heldout": { "n": 303, "escalation": 0.36303630363036304, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.8013698630136986, "hard": 0.5137614678899083 }, "accuracy": 0.6963696369636964, "ece_hard": 0.2241577981651376, "calibration": 62.89597018348624, "p50_s": 3.0780361533164977, "p95_s": 5.309960867464542, "cost_usd_per_1000": 0.017373444024177696, "axes": { "intelligence": 69.35120020108081, "calibration": 62.89597018348624, "speed": 67.866349629381, "cost": 62.80342241993066 }, "jevbench_score": 65.6641989765967 } }, { "threshold": 0.55, "public": { "n": 231, "escalation": 0.3463203463203463, "tiers": { "easy": 0.9791666666666666, "standard": 0.6111111111111112, "hard": 0.5585585585585585 }, "accuracy": 0.6623376623376623, "ece_hard": 0.19714554554554561, "calibration": 66.02394544544543, "p50_s": 1.5417469084262847, "p95_s": 5.071261623501778, "cost_usd_per_1000": 0.016706239359891042, "axes": { "intelligence": 66.07805722389055, "calibration": 66.02394544544543, "speed": 71.06870897634727, "cost": 63.31363901496965 }, "jevbench_score": 66.5631107831419 }, "heldout": { "n": 303, "escalation": 0.37293729372937295, "tiers": { "easy": 0.9166666666666666, "standard": 0.6666666666666666, "judge": 0.7876712328767124, "hard": 0.5229357798165137 }, "accuracy": 0.693069306930693, "ece_hard": 0.2175064220183486, "calibration": 63.561107798165146, "p50_s": 3.0780361533164977, "p95_s": 5.338054155558347, "cost_usd_per_1000": 0.01776863447917825, "axes": { "intelligence": 69.24286791504335, "calibration": 63.561107798165146, "speed": 67.8434330796185, "cost": 62.51037839422262 }, "jevbench_score": 65.7289375481791 } }, { "threshold": 0.56, "public": { "n": 231, "escalation": 0.354978354978355, "tiers": { "easy": 0.9791666666666666, "standard": 0.625, "hard": 0.5585585585585585 }, "accuracy": 0.6666666666666666, "ece_hard": 0.19714554554554561, "calibration": 66.02394544544543, "p50_s": 1.5832538619637488, "p95_s": 5.071261623501778, "cost_usd_per_1000": 0.01705181716036772, "axes": { "intelligence": 66.61818068068068, "calibration": 66.02394544544543, "speed": 70.95333427812263, "cost": 63.046879982839165 }, "jevbench_score": 66.60129326711187 }, "heldout": { "n": 303, "escalation": 0.39933993399339934, "tiers": { "easy": 0.9166666666666666, "standard": 0.7083333333333334, "judge": 0.7945205479452054, "hard": 0.5321100917431193 }, "accuracy": 0.7029702970297029, "ece_hard": 0.20954678899082568, "calibration": 64.35707110091744, "p50_s": 3.118002587556839, "p95_s": 5.371060359477997, "cost_usd_per_1000": 0.018822475692513066, "axes": { "intelligence": 70.876544761426, "calibration": 64.35707110091744, "speed": 67.76063495772257, "cost": 61.759697651143064 }, "jevbench_score": 66.09907153493513 } }, { "threshold": 0.57, "public": { "n": 231, "escalation": 0.36363636363636365, "tiers": { "easy": 0.9791666666666666, "standard": 0.6388888888888888, "hard": 0.5585585585585585 }, "accuracy": 0.670995670995671, "ece_hard": 0.20075365365365372, "calibration": 65.19313463463463, "p50_s": 1.600569875538349, "p95_s": 5.071261623501778, "cost_usd_per_1000": 0.017397394960844397, "axes": { "intelligence": 67.1583041374708, "calibration": 65.19313463463463, "speed": 70.90609345127447, "cost": 62.785473308124004 }, "jevbench_score": 66.44481229592837 }, "heldout": { "n": 303, "escalation": 0.40924092409240925, "tiers": { "easy": 0.9166666666666666, "standard": 0.7083333333333334, "judge": 0.8013698630136986, "hard": 0.5412844036697247 }, "accuracy": 0.7095709570957096, "ece_hard": 0.20431559633027518, "calibration": 64.88019036697249, "p50_s": 3.1302300930023192, "p95_s": 5.371060359477997, "cost_usd_per_1000": 0.019217666147513624, "axes": { "intelligence": 71.34355494114197, "calibration": 64.88019036697249, "speed": 67.74363704745622, "cost": 61.48898066568499 }, "jevbench_score": 66.26484189336807 } }, { "threshold": 0.58, "public": { "n": 231, "escalation": 0.36796536796536794, "tiers": { "easy": 0.9791666666666666, "standard": 0.6388888888888888, "hard": 0.5675675675675675 }, "accuracy": 0.6753246753246753, "ece_hard": 0.19558608608608616, "calibration": 65.70989139139138, "p50_s": 1.600569875538349, "p95_s": 5.094466378539801, "cost_usd_per_1000": 0.017570183861082733, "axes": { "intelligence": 67.53367951284618, "calibration": 65.70989139139138, "speed": 70.88626660843912, "cost": 62.656710815751495 }, "jevbench_score": 66.63006157342922 }, "heldout": { "n": 303, "escalation": 0.42244224422442245, "tiers": { "easy": 0.9166666666666666, "standard": 0.7083333333333334, "judge": 0.8082191780821918, "hard": 0.5504587155963303 }, "accuracy": 0.7161716171617162, "ece_hard": 0.19806788990825686, "calibration": 65.50496100917431, "p50_s": 3.1302300930023192, "p95_s": 5.371060359477997, "cost_usd_per_1000": 0.01974458675418103, "axes": { "intelligence": 71.81056512085794, "calibration": 65.50496100917431, "speed": 67.74363704745622, "cost": 61.13655854294383 }, "jevbench_score": 66.4366933155862 } }, { "threshold": 0.59, "public": { "n": 231, "escalation": 0.37662337662337664, "tiers": { "easy": 0.9791666666666666, "standard": 0.6388888888888888, "hard": 0.5675675675675675 }, "accuracy": 0.6753246753246753, "ece_hard": 0.19788788788788794, "calibration": 65.47971121121121, "p50_s": 1.600569875538349, "p95_s": 5.094466378539801, "cost_usd_per_1000": 0.01791576166155941, "axes": { "intelligence": 67.53367951284618, "calibration": 65.47971121121121, "speed": 70.88626660843912, "cost": 62.40294171193448 }, "jevbench_score": 66.50412498475238 }, "heldout": { "n": 303, "escalation": 0.43564356435643564, "tiers": { "easy": 0.9166666666666666, "standard": 0.7083333333333334, "judge": 0.8082191780821918, "hard": 0.5688073394495413 }, "accuracy": 0.7227722772277227, "ece_hard": 0.18613577981651375, "calibration": 66.69817201834863, "p50_s": 3.134873816370964, "p95_s": 5.371060359477997, "cost_usd_per_1000": 0.02027150736084844, "axes": { "intelligence": 72.36102383645428, "calibration": 66.69817201834863, "speed": 67.73719902557187, "cost": 60.793418697336364 }, "jevbench_score": 66.76911330556362 } }, { "threshold": 0.6, "public": { "n": 231, "escalation": 0.3939393939393939, "tiers": { "easy": 0.9791666666666666, "standard": 0.6666666666666666, "hard": 0.5765765765765766 }, "accuracy": 0.6883116883116883, "ece_hard": 0.18229419419419426, "calibration": 67.03908058058059, "p50_s": 1.600569875538349, "p95_s": 5.094466378539801, "cost_usd_per_1000": 0.018606917262512762, "axes": { "intelligence": 68.9893018018018, "calibration": 67.03908058058059, "speed": 70.88626660843912, "cost": 61.90976720496879 }, "jevbench_score": 67.12089958843899 }, "heldout": { "n": 303, "escalation": 0.44224422442244227, "tiers": { "easy": 0.9166666666666666, "standard": 0.7083333333333334, "judge": 0.8082191780821918, "hard": 0.5779816513761468 }, "accuracy": 0.7260726072607261, "ece_hard": 0.1859880733944954, "calibration": 66.71294266055045, "p50_s": 3.134873816370964, "p95_s": 5.371060359477997, "cost_usd_per_1000": 0.020534967664182145, "axes": { "intelligence": 72.63625319425245, "calibration": 66.71294266055045, "speed": 67.73719902557187, "cost": 60.62517930042136 }, "jevbench_score": 66.78992417574436 } }, { "threshold": 0.61, "public": { "n": 231, "escalation": 0.4199134199134199, "tiers": { "easy": 0.9791666666666666, "standard": 0.6666666666666666, "hard": 0.6036036036036037 }, "accuracy": 0.7012987012987013, "ece_hard": 0.16366806806806816, "calibration": 70.56069319319319, "p50_s": 1.6246334180235862, "p95_s": 5.094466378539801, "cost_usd_per_1000": 0.019643650663942795, "axes": { "intelligence": 70.11542792792794, "calibration": 70.56069319319319, "speed": 70.82145917341056, "cost": 61.20333393454989 }, "jevbench_score": 68.05015121323376 }, "heldout": { "n": 303, "escalation": 0.46534653465346537, "tiers": { "easy": 0.9166666666666666, "standard": 0.7083333333333334, "judge": 0.815068493150685, "hard": 0.5779816513761468 }, "accuracy": 0.7293729372937293, "ece_hard": 0.18801834862385322, "calibration": 66.50991513761468, "p50_s": 3.134873816370964, "p95_s": 5.371060359477997, "cost_usd_per_1000": 0.02145707872585011, "axes": { "intelligence": 72.82803401617025, "calibration": 66.50991513761468, "speed": 67.73719902557187, "cost": 60.05288216113367 }, "jevbench_score": 66.62489192045176 } }, { "threshold": 0.62, "public": { "n": 231, "escalation": 0.44155844155844154, "tiers": { "easy": 0.9791666666666666, "standard": 0.6666666666666666, "hard": 0.6216216216216216 }, "accuracy": 0.70995670995671, "ece_hard": 0.15622482482482491, "calibration": 71.30501751751751, "p50_s": 1.6246334180235862, "p95_s": 5.094466378539801, "cost_usd_per_1000": 0.020507595165134486, "axes": { "intelligence": 70.86617867867868, "calibration": 71.30501751751751, "speed": 70.82145917341056, "cost": 60.642557933092434 }, "jevbench_score": 68.25356999452754 }, "heldout": { "n": 303, "escalation": 0.4752475247524752, "tiers": { "easy": 0.9166666666666666, "standard": 0.75, "judge": 0.815068493150685, "hard": 0.5779816513761468 }, "accuracy": 0.7326732673267327, "ece_hard": 0.18827431192660551, "calibration": 66.48431880733946, "p50_s": 3.134873816370964, "p95_s": 5.371060359477997, "cost_usd_per_1000": 0.021852269180850665, "axes": { "intelligence": 73.99470068283692, "calibration": 66.48431880733946, "speed": 67.73719902557187, "cost": 59.81510375181035 }, "jevbench_score": 66.81738639984144 } }, { "threshold": 0.63, "public": { "n": 231, "escalation": 0.44155844155844154, "tiers": { "easy": 0.9791666666666666, "standard": 0.6666666666666666, "hard": 0.6216216216216216 }, "accuracy": 0.70995670995671, "ece_hard": 0.15622482482482491, "calibration": 71.30501751751751, "p50_s": 1.6246334180235862, "p95_s": 5.094466378539801, "cost_usd_per_1000": 0.020507595165134486, "axes": { "intelligence": 70.86617867867868, "calibration": 71.30501751751751, "speed": 70.82145917341056, "cost": 60.642557933092434 }, "jevbench_score": 68.25356999452754 }, "heldout": { "n": 303, "escalation": 0.49504950495049505, "tiers": { "easy": 0.9166666666666666, "standard": 0.7916666666666666, "judge": 0.815068493150685, "hard": 0.5779816513761468 }, "accuracy": 0.735973597359736, "ece_hard": 0.19052568807339448, "calibration": 66.25918119266055, "p50_s": 3.185190626978874, "p95_s": 5.371060359477997, "cost_usd_per_1000": 0.022642650090851778, "axes": { "intelligence": 75.16136734950358, "calibration": 66.25918119266055, "speed": 67.66804537972894, "cost": 59.35218234331272 }, "jevbench_score": 66.87522640050585 } }, { "threshold": 0.64, "public": { "n": 231, "escalation": 0.4588744588744589, "tiers": { "easy": 0.9791666666666666, "standard": 0.6666666666666666, "hard": 0.6126126126126126 }, "accuracy": 0.7056277056277056, "ece_hard": 0.16488248248248255, "calibration": 70.43925175175175, "p50_s": 1.6246334180235862, "p95_s": 5.106783525645733, "cost_usd_per_1000": 0.02119875076608784, "axes": { "intelligence": 70.49080330330331, "calibration": 70.43925175175175, "speed": 70.81097169065967, "cost": 60.2106919335243 }, "jevbench_score": 67.8313313879146 }, "heldout": { "n": 303, "escalation": 0.5148514851485149, "tiers": { "easy": 0.9166666666666666, "standard": 0.7916666666666666, "judge": 0.821917808219178, "hard": 0.5871559633027523 }, "accuracy": 0.7425742574257426, "ece_hard": 0.18019633027522927, "calibration": 67.29211697247707, "p50_s": 3.221156308054924, "p95_s": 5.371060359477997, "cost_usd_per_1000": 0.02343303100085289, "axes": { "intelligence": 75.62837752921956, "calibration": 67.29211697247707, "speed": 67.61928167370385, "cost": 58.905145987662976 }, "jevbench_score": 67.09933114070962 } }, { "threshold": 0.65, "public": { "n": 231, "escalation": 0.48917748917748916, "tiers": { "easy": 0.9791666666666666, "standard": 0.6666666666666666, "hard": 0.6216216216216216 }, "accuracy": 0.70995670995671, "ece_hard": 0.16041041041041051, "calibration": 70.88645895895895, "p50_s": 1.6485146522521972, "p95_s": 5.106783525645733, "cost_usd_per_1000": 0.02240827306775621, "axes": { "intelligence": 70.86617867867868, "calibration": 70.88645895895895, "speed": 70.74759739737742, "cost": 59.48774835487623 }, "jevbench_score": 67.80869416302681 }, "heldout": { "n": 303, "escalation": 0.528052805280528, "tiers": { "easy": 0.9583333333333334, "standard": 0.7916666666666666, "judge": 0.8356164383561644, "hard": 0.5963302752293578 }, "accuracy": 0.7557755775577558, "ece_hard": 0.1718211009174311, "calibration": 68.1296399082569, "p50_s": 3.221156308054924, "p95_s": 5.371060359477997, "cost_usd_per_1000": 0.023959951607520295, "axes": { "intelligence": 76.87050186418666, "calibration": 68.1296399082569, "speed": 67.61928167370385, "cost": 58.61542190309964 }, "jevbench_score": 67.49856882244008 } }, { "threshold": 0.66, "public": { "n": 231, "escalation": 0.5151515151515151, "tiers": { "easy": 0.9791666666666666, "standard": 0.6944444444444444, "hard": 0.6036036036036037 }, "accuracy": 0.70995670995671, "ece_hard": 0.1806347347347348, "calibration": 68.64902652652653, "p50_s": 1.6782644554972648, "p95_s": 5.106783525645733, "cost_usd_per_1000": 0.02344500646918624, "axes": { "intelligence": 71.19567484150818, "calibration": 68.64902652652653, "speed": 70.66992154702199, "cost": 58.89848929270405 }, "jevbench_score": 67.15938178194226 }, "heldout": { "n": 303, "escalation": 0.5478547854785478, "tiers": { "easy": 0.9583333333333334, "standard": 0.7916666666666666, "judge": 0.8493150684931506, "hard": 0.6055045871559633 }, "accuracy": 0.7656765676567657, "ece_hard": 0.16507981651376144, "calibration": 68.80376834862386, "p50_s": 3.224906811118126, "p95_s": 5.371060359477997, "cost_usd_per_1000": 0.024750332517521408, "axes": { "intelligence": 77.52929286582047, "calibration": 68.80376834862386, "speed": 67.61422797522653, "cost": 58.192568860028 }, "jevbench_score": 67.68554304709056 } }, { "threshold": 0.67, "public": { "n": 231, "escalation": 0.5194805194805194, "tiers": { "easy": 0.9791666666666666, "standard": 0.6944444444444444, "hard": 0.6126126126126126 }, "accuracy": 0.7142857142857143, "ece_hard": 0.17351581581581588, "calibration": 69.36091841841842, "p50_s": 1.6782644554972648, "p95_s": 5.106783525645733, "cost_usd_per_1000": 0.023617795369424575, "axes": { "intelligence": 71.57105021688355, "calibration": 69.36091841841842, "speed": 70.66992154702199, "cost": 58.80281933661971 }, "jevbench_score": 67.39400190096654 }, "heldout": { "n": 303, "escalation": 0.570957095709571, "tiers": { "easy": 0.9583333333333334, "standard": 0.7916666666666666, "judge": 0.8493150684931506, "hard": 0.6238532110091743 }, "accuracy": 0.7722772277227723, "ece_hard": 0.15095229357798157, "calibration": 70.21652064220186, "p50_s": 3.224906811118126, "p95_s": 5.371060359477997, "cost_usd_per_1000": 0.025672443579189377, "axes": { "intelligence": 78.07975158141679, "calibration": 70.21652064220186, "speed": 67.61422797522653, "cost": 57.71598375779501 }, "jevbench_score": 68.01081530561997 } }, { "threshold": 0.68, "public": { "n": 231, "escalation": 0.5238095238095238, "tiers": { "easy": 0.9791666666666666, "standard": 0.6944444444444444, "hard": 0.6126126126126126 }, "accuracy": 0.7142857142857143, "ece_hard": 0.17351581581581588, "calibration": 69.36091841841842, "p50_s": 1.6782644554972648, "p95_s": 5.106783525645733, "cost_usd_per_1000": 0.023790584269662917, "axes": { "intelligence": 71.57105021688355, "calibration": 69.36091841841842, "speed": 70.66992154702199, "cost": 58.70784676161256 }, "jevbench_score": 67.36677335041655 }, "heldout": { "n": 303, "escalation": 0.5907590759075908, "tiers": { "easy": 0.9583333333333334, "standard": 0.7916666666666666, "judge": 0.8561643835616438, "hard": 0.6330275229357798 }, "accuracy": 0.7788778877887789, "ece_hard": 0.14001376146788982, "calibration": 73.26612385321101, "p50_s": 3.2389582335948943, "p95_s": 5.389467868208885, "cost_usd_per_1000": 0.02646282448919049, "axes": { "intelligence": 78.54676176113276, "calibration": 73.26612385321101, "speed": 67.58048765164467, "cost": 57.32091410762562 }, "jevbench_score": 68.71340697500204 } }, { "threshold": 0.69, "public": { "n": 231, "escalation": 0.5238095238095238, "tiers": { "easy": 0.9791666666666666, "standard": 0.6944444444444444, "hard": 0.6126126126126126 }, "accuracy": 0.7142857142857143, "ece_hard": 0.17351581581581588, "calibration": 69.36091841841842, "p50_s": 1.6782644554972648, "p95_s": 5.106783525645733, "cost_usd_per_1000": 0.023790584269662917, "axes": { "intelligence": 71.57105021688355, "calibration": 69.36091841841842, "speed": 70.66992154702199, "cost": 58.70784676161256 }, "jevbench_score": 67.36677335041655 }, "heldout": { "n": 303, "escalation": 0.594059405940594, "tiers": { "easy": 0.9583333333333334, "standard": 0.7916666666666666, "judge": 0.863013698630137, "hard": 0.6330275229357798 }, "accuracy": 0.7821782178217822, "ece_hard": 0.14001376146788982, "calibration": 73.26612385321101, "p50_s": 3.2389582335948943, "p95_s": 5.389467868208885, "cost_usd_per_1000": 0.02659455464085734, "axes": { "intelligence": 78.73854258305057, "calibration": 73.26612385321101, "speed": 67.58048765164467, "cost": 57.25621834263437 }, "jevbench_score": 68.73590297774292 } }, { "threshold": 0.7, "public": { "n": 231, "escalation": 0.5367965367965368, "tiers": { "easy": 0.9791666666666666, "standard": 0.7083333333333334, "hard": 0.6216216216216216 }, "accuracy": 0.7229437229437229, "ece_hard": 0.16647977977977982, "calibration": 70.06452202202203, "p50_s": 1.6782644554972648, "p95_s": 5.106783525645733, "cost_usd_per_1000": 0.024308950970377934, "axes": { "intelligence": 72.48654904904906, "calibration": 70.06452202202203, "speed": 70.66992154702199, "cost": 58.4270134695082 }, "jevbench_score": 67.67074709541883 }, "heldout": { "n": 303, "escalation": 0.6105610561056105, "tiers": { "easy": 0.9583333333333334, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.6330275229357798 }, "accuracy": 0.7887788778877888, "ece_hard": 0.1400174311926605, "calibration": 73.26575688073395, "p50_s": 3.2389582335948943, "p95_s": 5.389467868208885, "cost_usd_per_1000": 0.027253205399191596, "axes": { "intelligence": 79.12210422688617, "calibration": 73.26575688073395, "speed": 67.58048765164467, "cost": 56.93747231870733 }, "jevbench_score": 68.72339302094086 } }, { "threshold": 0.71, "public": { "n": 231, "escalation": 0.5454545454545454, "tiers": { "easy": 0.9791666666666666, "standard": 0.7083333333333334, "hard": 0.6216216216216216 }, "accuracy": 0.7229437229437229, "ece_hard": 0.16647977977977982, "calibration": 70.06452202202203, "p50_s": 1.6797415181994437, "p95_s": 5.106783525645733, "cost_usd_per_1000": 0.024654528770854608, "axes": { "intelligence": 72.48654904904906, "calibration": 70.06452202202203, "speed": 70.66610094498984, "cost": 58.243098815490065 }, "jevbench_score": 67.61651726209439 }, "heldout": { "n": 303, "escalation": 0.6303630363036303, "tiers": { "easy": 0.9583333333333334, "standard": 0.7916666666666666, "judge": 0.8904109589041096, "hard": 0.6422018348623854 }, "accuracy": 0.7986798679867987, "ece_hard": 0.13176146788990822, "calibration": 74.09135321100918, "p50_s": 3.2389582335948943, "p95_s": 5.389467868208885, "cost_usd_per_1000": 0.02804358630919271, "axes": { "intelligence": 79.78089522851997, "calibration": 74.09135321100918, "speed": 67.58048765164467, "cost": 56.564993442953345 }, "jevbench_score": 68.94596802514619 } }, { "threshold": 0.72, "public": { "n": 231, "escalation": 0.5497835497835498, "tiers": { "easy": 0.9791666666666666, "standard": 0.7083333333333334, "hard": 0.6216216216216216 }, "accuracy": 0.7229437229437229, "ece_hard": 0.16829419419419417, "calibration": 69.88308058058058, "p50_s": 1.6797415181994437, "p95_s": 5.106783525645733, "cost_usd_per_1000": 0.024827317671092947, "axes": { "intelligence": 72.48654904904906, "calibration": 69.88308058058058, "speed": 70.66610094498984, "cost": 58.15210596499745 }, "jevbench_score": 67.54629160155962 }, "heldout": { "n": 303, "escalation": 0.6501650165016502, "tiers": { "easy": 0.9583333333333334, "standard": 0.7916666666666666, "judge": 0.8904109589041096, "hard": 0.6513761467889908 }, "accuracy": 0.801980198019802, "ece_hard": 0.1266119266055045, "calibration": 74.60630733944956, "p50_s": 3.2389582335948943, "p95_s": 5.389467868208885, "cost_usd_per_1000": 0.02883396721919382, "axes": { "intelligence": 80.05612458631813, "calibration": 74.60630733944956, "speed": 67.58048765164467, "cost": 56.202867988634125 }, "jevbench_score": 69.01404404442958 } }, { "threshold": 0.73, "public": { "n": 231, "escalation": 0.5584415584415584, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "hard": 0.6216216216216216 }, "accuracy": 0.7272727272727273, "ece_hard": 0.16829419419419417, "calibration": 69.88308058058058, "p50_s": 1.6816040739417075, "p95_s": 5.106783525645733, "cost_usd_per_1000": 0.02517289547156962, "axes": { "intelligence": 72.89164164164164, "calibration": 69.88308058058058, "speed": 70.66128800480433, "cost": 57.972004826752695 }, "jevbench_score": 67.58688183463718 }, "heldout": { "n": 303, "escalation": 0.6666666666666666, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9041095890410958, "hard": 0.6513761467889908 }, "accuracy": 0.8118811881188119, "ece_hard": 0.1266119266055045, "calibration": 74.60630733944956, "p50_s": 3.2389582335948943, "p95_s": 5.389467868208885, "cost_usd_per_1000": 0.02949261797752808, "axes": { "intelligence": 81.02301956348708, "calibration": 74.60630733944956, "speed": 67.58048765164467, "cost": 55.908600238776955 }, "jevbench_score": 69.13070365990912 } }, { "threshold": 0.74, "public": { "n": 231, "escalation": 0.5627705627705628, "tiers": { "easy": 1.0, "standard": 0.7222222222222222, "hard": 0.6216216216216216 }, "accuracy": 0.7316017316017316, "ece_hard": 0.16829419419419417, "calibration": 69.88308058058058, "p50_s": 1.6816040739417075, "p95_s": 5.106783525645733, "cost_usd_per_1000": 0.025345684371807963, "axes": { "intelligence": 73.43176509843178, "calibration": 69.88308058058058, "speed": 70.66128800480433, "cost": 57.88287933023409 }, "jevbench_score": 67.68569945472125 }, "heldout": { "n": 303, "escalation": 0.6897689768976898, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9178082191780822, "hard": 0.6513761467889908 }, "accuracy": 0.8184818481848185, "ece_hard": 0.12363394495412836, "calibration": 74.90410550458716, "p50_s": 3.2389582335948943, "p95_s": 5.389467868208885, "cost_usd_per_1000": 0.03041472903919605, "axes": { "intelligence": 81.4065812073227, "calibration": 74.90410550458716, "speed": 67.58048765164467, "cost": 55.507481447680085 }, "jevbench_score": 69.15673725621053 } }, { "threshold": 0.75, "public": { "n": 231, "escalation": 0.5844155844155844, "tiers": { "easy": 1.0, "standard": 0.7361111111111112, "hard": 0.6306306306306306 }, "accuracy": 0.7402597402597403, "ece_hard": 0.15926716716716713, "calibration": 70.78578328328328, "p50_s": 1.6816445007920264, "p95_s": 5.106783525645733, "cost_usd_per_1000": 0.026209628872999654, "axes": { "intelligence": 74.34726393059726, "calibration": 70.78578328328328, "speed": 70.66118359885499, "cost": 57.4461738579234 }, "jevbench_score": 67.98502509272693 }, "heldout": { "n": 303, "escalation": 0.7029702970297029, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9178082191780822, "hard": 0.6513761467889908 }, "accuracy": 0.8184818481848185, "ece_hard": 0.1257201834862385, "calibration": 74.69548165137616, "p50_s": 3.2389582335948943, "p95_s": 5.389467868208885, "cost_usd_per_1000": 0.030941649645863455, "axes": { "intelligence": 81.4065812073227, "calibration": 74.69548165137616, "speed": 67.58048765164467, "cost": 55.28369607172036 }, "jevbench_score": 69.03877233972415 } }, { "threshold": 0.76, "public": { "n": 231, "escalation": 0.5974025974025974, "tiers": { "easy": 1.0, "standard": 0.7361111111111112, "hard": 0.6306306306306306 }, "accuracy": 0.7402597402597403, "ece_hard": 0.16027887887887882, "calibration": 70.68461211211212, "p50_s": 1.6816445007920264, "p95_s": 5.106783525645733, "cost_usd_per_1000": 0.026727995573714667, "axes": { "intelligence": 74.34726393059726, "calibration": 70.68461211211212, "speed": 70.66118359885499, "cost": 57.19100827873557 }, "jevbench_score": 67.88512666379778 }, "heldout": { "n": 303, "escalation": 0.7227722772277227, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9178082191780822, "hard": 0.6605504587155964 }, "accuracy": 0.8217821782178217, "ece_hard": 0.1164247706422018, "calibration": 75.62502293577981, "p50_s": 3.2389582335948943, "p95_s": 5.389467868208885, "cost_usd_per_1000": 0.03173203055586457, "axes": { "intelligence": 81.68181056512087, "calibration": 75.62502293577981, "speed": 67.58048765164467, "cost": 54.95506408565655 }, "jevbench_score": 69.20778961047992 } }, { "threshold": 0.77, "public": { "n": 231, "escalation": 0.6190476190476191, "tiers": { "easy": 1.0, "standard": 0.7361111111111112, "hard": 0.6486486486486487 }, "accuracy": 0.7489177489177489, "ece_hard": 0.16094374374374362, "calibration": 70.61812562562564, "p50_s": 1.6816445007920264, "p95_s": 5.106783525645733, "cost_usd_per_1000": 0.02759194007490636, "axes": { "intelligence": 75.09801468134802, "calibration": 70.61812562562564, "speed": 70.66118359885499, "cost": 56.77653285564548 }, "jevbench_score": 67.9162353329055 }, "heldout": { "n": 303, "escalation": 0.735973597359736, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9246575342465754, "hard": 0.6605504587155964 }, "accuracy": 0.8250825082508251, "ece_hard": 0.1164247706422018, "calibration": 75.62502293577981, "p50_s": 3.3206340238451957, "p95_s": 5.389467868208885, "cost_usd_per_1000": 0.03225895116253198, "axes": { "intelligence": 81.87359138703867, "calibration": 75.62502293577981, "speed": 67.47233099524641, "cost": 54.740492708904235 }, "jevbench_score": 69.15298706257295 } }, { "threshold": 0.78, "public": { "n": 231, "escalation": 0.6406926406926406, "tiers": { "easy": 1.0, "standard": 0.7361111111111112, "hard": 0.6396396396396397 }, "accuracy": 0.7445887445887446, "ece_hard": 0.18034734734734723, "calibration": 68.67776526526528, "p50_s": 1.6991425320506095, "p95_s": 5.118455225229264, "cost_usd_per_1000": 0.028455884576098052, "axes": { "intelligence": 74.72263930597265, "calibration": 68.67776526526528, "speed": 70.60631280386866, "cost": 56.37483728264273 }, "jevbench_score": 67.22786170981375 }, "heldout": { "n": 303, "escalation": 0.7557755775577558, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6513761467889908 }, "accuracy": 0.8250825082508251, "ece_hard": 0.12746880733944943, "calibration": 74.52061926605506, "p50_s": 3.3560002908110618, "p95_s": 5.389467868208885, "cost_usd_per_1000": 0.03304933207253309, "axes": { "intelligence": 81.79014285115831, "calibration": 74.52061926605506, "speed": 67.42632123188174, "cost": 54.425119395617465 }, "jevbench_score": 68.77040301121508 } }, { "threshold": 0.79, "public": { "n": 231, "escalation": 0.645021645021645, "tiers": { "easy": 1.0, "standard": 0.75, "hard": 0.6396396396396397 }, "accuracy": 0.7489177489177489, "ece_hard": 0.18034734734734723, "calibration": 68.67776526526528, "p50_s": 1.6991425320506095, "p95_s": 5.118455225229264, "cost_usd_per_1000": 0.028628673476336394, "axes": { "intelligence": 75.26276276276278, "calibration": 68.67776526526528, "speed": 70.60631280386866, "cost": 56.29596324281763 }, "jevbench_score": 67.3254515021412 }, "heldout": { "n": 303, "escalation": 0.7755775577557755, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9383561643835616, "hard": 0.6513761467889908 }, "accuracy": 0.8283828382838284, "ece_hard": 0.12942018348623846, "calibration": 74.32548165137615, "p50_s": 3.3560002908110618, "p95_s": 5.389467868208885, "cost_usd_per_1000": 0.0338397129825342, "axes": { "intelligence": 81.98192367307612, "calibration": 74.32548165137615, "speed": 67.42632123188174, "cost": 54.11719987486828 }, "jevbench_score": 68.6681195975285 } }, { "threshold": 0.8, "public": { "n": 231, "escalation": 0.658008658008658, "tiers": { "easy": 1.0, "standard": 0.75, "hard": 0.6396396396396397 }, "accuracy": 0.7489177489177489, "ece_hard": 0.18034734734734723, "calibration": 68.67776526526528, "p50_s": 1.7019745722413062, "p95_s": 5.118455225229264, "cost_usd_per_1000": 0.029147040177051407, "axes": { "intelligence": 75.26276276276278, "calibration": 68.67776526526528, "speed": 70.59908024072018, "cost": 56.06216621175116 }, "jevbench_score": 67.25371936212116 }, "heldout": { "n": 303, "escalation": 0.7821782178217822, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9383561643835616, "hard": 0.6513761467889908 }, "accuracy": 0.8283828382838284, "ece_hard": 0.13039449541284395, "calibration": 74.2280504587156, "p50_s": 3.3560002908110618, "p95_s": 5.389467868208885, "cost_usd_per_1000": 0.034103173285867906, "axes": { "intelligence": 81.98192367307612, "calibration": 74.2280504587156, "speed": 67.42632123188174, "cost": 54.01615624627588 }, "jevbench_score": 68.61353979058752 } }, { "threshold": 0.81, "public": { "n": 231, "escalation": 0.6623376623376623, "tiers": { "easy": 1.0, "standard": 0.75, "hard": 0.6486486486486487 }, "accuracy": 0.7532467532467533, "ece_hard": 0.1775968968968968, "calibration": 68.95281031031033, "p50_s": 1.7019745722413062, "p95_s": 5.118455225229264, "cost_usd_per_1000": 0.029319829077289746, "axes": { "intelligence": 75.63813813813815, "calibration": 68.95281031031033, "speed": 70.59908024072018, "cost": 55.98515697352578 }, "jevbench_score": 67.38157935676284 }, "heldout": { "n": 303, "escalation": 0.7887788778877888, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9383561643835616, "hard": 0.6513761467889908 }, "accuracy": 0.8283828382838284, "ece_hard": 0.13108715596330267, "calibration": 74.55078440366972, "p50_s": 3.3769724905490874, "p95_s": 5.389467868208885, "cost_usd_per_1000": 0.03436663358920161, "axes": { "intelligence": 81.98192367307612, "calibration": 74.55078440366972, "speed": 67.39926590172655, "cost": 53.91589022317236 }, "jevbench_score": 68.6492136890111 } }, { "threshold": 0.82, "public": { "n": 231, "escalation": 0.670995670995671, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.6486486486486487 }, "accuracy": 0.7575757575757576, "ece_hard": 0.1775968968968968, "calibration": 68.95281031031033, "p50_s": 1.7019745722413062, "p95_s": 5.118455225229264, "cost_usd_per_1000": 0.029665406877766424, "axes": { "intelligence": 76.17826159492827, "calibration": 68.95281031031033, "speed": 70.59908024072018, "cost": 55.83249072076637 }, "jevbench_score": 67.45548492189039 }, "heldout": { "n": 303, "escalation": 0.801980198019802, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9452054794520548, "hard": 0.6513761467889908 }, "accuracy": 0.8316831683168316, "ece_hard": 0.13108715596330267, "calibration": 74.55078440366972, "p50_s": 3.381727920472622, "p95_s": 5.389467868208885, "cost_usd_per_1000": 0.03489355419586902, "axes": { "intelligence": 82.17370449499393, "calibration": 74.55078440366972, "speed": 67.39315449808362, "cost": 53.71764375487283 }, "jevbench_score": 68.62454149142391 } }, { "threshold": 0.83, "public": { "n": 231, "escalation": 0.696969696969697, "tiers": { "easy": 1.0, "standard": 0.7777777777777778, "hard": 0.6666666666666666 }, "accuracy": 0.7705627705627706, "ece_hard": 0.16287077077077075, "calibration": 70.42542292292293, "p50_s": 1.7500840589404105, "p95_s": 5.118455225229264, "cost_usd_per_1000": 0.030702140279196453, "axes": { "intelligence": 77.46913580246914, "calibration": 70.42542292292293, "speed": 70.4780218249594, "cost": 55.384940449972305 }, "jevbench_score": 67.93223346256725 }, "heldout": { "n": 303, "escalation": 0.8250825082508251, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9452054794520548, "hard": 0.6605504587155964 }, "accuracy": 0.834983498349835, "ece_hard": 0.12310275229357784, "calibration": 75.34922477064222, "p50_s": 3.381727920472622, "p95_s": 5.409488943219185, "cost_usd_per_1000": 0.03581566525753699, "axes": { "intelligence": 82.4489338527921, "calibration": 75.34922477064222, "speed": 67.37705099380456, "cost": 53.37780932902305 }, "jevbench_score": 68.7518112896614 } }, { "threshold": 0.84, "public": { "n": 231, "escalation": 0.7056277056277056, "tiers": { "easy": 1.0, "standard": 0.7777777777777778, "hard": 0.6666666666666666 }, "accuracy": 0.7705627705627706, "ece_hard": 0.16258788788788792, "calibration": 70.45371121121121, "p50_s": 1.7500840589404105, "p95_s": 5.118455225229264, "cost_usd_per_1000": 0.031047718079673128, "axes": { "intelligence": 77.46913580246914, "calibration": 70.45371121121121, "speed": 70.4780218249594, "cost": 55.239109412278076 }, "jevbench_score": 67.89428827140779 }, "heldout": { "n": 303, "escalation": 0.834983498349835, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9452054794520548, "hard": 0.6605504587155964 }, "accuracy": 0.834983498349835, "ece_hard": 0.1237522935779816, "calibration": 75.28427064220185, "p50_s": 3.381727920472622, "p95_s": 5.409488943219185, "cost_usd_per_1000": 0.03621085571253754, "axes": { "intelligence": 82.4489338527921, "calibration": 75.28427064220185, "speed": 67.37705099380456, "cost": 53.23483636249708 }, "jevbench_score": 68.69091530173435 } }, { "threshold": 0.85, "public": { "n": 231, "escalation": 0.70995670995671, "tiers": { "easy": 1.0, "standard": 0.7777777777777778, "hard": 0.6666666666666666 }, "accuracy": 0.7705627705627706, "ece_hard": 0.1599743743743744, "calibration": 70.71506256256257, "p50_s": 1.7500840589404105, "p95_s": 5.118455225229264, "cost_usd_per_1000": 0.03122050697991147, "axes": { "intelligence": 77.46913580246914, "calibration": 70.71506256256257, "speed": 70.4780218249594, "cost": 55.16680146537946 }, "jevbench_score": 67.93491524243011 }, "heldout": { "n": 303, "escalation": 0.8481848184818482, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9452054794520548, "hard": 0.6788990825688074 }, "accuracy": 0.8415841584158416, "ece_hard": 0.10491743119266046, "calibration": 77.16775688073395, "p50_s": 3.381727920472622, "p95_s": 5.409488943219185, "cost_usd_per_1000": 0.036737776319204954, "axes": { "intelligence": 82.99939256838842, "calibration": 77.16775688073395, "speed": 67.37705099380456, "cost": 53.046614032244385 }, "jevbench_score": 69.1703724633579 } }, { "threshold": 0.86, "public": { "n": 231, "escalation": 0.7229437229437229, "tiers": { "easy": 1.0, "standard": 0.7777777777777778, "hard": 0.6846846846846847 }, "accuracy": 0.7792207792207793, "ece_hard": 0.14639239239239246, "calibration": 72.07326076076076, "p50_s": 1.7500840589404105, "p95_s": 5.118455225229264, "cost_usd_per_1000": 0.03173887368062648, "axes": { "intelligence": 78.2198865532199, "calibration": 72.07326076076076, "speed": 70.4780218249594, "cost": 54.95225467428578 }, "jevbench_score": 68.35694478112248 }, "heldout": { "n": 303, "escalation": 0.8547854785478548, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9452054794520548, "hard": 0.6788990825688074 }, "accuracy": 0.8415841584158416, "ece_hard": 0.10351376146788983, "calibration": 77.21312385321102, "p50_s": 3.381727920472622, "p95_s": 5.409488943219185, "cost_usd_per_1000": 0.03700123662253865, "axes": { "intelligence": 82.99939256838842, "calibration": 77.21312385321102, "speed": 67.37705099380456, "cost": 52.95351283255494 }, "jevbench_score": 69.15016215833674 } }, { "threshold": 0.87, "public": { "n": 231, "escalation": 0.7532467532467533, "tiers": { "easy": 1.0, "standard": 0.7777777777777778, "hard": 0.7027027027027027 }, "accuracy": 0.7878787878787878, "ece_hard": 0.13028158158158162, "calibration": 73.68434184184184, "p50_s": 1.7500840589404105, "p95_s": 5.118455225229264, "cost_usd_per_1000": 0.032948395982294855, "axes": { "intelligence": 78.97063730397065, "calibration": 73.68434184184184, "speed": 70.4780218249594, "cost": 54.46497169582709 }, "jevbench_score": 68.74687476439891 }, "heldout": { "n": 303, "escalation": 0.8646864686468647, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9452054794520548, "hard": 0.6788990825688074 }, "accuracy": 0.8415841584158416, "ece_hard": 0.10351376146788983, "calibration": 77.21312385321102, "p50_s": 3.381727920472622, "p95_s": 5.409488943219185, "cost_usd_per_1000": 0.037396427077539214, "axes": { "intelligence": 82.99939256838842, "calibration": 77.21312385321102, "speed": 67.37705099380456, "cost": 52.815096672995836 }, "jevbench_score": 69.10492957664549 } }, { "threshold": 0.88, "public": { "n": 231, "escalation": 0.7792207792207793, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "hard": 0.7027027027027027 }, "accuracy": 0.7922077922077922, "ece_hard": 0.11023023023023025, "calibration": 75.68947697697698, "p50_s": 1.7500840589404105, "p95_s": 5.118455225229264, "cost_usd_per_1000": 0.03398512938372489, "axes": { "intelligence": 79.51076076076076, "calibration": 75.68947697697698, "speed": 70.4780218249594, "cost": 54.061332170488335 }, "jevbench_score": 69.1991031003942 }, "heldout": { "n": 303, "escalation": 0.8712871287128713, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9452054794520548, "hard": 0.6788990825688074 }, "accuracy": 0.8415841584158416, "ece_hard": 0.10351376146788983, "calibration": 77.21312385321102, "p50_s": 3.381727920472622, "p95_s": 5.409488943219185, "cost_usd_per_1000": 0.03765988738087291, "axes": { "intelligence": 82.99939256838842, "calibration": 77.21312385321102, "speed": 67.37705099380456, "cost": 52.723629491294844 }, "jevbench_score": 69.07499049342344 } }, { "threshold": 0.89, "public": { "n": 231, "escalation": 0.7878787878787878, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "hard": 0.7027027027027027 }, "accuracy": 0.7922077922077922, "ece_hard": 0.11023023023023025, "calibration": 75.68947697697698, "p50_s": 1.7500840589404105, "p95_s": 5.118455225229264, "cost_usd_per_1000": 0.03433070718420156, "axes": { "intelligence": 79.51076076076076, "calibration": 75.68947697697698, "speed": 70.4780218249594, "cost": 53.92951751123874 }, "jevbench_score": 69.15688342214489 }, "heldout": { "n": 303, "escalation": 0.8877887788778878, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6788990825688074 }, "accuracy": 0.8448844884488449, "ece_hard": 0.10351376146788983, "calibration": 77.21312385321102, "p50_s": 3.381727920472622, "p95_s": 5.409488943219185, "cost_usd_per_1000": 0.03831853813920717, "axes": { "intelligence": 84.16605923505509, "calibration": 77.21312385321102, "speed": 67.37705099380456, "cost": 52.49773203126971 }, "jevbench_score": 69.24208938234158 } }, { "threshold": 0.9, "public": { "n": 231, "escalation": 0.8008658008658008, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "hard": 0.7027027027027027 }, "accuracy": 0.7922077922077922, "ece_hard": 0.1099860860860861, "calibration": 75.7138913913914, "p50_s": 1.7699098750948905, "p95_s": 5.118455225229264, "cost_usd_per_1000": 0.03484907388491658, "axes": { "intelligence": 79.51076076076076, "calibration": 75.7138913913914, "speed": 70.4290993906221, "cost": 53.73426276400752 }, "jevbench_score": 69.08777822478241 }, "heldout": { "n": 303, "escalation": 0.9042904290429042, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6880733944954128 }, "accuracy": 0.8481848184818482, "ece_hard": 0.0952990825688073, "calibration": 78.03459174311928, "p50_s": 3.393709273636341, "p95_s": 5.44839422404766, "cost_usd_per_1000": 0.03897718889754143, "axes": { "intelligence": 84.44128859285325, "calibration": 78.03459174311928, "speed": 67.33056840950667, "cost": 52.275684584804516 }, "jevbench_score": 69.3966499912734 } }, { "threshold": 0.91, "public": { "n": 231, "escalation": 0.8138528138528138, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.7027027027027027 }, "accuracy": 0.7965367965367965, "ece_hard": 0.1099860860860861, "calibration": 75.7138913913914, "p50_s": 1.7699098750948905, "p95_s": 5.118455225229264, "cost_usd_per_1000": 0.03536744058563159, "axes": { "intelligence": 80.0508842175509, "calibration": 75.7138913913914, "speed": 70.4290993906221, "cost": 53.54189102129553 }, "jevbench_score": 69.14278769965603 }, "heldout": { "n": 303, "escalation": 0.9108910891089109, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6880733944954128 }, "accuracy": 0.8481848184818482, "ece_hard": 0.0952990825688073, "calibration": 78.03459174311928, "p50_s": 3.393709273636341, "p95_s": 5.44839422404766, "cost_usd_per_1000": 0.03924064920087514, "axes": { "intelligence": 84.44128859285325, "calibration": 78.03459174311928, "speed": 67.33056840950667, "cost": 52.18791448675545 }, "jevbench_score": 69.36750264694055 } }, { "threshold": 0.92, "public": { "n": 231, "escalation": 0.8225108225108225, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.7027027027027027 }, "accuracy": 0.7965367965367965, "ece_hard": 0.09483563563563568, "calibration": 77.93743643643643, "p50_s": 1.7699098750948905, "p95_s": 5.118455225229264, "cost_usd_per_1000": 0.03571301838610827, "axes": { "intelligence": 80.0508842175509, "calibration": 77.93743643643643, "speed": 70.4290993906221, "cost": 53.415203279122466 }, "jevbench_score": 69.60369857492185 }, "heldout": { "n": 303, "escalation": 0.9207920792079208, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6880733944954128 }, "accuracy": 0.8481848184818482, "ece_hard": 0.09555688073394496, "calibration": 78.0088119266055, "p50_s": 3.393709273636341, "p95_s": 5.44839422404766, "cost_usd_per_1000": 0.0396358396558757, "axes": { "intelligence": 84.44128859285325, "calibration": 78.0088119266055, "speed": 67.33056840950667, "cost": 52.057358114995985 }, "jevbench_score": 69.31835217353755 } }, { "threshold": 0.93, "public": { "n": 231, "escalation": 0.8398268398268398, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.7027027027027027 }, "accuracy": 0.7965367965367965, "ece_hard": 0.09483563563563568, "calibration": 77.93743643643643, "p50_s": 1.7699098750948905, "p95_s": 5.118455225229264, "cost_usd_per_1000": 0.036404173987061625, "axes": { "intelligence": 80.0508842175509, "calibration": 77.93743643643643, "speed": 70.4290993906221, "cost": 53.16546456006985 }, "jevbench_score": 69.52219884394702 }, "heldout": { "n": 303, "escalation": 0.9240924092409241, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6880733944954128 }, "accuracy": 0.8481848184818482, "ece_hard": 0.09555688073394496, "calibration": 78.0088119266055, "p50_s": 3.393709273636341, "p95_s": 5.44839422404766, "cost_usd_per_1000": 0.03976756980754255, "axes": { "intelligence": 84.44128859285325, "calibration": 78.0088119266055, "speed": 67.33056840950667, "cost": 52.014128436944496 }, "jevbench_score": 69.30395678458746 } }, { "threshold": 0.94, "public": { "n": 231, "escalation": 0.8528138528138528, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.7027027027027027 }, "accuracy": 0.7965367965367965, "ece_hard": 0.09483563563563568, "calibration": 77.93743643643643, "p50_s": 1.7860921561717986, "p95_s": 5.118455225229264, "cost_usd_per_1000": 0.03692254068777664, "axes": { "intelligence": 80.0508842175509, "calibration": 77.93743643643643, "speed": 70.38957228221139, "cost": 52.98125266758744 }, "jevbench_score": 69.45215081630607 }, "heldout": { "n": 303, "escalation": 0.9405940594059405, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6972477064220184 }, "accuracy": 0.8514851485148515, "ece_hard": 0.08064220183486236, "calibration": 81.94277981651376, "p50_s": 3.443461710214615, "p95_s": 5.44839422404766, "cost_usd_per_1000": 0.0404262205658768, "axes": { "intelligence": 84.71651795065142, "calibration": 81.94277981651376, "speed": 67.26736217960183, "cost": 51.80010576452236 }, "jevbench_score": 70.12993856665355 } }, { "threshold": 0.95, "public": { "n": 231, "escalation": 0.8614718614718615, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.7027027027027027 }, "accuracy": 0.7965367965367965, "ece_hard": 0.09483563563563568, "calibration": 77.93743643643643, "p50_s": 1.7860921561717986, "p95_s": 5.118455225229264, "cost_usd_per_1000": 0.037268118488253316, "axes": { "intelligence": 80.0508842175509, "calibration": 77.93743643643643, "speed": 70.38957228221139, "cost": 52.85987597225115 }, "jevbench_score": 69.41233897752889 }, "heldout": { "n": 303, "escalation": 0.9471947194719472, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6972477064220184 }, "accuracy": 0.8514851485148515, "ece_hard": 0.08064220183486236, "calibration": 81.94277981651376, "p50_s": 3.443461710214615, "p95_s": 5.44839422404766, "cost_usd_per_1000": 0.040689680869210515, "axes": { "intelligence": 84.71651795065142, "calibration": 81.94277981651376, "speed": 67.26736217960183, "cost": 51.71547148966887 }, "jevbench_score": 70.10127532124326 } }, { "threshold": 0.96, "public": { "n": 231, "escalation": 0.8874458874458875, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.7117117117117117 }, "accuracy": 0.8051948051948052, "ece_hard": 0.09203203203203206, "calibration": 78.2177967967968, "p50_s": 1.7860921561717986, "p95_s": 5.118455225229264, "cost_usd_per_1000": 0.03830485188968335, "axes": { "intelligence": 80.9663830497164, "calibration": 78.2177967967968, "speed": 70.38957228221139, "cost": 52.50238637718715 }, "jevbench_score": 69.55437009803765 }, "heldout": { "n": 303, "escalation": 0.9537953795379538, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6972477064220184 }, "accuracy": 0.8514851485148515, "ece_hard": 0.08064220183486236, "calibration": 81.94277981651376, "p50_s": 3.443461710214615, "p95_s": 5.44839422404766, "cost_usd_per_1000": 0.040953141172544213, "axes": { "intelligence": 84.71651795065142, "calibration": 81.94277981651376, "speed": 67.26736217960183, "cost": 51.63138344598353 }, "jevbench_score": 70.07276220626505 } }, { "threshold": 0.97, "public": { "n": 231, "escalation": 0.8961038961038961, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.7117117117117117 }, "accuracy": 0.8051948051948052, "ece_hard": 0.09203203203203206, "calibration": 78.2177967967968, "p50_s": 1.7860921561717986, "p95_s": 5.118455225229264, "cost_usd_per_1000": 0.03865042969016002, "axes": { "intelligence": 80.9663830497164, "calibration": 78.2177967967968, "speed": 70.38957228221139, "cost": 52.38537020551734 }, "jevbench_score": 69.51558234937795 }, "heldout": { "n": 303, "escalation": 0.9603960396039604, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6972477064220184 }, "accuracy": 0.8514851485148515, "ece_hard": 0.08064220183486236, "calibration": 81.94277981651376, "p50_s": 3.443461710214615, "p95_s": 5.44839422404766, "cost_usd_per_1000": 0.04121660147587792, "axes": { "intelligence": 84.71651795065142, "calibration": 81.94277981651376, "speed": 67.26736217960183, "cost": 51.54783462788429 }, "jevbench_score": 70.04439742094408 } }, { "threshold": 0.98, "public": { "n": 231, "escalation": 0.9090909090909091, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.7117117117117117 }, "accuracy": 0.8051948051948052, "ece_hard": 0.09203203203203206, "calibration": 78.2177967967968, "p50_s": 1.7860921561717986, "p95_s": 5.118455225229264, "cost_usd_per_1000": 0.03916879639087504, "axes": { "intelligence": 80.9663830497164, "calibration": 78.2177967967968, "speed": 70.38957228221139, "cost": 52.21179320753329 }, "jevbench_score": 69.45792632791425 }, "heldout": { "n": 303, "escalation": 0.9702970297029703, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6972477064220184 }, "accuracy": 0.8514851485148515, "ece_hard": 0.08064220183486236, "calibration": 81.94277981651376, "p50_s": 3.443461710214615, "p95_s": 5.44839422404766, "cost_usd_per_1000": 0.041611791930878474, "axes": { "intelligence": 84.71651795065142, "calibration": 81.94277981651376, "speed": 67.26736217960183, "cost": 51.42350745271782 }, "jevbench_score": 70.00212450295795 } }, { "threshold": 0.99, "public": { "n": 231, "escalation": 0.9393939393939394, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.7117117117117117 }, "accuracy": 0.8051948051948052, "ece_hard": 0.09203203203203206, "calibration": 78.2177967967968, "p50_s": 1.8066082835197448, "p95_s": 5.118455225229264, "cost_usd_per_1000": 0.04037831869254341, "axes": { "intelligence": 80.9663830497164, "calibration": 78.2177967967968, "speed": 70.33997094249276, "cost": 51.81555305638695 }, "jevbench_score": 69.31355280221626 }, "heldout": { "n": 303, "escalation": 0.9834983498349835, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6972477064220184 }, "accuracy": 0.8514851485148515, "ece_hard": 0.08064220183486236, "calibration": 81.94277981651376, "p50_s": 3.443461710214615, "p95_s": 5.44839422404766, "cost_usd_per_1000": 0.042138712537545885, "axes": { "intelligence": 84.71651795065142, "calibration": 81.94277981651376, "speed": 67.26736217960183, "cost": 51.25956212523175 }, "jevbench_score": 69.94626353512778 } }, { "threshold": 1.0, "public": { "n": 231, "escalation": 1.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.7117117117117117 }, "accuracy": 0.8051948051948052, "ece_hard": 0.09203203203203206, "calibration": 78.2177967967968, "p50_s": 1.8066082835197448, "p95_s": 5.118455225229264, "cost_usd_per_1000": 0.042797363295880145, "axes": { "intelligence": 80.9663830497164, "calibration": 78.2177967967968, "speed": 70.33997094249276, "cost": 51.05748959875674 }, "jevbench_score": 69.05863500149036 }, "heldout": { "n": 303, "escalation": 1.0, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6972477064220184 }, "accuracy": 0.8514851485148515, "ece_hard": 0.08064220183486236, "calibration": 81.94277981651376, "p50_s": 3.443461710214615, "p95_s": 5.44839422404766, "cost_usd_per_1000": 0.042797363295880145, "axes": { "intelligence": 84.71651795065142, "calibration": 81.94277981651376, "speed": 67.26736217960183, "cost": 51.05748959875674 }, "jevbench_score": 69.87722684982927 } } ] }, { "cheap": "open-alternative-jev", "backend": "djev", "chosen": { "threshold": 1.0, "public": { "n": 231, "escalation": 1.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6486486486486487 }, "accuracy": 0.7748917748917749, "ece_hard": 0.21041641822205612, "calibration": 61.35164946854124, "p50_s": 0.7970052972435951, "p95_s": 1.5881473660469054, "cost_usd_per_1000": 0.04812265917602997, "axes": { "intelligence": 78.33875542208877, "calibration": 61.35164946854124, "speed": 78.97647993431897, "cost": 49.52951146891361 }, "jevbench_score": 65.84778672780581 }, "heldout": { "n": 303, "escalation": 1.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6788990825688074 }, "accuracy": 0.834983498349835, "ece_hard": 0.1935286337386598, "calibration": 64.09002035150621, "p50_s": 0.8297482892870903, "p95_s": 1.4239212729036808, "cost_usd_per_1000": 0.04812265917602997, "axes": { "intelligence": 82.6158309245528, "calibration": 64.09002035150621, "speed": 79.27567656030779, "cost": 49.52951146891361 }, "jevbench_score": 67.52497336377385 } }, "full": { "tiers": { "easy": 1.0, "standard": 0.8125, "judge": 0.9315068493150684, "hard": 0.6636363636363637 }, "accuracy": 0.8089887640449438, "ece_hard": 0.20204928863710092, "calibration": 62.713158644349434, "p50_s": 0.8128920622169972, "p95_s": 1.4653228674083927, "cost_usd_per_1000": 0.04812265917602997, "axes": { "intelligence": 82.74128268991284, "calibration": 62.713158644349434, "speed": 79.24033790117065, "cost": 49.52951146891361 }, "jevbench_score": 67.1773428904274 }, "full_escalation": 1.0, "backend_heldout_accuracy": 0.834983498349835, "useful": false, "curve": [ { "threshold": 0.0, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 63.4379581919484, "calibration": 59.97952523584675, "speed": 81.86491236780878, "cost": 59.62620384142349 }, "jevbench_score": 65.64804472134733 } }, { "threshold": 0.01, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 63.4379581919484, "calibration": 59.97952523584675, "speed": 81.86491236780878, "cost": 59.62620384142349 }, "jevbench_score": 65.64804472134733 } }, { "threshold": 0.02, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 63.4379581919484, "calibration": 59.97952523584675, "speed": 81.86491236780878, "cost": 59.62620384142349 }, "jevbench_score": 65.64804472134733 } }, { "threshold": 0.03, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 63.4379581919484, "calibration": 59.97952523584675, "speed": 81.86491236780878, "cost": 59.62620384142349 }, "jevbench_score": 65.64804472134733 } }, { "threshold": 0.04, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 63.4379581919484, "calibration": 59.97952523584675, "speed": 81.86491236780878, "cost": 59.62620384142349 }, "jevbench_score": 65.64804472134733 } }, { "threshold": 0.05, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 63.4379581919484, "calibration": 59.97952523584675, "speed": 81.86491236780878, "cost": 59.62620384142349 }, "jevbench_score": 65.64804472134733 } }, { "threshold": 0.06, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 63.4379581919484, "calibration": 59.97952523584675, "speed": 81.86491236780878, "cost": 59.62620384142349 }, "jevbench_score": 65.64804472134733 } }, { "threshold": 0.07, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 63.4379581919484, "calibration": 59.97952523584675, "speed": 81.86491236780878, "cost": 59.62620384142349 }, "jevbench_score": 65.64804472134733 } }, { "threshold": 0.08, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 63.4379581919484, "calibration": 59.97952523584675, "speed": 81.86491236780878, "cost": 59.62620384142349 }, "jevbench_score": 65.64804472134733 } }, { "threshold": 0.09, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 63.4379581919484, "calibration": 59.97952523584675, "speed": 81.86491236780878, "cost": 59.62620384142349 }, "jevbench_score": 65.64804472134733 } }, { "threshold": 0.1, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 63.4379581919484, "calibration": 59.97952523584675, "speed": 81.86491236780878, "cost": 59.62620384142349 }, "jevbench_score": 65.64804472134733 } }, { "threshold": 0.11, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 63.4379581919484, "calibration": 59.97952523584675, "speed": 81.86491236780878, "cost": 59.62620384142349 }, "jevbench_score": 65.64804472134733 } }, { "threshold": 0.12, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 63.4379581919484, "calibration": 59.97952523584675, "speed": 81.86491236780878, "cost": 59.62620384142349 }, "jevbench_score": 65.64804472134733 } }, { "threshold": 0.13, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 63.4379581919484, "calibration": 59.97952523584675, "speed": 81.86491236780878, "cost": 59.62620384142349 }, "jevbench_score": 65.64804472134733 } }, { "threshold": 0.14, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 63.4379581919484, "calibration": 59.97952523584675, "speed": 81.86491236780878, "cost": 59.62620384142349 }, "jevbench_score": 65.64804472134733 } }, { "threshold": 0.15, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 63.4379581919484, "calibration": 59.97952523584675, "speed": 81.86491236780878, "cost": 59.62620384142349 }, "jevbench_score": 65.64804472134733 } }, { "threshold": 0.16, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 63.4379581919484, "calibration": 59.97952523584675, "speed": 81.86491236780878, "cost": 59.62620384142349 }, "jevbench_score": 65.64804472134733 } }, { "threshold": 0.17, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 63.4379581919484, "calibration": 59.97952523584675, "speed": 81.86491236780878, "cost": 59.62620384142349 }, "jevbench_score": 65.64804472134733 } }, { "threshold": 0.18, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 63.4379581919484, "calibration": 59.97952523584675, "speed": 81.86491236780878, "cost": 59.62620384142349 }, "jevbench_score": 65.64804472134733 } }, { "threshold": 0.19, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 63.4379581919484, "calibration": 59.97952523584675, "speed": 81.86491236780878, "cost": 59.62620384142349 }, "jevbench_score": 65.64804472134733 } }, { "threshold": 0.2, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 63.4379581919484, "calibration": 59.97952523584675, "speed": 81.86491236780878, "cost": 59.62620384142349 }, "jevbench_score": 65.64804472134733 } }, { "threshold": 0.21, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 63.4379581919484, "calibration": 59.97952523584675, "speed": 81.86491236780878, "cost": 59.62620384142349 }, "jevbench_score": 65.64804472134733 } }, { "threshold": 0.22, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 63.4379581919484, "calibration": 59.97952523584675, "speed": 81.86491236780878, "cost": 59.62620384142349 }, "jevbench_score": 65.64804472134733 } }, { "threshold": 0.23, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 63.4379581919484, "calibration": 59.97952523584675, "speed": 81.86491236780878, "cost": 59.62620384142349 }, "jevbench_score": 65.64804472134733 } }, { "threshold": 0.24, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 63.4379581919484, "calibration": 59.97952523584675, "speed": 81.86491236780878, "cost": 59.62620384142349 }, "jevbench_score": 65.64804472134733 } }, { "threshold": 0.25, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 63.4379581919484, "calibration": 59.97952523584675, "speed": 81.86491236780878, "cost": 59.62620384142349 }, "jevbench_score": 65.64804472134733 } }, { "threshold": 0.26, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 63.4379581919484, "calibration": 59.97952523584675, "speed": 81.86491236780878, "cost": 59.62620384142349 }, "jevbench_score": 65.64804472134733 } }, { "threshold": 0.27, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 63.4379581919484, "calibration": 59.97952523584675, "speed": 81.86491236780878, "cost": 59.62620384142349 }, "jevbench_score": 65.64804472134733 } }, { "threshold": 0.28, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 63.4379581919484, "calibration": 59.97952523584675, "speed": 81.86491236780878, "cost": 59.62620384142349 }, "jevbench_score": 65.64804472134733 } }, { "threshold": 0.29, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 63.4379581919484, "calibration": 59.97952523584675, "speed": 81.86491236780878, "cost": 59.62620384142349 }, "jevbench_score": 65.64804472134733 } }, { "threshold": 0.3, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 63.4379581919484, "calibration": 59.97952523584675, "speed": 81.86491236780878, "cost": 59.62620384142349 }, "jevbench_score": 65.64804472134733 } }, { "threshold": 0.31, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.0033003300330033004, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.27933610805262254, "calibration": 59.279361007745976, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022257052199601984, "axes": { "intelligence": 63.4379581919484, "calibration": 59.279361007745976, "speed": 81.86491236780878, "cost": 59.57597066933958 }, "jevbench_score": 65.44182611556691 } }, { "threshold": 0.32, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.006600660066006601, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5045871559633027 }, "accuracy": 0.5610561056105611, "ece_hard": 0.2852177387289042, "calibration": 58.69119794011781, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022342699904821944, "axes": { "intelligence": 63.16272883415023, "calibration": 58.69119794011781, "speed": 81.86491236780878, "cost": 59.52593042956179 }, "jevbench_score": 65.1942754517111 } }, { "threshold": 0.33, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.006600660066006601, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5045871559633027 }, "accuracy": 0.5610561056105611, "ece_hard": 0.2852177387289042, "calibration": 58.69119794011781, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022342699904821944, "axes": { "intelligence": 63.16272883415023, "calibration": 58.69119794011781, "speed": 81.86491236780878, "cost": 59.52593042956179 }, "jevbench_score": 65.1942754517111 } }, { "threshold": 0.34, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.013201320132013201, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5045871559633027 }, "accuracy": 0.5610561056105611, "ece_hard": 0.2877750155851336, "calibration": 58.435470254494874, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022513995315261864, "axes": { "intelligence": 63.16272883415023, "calibration": 58.435470254494874, "speed": 81.86491236780878, "cost": 59.426422858464164 }, "jevbench_score": 65.09591030073196 } }, { "threshold": 0.35, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.013201320132013201, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5045871559633027 }, "accuracy": 0.5610561056105611, "ece_hard": 0.2877750155851336, "calibration": 58.435470254494874, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022513995315261864, "axes": { "intelligence": 63.16272883415023, "calibration": 58.435470254494874, "speed": 81.86491236780878, "cost": 59.426422858464164 }, "jevbench_score": 65.09591030073196 } }, { "threshold": 0.36, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.0165016501650165, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5045871559633027 }, "accuracy": 0.5610561056105611, "ece_hard": 0.2943385794057959, "calibration": 57.77911387242864, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022599643020481824, "axes": { "intelligence": 63.16272883415023, "calibration": 57.77911387242864, "speed": 81.86491236780878, "cost": 59.37695262486027 }, "jevbench_score": 64.89882993124296 } }, { "threshold": 0.37, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.019801980198019802, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.29638039594731225, "calibration": 57.67462567634269, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022685290725701784, "axes": { "intelligence": 63.4379581919484, "calibration": 57.67462567634269, "speed": 81.86491236780878, "cost": 59.327669518486466 }, "jevbench_score": 64.9265412358167 } }, { "threshold": 0.38, "public": { "n": 231, "escalation": 0.004329004329004329, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.24100673876562734, "calibration": 57.15923831735846, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022283747588241972, "axes": { "intelligence": 66.27252252252254, "calibration": 57.15923831735846, "speed": 81.73312596625158, "cost": 59.56035308534193 }, "jevbench_score": 65.53055208136388 }, "heldout": { "n": 303, "escalation": 0.019801980198019802, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.29638039594731225, "calibration": 57.67462567634269, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022685290725701784, "axes": { "intelligence": 63.4379581919484, "calibration": 57.67462567634269, "speed": 81.86491236780878, "cost": 59.327669518486466 }, "jevbench_score": 64.9265412358167 } }, { "threshold": 0.39, "public": { "n": 231, "escalation": 0.008658008658008658, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.24044429994613517, "calibration": 57.21548219930767, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.02239609068210192, "axes": { "intelligence": 66.27252252252254, "calibration": 57.21548219930767, "speed": 81.73312596625158, "cost": 59.494833481261686 }, "jevbench_score": 65.52863276160195 }, "heldout": { "n": 303, "escalation": 0.019801980198019802, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.29638039594731225, "calibration": 57.67462567634269, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022685290725701784, "axes": { "intelligence": 63.4379581919484, "calibration": 57.67462567634269, "speed": 81.86491236780878, "cost": 59.327669518486466 }, "jevbench_score": 64.9265412358167 } }, { "threshold": 0.4, "public": { "n": 231, "escalation": 0.017316017316017316, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5495495495495496 }, "accuracy": 0.6666666666666666, "ece_hard": 0.23456480335928354, "calibration": 57.803431857992834, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022620776869821814, "axes": { "intelligence": 66.6478978978979, "calibration": 57.803431857992834, "speed": 81.73312596625158, "cost": 59.36477452279015 }, "jevbench_score": 65.75317873320142 }, "heldout": { "n": 303, "escalation": 0.019801980198019802, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.29638039594731225, "calibration": 57.67462567634269, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022685290725701784, "axes": { "intelligence": 63.4379581919484, "calibration": 57.67462567634269, "speed": 81.86491236780878, "cost": 59.327669518486466 }, "jevbench_score": 64.9265412358167 } }, { "threshold": 0.41, "public": { "n": 231, "escalation": 0.017316017316017316, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5495495495495496 }, "accuracy": 0.6666666666666666, "ece_hard": 0.23456480335928354, "calibration": 57.803431857992834, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022620776869821814, "axes": { "intelligence": 66.6478978978979, "calibration": 57.803431857992834, "speed": 81.73312596625158, "cost": 59.36477452279015 }, "jevbench_score": 65.75317873320142 }, "heldout": { "n": 303, "escalation": 0.0231023102310231, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5045871559633027 }, "accuracy": 0.5610561056105611, "ece_hard": 0.30920573881713476, "calibration": 56.392091389360445, "p50_s": 0.5870145052671433, "p95_s": 1.1089038744568824, "cost_usd_per_1000": 0.022770938430921744, "axes": { "intelligence": 63.16272883415023, "calibration": 56.392091389360445, "speed": 81.86457266219443, "cost": 59.27857212901238 }, "jevbench_score": 64.47898684488295 } }, { "threshold": 0.42, "public": { "n": 231, "escalation": 0.017316017316017316, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5495495495495496 }, "accuracy": 0.6666666666666666, "ece_hard": 0.23456480335928354, "calibration": 57.803431857992834, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022620776869821814, "axes": { "intelligence": 66.6478978978979, "calibration": 57.803431857992834, "speed": 81.73312596625158, "cost": 59.36477452279015 }, "jevbench_score": 65.75317873320142 }, "heldout": { "n": 303, "escalation": 0.0231023102310231, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5045871559633027 }, "accuracy": 0.5610561056105611, "ece_hard": 0.30920573881713476, "calibration": 56.392091389360445, "p50_s": 0.5870145052671433, "p95_s": 1.1089038744568824, "cost_usd_per_1000": 0.022770938430921744, "axes": { "intelligence": 63.16272883415023, "calibration": 56.392091389360445, "speed": 81.86457266219443, "cost": 59.27857212901238 }, "jevbench_score": 64.47898684488295 } }, { "threshold": 0.43, "public": { "n": 231, "escalation": 0.030303030303030304, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5495495495495496 }, "accuracy": 0.6666666666666666, "ece_hard": 0.239567964325421, "calibration": 57.30311576137909, "p50_s": 0.5530259400606156, "p95_s": 1.2135635629296302, "cost_usd_per_1000": 0.02295780615140166, "axes": { "intelligence": 66.6478978978979, "calibration": 57.30311576137909, "speed": 81.73191968991857, "cost": 59.17208846452324 }, "jevbench_score": 65.55688697785277 }, "heldout": { "n": 303, "escalation": 0.0231023102310231, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5045871559633027 }, "accuracy": 0.5610561056105611, "ece_hard": 0.30920573881713476, "calibration": 56.392091389360445, "p50_s": 0.5870145052671433, "p95_s": 1.1089038744568824, "cost_usd_per_1000": 0.022770938430921744, "axes": { "intelligence": 63.16272883415023, "calibration": 56.392091389360445, "speed": 81.86457266219443, "cost": 59.27857212901238 }, "jevbench_score": 64.47898684488295 } }, { "threshold": 0.44, "public": { "n": 231, "escalation": 0.030303030303030304, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5495495495495496 }, "accuracy": 0.6666666666666666, "ece_hard": 0.239567964325421, "calibration": 57.30311576137909, "p50_s": 0.5530259400606156, "p95_s": 1.2135635629296302, "cost_usd_per_1000": 0.02295780615140166, "axes": { "intelligence": 66.6478978978979, "calibration": 57.30311576137909, "speed": 81.73191968991857, "cost": 59.17208846452324 }, "jevbench_score": 65.55688697785277 }, "heldout": { "n": 303, "escalation": 0.026402640264026403, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5045871559633027 }, "accuracy": 0.5610561056105611, "ece_hard": 0.3143162286566719, "calibration": 55.88104240540673, "p50_s": 0.5878239214420319, "p95_s": 1.1089038744568824, "cost_usd_per_1000": 0.022856586136141704, "axes": { "intelligence": 63.16272883415023, "calibration": 55.88104240540673, "speed": 81.85858843447522, "cost": 59.22965906199177 }, "jevbench_score": 64.31795343456471 } }, { "threshold": 0.45, "public": { "n": 231, "escalation": 0.03463203463203463, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5495495495495496 }, "accuracy": 0.6666666666666666, "ece_hard": 0.2397812434851897, "calibration": 57.28178784540222, "p50_s": 0.5530259400606156, "p95_s": 1.2135635629296302, "cost_usd_per_1000": 0.023070149245261608, "axes": { "intelligence": 66.6478978978979, "calibration": 57.28178784540222, "speed": 81.73191968991857, "cost": 59.10848787804221 }, "jevbench_score": 65.53316487262794 }, "heldout": { "n": 303, "escalation": 0.0297029702970297, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5045871559633027 }, "accuracy": 0.5610561056105611, "ece_hard": 0.31929882881795, "calibration": 55.382782389278916, "p50_s": 0.5878239214420319, "p95_s": 1.1109892919659612, "cost_usd_per_1000": 0.022942233841361667, "axes": { "intelligence": 63.16272883415023, "calibration": 55.382782389278916, "speed": 81.85042871222566, "cost": 59.18092893862487 }, "jevbench_score": 64.15929717658179 } }, { "threshold": 0.46, "public": { "n": 231, "escalation": 0.04329004329004329, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.2516968282879506, "calibration": 56.090229365126135, "p50_s": 0.5530259400606156, "p95_s": 1.2245997205376624, "cost_usd_per_1000": 0.0232948354329815, "axes": { "intelligence": 66.27252252252254, "calibration": 56.090229365126135, "speed": 81.69260341346663, "cost": 58.98221059871426 }, "jevbench_score": 65.05506645465478 }, "heldout": { "n": 303, "escalation": 0.039603960396039604, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5045871559633027 }, "accuracy": 0.5610561056105611, "ece_hard": 0.3140727609314962, "calibration": 55.90538917792429, "p50_s": 0.5895052790641785, "p95_s": 1.1109892919659612, "cost_usd_per_1000": 0.023199176957021547, "axes": { "intelligence": 63.16272883415023, "calibration": 55.90538917792429, "speed": 81.83802428305651, "cost": 59.035822672278684 }, "jevbench_score": 64.26822846594878 } }, { "threshold": 0.47, "public": { "n": 231, "escalation": 0.047619047619047616, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.25355424436192187, "calibration": 55.904487757729, "p50_s": 0.5530259400606156, "p95_s": 1.2245997205376624, "cost_usd_per_1000": 0.02340717852684145, "axes": { "intelligence": 66.27252252252254, "calibration": 55.904487757729, "speed": 81.69260341346663, "cost": 58.91952797449864 }, "jevbench_score": 64.98386564688316 }, "heldout": { "n": 303, "escalation": 0.0462046204620462, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5136986301369864, "hard": 0.5045871559633027 }, "accuracy": 0.5643564356435643, "ece_hard": 0.3140727609314962, "calibration": 55.90538917792429, "p50_s": 0.5900527179241181, "p95_s": 1.1109892919659612, "cost_usd_per_1000": 0.023370472367461467, "axes": { "intelligence": 63.35450965606804, "calibration": 55.90538917792429, "speed": 81.83399311725155, "cost": 58.93997528483306 }, "jevbench_score": 64.29004436044627 } }, { "threshold": 0.48, "public": { "n": 231, "escalation": 0.05194805194805195, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.2582375323023387, "calibration": 55.43615896368732, "p50_s": 0.5530259400606156, "p95_s": 1.2804248012602328, "cost_usd_per_1000": 0.023519521620701398, "axes": { "intelligence": 66.27252252252254, "calibration": 55.43615896368732, "speed": 81.49900419692341, "cost": 58.85714547736994 }, "jevbench_score": 64.79172350350565 }, "heldout": { "n": 303, "escalation": 0.0462046204620462, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5136986301369864, "hard": 0.5045871559633027 }, "accuracy": 0.5643564356435643, "ece_hard": 0.3140727609314962, "calibration": 55.90538917792429, "p50_s": 0.5900527179241181, "p95_s": 1.1109892919659612, "cost_usd_per_1000": 0.023370472367461467, "axes": { "intelligence": 63.35450965606804, "calibration": 55.90538917792429, "speed": 81.83399311725155, "cost": 58.93997528483306 }, "jevbench_score": 64.29004436044627 } }, { "threshold": 0.49, "public": { "n": 231, "escalation": 0.05627705627705628, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5495495495495496 }, "accuracy": 0.6666666666666666, "ece_hard": 0.2499857186833643, "calibration": 56.26134032558477, "p50_s": 0.5530259400606156, "p95_s": 1.3686999656260013, "cost_usd_per_1000": 0.023631864714561347, "axes": { "intelligence": 66.6478978978979, "calibration": 56.26134032558477, "speed": 81.20946240988515, "cost": 58.79506024697553 }, "jevbench_score": 65.04830754981674 }, "heldout": { "n": 303, "escalation": 0.0627062706270627, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5205479452054794, "hard": 0.5045871559633027 }, "accuracy": 0.5676567656765676, "ece_hard": 0.31957796534055255, "calibration": 53.870712352012234, "p50_s": 0.5905896663665772, "p95_s": 1.1165145859122274, "cost_usd_per_1000": 0.023798710893561267, "axes": { "intelligence": 63.54629047798584, "calibration": 53.870712352012234, "speed": 81.80849755241792, "cost": 58.70339700297954 }, "jevbench_score": 63.67605409926032 } }, { "threshold": 0.5, "public": { "n": 231, "escalation": 0.06060606060606061, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5585585585585585 }, "accuracy": 0.670995670995671, "ece_hard": 0.24422013941522122, "calibration": 57.567901319016656, "p50_s": 0.5530259400606156, "p95_s": 1.3686999656260013, "cost_usd_per_1000": 0.02374420780842129, "axes": { "intelligence": 67.02327327327328, "calibration": 57.567901319016656, "speed": 81.20946240988515, "cost": 58.73326946365972 }, "jevbench_score": 65.49742269614522 }, "heldout": { "n": 303, "escalation": 0.06930693069306931, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5205479452054794, "hard": 0.5137614678899083 }, "accuracy": 0.570957095709571, "ece_hard": 0.316365686682092, "calibration": 55.42667769482901, "p50_s": 0.5923556029796601, "p95_s": 1.120798410475254, "cost_usd_per_1000": 0.023970006304001187, "axes": { "intelligence": 63.82151983578402, "calibration": 55.42667769482901, "speed": 81.77889992920498, "cost": 58.609955552466516 }, "jevbench_score": 64.1689098892973 } }, { "threshold": 0.51, "public": { "n": 231, "escalation": 0.06060606060606061, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5585585585585585 }, "accuracy": 0.670995670995671, "ece_hard": 0.24422013941522122, "calibration": 57.567901319016656, "p50_s": 0.5530259400606156, "p95_s": 1.3686999656260013, "cost_usd_per_1000": 0.02374420780842129, "axes": { "intelligence": 67.02327327327328, "calibration": 57.567901319016656, "speed": 81.20946240988515, "cost": 58.73326946365972 }, "jevbench_score": 65.49742269614522 }, "heldout": { "n": 303, "escalation": 0.0924092409240924, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.547945205479452, "hard": 0.5137614678899083 }, "accuracy": 0.5841584158415841, "ece_hard": 0.3109011668276338, "calibration": 55.97312968027483, "p50_s": 0.5930232793092728, "p95_s": 1.120798410475254, "cost_usd_per_1000": 0.024569540240540907, "axes": { "intelligence": 64.58864312345524, "calibration": 55.97312968027483, "speed": 81.77400751580626, "cost": 58.28808910640863 }, "jevbench_score": 64.42919613939127 } }, { "threshold": 0.52, "public": { "n": 231, "escalation": 0.06060606060606061, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5585585585585585 }, "accuracy": 0.670995670995671, "ece_hard": 0.24422013941522122, "calibration": 57.567901319016656, "p50_s": 0.5530259400606156, "p95_s": 1.3686999656260013, "cost_usd_per_1000": 0.02374420780842129, "axes": { "intelligence": 67.02327327327328, "calibration": 57.567901319016656, "speed": 81.20946240988515, "cost": 58.73326946365972 }, "jevbench_score": 65.49742269614522 }, "heldout": { "n": 303, "escalation": 0.0924092409240924, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.547945205479452, "hard": 0.5137614678899083 }, "accuracy": 0.5841584158415841, "ece_hard": 0.3109011668276338, "calibration": 55.97312968027483, "p50_s": 0.5930232793092728, "p95_s": 1.120798410475254, "cost_usd_per_1000": 0.024569540240540907, "axes": { "intelligence": 64.58864312345524, "calibration": 55.97312968027483, "speed": 81.77400751580626, "cost": 58.28808910640863 }, "jevbench_score": 64.42919613939127 } }, { "threshold": 0.53, "public": { "n": 231, "escalation": 0.06926406926406926, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5585585585585585 }, "accuracy": 0.670995670995671, "ece_hard": 0.250877932562358, "calibration": 56.902122004302974, "p50_s": 0.5530259400606156, "p95_s": 1.4122533962130546, "cost_usd_per_1000": 0.02396889399614119, "axes": { "intelligence": 67.02327327327328, "calibration": 56.902122004302974, "speed": 81.07341869528295, "cost": 58.610560158533204 }, "jevbench_score": 65.24573289024964 }, "heldout": { "n": 303, "escalation": 0.10561056105610561, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.547945205479452, "hard": 0.5229357798165137 }, "accuracy": 0.5874587458745875, "ece_hard": 0.309044022644981, "calibration": 56.403274951531145, "p50_s": 0.5963243931531906, "p95_s": 1.120798410475254, "cost_usd_per_1000": 0.02491213106142075, "axes": { "intelligence": 64.86387248125341, "calibration": 56.403274951531145, "speed": 81.74989918749694, "cost": 58.10767359908055 }, "jevbench_score": 64.56646018491533 } }, { "threshold": 0.54, "public": { "n": 231, "escalation": 0.08225108225108226, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5675675675675675 }, "accuracy": 0.6753246753246753, "ece_hard": 0.2472044747424858, "calibration": 57.60917146331262, "p50_s": 0.5530259400606156, "p95_s": 1.4225786752998828, "cost_usd_per_1000": 0.024305923277721034, "axes": { "intelligence": 67.39864864864866, "calibration": 57.60917146331262, "speed": 81.04178203075998, "cost": 58.428636318772945 }, "jevbench_score": 65.48161584696157 }, "heldout": { "n": 303, "escalation": 0.12871287128712872, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5684931506849316, "hard": 0.5229357798165137 }, "accuracy": 0.5973597359735974, "ece_hard": 0.3133258052649631, "calibration": 55.97509668953293, "p50_s": 0.5986908316612244, "p95_s": 1.120798410475254, "cost_usd_per_1000": 0.02551166499796047, "axes": { "intelligence": 65.43921494700685, "calibration": 55.97509668953293, "speed": 81.73269886221482, "cost": 57.79783589772764 }, "jevbench_score": 64.496342579052 } }, { "threshold": 0.55, "public": { "n": 231, "escalation": 0.08225108225108226, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5675675675675675 }, "accuracy": 0.6753246753246753, "ece_hard": 0.2472044747424858, "calibration": 57.60917146331262, "p50_s": 0.5530259400606156, "p95_s": 1.4225786752998828, "cost_usd_per_1000": 0.024305923277721034, "axes": { "intelligence": 67.39864864864866, "calibration": 57.60917146331262, "speed": 81.04178203075998, "cost": 58.428636318772945 }, "jevbench_score": 65.48161584696157 }, "heldout": { "n": 303, "escalation": 0.12871287128712872, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5684931506849316, "hard": 0.5229357798165137 }, "accuracy": 0.5973597359735974, "ece_hard": 0.3133258052649631, "calibration": 55.97509668953293, "p50_s": 0.5986908316612244, "p95_s": 1.120798410475254, "cost_usd_per_1000": 0.02551166499796047, "axes": { "intelligence": 65.43921494700685, "calibration": 55.97509668953293, "speed": 81.73269886221482, "cost": 57.79783589772764 }, "jevbench_score": 64.496342579052 } }, { "threshold": 0.56, "public": { "n": 231, "escalation": 0.09090909090909091, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5675675675675675 }, "accuracy": 0.6753246753246753, "ece_hard": 0.24855170156324782, "calibration": 57.47444878123642, "p50_s": 0.5530259400606156, "p95_s": 1.4364962704479693, "cost_usd_per_1000": 0.024530609465440927, "axes": { "intelligence": 67.39864864864866, "calibration": 57.47444878123642, "speed": 80.99949994559617, "cost": 58.30874984709447 }, "jevbench_score": 65.4011700252847 }, "heldout": { "n": 303, "escalation": 0.12871287128712872, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5684931506849316, "hard": 0.5229357798165137 }, "accuracy": 0.5973597359735974, "ece_hard": 0.3133258052649631, "calibration": 55.97509668953293, "p50_s": 0.5986908316612244, "p95_s": 1.120798410475254, "cost_usd_per_1000": 0.02551166499796047, "axes": { "intelligence": 65.43921494700685, "calibration": 55.97509668953293, "speed": 81.73269886221482, "cost": 57.79783589772764 }, "jevbench_score": 64.496342579052 } }, { "threshold": 0.57, "public": { "n": 231, "escalation": 0.11255411255411256, "tiers": { "easy": 1.0, "standard": 0.6388888888888888, "hard": 0.5585585585585585 }, "accuracy": 0.6753246753246753, "ece_hard": 0.2640169926404773, "calibration": 54.248400364363064, "p50_s": 0.5537255644798279, "p95_s": 1.4364962704479693, "cost_usd_per_1000": 0.025092324934740666, "axes": { "intelligence": 67.5633967300634, "calibration": 54.248400364363064, "speed": 80.99400922678721, "cost": 58.013772915185115 }, "jevbench_score": 64.41998286588482 }, "heldout": { "n": 303, "escalation": 0.13861386138613863, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5821917808219178, "hard": 0.5321100917431193 }, "accuracy": 0.6072607260726073, "ece_hard": 0.30815389021911027, "calibration": 56.49228819411822, "p50_s": 0.6008343935012818, "p95_s": 1.120798410475254, "cost_usd_per_1000": 0.025768608113620355, "axes": { "intelligence": 66.09800594864062, "calibration": 56.49228819411822, "speed": 81.71717708659592, "cost": 57.667271174438504 }, "jevbench_score": 64.76719223800568 } }, { "threshold": 0.58, "public": { "n": 231, "escalation": 0.12121212121212122, "tiers": { "easy": 1.0, "standard": 0.6388888888888888, "hard": 0.5585585585585585 }, "accuracy": 0.6753246753246753, "ece_hard": 0.2643256322526132, "calibration": 54.217536403149474, "p50_s": 0.5537255644798279, "p95_s": 1.4364962704479693, "cost_usd_per_1000": 0.025317011122460563, "axes": { "intelligence": 67.5633967300634, "calibration": 54.217536403149474, "speed": 80.99400922678721, "cost": 57.897627027937 }, "jevbench_score": 64.37855568240049 }, "heldout": { "n": 303, "escalation": 0.1419141914191419, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5821917808219178, "hard": 0.5412844036697247 }, "accuracy": 0.6105610561056105, "ece_hard": 0.2985951361222342, "calibration": 57.448163603805824, "p50_s": 0.6008917182683945, "p95_s": 1.120798410475254, "cost_usd_per_1000": 0.02585425581884031, "axes": { "intelligence": 66.37323530643879, "calibration": 57.448163603805824, "speed": 81.71676275208478, "cost": 57.62403874943091 }, "jevbench_score": 65.09475493501058 } }, { "threshold": 0.59, "public": { "n": 231, "escalation": 0.12121212121212122, "tiers": { "easy": 1.0, "standard": 0.6388888888888888, "hard": 0.5585585585585585 }, "accuracy": 0.6753246753246753, "ece_hard": 0.2643256322526132, "calibration": 54.217536403149474, "p50_s": 0.5537255644798279, "p95_s": 1.4364962704479693, "cost_usd_per_1000": 0.025317011122460563, "axes": { "intelligence": 67.5633967300634, "calibration": 54.217536403149474, "speed": 80.99400922678721, "cost": 57.897627027937 }, "jevbench_score": 64.37855568240049 }, "heldout": { "n": 303, "escalation": 0.1419141914191419, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5821917808219178, "hard": 0.5412844036697247 }, "accuracy": 0.6105610561056105, "ece_hard": 0.2985951361222342, "calibration": 57.448163603805824, "p50_s": 0.6008917182683945, "p95_s": 1.120798410475254, "cost_usd_per_1000": 0.02585425581884031, "axes": { "intelligence": 66.37323530643879, "calibration": 57.448163603805824, "speed": 81.71676275208478, "cost": 57.62403874943091 }, "jevbench_score": 65.09475493501058 } }, { "threshold": 0.6, "public": { "n": 231, "escalation": 0.12987012987012986, "tiers": { "easy": 1.0, "standard": 0.6527777777777778, "hard": 0.5585585585585585 }, "accuracy": 0.6796536796536796, "ece_hard": 0.267553286864166, "calibration": 53.894770941994196, "p50_s": 0.5537255644798279, "p95_s": 1.4364962704479693, "cost_usd_per_1000": 0.025541697310180456, "axes": { "intelligence": 68.10352018685353, "calibration": 53.894770941994196, "speed": 80.99400922678721, "cost": 57.78250738451238 }, "jevbench_score": 64.37857625462823 }, "heldout": { "n": 303, "escalation": 0.1617161716171617, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.6027397260273972, "hard": 0.5321100917431193 }, "accuracy": 0.6171617161716172, "ece_hard": 0.31242974877100405, "calibration": 54.53150644626791, "p50_s": 0.6067677825689316, "p95_s": 1.120798410475254, "cost_usd_per_1000": 0.026368142050160075, "axes": { "intelligence": 66.67334841439406, "calibration": 54.53150644626791, "speed": 81.67449980564413, "cost": 57.36761411050895 }, "jevbench_score": 64.24484096098857 } }, { "threshold": 0.61, "public": { "n": 231, "escalation": 0.13852813852813853, "tiers": { "easy": 1.0, "standard": 0.6527777777777778, "hard": 0.5585585585585585 }, "accuracy": 0.6796536796536796, "ece_hard": 0.27173345718046416, "calibration": 53.47675391036438, "p50_s": 0.5537255644798279, "p95_s": 1.4364962704479693, "cost_usd_per_1000": 0.025766383497900353, "axes": { "intelligence": 68.10352018685353, "calibration": 53.47675391036438, "speed": 80.99400922678721, "cost": 57.66839600822845 }, "jevbench_score": 64.22163261317095 }, "heldout": { "n": 303, "escalation": 0.1716171617161716, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.6095890410958904, "hard": 0.5229357798165137 }, "accuracy": 0.6171617161716172, "ece_hard": 0.3265447760258394, "calibration": 49.47356402654596, "p50_s": 0.6076879739761353, "p95_s": 1.120798410475254, "cost_usd_per_1000": 0.026625085165819955, "axes": { "intelligence": 66.58989987851368, "calibration": 49.47356402654596, "speed": 81.66791851810413, "cost": 57.24126983111844 }, "jevbench_score": 62.64487925108933 } }, { "threshold": 0.62, "public": { "n": 231, "escalation": 0.14285714285714285, "tiers": { "easy": 1.0, "standard": 0.6527777777777778, "hard": 0.5585585585585585 }, "accuracy": 0.6796536796536796, "ece_hard": 0.27173345718046416, "calibration": 53.47675391036438, "p50_s": 0.5538872867822647, "p95_s": 1.4364962704479693, "cost_usd_per_1000": 0.0258787265917603, "axes": { "intelligence": 68.10352018685353, "calibration": 53.47675391036438, "speed": 80.99274100199246, "cost": 57.61171293103288 }, "jevbench_score": 64.20559436730154 }, "heldout": { "n": 303, "escalation": 0.1716171617161716, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.6095890410958904, "hard": 0.5229357798165137 }, "accuracy": 0.6171617161716172, "ece_hard": 0.3265447760258394, "calibration": 49.47356402654596, "p50_s": 0.6076879739761353, "p95_s": 1.120798410475254, "cost_usd_per_1000": 0.026625085165819955, "axes": { "intelligence": 66.58989987851368, "calibration": 49.47356402654596, "speed": 81.66791851810413, "cost": 57.24126983111844 }, "jevbench_score": 62.64487925108933 } }, { "threshold": 0.63, "public": { "n": 231, "escalation": 0.16017316017316016, "tiers": { "easy": 1.0, "standard": 0.6666666666666666, "hard": 0.5585585585585585 }, "accuracy": 0.683982683982684, "ece_hard": 0.27485410783189745, "calibration": 53.009830967379, "p50_s": 0.5563413441181183, "p95_s": 1.4364962704479693, "cost_usd_per_1000": 0.026328098967200092, "axes": { "intelligence": 68.64364364364363, "calibration": 53.009830967379, "speed": 80.9735416179134, "cost": 57.38741494599104 }, "jevbench_score": 64.12525957193229 }, "heldout": { "n": 303, "escalation": 0.18151815181518152, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.6301369863013698, "hard": 0.5229357798165137 }, "accuracy": 0.6270627062706271, "ece_hard": 0.3265447760258394, "calibration": 49.47356402654596, "p50_s": 0.6084316313266754, "p95_s": 1.120798410475254, "cost_usd_per_1000": 0.026882028281479835, "axes": { "intelligence": 67.16524234426711, "calibration": 49.47356402654596, "speed": 81.662607094415, "cost": 57.11613899017338 }, "jevbench_score": 62.74439940657516 } }, { "threshold": 0.64, "public": { "n": 231, "escalation": 0.1645021645021645, "tiers": { "easy": 1.0, "standard": 0.6666666666666666, "hard": 0.5675675675675675 }, "accuracy": 0.6883116883116883, "ece_hard": 0.26640718594985546, "calibration": 53.85452315558321, "p50_s": 0.5563413441181183, "p95_s": 1.4364962704479693, "cost_usd_per_1000": 0.02644044206106004, "axes": { "intelligence": 69.01901901901901, "calibration": 53.85452315558321, "speed": 80.9735416179134, "cost": 57.331938643056986 }, "jevbench_score": 64.45144851462179 }, "heldout": { "n": 303, "escalation": 0.1848184818481848, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.636986301369863, "hard": 0.5229357798165137 }, "accuracy": 0.6303630363036303, "ece_hard": 0.3265447760258394, "calibration": 49.47356402654596, "p50_s": 0.6108522057533264, "p95_s": 1.120798410475254, "cost_usd_per_1000": 0.026967675986699795, "axes": { "intelligence": 67.3570231661849, "calibration": 49.47356402654596, "speed": 81.64536347212439, "cost": 57.07469435351088 }, "jevbench_score": 62.77443333826848 } }, { "threshold": 0.65, "public": { "n": 231, "escalation": 0.16883116883116883, "tiers": { "easy": 1.0, "standard": 0.6666666666666666, "hard": 0.5675675675675675 }, "accuracy": 0.6883116883116883, "ece_hard": 0.26661263466860685, "calibration": 53.83397828370806, "p50_s": 0.5563413441181183, "p95_s": 1.4364962704479693, "cost_usd_per_1000": 0.02655278515491999, "axes": { "intelligence": 69.01901901901901, "calibration": 53.83397828370806, "speed": 80.9735416179134, "cost": 57.27669755501917 }, "jevbench_score": 64.42977139614453 }, "heldout": { "n": 303, "escalation": 0.19141914191419143, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.636986301369863, "hard": 0.5229357798165137 }, "accuracy": 0.6303630363036303, "ece_hard": 0.3295998183975264, "calibration": 49.16805978937725, "p50_s": 0.6108522057533264, "p95_s": 1.120798410475254, "cost_usd_per_1000": 0.027138971397139715, "axes": { "intelligence": 67.3570231661849, "calibration": 49.16805978937725, "speed": 81.64536347212439, "cost": 56.992198500964676 }, "jevbench_score": 62.65463775277928 } }, { "threshold": 0.66, "public": { "n": 231, "escalation": 0.1774891774891775, "tiers": { "easy": 1.0, "standard": 0.6666666666666666, "hard": 0.5765765765765766 }, "accuracy": 0.6926406926406926, "ece_hard": 0.26373654476627173, "calibration": 52.82225743958769, "p50_s": 0.5563752442598343, "p95_s": 1.4364962704479693, "cost_usd_per_1000": 0.026777471342639886, "axes": { "intelligence": 69.3943943943944, "calibration": 52.82225743958769, "speed": 80.97327699267744, "cost": 57.166913104023955 }, "jevbench_score": 64.18106970929136 }, "heldout": { "n": 303, "escalation": 0.21782178217821782, "tiers": { "easy": 1.0, "standard": 0.75, "judge": 0.6575342465753424, "hard": 0.5229357798165137 }, "accuracy": 0.6435643564356436, "ece_hard": 0.3286472536292751, "calibration": 49.26331626620238, "p50_s": 0.6203177213668823, "p95_s": 1.120798410475254, "cost_usd_per_1000": 0.0278241530388994, "axes": { "intelligence": 69.09903229860501, "calibration": 49.26331626620238, "speed": 81.5785830528343, "cost": 56.667341398617744 }, "jevbench_score": 62.98340724737732 } }, { "threshold": 0.67, "public": { "n": 231, "escalation": 0.18614718614718614, "tiers": { "easy": 1.0, "standard": 0.6666666666666666, "hard": 0.5855855855855856 }, "accuracy": 0.696969696969697, "ece_hard": 0.26038638098739647, "calibration": 53.157273817475215, "p50_s": 0.5563752442598343, "p95_s": 1.4364962704479693, "cost_usd_per_1000": 0.02700215753035978, "axes": { "intelligence": 69.76976976976977, "calibration": 53.157273817475215, "speed": 80.97327699267744, "cost": 57.05804600179206 }, "jevbench_score": 64.33868052784132 }, "heldout": { "n": 303, "escalation": 0.23102310231023102, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.6643835616438356, "hard": 0.5229357798165137 }, "accuracy": 0.6501650165016502, "ece_hard": 0.3322089472417288, "calibration": 51.03041822398225, "p50_s": 0.6225624173879624, "p95_s": 1.1283639609813687, "cost_usd_per_1000": 0.02816674385977924, "axes": { "intelligence": 70.45747978718948, "calibration": 51.03041822398225, "speed": 81.53367893491264, "cost": 56.50790066653437 }, "jevbench_score": 63.79705203126036 } }, { "threshold": 0.68, "public": { "n": 231, "escalation": 0.19047619047619047, "tiers": { "easy": 1.0, "standard": 0.6805555555555556, "hard": 0.5855855855855856 }, "accuracy": 0.7012987012987013, "ece_hard": 0.26038638098739647, "calibration": 53.157273817475215, "p50_s": 0.5565016955137253, "p95_s": 1.4364962704479693, "cost_usd_per_1000": 0.027114500624219728, "axes": { "intelligence": 70.30989322655991, "calibration": 53.157273817475215, "speed": 80.97229005368952, "cost": 57.003951690545136 }, "jevbench_score": 64.4473598819897 }, "heldout": { "n": 303, "escalation": 0.264026402640264, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.7054794520547946, "hard": 0.5229357798165137 }, "accuracy": 0.66996699669967, "ece_hard": 0.3351163464543423, "calibration": 50.739678302720904, "p50_s": 0.635247477889061, "p95_s": 1.1286718428134916, "cost_usd_per_1000": 0.02902322091197884, "axes": { "intelligence": 71.60816471869633, "calibration": 50.739678302720904, "speed": 81.44489359790727, "cost": 56.11763177485594 }, "jevbench_score": 63.83639626570423 } }, { "threshold": 0.69, "public": { "n": 231, "escalation": 0.19047619047619047, "tiers": { "easy": 1.0, "standard": 0.6805555555555556, "hard": 0.5855855855855856 }, "accuracy": 0.7012987012987013, "ece_hard": 0.26038638098739647, "calibration": 53.157273817475215, "p50_s": 0.5565016955137253, "p95_s": 1.4364962704479693, "cost_usd_per_1000": 0.027114500624219728, "axes": { "intelligence": 70.30989322655991, "calibration": 53.157273817475215, "speed": 80.97229005368952, "cost": 57.003951690545136 }, "jevbench_score": 64.4473598819897 }, "heldout": { "n": 303, "escalation": 0.26732673267326734, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.7054794520547946, "hard": 0.5229357798165137 }, "accuracy": 0.66996699669967, "ece_hard": 0.3351163464543423, "calibration": 50.739678302720904, "p50_s": 0.635247477889061, "p95_s": 1.1286718428134916, "cost_usd_per_1000": 0.029108868617198802, "axes": { "intelligence": 71.60816471869633, "calibration": 50.739678302720904, "speed": 81.44489359790727, "cost": 56.0792402224655 }, "jevbench_score": 63.82547542288734 } }, { "threshold": 0.7, "public": { "n": 231, "escalation": 0.19913419913419914, "tiers": { "easy": 1.0, "standard": 0.6805555555555556, "hard": 0.5945945945945946 }, "accuracy": 0.7056277056277056, "ece_hard": 0.2527909015390277, "calibration": 53.91682176231209, "p50_s": 0.5565016955137253, "p95_s": 1.4747059546411037, "cost_usd_per_1000": 0.02733918681193962, "axes": { "intelligence": 70.68526860193528, "calibration": 53.91682176231209, "speed": 80.85828074278152, "cost": 56.89643222231397 }, "jevbench_score": 64.70914825569797 }, "heldout": { "n": 303, "escalation": 0.2739273927392739, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.7123287671232876, "hard": 0.5229357798165137 }, "accuracy": 0.6732673267326733, "ece_hard": 0.33288186852422397, "calibration": 50.96312609573273, "p50_s": 0.6424911558628082, "p95_s": 1.1286718428134916, "cost_usd_per_1000": 0.029280164027638722, "axes": { "intelligence": 71.79994554061413, "calibration": 50.96312609573273, "speed": 81.39565155431104, "cost": 56.00279484060318 }, "jevbench_score": 63.90690275101672 } }, { "threshold": 0.71, "public": { "n": 231, "escalation": 0.22077922077922077, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "hard": 0.5855855855855856 }, "accuracy": 0.70995670995671, "ece_hard": 0.26710066926452947, "calibration": 52.485844989761915, "p50_s": 0.5587625443935395, "p95_s": 1.4747059546411037, "cost_usd_per_1000": 0.027900902281239363, "axes": { "intelligence": 71.39014014014015, "calibration": 52.485844989761915, "speed": 80.84067279835384, "cost": 56.63145255832192 }, "jevbench_score": 64.35643997531017 }, "heldout": { "n": 303, "escalation": 0.297029702970297, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.7534246575342466, "hard": 0.5229357798165137 }, "accuracy": 0.693069306930693, "ece_hard": 0.33288186852422397, "calibration": 50.96312609573273, "p50_s": 0.6549014985561371, "p95_s": 1.1286718428134916, "cost_usd_per_1000": 0.029879697964178442, "axes": { "intelligence": 72.95063047212098, "calibration": 50.96312609573273, "speed": 81.3125632505591, "cost": 55.73871390565259 }, "jevbench_score": 64.06929220942753 } }, { "threshold": 0.72, "public": { "n": 231, "escalation": 0.22077922077922077, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "hard": 0.5855855855855856 }, "accuracy": 0.70995670995671, "ece_hard": 0.26710066926452947, "calibration": 52.485844989761915, "p50_s": 0.5587625443935395, "p95_s": 1.4747059546411037, "cost_usd_per_1000": 0.027900902281239363, "axes": { "intelligence": 71.39014014014015, "calibration": 52.485844989761915, "speed": 80.84067279835384, "cost": 56.63145255832192 }, "jevbench_score": 64.35643997531017 }, "heldout": { "n": 303, "escalation": 0.297029702970297, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.7534246575342466, "hard": 0.5229357798165137 }, "accuracy": 0.693069306930693, "ece_hard": 0.33288186852422397, "calibration": 50.96312609573273, "p50_s": 0.6549014985561371, "p95_s": 1.1286718428134916, "cost_usd_per_1000": 0.029879697964178442, "axes": { "intelligence": 72.95063047212098, "calibration": 50.96312609573273, "speed": 81.3125632505591, "cost": 55.73871390565259 }, "jevbench_score": 64.06929220942753 } }, { "threshold": 0.73, "public": { "n": 231, "escalation": 0.22077922077922077, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "hard": 0.5855855855855856 }, "accuracy": 0.70995670995671, "ece_hard": 0.26710066926452947, "calibration": 52.485844989761915, "p50_s": 0.5587625443935395, "p95_s": 1.4747059546411037, "cost_usd_per_1000": 0.027900902281239363, "axes": { "intelligence": 71.39014014014015, "calibration": 52.485844989761915, "speed": 80.84067279835384, "cost": 56.63145255832192 }, "jevbench_score": 64.35643997531017 }, "heldout": { "n": 303, "escalation": 0.30363036303630364, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.7602739726027398, "hard": 0.5229357798165137 }, "accuracy": 0.6963696369636964, "ece_hard": 0.33288186852422397, "calibration": 50.96312609573273, "p50_s": 0.655505919456482, "p95_s": 1.1286718428134916, "cost_usd_per_1000": 0.030050993374618362, "axes": { "intelligence": 73.1424112940388, "calibration": 50.96312609573273, "speed": 81.30855691364778, "cost": 55.664235017672915 }, "jevbench_score": 64.08914207112447 } }, { "threshold": 0.74, "public": { "n": 231, "escalation": 0.22510822510822512, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "hard": 0.5945945945945946 }, "accuracy": 0.7142857142857143, "ece_hard": 0.25989188542934216, "calibration": 53.206723373280646, "p50_s": 0.5587625443935395, "p95_s": 1.5043965779244899, "cost_usd_per_1000": 0.028013245375099312, "axes": { "intelligence": 71.76551551551552, "calibration": 53.206723373280646, "speed": 80.75410376849155, "cost": 56.57909723848732 }, "jevbench_score": 64.62874679723933 }, "heldout": { "n": 303, "escalation": 0.31683168316831684, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.7808219178082192, "hard": 0.5229357798165137 }, "accuracy": 0.7062706270627063, "ece_hard": 0.33517888581453914, "calibration": 50.733424366701215, "p50_s": 0.6719379365444184, "p95_s": 1.1286718428134916, "cost_usd_per_1000": 0.030393584195498202, "axes": { "intelligence": 73.71775375979222, "calibration": 50.733424366701215, "speed": 81.20103147889616, "cost": 55.516542467945555 }, "jevbench_score": 64.0785324465737 } }, { "threshold": 0.75, "public": { "n": 231, "escalation": 0.23376623376623376, "tiers": { "easy": 1.0, "standard": 0.7222222222222222, "hard": 0.6036036036036037 }, "accuracy": 0.7229437229437229, "ece_hard": 0.24811747118757388, "calibration": 54.38416479745747, "p50_s": 0.5630453020334244, "p95_s": 1.5043965779244899, "cost_usd_per_1000": 0.028237931562819205, "axes": { "intelligence": 72.68101434768103, "calibration": 54.38416479745747, "speed": 80.7209432401041, "cost": 56.475013559790185 }, "jevbench_score": 65.15293788287313 }, "heldout": { "n": 303, "escalation": 0.31683168316831684, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.7808219178082192, "hard": 0.5229357798165137 }, "accuracy": 0.7062706270627063, "ece_hard": 0.33517888581453914, "calibration": 50.733424366701215, "p50_s": 0.6719379365444184, "p95_s": 1.1286718428134916, "cost_usd_per_1000": 0.030393584195498202, "axes": { "intelligence": 73.71775375979222, "calibration": 50.733424366701215, "speed": 81.20103147889616, "cost": 55.516542467945555 }, "jevbench_score": 64.0785324465737 } }, { "threshold": 0.76, "public": { "n": 231, "escalation": 0.24675324675324675, "tiers": { "easy": 1.0, "standard": 0.7222222222222222, "hard": 0.6036036036036037 }, "accuracy": 0.7229437229437229, "ece_hard": 0.2476327117698351, "calibration": 54.75166494624854, "p50_s": 0.5630453020334244, "p95_s": 1.5043965779244899, "cost_usd_per_1000": 0.028574960844399047, "axes": { "intelligence": 72.68101434768103, "calibration": 54.75166494624854, "speed": 80.7209432401041, "cost": 56.320430681075514 }, "jevbench_score": 65.21802250604368 }, "heldout": { "n": 303, "escalation": 0.3333333333333333, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8013698630136986, "hard": 0.5321100917431193 }, "accuracy": 0.7194719471947195, "ece_hard": 0.3280982234794359, "calibration": 51.44149060021154, "p50_s": 0.6774831205606461, "p95_s": 1.1286718428134916, "cost_usd_per_1000": 0.030821822721598002, "axes": { "intelligence": 74.5683255833438, "calibration": 51.44149060021154, "speed": 81.16533829977527, "cost": 55.33425045471382 }, "jevbench_score": 64.42555115271709 } }, { "threshold": 0.77, "public": { "n": 231, "escalation": 0.2510822510822511, "tiers": { "easy": 1.0, "standard": 0.7222222222222222, "hard": 0.6036036036036037 }, "accuracy": 0.7229437229437229, "ece_hard": 0.24955544327053508, "calibration": 54.55939179617853, "p50_s": 0.5630453020334244, "p95_s": 1.5043965779244899, "cost_usd_per_1000": 0.028687303938258996, "axes": { "intelligence": 72.68101434768103, "calibration": 54.55939179617853, "speed": 80.7209432401041, "cost": 56.2693079580809 }, "jevbench_score": 65.1458980836118 }, "heldout": { "n": 303, "escalation": 0.3333333333333333, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8013698630136986, "hard": 0.5321100917431193 }, "accuracy": 0.7194719471947195, "ece_hard": 0.3280982234794359, "calibration": 51.44149060021154, "p50_s": 0.6774831205606461, "p95_s": 1.1286718428134916, "cost_usd_per_1000": 0.030821822721598002, "axes": { "intelligence": 74.5683255833438, "calibration": 51.44149060021154, "speed": 81.16533829977527, "cost": 55.33425045471382 }, "jevbench_score": 64.42555115271709 } }, { "threshold": 0.78, "public": { "n": 231, "escalation": 0.2727272727272727, "tiers": { "easy": 1.0, "standard": 0.7222222222222222, "hard": 0.6126126126126126 }, "accuracy": 0.7272727272727273, "ece_hard": 0.23938206909436277, "calibration": 55.576729213795765, "p50_s": 0.5651318520307541, "p95_s": 1.5070105396211146, "cost_usd_per_1000": 0.029249019407558734, "axes": { "intelligence": 73.0563897230564, "calibration": 55.576729213795765, "speed": 80.69733924590882, "cost": 56.01666068030005 }, "jevbench_score": 65.45335394438592 }, "heldout": { "n": 303, "escalation": 0.35313531353135313, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.815068493150685, "hard": 0.5412844036697247 }, "accuracy": 0.7293729372937293, "ece_hard": 0.3241917435366488, "calibration": 51.02370937170731, "p50_s": 0.6951174080371857, "p95_s": 1.2148109033703782, "cost_usd_per_1000": 0.03133570895291776, "axes": { "intelligence": 75.2271165849776, "calibration": 51.02370937170731, "speed": 80.73433153970151, "cost": 55.118814255481226 }, "jevbench_score": 64.2874413346338 } }, { "threshold": 0.79, "public": { "n": 231, "escalation": 0.27705627705627706, "tiers": { "easy": 1.0, "standard": 0.7222222222222222, "hard": 0.6126126126126126 }, "accuracy": 0.7272727272727273, "ece_hard": 0.23862440905451748, "calibration": 55.6524952177803, "p50_s": 0.5651318520307541, "p95_s": 1.5138229928910731, "cost_usd_per_1000": 0.029361362501418683, "axes": { "intelligence": 73.0563897230564, "calibration": 55.6524952177803, "speed": 80.67775116965697, "cost": 55.96671385118233 }, "jevbench_score": 65.45707728890314 }, "heldout": { "n": 303, "escalation": 0.35973597359735976, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.815068493150685, "hard": 0.5412844036697247 }, "accuracy": 0.7293729372937293, "ece_hard": 0.3281465079488912, "calibration": 50.62823293048308, "p50_s": 0.6960485965013504, "p95_s": 1.2787909343838677, "cost_usd_per_1000": 0.03150700436335769, "axes": { "intelligence": 75.2271165849776, "calibration": 50.62823293048308, "speed": 80.50560889089522, "cost": 55.0477866070741 }, "jevbench_score": 64.09634960805757 } }, { "threshold": 0.8, "public": { "n": 231, "escalation": 0.2857142857142857, "tiers": { "easy": 1.0, "standard": 0.7222222222222222, "hard": 0.6216216216216216 }, "accuracy": 0.7316017316017316, "ece_hard": 0.2332254864015691, "calibration": 56.19238748307514, "p50_s": 0.5651318520307541, "p95_s": 1.5138229928910731, "cost_usd_per_1000": 0.029586048689138576, "axes": { "intelligence": 73.43176509843178, "calibration": 56.19238748307514, "speed": 80.67775116965697, "cost": 55.86739097125985 }, "jevbench_score": 65.67021023144495 }, "heldout": { "n": 303, "escalation": 0.38613861386138615, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8287671232876712, "hard": 0.5412844036697247 }, "accuracy": 0.735973597359736, "ece_hard": 0.32979385645745596, "calibration": 50.4634980796266, "p50_s": 0.7260566294193268, "p95_s": 1.2861934348940849, "cost_usd_per_1000": 0.03219218600511737, "axes": { "intelligence": 75.61067822881321, "calibration": 50.4634980796266, "speed": 80.2972321631552, "cost": 54.76748594852223 }, "jevbench_score": 64.00235681501232 } }, { "threshold": 0.81, "public": { "n": 231, "escalation": 0.2857142857142857, "tiers": { "easy": 1.0, "standard": 0.7222222222222222, "hard": 0.6216216216216216 }, "accuracy": 0.7316017316017316, "ece_hard": 0.2332254864015691, "calibration": 56.19238748307514, "p50_s": 0.5651318520307541, "p95_s": 1.5138229928910731, "cost_usd_per_1000": 0.029586048689138576, "axes": { "intelligence": 73.43176509843178, "calibration": 56.19238748307514, "speed": 80.67775116965697, "cost": 55.86739097125985 }, "jevbench_score": 65.67021023144495 }, "heldout": { "n": 303, "escalation": 0.38943894389438943, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8287671232876712, "hard": 0.5412844036697247 }, "accuracy": 0.735973597359736, "ece_hard": 0.3313364755281123, "calibration": 50.30923617256097, "p50_s": 0.7260566294193268, "p95_s": 1.2974561408162113, "cost_usd_per_1000": 0.03227783371033733, "axes": { "intelligence": 75.61067822881321, "calibration": 50.30923617256097, "speed": 80.25936818916537, "cost": 54.73286860438806 }, "jevbench_score": 63.93574073646394 } }, { "threshold": 0.82, "public": { "n": 231, "escalation": 0.2857142857142857, "tiers": { "easy": 1.0, "standard": 0.7222222222222222, "hard": 0.6216216216216216 }, "accuracy": 0.7316017316017316, "ece_hard": 0.2332254864015691, "calibration": 56.19238748307514, "p50_s": 0.5651318520307541, "p95_s": 1.5138229928910731, "cost_usd_per_1000": 0.029586048689138576, "axes": { "intelligence": 73.43176509843178, "calibration": 56.19238748307514, "speed": 80.67775116965697, "cost": 55.86739097125985 }, "jevbench_score": 65.67021023144495 }, "heldout": { "n": 303, "escalation": 0.429042904290429, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8561643835616438, "hard": 0.5504587155963303 }, "accuracy": 0.7524752475247525, "ece_hard": 0.33132639481871223, "calibration": 50.310244243500975, "p50_s": 0.769572751224041, "p95_s": 1.3029031932353972, "cost_usd_per_1000": 0.03330560617297685, "axes": { "intelligence": 76.6530308742826, "calibration": 50.310244243500975, "speed": 79.98838169573686, "cost": 54.32447972898722 }, "jevbench_score": 63.98115279983164 } }, { "threshold": 0.83, "public": { "n": 231, "escalation": 0.29004329004329005, "tiers": { "easy": 1.0, "standard": 0.7222222222222222, "hard": 0.6216216216216216 }, "accuracy": 0.7316017316017316, "ece_hard": 0.23419887131490175, "calibration": 56.09504899174187, "p50_s": 0.5651318520307541, "p95_s": 1.5138229928910731, "cost_usd_per_1000": 0.029698391782998525, "axes": { "intelligence": 73.43176509843178, "calibration": 56.09504899174187, "speed": 80.67775116965697, "cost": 55.818012034301745 }, "jevbench_score": 65.6272433237828 }, "heldout": { "n": 303, "escalation": 0.43564356435643564, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8561643835616438, "hard": 0.5504587155963303 }, "accuracy": 0.7524752475247525, "ece_hard": 0.33132639481871223, "calibration": 50.310244243500975, "p50_s": 0.7696364015340805, "p95_s": 1.3029031932353972, "cost_usd_per_1000": 0.03347690158341677, "axes": { "intelligence": 76.6530308742826, "calibration": 50.310244243500975, "speed": 79.98802251156248, "cost": 54.25764233353019 }, "jevbench_score": 63.9613923210539 } }, { "threshold": 0.84, "public": { "n": 231, "escalation": 0.30735930735930733, "tiers": { "easy": 1.0, "standard": 0.7222222222222222, "hard": 0.6216216216216216 }, "accuracy": 0.7316017316017316, "ece_hard": 0.24011144846914487, "calibration": 55.50379127631756, "p50_s": 0.5724019199609757, "p95_s": 1.5138229928910731, "cost_usd_per_1000": 0.03014776415843832, "axes": { "intelligence": 73.43176509843178, "calibration": 55.50379127631756, "speed": 80.62223820903617, "cost": 55.622346724280355 }, "jevbench_score": 65.38493463220308 }, "heldout": { "n": 303, "escalation": 0.44224422442244227, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8698630136986302, "hard": 0.5504587155963303 }, "accuracy": 0.759075907590759, "ece_hard": 0.33132639481871223, "calibration": 50.310244243500975, "p50_s": 0.772303269803524, "p95_s": 1.3029031932353972, "cost_usd_per_1000": 0.03364819699385669, "axes": { "intelligence": 77.03659251811821, "calibration": 50.310244243500975, "speed": 79.97299977948309, "cost": 54.19114606200929 }, "jevbench_score": 64.01861927689347 } }, { "threshold": 0.85, "public": { "n": 231, "escalation": 0.31601731601731603, "tiers": { "easy": 1.0, "standard": 0.7222222222222222, "hard": 0.6126126126126126 }, "accuracy": 0.7272727272727273, "ece_hard": 0.2485662610244598, "calibration": 54.65831002078606, "p50_s": 0.5724860668182373, "p95_s": 1.5295075267553329, "cost_usd_per_1000": 0.030372450346158215, "axes": { "intelligence": 73.0563897230564, "calibration": 54.65831002078606, "speed": 80.57683460227634, "cost": 55.52560507792729 }, "jevbench_score": 65.01363925572078 }, "heldout": { "n": 303, "escalation": 0.4521452145214521, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8698630136986302, "hard": 0.5504587155963303 }, "accuracy": 0.759075907590759, "ece_hard": 0.3330161651703568, "calibration": 50.141267208336515, "p50_s": 0.7730389326810837, "p95_s": 1.3029031932353972, "cost_usd_per_1000": 0.03390514010951657, "axes": { "intelligence": 77.03659251811821, "calibration": 50.141267208336515, "speed": 79.96886484610863, "cost": 54.09203369817483 }, "jevbench_score": 63.93470312367226 } }, { "threshold": 0.86, "public": { "n": 231, "escalation": 0.3203463203463203, "tiers": { "easy": 1.0, "standard": 0.7222222222222222, "hard": 0.6126126126126126 }, "accuracy": 0.7272727272727273, "ece_hard": 0.24987647529254448, "calibration": 54.527288593977595, "p50_s": 0.5724860668182373, "p95_s": 1.5295075267553329, "cost_usd_per_1000": 0.03048479344001816, "axes": { "intelligence": 73.0563897230564, "calibration": 54.527288593977595, "speed": 80.57683460227634, "cost": 55.477502300217274 }, "jevbench_score": 64.96056640384292 }, "heldout": { "n": 303, "escalation": 0.47194719471947194, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8904109589041096, "hard": 0.5688073394495413 }, "accuracy": 0.7755775577557755, "ece_hard": 0.317352532876582, "calibration": 51.707630437714, "p50_s": 0.7797016367316246, "p95_s": 1.3029031932353972, "cost_usd_per_1000": 0.03441902634083634, "axes": { "intelligence": 78.16239369946798, "calibration": 51.707630437714, "speed": 79.93159405638585, "cost": 53.89604257991348 }, "jevbench_score": 64.59619839104852 } }, { "threshold": 0.87, "public": { "n": 231, "escalation": 0.3333333333333333, "tiers": { "easy": 1.0, "standard": 0.7222222222222222, "hard": 0.6126126126126126 }, "accuracy": 0.7272727272727273, "ece_hard": 0.25086233184935813, "calibration": 54.428702938296226, "p50_s": 0.5749294698238373, "p95_s": 1.5295075267553329, "cost_usd_per_1000": 0.030821822721598002, "axes": { "intelligence": 73.0563897230564, "calibration": 54.428702938296226, "speed": 80.5583381114668, "cost": 55.33425045471382 }, "jevbench_score": 64.8855036951728 }, "heldout": { "n": 303, "escalation": 0.5016501650165016, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8972602739726028, "hard": 0.5871559633027523 }, "accuracy": 0.7854785478547854, "ece_hard": 0.2995729388995245, "calibration": 53.485589835419745, "p50_s": 0.7821924328804016, "p95_s": 1.3428442820906634, "cost_usd_per_1000": 0.03518985568781598, "axes": { "intelligence": 78.90463323698211, "calibration": 53.485589835419745, "speed": 79.78660735578929, "cost": 53.60747542556122 }, "jevbench_score": 65.18139669426918 } }, { "threshold": 0.88, "public": { "n": 231, "escalation": 0.3463203463203463, "tiers": { "easy": 1.0, "standard": 0.7222222222222222, "hard": 0.6216216216216216 }, "accuracy": 0.7316017316017316, "ece_hard": 0.24475814431504095, "calibration": 55.03912169172795, "p50_s": 0.5749294698238373, "p95_s": 1.5505021527409553, "cost_usd_per_1000": 0.031158852003177848, "axes": { "intelligence": 73.43176509843178, "calibration": 55.03912169172795, "speed": 80.49913056094756, "cost": 55.192556547188424 }, "jevbench_score": 65.09637290690682 }, "heldout": { "n": 303, "escalation": 0.5016501650165016, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8972602739726028, "hard": 0.5871559633027523 }, "accuracy": 0.7854785478547854, "ece_hard": 0.2995729388995245, "calibration": 53.485589835419745, "p50_s": 0.7821924328804016, "p95_s": 1.3428442820906634, "cost_usd_per_1000": 0.03518985568781598, "axes": { "intelligence": 78.90463323698211, "calibration": 53.485589835419745, "speed": 79.78660735578929, "cost": 53.60747542556122 }, "jevbench_score": 65.18139669426918 } }, { "threshold": 0.89, "public": { "n": 231, "escalation": 0.3593073593073593, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "hard": 0.6216216216216216 }, "accuracy": 0.7272727272727273, "ece_hard": 0.24475814431504095, "calibration": 55.03912169172795, "p50_s": 0.5810501962900162, "p95_s": 1.5505021527409553, "cost_usd_per_1000": 0.03149588128475769, "axes": { "intelligence": 72.89164164164164, "calibration": 55.03912169172795, "speed": 80.4531397420952, "cost": 55.05238705512493 }, "jevbench_score": 64.92576785177982 }, "heldout": { "n": 303, "escalation": 0.5247524752475248, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9041095890410958, "hard": 0.5963302752293578 }, "accuracy": 0.7920792079207921, "ece_hard": 0.28897457131327225, "calibration": 54.54542659404497, "p50_s": 0.7854591920971871, "p95_s": 1.3428442820906634, "cost_usd_per_1000": 0.0357893896243557, "axes": { "intelligence": 79.37164341669809, "calibration": 54.54542659404497, "speed": 79.76850719204685, "cost": 53.387371248664195 }, "jevbench_score": 65.52747437170514 } }, { "threshold": 0.9, "public": { "n": 231, "escalation": 0.3722943722943723, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "hard": 0.6126126126126126 }, "accuracy": 0.7229437229437229, "ece_hard": 0.253147430415549, "calibration": 54.20019308167714, "p50_s": 0.5851256638765335, "p95_s": 1.5505021527409553, "cost_usd_per_1000": 0.03183291056633754, "axes": { "intelligence": 72.51626626626627, "calibration": 54.20019308167714, "speed": 80.42278479564052, "cost": 54.91370952646843 }, "jevbench_score": 64.5466994033858 }, "heldout": { "n": 303, "escalation": 0.5346534653465347, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9041095890410958, "hard": 0.5963302752293578 }, "accuracy": 0.7920792079207921, "ece_hard": 0.2899519510423916, "calibration": 54.44768862113304, "p50_s": 0.7884413316845894, "p95_s": 1.3428442820906634, "cost_usd_per_1000": 0.03604633274001558, "axes": { "intelligence": 79.37164341669809, "calibration": 54.44768862113304, "speed": 79.75204962948342, "cost": 53.29416738058839 }, "jevbench_score": 65.46611798877501 } }, { "threshold": 0.91, "public": { "n": 231, "escalation": 0.3939393939393939, "tiers": { "easy": 1.0, "standard": 0.7222222222222222, "hard": 0.6126126126126126 }, "accuracy": 0.7272727272727273, "ece_hard": 0.2563939727178648, "calibration": 53.87553885144557, "p50_s": 0.5868345886468888, "p95_s": 1.5729120425879954, "cost_usd_per_1000": 0.032394626035637274, "axes": { "intelligence": 73.0563897230564, "calibration": 53.87553885144557, "speed": 80.34779858793763, "cost": 54.68581087587814 }, "jevbench_score": 64.48736295951903 }, "heldout": { "n": 303, "escalation": 0.5445544554455446, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9178082191780822, "hard": 0.5963302752293578 }, "accuracy": 0.7986798679867987, "ece_hard": 0.29076300585982884, "calibration": 54.36658313938931, "p50_s": 0.7916510790586472, "p95_s": 1.3428442820906634, "cost_usd_per_1000": 0.03630327585567546, "axes": { "intelligence": 79.75520506053371, "calibration": 54.36658313938931, "speed": 79.73440537677043, "cost": 53.20162552837711 }, "jevbench_score": 65.48855897429857 } }, { "threshold": 0.92, "public": { "n": 231, "escalation": 0.4155844155844156, "tiers": { "easy": 1.0, "standard": 0.7222222222222222, "hard": 0.6126126126126126 }, "accuracy": 0.7272727272727273, "ece_hard": 0.25309935388516536, "calibration": 56.66444195372783, "p50_s": 0.6046394348144531, "p95_s": 1.5729120425879954, "cost_usd_per_1000": 0.032956341504937016, "axes": { "intelligence": 73.0563897230564, "calibration": 56.66444195372783, "speed": 80.21799093875035, "cost": 54.46183016447051 }, "jevbench_score": 65.21285422662278 }, "heldout": { "n": 303, "escalation": 0.570957095709571, "tiers": { "easy": 1.0, "standard": 0.75, "judge": 0.9178082191780822, "hard": 0.6055045871559633 }, "accuracy": 0.7986798679867987, "ece_hard": 0.2836325915330308, "calibration": 55.07962457206912, "p50_s": 0.7934435546398163, "p95_s": 1.3441746316850183, "cost_usd_per_1000": 0.03698845749743514, "axes": { "intelligence": 78.86376775166522, "calibration": 55.07962457206912, "speed": 79.72028268605419, "cost": 52.95801338115475 }, "jevbench_score": 65.43984350390694 } }, { "threshold": 0.93, "public": { "n": 231, "escalation": 0.43722943722943725, "tiers": { "easy": 1.0, "standard": 0.7361111111111112, "hard": 0.6126126126126126 }, "accuracy": 0.7316017316017316, "ece_hard": 0.2550618727169931, "calibration": 56.092134968249596, "p50_s": 0.635255092382431, "p95_s": 1.5729120425879954, "cost_usd_per_1000": 0.03351805697423675, "axes": { "intelligence": 73.59651317984653, "calibration": 56.092134968249596, "speed": 80.00347407329882, "cost": 54.241634954676485 }, "jevbench_score": 65.0579248957372 }, "heldout": { "n": 303, "escalation": 0.5874587458745875, "tiers": { "easy": 1.0, "standard": 0.75, "judge": 0.9178082191780822, "hard": 0.6055045871559633 }, "accuracy": 0.7986798679867987, "ece_hard": 0.28174864837722746, "calibration": 55.26801888764945, "p50_s": 0.7937427327036858, "p95_s": 1.3441746316850183, "cost_usd_per_1000": 0.03741669602353494, "axes": { "intelligence": 78.86376775166522, "calibration": 55.26801888764945, "speed": 79.71864543169005, "cost": 52.80803692899604 }, "jevbench_score": 65.44897332454669 } }, { "threshold": 0.94, "public": { "n": 231, "escalation": 0.4588744588744589, "tiers": { "easy": 1.0, "standard": 0.75, "hard": 0.6216216216216216 }, "accuracy": 0.7402597402597403, "ece_hard": 0.24779820956123877, "calibration": 56.81850128382503, "p50_s": 0.7361026495695114, "p95_s": 1.5729120425879954, "cost_usd_per_1000": 0.034079772443536493, "axes": { "intelligence": 74.51201201201201, "calibration": 56.81850128382503, "speed": 79.36357181533658, "cost": 54.025099412623476 }, "jevbench_score": 65.27294551569697 }, "heldout": { "n": 303, "escalation": 0.6204620462046204, "tiers": { "easy": 1.0, "standard": 0.75, "judge": 0.9178082191780822, "hard": 0.6238532110091743 }, "accuracy": 0.8052805280528053, "ece_hard": 0.26423412864658846, "calibration": 57.019470860713355, "p50_s": 0.7992249444127083, "p95_s": 1.4028187438845618, "cost_usd_per_1000": 0.03827317307573454, "axes": { "intelligence": 79.41422646726154, "calibration": 57.019470860713355, "speed": 79.50329409800838, "cost": 52.513165919920645 }, "jevbench_score": 65.9392046598166 } }, { "threshold": 0.95, "public": { "n": 231, "escalation": 0.47186147186147187, "tiers": { "easy": 1.0, "standard": 0.75, "hard": 0.6126126126126126 }, "accuracy": 0.7359307359307359, "ece_hard": 0.2552117031899332, "calibration": 56.077151920955586, "p50_s": 0.7458489030599594, "p95_s": 1.5729120425879954, "cost_usd_per_1000": 0.03441680172511634, "axes": { "intelligence": 74.13663663663664, "calibration": 56.077151920955586, "speed": 79.30644707155861, "cost": 53.89688470387046 }, "jevbench_score": 64.92661276592354 }, "heldout": { "n": 303, "escalation": 0.636963696369637, "tiers": { "easy": 1.0, "standard": 0.75, "judge": 0.9246575342465754, "hard": 0.6422018348623854 }, "accuracy": 0.8151815181518152, "ece_hard": 0.24187768395658696, "calibration": 59.255115329713504, "p50_s": 0.7995885103940964, "p95_s": 1.4093246273696423, "cost_usd_per_1000": 0.038701411601834344, "axes": { "intelligence": 80.15646600477568, "calibration": 59.255115329713504, "speed": 79.48122413946797, "cost": 52.368195824853665 }, "jevbench_score": 66.68054520468542 } }, { "threshold": 0.96, "public": { "n": 231, "escalation": 0.5064935064935064, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.6306306306306306 }, "accuracy": 0.7489177489177489, "ece_hard": 0.2331785445062401, "calibration": 59.07543684012285, "p50_s": 0.754132778942585, "p95_s": 1.5729120425879954, "cost_usd_per_1000": 0.03531554647599591, "axes": { "intelligence": 75.4275108441775, "calibration": 59.07543684012285, "speed": 79.25847744360823, "cost": 53.56102207144861 }, "jevbench_score": 65.94894078801191 }, "heldout": { "n": 303, "escalation": 0.6501650165016502, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9246575342465754, "hard": 0.6605504587155964 }, "accuracy": 0.8250825082508251, "ece_hard": 0.2243717028724463, "calibration": 61.00571343812757, "p50_s": 0.8045990258455277, "p95_s": 1.4093246273696423, "cost_usd_per_1000": 0.039044002422714184, "axes": { "intelligence": 81.87359138703867, "calibration": 61.00571343812757, "speed": 79.45409456466291, "cost": 52.25337006852936 }, "jevbench_score": 67.4817342090295 } }, { "threshold": 0.97, "public": { "n": 231, "escalation": 0.5194805194805194, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.6306306306306306 }, "accuracy": 0.7489177489177489, "ece_hard": 0.2331785445062401, "calibration": 59.07543684012285, "p50_s": 0.7583087027072907, "p95_s": 1.5729120425879954, "cost_usd_per_1000": 0.03565257575757576, "axes": { "intelligence": 75.4275108441775, "calibration": 59.07543684012285, "speed": 79.23449522309356, "cost": 53.437272658605536 }, "jevbench_score": 65.90582857805916 }, "heldout": { "n": 303, "escalation": 0.66996699669967, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9246575342465754, "hard": 0.6605504587155964 }, "accuracy": 0.8250825082508251, "ece_hard": 0.2215725717457204, "calibration": 61.28562655080016, "p50_s": 0.8049376547336579, "p95_s": 1.4093246273696423, "cost_usd_per_1000": 0.039557888654033944, "axes": { "intelligence": 81.87359138703867, "calibration": 61.28562655080016, "speed": 79.45226714859376, "cost": 52.08300688917119 }, "jevbench_score": 67.50348640556179 } }, { "threshold": 0.98, "public": { "n": 231, "escalation": 0.5670995670995671, "tiers": { "easy": 1.0, "standard": 0.7777777777777778, "hard": 0.6396396396396397 }, "accuracy": 0.7575757575757576, "ece_hard": 0.22108134892973885, "calibration": 60.285156397772965, "p50_s": 0.767550015449524, "p95_s": 1.5729120425879954, "cost_usd_per_1000": 0.036888349790035184, "axes": { "intelligence": 76.34300967634302, "calibration": 60.285156397772965, "speed": 79.18188877722798, "cost": 52.99332317847252 }, "jevbench_score": 66.29132051818229 }, "heldout": { "n": 303, "escalation": 0.6864686468646864, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9246575342465754, "hard": 0.6605504587155964 }, "accuracy": 0.8250825082508251, "ece_hard": 0.21633695028446237, "calibration": 61.80918869692596, "p50_s": 0.8052033945918083, "p95_s": 1.418343103677034, "cost_usd_per_1000": 0.03998612718013375, "axes": { "intelligence": 81.87359138703867, "calibration": 61.80918869692596, "speed": 79.42313101497078, "cost": 51.94271971076275 }, "jevbench_score": 67.59540015990072 } }, { "threshold": 0.99, "public": { "n": 231, "escalation": 0.6320346320346321, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "hard": 0.6486486486486487 }, "accuracy": 0.7662337662337663, "ece_hard": 0.2101527114369838, "calibration": 61.378020147048474, "p50_s": 0.7777980506420136, "p95_s": 1.5729120425879954, "cost_usd_per_1000": 0.038573496197934404, "axes": { "intelligence": 77.25850850850851, "calibration": 61.378020147048474, "speed": 79.1242871213088, "cost": 52.41132988199057 }, "jevbench_score": 66.59222811712418 }, "heldout": { "n": 303, "escalation": 0.7326732673267327, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6605504587155964 }, "accuracy": 0.8283828382838284, "ece_hard": 0.21654588637609265, "calibration": 61.78829508776293, "p50_s": 0.8088261872529984, "p95_s": 1.4193350210785864, "cost_usd_per_1000": 0.04118519505321319, "axes": { "intelligence": 82.06537220895648, "calibration": 61.78829508776293, "speed": 79.40059877290577, "cost": 51.55776618566186 }, "jevbench_score": 67.4987929229066 } }, { "threshold": 1.0, "public": { "n": 231, "escalation": 1.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6486486486486487 }, "accuracy": 0.7748917748917749, "ece_hard": 0.21041641822205612, "calibration": 61.35164946854124, "p50_s": 0.7970052972435951, "p95_s": 1.5881473660469054, "cost_usd_per_1000": 0.04812265917602997, "axes": { "intelligence": 78.33875542208877, "calibration": 61.35164946854124, "speed": 78.97647993431897, "cost": 49.52951146891361 }, "jevbench_score": 65.84778672780581 }, "heldout": { "n": 303, "escalation": 1.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.6788990825688074 }, "accuracy": 0.834983498349835, "ece_hard": 0.1935286337386598, "calibration": 64.09002035150621, "p50_s": 0.8297482892870903, "p95_s": 1.4239212729036808, "cost_usd_per_1000": 0.04812265917602997, "axes": { "intelligence": 82.6158309245528, "calibration": 64.09002035150621, "speed": 79.27567656030779, "cost": 49.52951146891361 }, "jevbench_score": 67.52497336377385 } } ] }, { "cheap": "system-one-open", "backend": "jev-1.13.0", "chosen": { "threshold": 0.99, "public": { "n": 231, "escalation": 0.7316017316017316, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.7027027027027027 }, "accuracy": 0.8008658008658008, "ece_hard": 0.10777876153483887, "calibration": 76.64312384651612, "p50_s": 1.961857296526432, "p95_s": 2.823735985904932, "cost_usd_per_1000": 0.044082260469867206, "axes": { "intelligence": 80.59100767434103, "calibration": 76.64312384651612, "speed": 72.56508497392635, "cost": 50.67208430915788 }, "jevbench_score": 69.0341632296934 }, "heldout": { "n": 303, "escalation": 0.8547854785478548, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6788990825688074 }, "accuracy": 0.8448844884488449, "ece_hard": 0.1088133754767477, "calibration": 79.12566245232523, "p50_s": 1.9719288051128387, "p95_s": 2.66972187384963, "cost_usd_per_1000": 0.048999045611302694, "axes": { "intelligence": 84.16605923505509, "calibration": 79.12566245232523, "speed": 72.78642749148585, "cost": 49.29437136839852 }, "jevbench_score": 69.91583072560415 } }, "full": { "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9452054794520548, "hard": 0.6909090909090909 }, "accuracy": 0.8258426966292135, "ece_hard": 0.10692772935151185, "calibration": 78.02122706484883, "p50_s": 1.9666467420756817, "p95_s": 2.7358388897031545, "cost_usd_per_1000": 0.04687212170180532, "axes": { "intelligence": 84.23469281859694, "calibration": 78.02122706484883, "speed": 72.69183124285641, "cost": 49.87256162106466 }, "jevbench_score": 69.86549887974088 }, "full_escalation": 0.8014981273408239, "backend_heldout_accuracy": 0.8514851485148515, "useful": false, "curve": [ { "threshold": 0.0, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 1.29877570271492, "p95_s": 2.095727913081646, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 68.67075408742076, "calibration": 50.471910537865476, "speed": 75.65130946864079, "cost": 64.82109224519658 }, "jevbench_score": 64.2079181671165 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28023325293453055, "calibration": 55.17142992768658, "p50_s": 1.3129856139421463, "p95_s": 1.9534294441342355, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 74.75130911985254, "calibration": 55.17142992768658, "speed": 75.90942302768094, "cost": 64.82109224519658 }, "jevbench_score": 67.11757697107404 } }, { "threshold": 0.01, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 1.29877570271492, "p95_s": 2.095727913081646, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 68.67075408742076, "calibration": 50.471910537865476, "speed": 75.65130946864079, "cost": 64.82109224519658 }, "jevbench_score": 64.2079181671165 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28023325293453055, "calibration": 55.17142992768658, "p50_s": 1.3129856139421463, "p95_s": 1.9534294441342355, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 74.75130911985254, "calibration": 55.17142992768658, "speed": 75.90942302768094, "cost": 64.82109224519658 }, "jevbench_score": 67.11757697107404 } }, { "threshold": 0.02, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 1.29877570271492, "p95_s": 2.095727913081646, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 68.67075408742076, "calibration": 50.471910537865476, "speed": 75.65130946864079, "cost": 64.82109224519658 }, "jevbench_score": 64.2079181671165 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28023325293453055, "calibration": 55.17142992768658, "p50_s": 1.3129856139421463, "p95_s": 1.9534294441342355, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 74.75130911985254, "calibration": 55.17142992768658, "speed": 75.90942302768094, "cost": 64.82109224519658 }, "jevbench_score": 67.11757697107404 } }, { "threshold": 0.03, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 1.29877570271492, "p95_s": 2.095727913081646, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 68.67075408742076, "calibration": 50.471910537865476, "speed": 75.65130946864079, "cost": 64.82109224519658 }, "jevbench_score": 64.2079181671165 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28023325293453055, "calibration": 55.17142992768658, "p50_s": 1.3129856139421463, "p95_s": 1.9534294441342355, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 74.75130911985254, "calibration": 55.17142992768658, "speed": 75.90942302768094, "cost": 64.82109224519658 }, "jevbench_score": 67.11757697107404 } }, { "threshold": 0.04, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 1.29877570271492, "p95_s": 2.095727913081646, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 68.67075408742076, "calibration": 50.471910537865476, "speed": 75.65130946864079, "cost": 64.82109224519658 }, "jevbench_score": 64.2079181671165 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28023325293453055, "calibration": 55.17142992768658, "p50_s": 1.3129856139421463, "p95_s": 1.9534294441342355, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 74.75130911985254, "calibration": 55.17142992768658, "speed": 75.90942302768094, "cost": 64.82109224519658 }, "jevbench_score": 67.11757697107404 } }, { "threshold": 0.05, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 1.29877570271492, "p95_s": 2.095727913081646, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 68.67075408742076, "calibration": 50.471910537865476, "speed": 75.65130946864079, "cost": 64.82109224519658 }, "jevbench_score": 64.2079181671165 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28023325293453055, "calibration": 55.17142992768658, "p50_s": 1.3129856139421463, "p95_s": 1.9534294441342355, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 74.75130911985254, "calibration": 55.17142992768658, "speed": 75.90942302768094, "cost": 64.82109224519658 }, "jevbench_score": 67.11757697107404 } }, { "threshold": 0.06, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 1.29877570271492, "p95_s": 2.095727913081646, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 68.67075408742076, "calibration": 50.471910537865476, "speed": 75.65130946864079, "cost": 64.82109224519658 }, "jevbench_score": 64.2079181671165 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28023325293453055, "calibration": 55.17142992768658, "p50_s": 1.3129856139421463, "p95_s": 1.9534294441342355, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 74.75130911985254, "calibration": 55.17142992768658, "speed": 75.90942302768094, "cost": 64.82109224519658 }, "jevbench_score": 67.11757697107404 } }, { "threshold": 0.07, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 1.29877570271492, "p95_s": 2.095727913081646, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 68.67075408742076, "calibration": 50.471910537865476, "speed": 75.65130946864079, "cost": 64.82109224519658 }, "jevbench_score": 64.2079181671165 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28023325293453055, "calibration": 55.17142992768658, "p50_s": 1.3129856139421463, "p95_s": 1.9534294441342355, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 74.75130911985254, "calibration": 55.17142992768658, "speed": 75.90942302768094, "cost": 64.82109224519658 }, "jevbench_score": 67.11757697107404 } }, { "threshold": 0.08, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 1.29877570271492, "p95_s": 2.095727913081646, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 68.67075408742076, "calibration": 50.471910537865476, "speed": 75.65130946864079, "cost": 64.82109224519658 }, "jevbench_score": 64.2079181671165 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28023325293453055, "calibration": 55.17142992768658, "p50_s": 1.3129856139421463, "p95_s": 1.9534294441342355, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 74.75130911985254, "calibration": 55.17142992768658, "speed": 75.90942302768094, "cost": 64.82109224519658 }, "jevbench_score": 67.11757697107404 } }, { "threshold": 0.09, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 1.29877570271492, "p95_s": 2.095727913081646, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 68.67075408742076, "calibration": 50.471910537865476, "speed": 75.65130946864079, "cost": 64.82109224519658 }, "jevbench_score": 64.2079181671165 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28023325293453055, "calibration": 55.17142992768658, "p50_s": 1.3129856139421463, "p95_s": 1.9534294441342355, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 74.75130911985254, "calibration": 55.17142992768658, "speed": 75.90942302768094, "cost": 64.82109224519658 }, "jevbench_score": 67.11757697107404 } }, { "threshold": 0.1, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 1.29877570271492, "p95_s": 2.095727913081646, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 68.67075408742076, "calibration": 50.471910537865476, "speed": 75.65130946864079, "cost": 64.82109224519658 }, "jevbench_score": 64.2079181671165 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28023325293453055, "calibration": 55.17142992768658, "p50_s": 1.3129856139421463, "p95_s": 1.9534294441342355, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 74.75130911985254, "calibration": 55.17142992768658, "speed": 75.90942302768094, "cost": 64.82109224519658 }, "jevbench_score": 67.11757697107404 } }, { "threshold": 0.11, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 1.29877570271492, "p95_s": 2.095727913081646, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 68.67075408742076, "calibration": 50.471910537865476, "speed": 75.65130946864079, "cost": 64.82109224519658 }, "jevbench_score": 64.2079181671165 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28023325293453055, "calibration": 55.17142992768658, "p50_s": 1.3129856139421463, "p95_s": 1.9534294441342355, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 74.75130911985254, "calibration": 55.17142992768658, "speed": 75.90942302768094, "cost": 64.82109224519658 }, "jevbench_score": 67.11757697107404 } }, { "threshold": 0.12, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 1.29877570271492, "p95_s": 2.095727913081646, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 68.67075408742076, "calibration": 50.471910537865476, "speed": 75.65130946864079, "cost": 64.82109224519658 }, "jevbench_score": 64.2079181671165 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28023325293453055, "calibration": 55.17142992768658, "p50_s": 1.3129856139421463, "p95_s": 1.9534294441342355, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 74.75130911985254, "calibration": 55.17142992768658, "speed": 75.90942302768094, "cost": 64.82109224519658 }, "jevbench_score": 67.11757697107404 } }, { "threshold": 0.13, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 1.29877570271492, "p95_s": 2.095727913081646, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 68.67075408742076, "calibration": 50.471910537865476, "speed": 75.65130946864079, "cost": 64.82109224519658 }, "jevbench_score": 64.2079181671165 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28023325293453055, "calibration": 55.17142992768658, "p50_s": 1.3129856139421463, "p95_s": 1.9534294441342355, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 74.75130911985254, "calibration": 55.17142992768658, "speed": 75.90942302768094, "cost": 64.82109224519658 }, "jevbench_score": 67.11757697107404 } }, { "threshold": 0.14, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 1.29877570271492, "p95_s": 2.095727913081646, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 68.67075408742076, "calibration": 50.471910537865476, "speed": 75.65130946864079, "cost": 64.82109224519658 }, "jevbench_score": 64.2079181671165 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28023325293453055, "calibration": 55.17142992768658, "p50_s": 1.3129856139421463, "p95_s": 1.9534294441342355, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 74.75130911985254, "calibration": 55.17142992768658, "speed": 75.90942302768094, "cost": 64.82109224519658 }, "jevbench_score": 67.11757697107404 } }, { "threshold": 0.15, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 1.29877570271492, "p95_s": 2.095727913081646, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 68.67075408742076, "calibration": 50.471910537865476, "speed": 75.65130946864079, "cost": 64.82109224519658 }, "jevbench_score": 64.2079181671165 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28023325293453055, "calibration": 55.17142992768658, "p50_s": 1.3129856139421463, "p95_s": 1.9534294441342355, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 74.75130911985254, "calibration": 55.17142992768658, "speed": 75.90942302768094, "cost": 64.82109224519658 }, "jevbench_score": 67.11757697107404 } }, { "threshold": 0.16, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 1.29877570271492, "p95_s": 2.095727913081646, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 68.67075408742076, "calibration": 50.471910537865476, "speed": 75.65130946864079, "cost": 64.82109224519658 }, "jevbench_score": 64.2079181671165 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28023325293453055, "calibration": 55.17142992768658, "p50_s": 1.3129856139421463, "p95_s": 1.9534294441342355, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 74.75130911985254, "calibration": 55.17142992768658, "speed": 75.90942302768094, "cost": 64.82109224519658 }, "jevbench_score": 67.11757697107404 } }, { "threshold": 0.17, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 1.29877570271492, "p95_s": 2.095727913081646, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 68.67075408742076, "calibration": 50.471910537865476, "speed": 75.65130946864079, "cost": 64.82109224519658 }, "jevbench_score": 64.2079181671165 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28023325293453055, "calibration": 55.17142992768658, "p50_s": 1.3129856139421463, "p95_s": 1.9534294441342355, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 74.75130911985254, "calibration": 55.17142992768658, "speed": 75.90942302768094, "cost": 64.82109224519658 }, "jevbench_score": 67.11757697107404 } }, { "threshold": 0.18, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 1.29877570271492, "p95_s": 2.095727913081646, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 68.67075408742076, "calibration": 50.471910537865476, "speed": 75.65130946864079, "cost": 64.82109224519658 }, "jevbench_score": 64.2079181671165 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28023325293453055, "calibration": 55.17142992768658, "p50_s": 1.3129856139421463, "p95_s": 1.9534294441342355, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 74.75130911985254, "calibration": 55.17142992768658, "speed": 75.90942302768094, "cost": 64.82109224519658 }, "jevbench_score": 67.11757697107404 } }, { "threshold": 0.19, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 1.29877570271492, "p95_s": 2.095727913081646, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 68.67075408742076, "calibration": 50.471910537865476, "speed": 75.65130946864079, "cost": 64.82109224519658 }, "jevbench_score": 64.2079181671165 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28023325293453055, "calibration": 55.17142992768658, "p50_s": 1.3129856139421463, "p95_s": 1.9534294441342355, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 74.75130911985254, "calibration": 55.17142992768658, "speed": 75.90942302768094, "cost": 64.82109224519658 }, "jevbench_score": 67.11757697107404 } }, { "threshold": 0.2, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 1.29877570271492, "p95_s": 2.095727913081646, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 68.67075408742076, "calibration": 50.471910537865476, "speed": 75.65130946864079, "cost": 64.82109224519658 }, "jevbench_score": 64.2079181671165 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28023325293453055, "calibration": 55.17142992768658, "p50_s": 1.3129856139421463, "p95_s": 1.9534294441342355, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 74.75130911985254, "calibration": 55.17142992768658, "speed": 75.90942302768094, "cost": 64.82109224519658 }, "jevbench_score": 67.11757697107404 } }, { "threshold": 0.21, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 1.29877570271492, "p95_s": 2.095727913081646, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 68.67075408742076, "calibration": 50.471910537865476, "speed": 75.65130946864079, "cost": 64.82109224519658 }, "jevbench_score": 64.2079181671165 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28023325293453055, "calibration": 55.17142992768658, "p50_s": 1.3129856139421463, "p95_s": 1.9534294441342355, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 74.75130911985254, "calibration": 55.17142992768658, "speed": 75.90942302768094, "cost": 64.82109224519658 }, "jevbench_score": 67.11757697107404 } }, { "threshold": 0.22, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 1.29877570271492, "p95_s": 2.095727913081646, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 68.67075408742076, "calibration": 50.471910537865476, "speed": 75.65130946864079, "cost": 64.82109224519658 }, "jevbench_score": 64.2079181671165 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28023325293453055, "calibration": 55.17142992768658, "p50_s": 1.3129856139421463, "p95_s": 1.9534294441342355, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 74.75130911985254, "calibration": 55.17142992768658, "speed": 75.90942302768094, "cost": 64.82109224519658 }, "jevbench_score": 67.11757697107404 } }, { "threshold": 0.23, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3114836765033705, "calibration": 50.471910537865476, "p50_s": 1.29877570271492, "p95_s": 2.095727913081646, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 68.67075408742076, "calibration": 50.471910537865476, "speed": 75.65130946864079, "cost": 64.82109224519658 }, "jevbench_score": 64.2079181671165 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28023325293453055, "calibration": 55.17142992768658, "p50_s": 1.3129856139421463, "p95_s": 1.9534294441342355, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 74.75130911985254, "calibration": 55.17142992768658, "speed": 75.90942302768094, "cost": 64.82109224519658 }, "jevbench_score": 67.11757697107404 } }, { "threshold": 0.24, "public": { "n": 231, "escalation": 0.004329004329004329, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.31064198216100836, "calibration": 50.55607997210169, "p50_s": 1.29877570271492, "p95_s": 2.124082013964653, "cost_usd_per_1000": 0.015053725229826353, "axes": { "intelligence": 68.67075408742076, "calibration": 50.55607997210169, "speed": 75.59294563191898, "cost": 64.6706804574606 }, "jevbench_score": 64.18498987266955 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28023325293453055, "calibration": 55.17142992768658, "p50_s": 1.3129856139421463, "p95_s": 1.9534294441342355, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 74.75130911985254, "calibration": 55.17142992768658, "speed": 75.90942302768094, "cost": 64.82109224519658 }, "jevbench_score": 67.11757697107404 } }, { "threshold": 0.25, "public": { "n": 231, "escalation": 0.004329004329004329, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.31064198216100836, "calibration": 50.55607997210169, "p50_s": 1.29877570271492, "p95_s": 2.124082013964653, "cost_usd_per_1000": 0.015053725229826353, "axes": { "intelligence": 68.67075408742076, "calibration": 50.55607997210169, "speed": 75.59294563191898, "cost": 64.6706804574606 }, "jevbench_score": 64.18498987266955 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.28023325293453055, "calibration": 55.17142992768658, "p50_s": 1.3129856139421463, "p95_s": 1.9534294441342355, "cost_usd_per_1000": 0.014880936329588014, "axes": { "intelligence": 74.75130911985254, "calibration": 55.17142992768658, "speed": 75.90942302768094, "cost": 64.82109224519658 }, "jevbench_score": 67.11757697107404 } }, { "threshold": 0.26, "public": { "n": 231, "escalation": 0.004329004329004329, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.31064198216100836, "calibration": 50.55607997210169, "p50_s": 1.29877570271492, "p95_s": 2.124082013964653, "cost_usd_per_1000": 0.015053725229826353, "axes": { "intelligence": 68.67075408742076, "calibration": 50.55607997210169, "speed": 75.59294563191898, "cost": 64.6706804574606 }, "jevbench_score": 64.18498987266955 }, "heldout": { "n": 303, "escalation": 0.0033003300330033004, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.2805812954685626, "calibration": 55.13662567428338, "p50_s": 1.3129856139421463, "p95_s": 1.9764527156949037, "cost_usd_per_1000": 0.015012666481254867, "axes": { "intelligence": 74.75130911985254, "calibration": 55.13662567428338, "speed": 75.85853603522497, "cost": 64.7062649050944 }, "jevbench_score": 67.06600606370351 } }, { "threshold": 0.27, "public": { "n": 231, "escalation": 0.004329004329004329, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.31064198216100836, "calibration": 50.55607997210169, "p50_s": 1.29877570271492, "p95_s": 2.124082013964653, "cost_usd_per_1000": 0.015053725229826353, "axes": { "intelligence": 68.67075408742076, "calibration": 50.55607997210169, "speed": 75.59294563191898, "cost": 64.6706804574606 }, "jevbench_score": 64.18498987266955 }, "heldout": { "n": 303, "escalation": 0.0033003300330033004, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.2805812954685626, "calibration": 55.13662567428338, "p50_s": 1.3129856139421463, "p95_s": 1.9764527156949037, "cost_usd_per_1000": 0.015012666481254867, "axes": { "intelligence": 74.75130911985254, "calibration": 55.13662567428338, "speed": 75.85853603522497, "cost": 64.7062649050944 }, "jevbench_score": 67.06600606370351 } }, { "threshold": 0.28, "public": { "n": 231, "escalation": 0.004329004329004329, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.31064198216100836, "calibration": 50.55607997210169, "p50_s": 1.29877570271492, "p95_s": 2.124082013964653, "cost_usd_per_1000": 0.015053725229826353, "axes": { "intelligence": 68.67075408742076, "calibration": 50.55607997210169, "speed": 75.59294563191898, "cost": 64.6706804574606 }, "jevbench_score": 64.18498987266955 }, "heldout": { "n": 303, "escalation": 0.0033003300330033004, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.2805812954685626, "calibration": 55.13662567428338, "p50_s": 1.3129856139421463, "p95_s": 1.9764527156949037, "cost_usd_per_1000": 0.015012666481254867, "axes": { "intelligence": 74.75130911985254, "calibration": 55.13662567428338, "speed": 75.85853603522497, "cost": 64.7062649050944 }, "jevbench_score": 67.06600606370351 } }, { "threshold": 0.29, "public": { "n": 231, "escalation": 0.004329004329004329, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.31064198216100836, "calibration": 50.55607997210169, "p50_s": 1.29877570271492, "p95_s": 2.124082013964653, "cost_usd_per_1000": 0.015053725229826353, "axes": { "intelligence": 68.67075408742076, "calibration": 50.55607997210169, "speed": 75.59294563191898, "cost": 64.6706804574606 }, "jevbench_score": 64.18498987266955 }, "heldout": { "n": 303, "escalation": 0.0033003300330033004, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.2805812954685626, "calibration": 55.13662567428338, "p50_s": 1.3129856139421463, "p95_s": 1.9764527156949037, "cost_usd_per_1000": 0.015012666481254867, "axes": { "intelligence": 74.75130911985254, "calibration": 55.13662567428338, "speed": 75.85853603522497, "cost": 64.7062649050944 }, "jevbench_score": 67.06600606370351 } }, { "threshold": 0.3, "public": { "n": 231, "escalation": 0.004329004329004329, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.31064198216100836, "calibration": 50.55607997210169, "p50_s": 1.29877570271492, "p95_s": 2.124082013964653, "cost_usd_per_1000": 0.015053725229826353, "axes": { "intelligence": 68.67075408742076, "calibration": 50.55607997210169, "speed": 75.59294563191898, "cost": 64.6706804574606 }, "jevbench_score": 64.18498987266955 }, "heldout": { "n": 303, "escalation": 0.0033003300330033004, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.2805812954685626, "calibration": 55.13662567428338, "p50_s": 1.3129856139421463, "p95_s": 1.9764527156949037, "cost_usd_per_1000": 0.015012666481254867, "axes": { "intelligence": 74.75130911985254, "calibration": 55.13662567428338, "speed": 75.85853603522497, "cost": 64.7062649050944 }, "jevbench_score": 67.06600606370351 } }, { "threshold": 0.31, "public": { "n": 231, "escalation": 0.004329004329004329, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.31064198216100836, "calibration": 50.55607997210169, "p50_s": 1.29877570271492, "p95_s": 2.124082013964653, "cost_usd_per_1000": 0.015053725229826353, "axes": { "intelligence": 68.67075408742076, "calibration": 50.55607997210169, "speed": 75.59294563191898, "cost": 64.6706804574606 }, "jevbench_score": 64.18498987266955 }, "heldout": { "n": 303, "escalation": 0.006600660066006601, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.2816151944761498, "calibration": 55.03323577352466, "p50_s": 1.3150212317705154, "p95_s": 1.9764527156949037, "cost_usd_per_1000": 0.015144396632921718, "axes": { "intelligence": 74.75130911985254, "calibration": 55.03323577352466, "speed": 75.8518080636569, "cost": 64.59244074067499 }, "jevbench_score": 67.00355893002215 } }, { "threshold": 0.32, "public": { "n": 231, "escalation": 0.004329004329004329, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.31064198216100836, "calibration": 50.55607997210169, "p50_s": 1.29877570271492, "p95_s": 2.124082013964653, "cost_usd_per_1000": 0.015053725229826353, "axes": { "intelligence": 68.67075408742076, "calibration": 50.55607997210169, "speed": 75.59294563191898, "cost": 64.6706804574606 }, "jevbench_score": 64.18498987266955 }, "heldout": { "n": 303, "escalation": 0.0165016501650165, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.2812574962849533, "calibration": 55.06900559264431, "p50_s": 1.315921351313591, "p95_s": 1.9910572722554205, "cost_usd_per_1000": 0.015539587087922274, "axes": { "intelligence": 74.75130911985254, "calibration": 55.06900559264431, "speed": 75.81686313852413, "cost": 64.25681575875456 }, "jevbench_score": 66.91951154702434 } }, { "threshold": 0.33, "public": { "n": 231, "escalation": 0.012987012987012988, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.304721239904861, "calibration": 51.14815419771642, "p50_s": 1.29877570271492, "p95_s": 2.160557709634304, "cost_usd_per_1000": 0.015399303030303029, "axes": { "intelligence": 68.67075408742076, "calibration": 51.14815419771642, "speed": 75.51899973644217, "cost": 64.37496804427596 }, "jevbench_score": 64.28264815618768 }, "heldout": { "n": 303, "escalation": 0.0165016501650165, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.2812574962849533, "calibration": 55.06900559264431, "p50_s": 1.315921351313591, "p95_s": 1.9910572722554205, "cost_usd_per_1000": 0.015539587087922274, "axes": { "intelligence": 74.75130911985254, "calibration": 55.06900559264431, "speed": 75.81686313852413, "cost": 64.25681575875456 }, "jevbench_score": 66.91951154702434 } }, { "threshold": 0.34, "public": { "n": 231, "escalation": 0.017316017316017316, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3044823064919826, "calibration": 51.17204753900427, "p50_s": 1.29877570271492, "p95_s": 2.182773359119892, "cost_usd_per_1000": 0.015572091930541368, "axes": { "intelligence": 68.67075408742076, "calibration": 51.17204753900427, "speed": 75.47457200127542, "cost": 64.22959123208213 }, "jevbench_score": 64.24437479279123 }, "heldout": { "n": 303, "escalation": 0.019801980198019802, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.45871559633027525 }, "accuracy": 0.7293729372937293, "ece_hard": 0.2908935529487464, "calibration": 54.105399926264994, "p50_s": 1.31616772711277, "p95_s": 2.017643444240093, "cost_usd_per_1000": 0.015671317239589127, "axes": { "intelligence": 74.47607976205437, "calibration": 54.105399926264994, "speed": 75.75844341824187, "cost": 64.14683493202692 }, "jevbench_score": 66.52209587726212 } }, { "threshold": 0.35, "public": { "n": 231, "escalation": 0.017316017316017316, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3044823064919826, "calibration": 51.17204753900427, "p50_s": 1.29877570271492, "p95_s": 2.182773359119892, "cost_usd_per_1000": 0.015572091930541368, "axes": { "intelligence": 68.67075408742076, "calibration": 51.17204753900427, "speed": 75.47457200127542, "cost": 64.22959123208213 }, "jevbench_score": 64.24437479279123 }, "heldout": { "n": 303, "escalation": 0.019801980198019802, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.45871559633027525 }, "accuracy": 0.7293729372937293, "ece_hard": 0.2908935529487464, "calibration": 54.105399926264994, "p50_s": 1.31616772711277, "p95_s": 2.017643444240093, "cost_usd_per_1000": 0.015671317239589127, "axes": { "intelligence": 74.47607976205437, "calibration": 54.105399926264994, "speed": 75.75844341824187, "cost": 64.14683493202692 }, "jevbench_score": 66.52209587726212 } }, { "threshold": 0.36, "public": { "n": 231, "escalation": 0.021645021645021644, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.4774774774774775 }, "accuracy": 0.6753246753246753, "ece_hard": 0.3076807610008929, "calibration": 50.85220208811323, "p50_s": 1.29877570271492, "p95_s": 2.182773359119892, "cost_usd_per_1000": 0.015744880830779705, "axes": { "intelligence": 69.04612946279613, "calibration": 50.85220208811323, "speed": 75.47457200127542, "cost": 64.08581866256776 }, "jevbench_score": 64.19525443243904 }, "heldout": { "n": 303, "escalation": 0.026402640264026403, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8767123287671232, "hard": 0.46788990825688076 }, "accuracy": 0.7326732673267327, "ece_hard": 0.2832437750398088, "calibration": 54.87037771715876, "p50_s": 1.3169443011283875, "p95_s": 2.0189615026116368, "cost_usd_per_1000": 0.01593477754292283, "axes": { "intelligence": 74.75130911985254, "calibration": 54.87037771715876, "speed": 75.75304554778407, "cost": 63.92961985316673 }, "jevbench_score": 66.75975636733942 } }, { "threshold": 0.37, "public": { "n": 231, "escalation": 0.030303030303030304, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.3132753709552286, "calibration": 50.29274109267966, "p50_s": 1.2999516874551773, "p95_s": 2.182773359119892, "cost_usd_per_1000": 0.016090458631256382, "axes": { "intelligence": 68.67075408742076, "calibration": 50.29274109267966, "speed": 75.47064143341139, "cost": 63.802947306888925 }, "jevbench_score": 63.859273914783024 }, "heldout": { "n": 303, "escalation": 0.033003300330033, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8835616438356164, "hard": 0.47706422018348627 }, "accuracy": 0.7392739273927392, "ece_hard": 0.2737169911994368, "calibration": 55.82305610119596, "p50_s": 1.3169443011283875, "p95_s": 2.0295264974236487, "cost_usd_per_1000": 0.016198237846256534, "axes": { "intelligence": 75.21831929956852, "calibration": 55.82305610119596, "speed": 75.73037866887115, "cost": 63.71596685126264 }, "jevbench_score": 67.09094513800281 } }, { "threshold": 0.38, "public": { "n": 231, "escalation": 0.03463203463203463, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.30401278236189166, "calibration": 51.21899995201336, "p50_s": 1.3007893413305283, "p95_s": 2.182773359119892, "cost_usd_per_1000": 0.01626324753149472, "axes": { "intelligence": 68.67075408742076, "calibration": 51.21899995201336, "speed": 75.46784385791761, "cost": 63.66378183572344 }, "jevbench_score": 64.11569019375968 }, "heldout": { "n": 303, "escalation": 0.036303630363036306, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8835616438356164, "hard": 0.48623853211009177 }, "accuracy": 0.7425742574257426, "ece_hard": 0.2692179407768909, "calibration": 56.27296114345055, "p50_s": 1.3169443011283875, "p95_s": 2.031159219145775, "cost_usd_per_1000": 0.016329967997923387, "axes": { "intelligence": 75.49354865736669, "calibration": 56.27296114345055, "speed": 75.72688624359076, "cost": 63.61043999067198 }, "jevbench_score": 67.25847671966116 } }, { "threshold": 0.39, "public": { "n": 231, "escalation": 0.03896103896103896, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.46846846846846846 }, "accuracy": 0.670995670995671, "ece_hard": 0.29791612639381837, "calibration": 51.82866554882069, "p50_s": 1.3007893413305283, "p95_s": 2.194945380091667, "cost_usd_per_1000": 0.01643603643173306, "axes": { "intelligence": 68.67075408742076, "calibration": 51.82866554882069, "speed": 75.44369312791835, "cost": 63.52608714286394 }, "jevbench_score": 64.26569704023849 }, "heldout": { "n": 303, "escalation": 0.0429042904290429, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8835616438356164, "hard": 0.4954128440366973 }, "accuracy": 0.7458745874587459, "ece_hard": 0.25981230013978185, "calibration": 57.21352520716145, "p50_s": 1.3177975565195084, "p95_s": 2.031159219145775, "cost_usd_per_1000": 0.01659342830125709, "axes": { "intelligence": 75.76877801516484, "calibration": 57.21352520716145, "speed": 75.72407333712027, "cost": 63.40191630624812 }, "jevbench_score": 67.54315385051389 } }, { "threshold": 0.4, "public": { "n": 231, "escalation": 0.04329004329004329, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.4774774774774775 }, "accuracy": 0.6753246753246753, "ece_hard": 0.307475731816139, "calibration": 50.872705006588625, "p50_s": 1.3007893413305283, "p95_s": 2.194945380091667, "cost_usd_per_1000": 0.0166088253319714, "axes": { "intelligence": 69.04612946279613, "calibration": 50.872705006588625, "speed": 75.44369312791835, "cost": 63.389832465138795 }, "jevbench_score": 64.02014864351527 }, "heldout": { "n": 303, "escalation": 0.0429042904290429, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8835616438356164, "hard": 0.4954128440366973 }, "accuracy": 0.7458745874587459, "ece_hard": 0.25981230013978185, "calibration": 57.21352520716145, "p50_s": 1.3177975565195084, "p95_s": 2.031159219145775, "cost_usd_per_1000": 0.01659342830125709, "axes": { "intelligence": 75.76877801516484, "calibration": 57.21352520716145, "speed": 75.72407333712027, "cost": 63.40191630624812 }, "jevbench_score": 67.54315385051389 } }, { "threshold": 0.41, "public": { "n": 231, "escalation": 0.04329004329004329, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.4774774774774775 }, "accuracy": 0.6753246753246753, "ece_hard": 0.307475731816139, "calibration": 50.872705006588625, "p50_s": 1.3007893413305283, "p95_s": 2.194945380091667, "cost_usd_per_1000": 0.0166088253319714, "axes": { "intelligence": 69.04612946279613, "calibration": 50.872705006588625, "speed": 75.44369312791835, "cost": 63.389832465138795 }, "jevbench_score": 64.02014864351527 }, "heldout": { "n": 303, "escalation": 0.0429042904290429, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8835616438356164, "hard": 0.4954128440366973 }, "accuracy": 0.7458745874587459, "ece_hard": 0.25981230013978185, "calibration": 57.21352520716145, "p50_s": 1.3177975565195084, "p95_s": 2.031159219145775, "cost_usd_per_1000": 0.01659342830125709, "axes": { "intelligence": 75.76877801516484, "calibration": 57.21352520716145, "speed": 75.72407333712027, "cost": 63.40191630624812 }, "jevbench_score": 67.54315385051389 } }, { "threshold": 0.42, "public": { "n": 231, "escalation": 0.047619047619047616, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.4774774774774775 }, "accuracy": 0.6753246753246753, "ece_hard": 0.3079581809503299, "calibration": 50.824460093169535, "p50_s": 1.3007893413305283, "p95_s": 2.194945380091667, "cost_usd_per_1000": 0.016781614232209738, "axes": { "intelligence": 69.04612946279613, "calibration": 50.824460093169535, "speed": 75.44369312791835, "cost": 63.25498799457087 }, "jevbench_score": 63.97089949835608 }, "heldout": { "n": 303, "escalation": 0.0429042904290429, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8835616438356164, "hard": 0.4954128440366973 }, "accuracy": 0.7458745874587459, "ece_hard": 0.25981230013978185, "calibration": 57.21352520716145, "p50_s": 1.3177975565195084, "p95_s": 2.031159219145775, "cost_usd_per_1000": 0.01659342830125709, "axes": { "intelligence": 75.76877801516484, "calibration": 57.21352520716145, "speed": 75.72407333712027, "cost": 63.40191630624812 }, "jevbench_score": 67.54315385051389 } }, { "threshold": 0.43, "public": { "n": 231, "escalation": 0.05194805194805195, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.4774774774774775 }, "accuracy": 0.6753246753246753, "ece_hard": 0.31316801218597495, "calibration": 50.30347696960503, "p50_s": 1.3012879490852356, "p95_s": 2.194945380091667, "cost_usd_per_1000": 0.016954403132448073, "axes": { "intelligence": 69.04612946279613, "calibration": 50.30347696960503, "speed": 75.44202874538865, "cost": 63.12152483923731 }, "jevbench_score": 63.7722949063986 }, "heldout": { "n": 303, "escalation": 0.0429042904290429, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8835616438356164, "hard": 0.4954128440366973 }, "accuracy": 0.7458745874587459, "ece_hard": 0.25981230013978185, "calibration": 57.21352520716145, "p50_s": 1.3177975565195084, "p95_s": 2.031159219145775, "cost_usd_per_1000": 0.01659342830125709, "axes": { "intelligence": 75.76877801516484, "calibration": 57.21352520716145, "speed": 75.72407333712027, "cost": 63.40191630624812 }, "jevbench_score": 67.54315385051389 } }, { "threshold": 0.44, "public": { "n": 231, "escalation": 0.05194805194805195, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.4774774774774775 }, "accuracy": 0.6753246753246753, "ece_hard": 0.31316801218597495, "calibration": 50.30347696960503, "p50_s": 1.3012879490852356, "p95_s": 2.194945380091667, "cost_usd_per_1000": 0.016954403132448073, "axes": { "intelligence": 69.04612946279613, "calibration": 50.30347696960503, "speed": 75.44202874538865, "cost": 63.12152483923731 }, "jevbench_score": 63.7722949063986 }, "heldout": { "n": 303, "escalation": 0.0429042904290429, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8835616438356164, "hard": 0.4954128440366973 }, "accuracy": 0.7458745874587459, "ece_hard": 0.25981230013978185, "calibration": 57.21352520716145, "p50_s": 1.3177975565195084, "p95_s": 2.031159219145775, "cost_usd_per_1000": 0.01659342830125709, "axes": { "intelligence": 75.76877801516484, "calibration": 57.21352520716145, "speed": 75.72407333712027, "cost": 63.40191630624812 }, "jevbench_score": 67.54315385051389 } }, { "threshold": 0.45, "public": { "n": 231, "escalation": 0.05627705627705628, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.4774774774774775 }, "accuracy": 0.6753246753246753, "ece_hard": 0.31730690094026126, "calibration": 49.8895880941764, "p50_s": 1.3016724586486816, "p95_s": 2.194945380091667, "cost_usd_per_1000": 0.017127192032686412, "axes": { "intelligence": 69.04612946279613, "calibration": 49.8895880941764, "speed": 75.44074566492104, "cost": 62.98941498611381 }, "jevbench_score": 63.60711544138155 }, "heldout": { "n": 303, "escalation": 0.0462046204620462, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8835616438356164, "hard": 0.4954128440366973 }, "accuracy": 0.7458745874587459, "ece_hard": 0.25981230013978185, "calibration": 57.21352520716145, "p50_s": 1.3177975565195084, "p95_s": 2.0385801903903484, "cost_usd_per_1000": 0.016725158452923942, "axes": { "intelligence": 75.76877801516484, "calibration": 57.21352520716145, "speed": 75.70823502360889, "cost": 63.29889277199784 }, "jevbench_score": 67.5121682856777 } }, { "threshold": 0.46, "public": { "n": 231, "escalation": 0.06060606060606061, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.4774774774774775 }, "accuracy": 0.6753246753246753, "ece_hard": 0.31730690094026126, "calibration": 49.8895880941764, "p50_s": 1.3016724586486816, "p95_s": 2.194945380091667, "cost_usd_per_1000": 0.01729998093292475, "axes": { "intelligence": 69.04612946279613, "calibration": 49.8895880941764, "speed": 75.44074566492104, "cost": 62.85863126578376 }, "jevbench_score": 63.574073143332285 }, "heldout": { "n": 303, "escalation": 0.04950495049504951, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8835616438356164, "hard": 0.5045871559633027 }, "accuracy": 0.7491749174917491, "ece_hard": 0.25536173506631354, "calibration": 57.65858171450829, "p50_s": 1.3177975565195084, "p95_s": 2.05896622762084, "cost_usd_per_1000": 0.016856888604590795, "axes": { "intelligence": 76.04400737296301, "calibration": 57.65858171450829, "speed": 75.66502078941178, "cost": 63.19667749375031 }, "jevbench_score": 67.66741532590365 } }, { "threshold": 0.47, "public": { "n": 231, "escalation": 0.06493506493506493, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.4864864864864865 }, "accuracy": 0.6796536796536796, "ece_hard": 0.3121326484366342, "calibration": 50.40701334453911, "p50_s": 1.3016724586486816, "p95_s": 2.249755024909973, "cost_usd_per_1000": 0.01747276983316309, "axes": { "intelligence": 69.42150483817151, "calibration": 50.40701334453911, "speed": 75.33363053543655, "cost": 62.72914731890118 }, "jevbench_score": 63.76917785289303 }, "heldout": { "n": 303, "escalation": 0.04950495049504951, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8835616438356164, "hard": 0.5045871559633027 }, "accuracy": 0.7491749174917491, "ece_hard": 0.25536173506631354, "calibration": 57.65858171450829, "p50_s": 1.3177975565195084, "p95_s": 2.05896622762084, "cost_usd_per_1000": 0.016856888604590795, "axes": { "intelligence": 76.04400737296301, "calibration": 57.65858171450829, "speed": 75.66502078941178, "cost": 63.19667749375031 }, "jevbench_score": 67.66741532590365 } }, { "threshold": 0.48, "public": { "n": 231, "escalation": 0.08225108225108226, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.4954954954954955 }, "accuracy": 0.683982683982684, "ece_hard": 0.31403554082215135, "calibration": 50.216724105987396, "p50_s": 1.302245020866394, "p95_s": 2.392199069261551, "cost_usd_per_1000": 0.018163925434116445, "axes": { "intelligence": 69.7968802135469, "calibration": 50.216724105987396, "speed": 75.06509977342857, "cost": 62.2237086885179 }, "jevbench_score": 63.60914922879819 }, "heldout": { "n": 303, "escalation": 0.056105610561056105, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8835616438356164, "hard": 0.5045871559633027 }, "accuracy": 0.7491749174917491, "ece_hard": 0.26335743884444923, "calibration": 56.2757733593688, "p50_s": 1.3178558647632599, "p95_s": 2.0655703842639923, "cost_usd_per_1000": 0.0171203489079245, "axes": { "intelligence": 76.04400737296301, "calibration": 56.2757733593688, "speed": 75.65092088234661, "cost": 62.99462166305644 }, "jevbench_score": 67.20104701075917 } }, { "threshold": 0.49, "public": { "n": 231, "escalation": 0.08225108225108226, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.4954954954954955 }, "accuracy": 0.683982683982684, "ece_hard": 0.31403554082215135, "calibration": 50.216724105987396, "p50_s": 1.302245020866394, "p95_s": 2.392199069261551, "cost_usd_per_1000": 0.018163925434116445, "axes": { "intelligence": 69.7968802135469, "calibration": 50.216724105987396, "speed": 75.06509977342857, "cost": 62.2237086885179 }, "jevbench_score": 63.60914922879819 }, "heldout": { "n": 303, "escalation": 0.066006600660066, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8972602739726028, "hard": 0.5045871559633027 }, "accuracy": 0.7557755775577558, "ece_hard": 0.26647609528068694, "calibration": 58.34821169554418, "p50_s": 1.3178558647632599, "p95_s": 2.1734318621456605, "cost_usd_per_1000": 0.017515539362925055, "axes": { "intelligence": 76.42756901679863, "calibration": 58.34821169554418, "speed": 75.42986056918713, "cost": 62.697294542163434 }, "jevbench_score": 67.76686975414808 } }, { "threshold": 0.5, "public": { "n": 231, "escalation": 0.08225108225108226, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.4954954954954955 }, "accuracy": 0.683982683982684, "ece_hard": 0.31403554082215135, "calibration": 50.216724105987396, "p50_s": 1.302245020866394, "p95_s": 2.392199069261551, "cost_usd_per_1000": 0.018163925434116445, "axes": { "intelligence": 69.7968802135469, "calibration": 50.216724105987396, "speed": 75.06509977342857, "cost": 62.2237086885179 }, "jevbench_score": 63.60914922879819 }, "heldout": { "n": 303, "escalation": 0.06930693069306931, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8972602739726028, "hard": 0.5137614678899083 }, "accuracy": 0.759075907590759, "ece_hard": 0.2614714140849992, "calibration": 60.16324660617054, "p50_s": 1.3183358311653137, "p95_s": 2.1734318621456605, "cost_usd_per_1000": 0.017647269514591907, "axes": { "intelligence": 76.7027983745968, "calibration": 60.16324660617054, "speed": 75.42827914578353, "cost": 62.599674447345016 }, "jevbench_score": 68.3222541888607 } }, { "threshold": 0.51, "public": { "n": 231, "escalation": 0.09090909090909091, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.4954954954954955 }, "accuracy": 0.683982683982684, "ece_hard": 0.3168593534400633, "calibration": 49.934342844196195, "p50_s": 1.3028961420059204, "p95_s": 2.4835226498544216, "cost_usd_per_1000": 0.01850950323459312, "axes": { "intelligence": 69.7968802135469, "calibration": 49.934342844196195, "speed": 74.90022076764362, "cost": 61.97815710575984 }, "jevbench_score": 63.42190408525422 }, "heldout": { "n": 303, "escalation": 0.07920792079207921, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8972602739726028, "hard": 0.5229357798165137 }, "accuracy": 0.7623762376237624, "ece_hard": 0.26614706052569764, "calibration": 59.69568196210069, "p50_s": 1.3189159780740738, "p95_s": 2.1768629007041445, "cost_usd_per_1000": 0.018042459969592462, "axes": { "intelligence": 76.97802773239495, "calibration": 59.69568196210069, "speed": 75.4195179248661, "cost": 62.311127487660926 }, "jevbench_score": 68.16944604409876 } }, { "threshold": 0.52, "public": { "n": 231, "escalation": 0.1038961038961039, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.4954954954954955 }, "accuracy": 0.683982683982684, "ece_hard": 0.3248893091944723, "calibration": 49.1313472687553, "p50_s": 1.304066225886345, "p95_s": 2.4835226498544216, "cost_usd_per_1000": 0.019027869935308136, "axes": { "intelligence": 69.7968802135469, "calibration": 49.1313472687553, "speed": 74.89632227633425, "cost": 61.61829477564326 }, "jevbench_score": 63.072669579512855 }, "heldout": { "n": 303, "escalation": 0.08250825082508251, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8972602739726028, "hard": 0.5229357798165137 }, "accuracy": 0.7623762376237624, "ece_hard": 0.26614706052569764, "calibration": 59.69568196210069, "p50_s": 1.319311037659645, "p95_s": 2.1768629007041445, "cost_usd_per_1000": 0.018174190121259315, "axes": { "intelligence": 76.97802773239495, "calibration": 59.69568196210069, "speed": 75.41821726197169, "cost": 62.2163479923975 }, "jevbench_score": 68.14321484723463 } }, { "threshold": 0.53, "public": { "n": 231, "escalation": 0.11255411255411256, "tiers": { "easy": 1.0, "standard": 0.7777777777777778, "hard": 0.4864864864864865 }, "accuracy": 0.683982683982684, "ece_hard": 0.33511677813183927, "calibration": 48.1086003750186, "p50_s": 1.3058469742536545, "p95_s": 2.4835226498544216, "cost_usd_per_1000": 0.019373447735784814, "axes": { "intelligence": 69.96162829496163, "calibration": 48.1086003750186, "speed": 74.89039587763601, "cost": 61.38379253580756 }, "jevbench_score": 62.71777151854593 }, "heldout": { "n": 303, "escalation": 0.09570957095709572, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8972602739726028, "hard": 0.5321100917431193 }, "accuracy": 0.7656765676567657, "ece_hard": 0.2563562887963421, "calibration": 60.674759135036254, "p50_s": 1.322104200720787, "p95_s": 2.1943851932883263, "cost_usd_per_1000": 0.018701110727926722, "axes": { "intelligence": 77.25325709019313, "calibration": 60.674759135036254, "speed": 75.37421450699631, "cost": 61.843977950401396 }, "jevbench_score": 68.36932133843797 } }, { "threshold": 0.54, "public": { "n": 231, "escalation": 0.11688311688311688, "tiers": { "easy": 1.0, "standard": 0.7777777777777778, "hard": 0.4864864864864865 }, "accuracy": 0.683982683982684, "ece_hard": 0.33511677813183927, "calibration": 48.1086003750186, "p50_s": 1.307793453335762, "p95_s": 2.4835226498544216, "cost_usd_per_1000": 0.019546236636023152, "axes": { "intelligence": 69.96162829496163, "calibration": 48.1086003750186, "speed": 74.8839271585891, "cost": 61.26810543275963 }, "jevbench_score": 62.686846583694845 }, "heldout": { "n": 303, "escalation": 0.11221122112211221, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8972602739726028, "hard": 0.5229357798165137 }, "accuracy": 0.7623762376237624, "ece_hard": 0.26480186582722387, "calibration": 59.83020143194807, "p50_s": 1.323771134018898, "p95_s": 2.202232655882835, "cost_usd_per_1000": 0.019359761486260983, "axes": { "intelligence": 76.97802773239495, "calibration": 59.83020143194807, "speed": 75.35323895013562, "cost": 61.39299992607483 }, "jevbench_score": 67.94022897006589 } }, { "threshold": 0.55, "public": { "n": 231, "escalation": 0.12987012987012986, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "hard": 0.4954954954954955 }, "accuracy": 0.6926406926406926, "ece_hard": 0.32993759155571767, "calibration": 48.62651903263076, "p50_s": 1.3082244992256165, "p95_s": 2.5310783348977566, "cost_usd_per_1000": 0.020064603336738165, "axes": { "intelligence": 70.87712712712712, "calibration": 48.62651903263076, "speed": 74.80012135618844, "cost": 60.92708264635968 }, "jevbench_score": 62.9539512326207 }, "heldout": { "n": 303, "escalation": 0.13861386138613863, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.910958904109589, "hard": 0.5321100917431193 }, "accuracy": 0.7722772277227723, "ece_hard": 0.25944148306469994, "calibration": 60.36623970820047, "p50_s": 1.3345372080802917, "p95_s": 2.205136180669069, "cost_usd_per_1000": 0.0204136026995958, "axes": { "intelligence": 77.63681873402875, "calibration": 60.36623970820047, "speed": 75.31233898215517, "cost": 60.70241025859773 }, "jevbench_score": 68.03517114979368 } }, { "threshold": 0.56, "public": { "n": 231, "escalation": 0.13852813852813853, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "hard": 0.5045045045045045 }, "accuracy": 0.696969696969697, "ece_hard": 0.3265388701013378, "calibration": 48.96639117806875, "p50_s": 1.309425413608551, "p95_s": 2.5310783348977566, "cost_usd_per_1000": 0.020410181137214843, "axes": { "intelligence": 71.25250250250251, "calibration": 48.96639117806875, "speed": 74.79613648010718, "cost": 60.704594229105155 }, "jevbench_score": 63.08843241077166 }, "heldout": { "n": 303, "escalation": 0.1551155115511551, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.910958904109589, "hard": 0.5321100917431193 }, "accuracy": 0.7722772277227723, "ece_hard": 0.2683321712543727, "calibration": 59.47717088923319, "p50_s": 1.336438462138176, "p95_s": 2.230665914714336, "cost_usd_per_1000": 0.02107225345793006, "axes": { "intelligence": 77.63681873402875, "calibration": 59.47717088923319, "speed": 75.25616502870035, "cost": 60.288670559170306 }, "jevbench_score": 67.65485321369933 } }, { "threshold": 0.57, "public": { "n": 231, "escalation": 0.14285714285714285, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "hard": 0.5135135135135135 }, "accuracy": 0.7012987012987013, "ece_hard": 0.3269340084342508, "calibration": 48.926877344777445, "p50_s": 1.309425413608551, "p95_s": 2.5801136046648026, "cost_usd_per_1000": 0.020582970037453182, "axes": { "intelligence": 71.62787787787788, "calibration": 48.926877344777445, "speed": 74.71280405700864, "cost": 60.594758744625985 }, "jevbench_score": 63.112432620008704 }, "heldout": { "n": 303, "escalation": 0.16501650165016502, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.910958904109589, "hard": 0.5321100917431193 }, "accuracy": 0.7722772277227723, "ece_hard": 0.2738721928685699, "calibration": 58.92316872781347, "p50_s": 1.3393714874982834, "p95_s": 2.230665914714336, "cost_usd_per_1000": 0.021467443912930616, "axes": { "intelligence": 77.63681873402875, "calibration": 58.92316872781347, "speed": 75.24664419378827, "cost": 60.04658989270939 }, "jevbench_score": 67.42676560445072 } }, { "threshold": 0.58, "public": { "n": 231, "escalation": 0.1471861471861472, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "hard": 0.5135135135135135 }, "accuracy": 0.7012987012987013, "ece_hard": 0.33471237572165463, "calibration": 48.14904061603706, "p50_s": 1.309425413608551, "p95_s": 2.5801136046648026, "cost_usd_per_1000": 0.02075575893769152, "axes": { "intelligence": 71.62787787787788, "calibration": 48.14904061603706, "speed": 74.71280405700864, "cost": 60.48584145840898 }, "jevbench_score": 62.83181734496725 }, "heldout": { "n": 303, "escalation": 0.16501650165016502, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.910958904109589, "hard": 0.5321100917431193 }, "accuracy": 0.7722772277227723, "ece_hard": 0.2738721928685699, "calibration": 58.92316872781347, "p50_s": 1.3393714874982834, "p95_s": 2.230665914714336, "cost_usd_per_1000": 0.021467443912930616, "axes": { "intelligence": 77.63681873402875, "calibration": 58.92316872781347, "speed": 75.24664419378827, "cost": 60.04658989270939 }, "jevbench_score": 67.42676560445072 } }, { "threshold": 0.59, "public": { "n": 231, "escalation": 0.15584415584415584, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.5135135135135135 }, "accuracy": 0.7056277056277056, "ece_hard": 0.33471237572165463, "calibration": 48.14904061603706, "p50_s": 1.3110789507627487, "p95_s": 2.5801136046648026, "cost_usd_per_1000": 0.0211013367381682, "axes": { "intelligence": 72.16800133466802, "calibration": 48.14904061603706, "speed": 74.70732326356473, "cost": 60.270700957720834 }, "jevbench_score": 62.89272810916393 }, "heldout": { "n": 303, "escalation": 0.16831683168316833, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.910958904109589, "hard": 0.5321100917431193 }, "accuracy": 0.7722772277227723, "ece_hard": 0.27483151543048134, "calibration": 58.82723647162233, "p50_s": 1.3393714874982834, "p95_s": 2.230665914714336, "cost_usd_per_1000": 0.02159917406459747, "axes": { "intelligence": 77.63681873402875, "calibration": 58.82723647162233, "speed": 75.24664419378827, "cost": 59.9668856683135 }, "jevbench_score": 67.37692746578533 } }, { "threshold": 0.6, "public": { "n": 231, "escalation": 0.17316017316017315, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.5135135135135135 }, "accuracy": 0.7056277056277056, "ece_hard": 0.32563935576556813, "calibration": 48.00420146045591, "p50_s": 1.3143753111362457, "p95_s": 2.6578924618661404, "cost_usd_per_1000": 0.02179249233912155, "axes": { "intelligence": 72.16800133466802, "calibration": 48.00420146045591, "speed": 74.56743201724963, "cost": 59.850792940392104 }, "jevbench_score": 62.70623934070211 }, "heldout": { "n": 303, "escalation": 0.17491749174917492, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.910958904109589, "hard": 0.5321100917431193 }, "accuracy": 0.7722772277227723, "ece_hard": 0.27256407282785133, "calibration": 59.053980731885325, "p50_s": 1.3393714874982834, "p95_s": 2.230665914714336, "cost_usd_per_1000": 0.021862634367931174, "axes": { "intelligence": 77.63681873402875, "calibration": 59.053980731885325, "speed": 75.24664419378827, "cost": 59.808925249827894 }, "jevbench_score": 67.39730191707959 } }, { "threshold": 0.61, "public": { "n": 231, "escalation": 0.17316017316017315, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.5135135135135135 }, "accuracy": 0.7056277056277056, "ece_hard": 0.32563935576556813, "calibration": 48.00420146045591, "p50_s": 1.3143753111362457, "p95_s": 2.6578924618661404, "cost_usd_per_1000": 0.02179249233912155, "axes": { "intelligence": 72.16800133466802, "calibration": 48.00420146045591, "speed": 74.56743201724963, "cost": 59.850792940392104 }, "jevbench_score": 62.70623934070211 }, "heldout": { "n": 303, "escalation": 0.1848184818481848, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9178082191780822, "hard": 0.5321100917431193 }, "accuracy": 0.7755775577557755, "ece_hard": 0.27256407282785133, "calibration": 59.053980731885325, "p50_s": 1.3393714874982834, "p95_s": 2.260821652412414, "cost_usd_per_1000": 0.02225782482293173, "axes": { "intelligence": 77.82859955594657, "calibration": 59.053980731885325, "speed": 75.18832646686673, "cost": 59.575518398901686 }, "jevbench_score": 67.35993518348614 } }, { "threshold": 0.62, "public": { "n": 231, "escalation": 0.1774891774891775, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.5225225225225225 }, "accuracy": 0.70995670995671, "ece_hard": 0.31767922562820183, "calibration": 50.31808670043079, "p50_s": 1.3143753111362457, "p95_s": 2.6578924618661404, "cost_usd_per_1000": 0.02196528123935989, "axes": { "intelligence": 72.54337671004339, "calibration": 50.31808670043079, "speed": 74.56743201724963, "cost": 59.74789695212805 }, "jevbench_score": 63.50361141625562 }, "heldout": { "n": 303, "escalation": 0.19471947194719472, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9178082191780822, "hard": 0.5504587155963303 }, "accuracy": 0.7821782178217822, "ece_hard": 0.2591018608077472, "calibration": 60.40020193389574, "p50_s": 1.3400312662124634, "p95_s": 2.2951173394918434, "cost_usd_per_1000": 0.022653015277932287, "axes": { "intelligence": 78.37905827154289, "calibration": 60.40020193389574, "speed": 75.12080174310293, "cost": 59.34621946329554 }, "jevbench_score": 67.77943215799725 } }, { "threshold": 0.63, "public": { "n": 231, "escalation": 0.19047619047619047, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.5405405405405406 }, "accuracy": 0.7186147186147186, "ece_hard": 0.301259632822391, "calibration": 52.251880204654825, "p50_s": 1.3146315664052963, "p95_s": 2.6578924618661404, "cost_usd_per_1000": 0.022483647940074902, "axes": { "intelligence": 73.29412746079414, "calibration": 52.251880204654825, "speed": 74.56658538386415, "cost": 59.44399671162006 }, "jevbench_score": 64.18829497902954 }, "heldout": { "n": 303, "escalation": 0.20462046204620463, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9246575342465754, "hard": 0.5504587155963303 }, "accuracy": 0.7854785478547854, "ece_hard": 0.2618327048723373, "calibration": 60.12711752743673, "p50_s": 1.3414461016654968, "p95_s": 2.3032229147851466, "cost_usd_per_1000": 0.02304820573293284, "axes": { "intelligence": 78.57083909346069, "calibration": 60.12711752743673, "speed": 75.10090799588625, "cost": 59.12088634330617 }, "jevbench_score": 67.67518863151629 } }, { "threshold": 0.64, "public": { "n": 231, "escalation": 0.19913419913419914, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.5405405405405406 }, "accuracy": 0.7186147186147186, "ece_hard": 0.29534045785695345, "calibration": 52.84379770119858, "p50_s": 1.3177027255296707, "p95_s": 2.6578924618661404, "cost_usd_per_1000": 0.02282922574055158, "axes": { "intelligence": 73.29412746079414, "calibration": 52.84379770119858, "speed": 74.55645150330015, "cost": 59.24526452429171 }, "jevbench_score": 64.3132591093288 }, "heldout": { "n": 303, "escalation": 0.22112211221122113, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9315068493150684, "hard": 0.5688073394495413 }, "accuracy": 0.7953795379537953, "ece_hard": 0.2495068951704702, "calibration": 61.35969849762344, "p50_s": 1.3426922708749771, "p95_s": 2.3393441632390015, "cost_usd_per_1000": 0.023706856491267102, "axes": { "intelligence": 79.31307863097483, "calibration": 61.35969849762344, "speed": 75.02929391350902, "cost": 58.7537808783413 }, "jevbench_score": 68.05713706943828 } }, { "threshold": 0.65, "public": { "n": 231, "escalation": 0.20346320346320346, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.5315315315315315 }, "accuracy": 0.7142857142857143, "ece_hard": 0.3051964887872234, "calibration": 51.85819460817159, "p50_s": 1.319142371416092, "p95_s": 2.6578924618661404, "cost_usd_per_1000": 0.02300201464078992, "axes": { "intelligence": 72.91875208541877, "calibration": 51.85819460817159, "speed": 74.55170924273115, "cost": 59.14702373374093 }, "jevbench_score": 63.901607067956164 }, "heldout": { "n": 303, "escalation": 0.23102310231023102, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9383561643835616, "hard": 0.5688073394495413 }, "accuracy": 0.7986798679867987, "ece_hard": 0.2474191121881777, "calibration": 61.56847679585269, "p50_s": 1.3426922708749771, "p95_s": 2.3552090190351005, "cost_usd_per_1000": 0.024102046946267656, "axes": { "intelligence": 79.50485945289263, "calibration": 61.56847679585269, "speed": 74.99994055317771, "cost": 58.538382158789574 }, "jevbench_score": 68.08687936246193 } }, { "threshold": 0.66, "public": { "n": 231, "escalation": 0.2077922077922078, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.5315315315315315 }, "accuracy": 0.7142857142857143, "ece_hard": 0.2960743427736027, "calibration": 52.77040920953365, "p50_s": 1.319142371416092, "p95_s": 2.6578924618661404, "cost_usd_per_1000": 0.023174803541028258, "axes": { "intelligence": 72.91875208541877, "calibration": 52.77040920953365, "speed": 74.55170924273115, "cost": 59.04951816392984 }, "jevbench_score": 64.15432105128085 }, "heldout": { "n": 303, "escalation": 0.24092409240924093, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9452054794520548, "hard": 0.5779816513761468 }, "accuracy": 0.8052805280528053, "ece_hard": 0.24038258386331557, "calibration": 63.38003752813587, "p50_s": 1.3426922708749771, "p95_s": 2.3552090190351005, "cost_usd_per_1000": 0.024497237401268215, "axes": { "intelligence": 79.9718696326086, "calibration": 63.38003752813587, "speed": 74.99994055317771, "cost": 58.326486671961774 }, "jevbench_score": 68.62053811741653 } }, { "threshold": 0.67, "public": { "n": 231, "escalation": 0.2077922077922078, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.5315315315315315 }, "accuracy": 0.7142857142857143, "ece_hard": 0.2960743427736027, "calibration": 52.77040920953365, "p50_s": 1.319142371416092, "p95_s": 2.6578924618661404, "cost_usd_per_1000": 0.023174803541028258, "axes": { "intelligence": 72.91875208541877, "calibration": 52.77040920953365, "speed": 74.55170924273115, "cost": 59.04951816392984 }, "jevbench_score": 64.15432105128085 }, "heldout": { "n": 303, "escalation": 0.24422442244224424, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9452054794520548, "hard": 0.5779816513761468 }, "accuracy": 0.8052805280528053, "ece_hard": 0.24038258386331557, "calibration": 63.38003752813587, "p50_s": 1.3426922708749771, "p95_s": 2.3552090190351005, "cost_usd_per_1000": 0.024628967552935067, "axes": { "intelligence": 79.9718696326086, "calibration": 63.38003752813587, "speed": 74.99994055317771, "cost": 58.256613802514124 }, "jevbench_score": 68.59997769265514 } }, { "threshold": 0.68, "public": { "n": 231, "escalation": 0.2077922077922078, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.5315315315315315 }, "accuracy": 0.7142857142857143, "ece_hard": 0.2960743427736027, "calibration": 52.77040920953365, "p50_s": 1.319142371416092, "p95_s": 2.6578924618661404, "cost_usd_per_1000": 0.023174803541028258, "axes": { "intelligence": 72.91875208541877, "calibration": 52.77040920953365, "speed": 74.55170924273115, "cost": 59.04951816392984 }, "jevbench_score": 64.15432105128085 }, "heldout": { "n": 303, "escalation": 0.25412541254125415, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9452054794520548, "hard": 0.5779816513761468 }, "accuracy": 0.8052805280528053, "ece_hard": 0.24103562711592802, "calibration": 63.31473320287463, "p50_s": 1.3468152582645416, "p95_s": 2.4394075006246547, "cost_usd_per_1000": 0.025024158007935622, "axes": { "intelligence": 79.9718696326086, "calibration": 63.31473320287463, "speed": 74.83407617230978, "cost": 58.04921579147436 }, "jevbench_score": 68.48326338780231 } }, { "threshold": 0.69, "public": { "n": 231, "escalation": 0.2077922077922078, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.5315315315315315 }, "accuracy": 0.7142857142857143, "ece_hard": 0.2960743427736027, "calibration": 52.77040920953365, "p50_s": 1.319142371416092, "p95_s": 2.6578924618661404, "cost_usd_per_1000": 0.023174803541028258, "axes": { "intelligence": 72.91875208541877, "calibration": 52.77040920953365, "speed": 74.55170924273115, "cost": 59.04951816392984 }, "jevbench_score": 64.15432105128085 }, "heldout": { "n": 303, "escalation": 0.26732673267326734, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9452054794520548, "hard": 0.5871559633027523 }, "accuracy": 0.8085808580858086, "ece_hard": 0.23385204002414964, "calibration": 65.21470588567001, "p50_s": 1.3563375920057297, "p95_s": 2.4394075006246547, "cost_usd_per_1000": 0.025551078614603033, "axes": { "intelligence": 80.24709899040677, "calibration": 65.21470588567001, "speed": 74.80347846412252, "cost": 57.77772285435231 }, "jevbench_score": 68.96270278927393 } }, { "threshold": 0.7, "public": { "n": 231, "escalation": 0.21212121212121213, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.5315315315315315 }, "accuracy": 0.7142857142857143, "ece_hard": 0.2960743427736027, "calibration": 52.77040920953365, "p50_s": 1.3191694766283035, "p95_s": 2.6578924618661404, "cost_usd_per_1000": 0.023347592441266597, "axes": { "intelligence": 72.91875208541877, "calibration": 52.77040920953365, "speed": 74.55162000654701, "cost": 58.95273689196884 }, "jevbench_score": 64.1279986977054 }, "heldout": { "n": 303, "escalation": 0.2706270627062706, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9452054794520548, "hard": 0.5871559633027523 }, "accuracy": 0.8085808580858086, "ece_hard": 0.23385204002414964, "calibration": 65.21470588567001, "p50_s": 1.3563375920057297, "p95_s": 2.5285170599818207, "cost_usd_per_1000": 0.025682808766269882, "axes": { "intelligence": 80.24709899040677, "calibration": 65.21470588567001, "speed": 74.64766313075873, "cost": 57.71072445899868 }, "jevbench_score": 68.9067720789428 } }, { "threshold": 0.71, "public": { "n": 231, "escalation": 0.22077922077922077, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.5495495495495496 }, "accuracy": 0.7229437229437229, "ece_hard": 0.2822258903129442, "calibration": 54.155254455599504, "p50_s": 1.3192902952432632, "p95_s": 2.705424565821886, "cost_usd_per_1000": 0.02369317024174327, "axes": { "intelligence": 73.66950283616951, "calibration": 54.155254455599504, "speed": 74.47424203046299, "cost": 58.76130475107429 }, "jevbench_score": 64.64076172352922 }, "heldout": { "n": 303, "escalation": 0.2871287128712871, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9452054794520548, "hard": 0.5963302752293578 }, "accuracy": 0.8118811881188119, "ece_hard": 0.22462011039820967, "calibration": 66.13789884826402, "p50_s": 1.3565552085638046, "p95_s": 2.549693354964256, "cost_usd_per_1000": 0.026341459524604142, "axes": { "intelligence": 80.52232834820494, "calibration": 66.13789884826402, "speed": 74.61074574693677, "cost": 57.380804961043744 }, "jevbench_score": 69.10089680660188 } }, { "threshold": 0.72, "public": { "n": 231, "escalation": 0.23376623376623376, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.5495495495495496 }, "accuracy": 0.7229437229437229, "ece_hard": 0.2892554206706144, "calibration": 54.057104087182054, "p50_s": 1.3224025070667267, "p95_s": 2.705424565821886, "cost_usd_per_1000": 0.024211536942458287, "axes": { "intelligence": 73.66950283616951, "calibration": 54.057104087182054, "speed": 74.46400906888935, "cost": 58.479329223263164 }, "jevbench_score": 64.53158438209522 }, "heldout": { "n": 303, "escalation": 0.297029702970297, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9452054794520548, "hard": 0.5963302752293578 }, "accuracy": 0.8118811881188119, "ece_hard": 0.22462011039820967, "calibration": 66.13789884826402, "p50_s": 1.3586038798093796, "p95_s": 2.549693354964256, "cost_usd_per_1000": 0.026736649979604697, "axes": { "intelligence": 80.52232834820494, "calibration": 66.13789884826402, "speed": 74.60419197415901, "cost": 57.1867902830308 }, "jevbench_score": 69.04089578900803 } }, { "threshold": 0.73, "public": { "n": 231, "escalation": 0.24242424242424243, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.5585585585585585 }, "accuracy": 0.7272727272727273, "ece_hard": 0.2716077530736083, "calibration": 55.82187084688267, "p50_s": 1.3224025070667267, "p95_s": 2.7129402682185173, "cost_usd_per_1000": 0.024557114742934965, "axes": { "intelligence": 74.04487821154488, "calibration": 55.82187084688267, "speed": 74.45196104232865, "cost": 58.29467981586611 }, "jevbench_score": 65.0805355881139 }, "heldout": { "n": 303, "escalation": 0.3069306930693069, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9452054794520548, "hard": 0.5963302752293578 }, "accuracy": 0.8118811881188119, "ece_hard": 0.22596800541676101, "calibration": 66.00310934640888, "p50_s": 1.3768931925296783, "p95_s": 2.549693354964256, "cost_usd_per_1000": 0.027131840434605255, "axes": { "intelligence": 80.52232834820494, "calibration": 66.00310934640888, "speed": 74.54611795149118, "cost": 56.99562237202096 }, "jevbench_score": 68.93452906603929 } }, { "threshold": 0.74, "public": { "n": 231, "escalation": 0.2510822510822511, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.5585585585585585 }, "accuracy": 0.7316017316017316, "ece_hard": 0.272210138985464, "calibration": 55.761632255697094, "p50_s": 1.3224025070667267, "p95_s": 2.7129402682185173, "cost_usd_per_1000": 0.02490269254341164, "axes": { "intelligence": 74.58500166833501, "calibration": 55.761632255697094, "speed": 74.45196104232865, "cost": 58.112610799746285 }, "jevbench_score": 65.13034483946844 }, "heldout": { "n": 303, "escalation": 0.3201320132013201, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9452054794520548, "hard": 0.5963302752293578 }, "accuracy": 0.8118811881188119, "ece_hard": 0.2269119053588616, "calibration": 65.90871935219882, "p50_s": 1.3797449171543121, "p95_s": 2.549693354964256, "cost_usd_per_1000": 0.027658761041272663, "axes": { "intelligence": 80.52232834820494, "calibration": 65.90871935219882, "speed": 74.53713245059546, "cost": 56.7450183330808 }, "jevbench_score": 68.83192329082638 } }, { "threshold": 0.75, "public": { "n": 231, "escalation": 0.26406926406926406, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.5495495495495496 }, "accuracy": 0.7272727272727273, "ece_hard": 0.2792555521659784, "calibration": 55.057090937645654, "p50_s": 1.3238774687051773, "p95_s": 2.716652635484934, "cost_usd_per_1000": 0.02542105924412666, "axes": { "intelligence": 74.20962629295963, "calibration": 55.057090937645654, "speed": 74.44118098153855, "cost": 57.844190716967475 }, "jevbench_score": 64.76444188216044 }, "heldout": { "n": 303, "escalation": 0.3234323432343234, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9452054794520548, "hard": 0.6055045871559633 }, "accuracy": 0.8151815181518152, "ece_hard": 0.2266353941621652, "calibration": 65.93637047186846, "p50_s": 1.3797449171543121, "p95_s": 2.5530540913343427, "cost_usd_per_1000": 0.027790491192939512, "axes": { "intelligence": 80.7975577060031, "calibration": 65.93637047186846, "speed": 74.53141180897228, "cost": 56.68311331186638 }, "jevbench_score": 68.87777011756367 } }, { "threshold": 0.76, "public": { "n": 231, "escalation": 0.2683982683982684, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.5495495495495496 }, "accuracy": 0.7272727272727273, "ece_hard": 0.2792555521659784, "calibration": 55.057090937645654, "p50_s": 1.3238774687051773, "p95_s": 2.716652635484934, "cost_usd_per_1000": 0.025593848144364995, "axes": { "intelligence": 74.20962629295963, "calibration": 55.057090937645654, "speed": 74.44118098153855, "cost": 57.755932335202885 }, "jevbench_score": 64.73972341767838 }, "heldout": { "n": 303, "escalation": 0.33993399339933994, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9452054794520548, "hard": 0.6055045871559633 }, "accuracy": 0.8151815181518152, "ece_hard": 0.23016524078666206, "calibration": 65.58338580941877, "p50_s": 1.384657546877861, "p95_s": 2.571992296725511, "cost_usd_per_1000": 0.028449141951273776, "axes": { "intelligence": 80.7975577060031, "calibration": 65.58338580941877, "speed": 74.48387959395252, "cost": 56.37792483214104 }, "jevbench_score": 68.6816722443907 } }, { "threshold": 0.77, "public": { "n": 231, "escalation": 0.27705627705627706, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.5495495495495496 }, "accuracy": 0.7272727272727273, "ece_hard": 0.28684046368639515, "calibration": 54.29859978560398, "p50_s": 1.325474590063095, "p95_s": 2.716652635484934, "cost_usd_per_1000": 0.025939425944841672, "axes": { "intelligence": 74.20962629295963, "calibration": 54.29859978560398, "speed": 74.43594482521571, "cost": 57.581189180341 }, "jevbench_score": 64.46560347352965 }, "heldout": { "n": 303, "escalation": 0.34983498349834985, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9452054794520548, "hard": 0.6055045871559633 }, "accuracy": 0.8151815181518152, "ece_hard": 0.22423419937098382, "calibration": 66.17648995098659, "p50_s": 1.4045170545578003, "p95_s": 2.571992296725511, "cost_usd_per_1000": 0.028844332406274334, "axes": { "intelligence": 80.7975577060031, "calibration": 66.17648995098659, "speed": 74.42203319429316, "cost": 56.19818524628841 }, "jevbench_score": 68.76721653858543 } }, { "threshold": 0.78, "public": { "n": 231, "escalation": 0.29004329004329005, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.5585585585585585 }, "accuracy": 0.7316017316017316, "ece_hard": 0.28016441535603664, "calibration": 56.195083865128055, "p50_s": 1.3266585320234299, "p95_s": 2.716652635484934, "cost_usd_per_1000": 0.026457792645556685, "axes": { "intelligence": 74.58500166833501, "calibration": 56.195083865128055, "speed": 74.43206734644106, "cost": 57.32339174532019 }, "jevbench_score": 65.02950482824353 }, "heldout": { "n": 303, "escalation": 0.3564356435643564, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9452054794520548, "hard": 0.6055045871559633 }, "accuracy": 0.8151815181518152, "ece_hard": 0.22114030503368842, "calibration": 66.48587938471614, "p50_s": 1.438198447227478, "p95_s": 2.571992296725511, "cost_usd_per_1000": 0.029107792709608036, "axes": { "intelligence": 80.7975577060031, "calibration": 66.48587938471614, "speed": 74.31911520959446, "cost": 56.07972179669437 }, "jevbench_score": 68.78733878507389 } }, { "threshold": 0.79, "public": { "n": 231, "escalation": 0.2987012987012987, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.5585585585585585 }, "accuracy": 0.7272727272727273, "ece_hard": 0.2818042070723631, "calibration": 56.03110469349541, "p50_s": 1.3277880102396011, "p95_s": 2.716652635484934, "cost_usd_per_1000": 0.026803370446033363, "axes": { "intelligence": 74.04487821154488, "calibration": 56.03110469349541, "speed": 74.42837146348762, "cost": 57.15431773800446 }, "jevbench_score": 64.8153607185516 }, "heldout": { "n": 303, "escalation": 0.3696369636963696, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9452054794520548, "hard": 0.6055045871559633 }, "accuracy": 0.8151815181518152, "ece_hard": 0.22114030503368842, "calibration": 66.48587938471614, "p50_s": 1.4431772530078888, "p95_s": 2.571992296725511, "cost_usd_per_1000": 0.029634713316275443, "axes": { "intelligence": 80.7975577060031, "calibration": 66.48587938471614, "speed": 74.3041066147159, "cost": 55.84597809198816 }, "jevbench_score": 68.71207951706185 } }, { "threshold": 0.8, "public": { "n": 231, "escalation": 0.30735930735930733, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.5585585585585585 }, "accuracy": 0.7272727272727273, "ece_hard": 0.2818042070723631, "calibration": 56.03110469349541, "p50_s": 1.3289764374494553, "p95_s": 2.716652635484934, "cost_usd_per_1000": 0.02714894824651004, "axes": { "intelligence": 74.04487821154488, "calibration": 56.03110469349541, "speed": 74.4244860801654, "cost": 56.98740971107676 }, "jevbench_score": 64.76714337012491 }, "heldout": { "n": 303, "escalation": 0.3795379537953795, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9452054794520548, "hard": 0.6146788990825688 }, "accuracy": 0.8184818481848185, "ece_hard": 0.22787172061604782, "calibration": 65.8127378264802, "p50_s": 1.4465409815311432, "p95_s": 2.571992296725511, "cost_usd_per_1000": 0.030029903771275998, "axes": { "intelligence": 81.07278706380127, "calibration": 65.8127378264802, "speed": 74.29399594455704, "cost": 55.67338178395144 }, "jevbench_score": 68.54039380098389 } }, { "threshold": 0.81, "public": { "n": 231, "escalation": 0.31601731601731603, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.5585585585585585 }, "accuracy": 0.7272727272727273, "ece_hard": 0.2807887479988066, "calibration": 56.13265060085106, "p50_s": 1.3289764374494553, "p95_s": 2.72840940579772, "cost_usd_per_1000": 0.02749452604698672, "axes": { "intelligence": 74.04487821154488, "calibration": 56.13265060085106, "speed": 74.40573180935013, "cost": 56.822612869700315 }, "jevbench_score": 64.74549290090691 }, "heldout": { "n": 303, "escalation": 0.3927392739273927, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6146788990825688 }, "accuracy": 0.8217821782178217, "ece_hard": 0.23179135683496743, "calibration": 65.42077420458824, "p50_s": 1.4572032541036606, "p95_s": 2.571992296725511, "cost_usd_per_1000": 0.03055682437794341, "axes": { "intelligence": 82.23945373046794, "calibration": 65.42077420458824, "speed": 74.262102040836, "cost": 55.44675345589869 }, "jevbench_score": 68.60563906056122 } }, { "threshold": 0.82, "public": { "n": 231, "escalation": 0.3203463203463203, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.5585585585585585 }, "accuracy": 0.7272727272727273, "ece_hard": 0.2807887479988066, "calibration": 56.13265060085106, "p50_s": 1.3397867381572723, "p95_s": 2.72840940579772, "cost_usd_per_1000": 0.027667314947225054, "axes": { "intelligence": 74.04487821154488, "calibration": 56.13265060085106, "speed": 74.37054787346555, "cost": 56.74098958448397 }, "jevbench_score": 64.71457673223789 }, "heldout": { "n": 303, "escalation": 0.39603960396039606, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6146788990825688 }, "accuracy": 0.8217821782178217, "ece_hard": 0.2331485356542563, "calibration": 65.28505632265936, "p50_s": 1.4642701894044876, "p95_s": 2.571992296725511, "cost_usd_per_1000": 0.03068855452961026, "axes": { "intelligence": 82.23945373046794, "calibration": 65.28505632265936, "speed": 74.24109115676876, "cost": 55.390707007453855 }, "jevbench_score": 68.54784621077677 } }, { "threshold": 0.83, "public": { "n": 231, "escalation": 0.33766233766233766, "tiers": { "easy": 1.0, "standard": 0.8055555555555556, "hard": 0.5585585585585585 }, "accuracy": 0.7272727272727273, "ece_hard": 0.28217007372054737, "calibration": 55.99451802867698, "p50_s": 1.349913328886032, "p95_s": 2.72840940579772, "cost_usd_per_1000": 0.02835847054817841, "axes": { "intelligence": 74.04487821154488, "calibration": 55.99451802867698, "speed": 74.3378457643491, "cost": 56.41951586729053 }, "jevbench_score": 64.5758252791851 }, "heldout": { "n": 303, "escalation": 0.40264026402640263, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6238532110091743 }, "accuracy": 0.8250825082508251, "ece_hard": 0.22420814658540783, "calibration": 66.1790952295442, "p50_s": 1.4727533608675003, "p95_s": 2.571992296725511, "cost_usd_per_1000": 0.030952014832943964, "axes": { "intelligence": 82.51468308826611, "calibration": 66.1790952295442, "speed": 74.21600314002204, "cost": 55.27933225487453 }, "jevbench_score": 68.7983627228977 } }, { "threshold": 0.84, "public": { "n": 231, "escalation": 0.3593073593073593, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.5765765765765766 }, "accuracy": 0.7402597402597403, "ece_hard": 0.2643081802589161, "calibration": 57.780707374840105, "p50_s": 1.3694269061088562, "p95_s": 2.72840940579772, "cost_usd_per_1000": 0.0292224150493701, "axes": { "intelligence": 75.33575241908576, "calibration": 57.780707374840105, "speed": 74.27551605461642, "cost": 56.02851685608284 }, "jevbench_score": 65.23936078591521 }, "heldout": { "n": 303, "escalation": 0.41914191419141916, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6330275229357798 }, "accuracy": 0.8283828382838284, "ece_hard": 0.20584607981777647, "calibration": 68.01530190630733, "p50_s": 1.4812378883361816, "p95_s": 2.571992296725511, "cost_usd_per_1000": 0.03161066559127823, "axes": { "intelligence": 82.78991244606426, "calibration": 68.01530190630733, "speed": 74.19105524219056, "cost": 55.004990787979885 }, "jevbench_score": 69.23639141941668 } }, { "threshold": 0.85, "public": { "n": 231, "escalation": 0.36796536796536794, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.5765765765765766 }, "accuracy": 0.7402597402597403, "ece_hard": 0.264127000460515, "calibration": 58.499380142792845, "p50_s": 1.377013847231865, "p95_s": 2.72840940579772, "cost_usd_per_1000": 0.029567992849846778, "axes": { "intelligence": 75.33575241908576, "calibration": 58.499380142792845, "speed": 74.25152154432672, "cost": 55.87534466444132 }, "jevbench_score": 65.39122778959508 }, "heldout": { "n": 303, "escalation": 0.43234323432343236, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6330275229357798 }, "accuracy": 0.8283828382838284, "ece_hard": 0.20277732820702932, "calibration": 68.32217706738206, "p50_s": 1.5059185326099396, "p95_s": 2.571992296725511, "cost_usd_per_1000": 0.032137586197945635, "axes": { "intelligence": 82.78991244606426, "calibration": 68.32217706738206, "speed": 74.1192885850752, "cost": 54.78960236508912 }, "jevbench_score": 69.22964867369186 } }, { "threshold": 0.86, "public": { "n": 231, "escalation": 0.38095238095238093, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.5765765765765766 }, "accuracy": 0.7402597402597403, "ece_hard": 0.2642288314627934, "calibration": 58.48919704256501, "p50_s": 1.384503349661827, "p95_s": 2.72840940579772, "cost_usd_per_1000": 0.03008635955056179, "axes": { "intelligence": 75.33575241908576, "calibration": 58.48919704256501, "speed": 74.22796451157566, "cost": 55.64891076136781 }, "jevbench_score": 65.31685303289808 }, "heldout": { "n": 303, "escalation": 0.44554455445544555, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6330275229357798 }, "accuracy": 0.8283828382838284, "ece_hard": 0.1930111077074132, "calibration": 68.9230285805335, "p50_s": 1.5078626871109009, "p95_s": 2.5873304694890975, "cost_usd_per_1000": 0.03266450680461304, "axes": { "intelligence": 82.78991244606426, "calibration": 68.9230285805335, "speed": 74.08786302536055, "cost": 54.5777168427177 }, "jevbench_score": 69.3068326759638 } }, { "threshold": 0.87, "public": { "n": 231, "escalation": 0.39826839826839827, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.5945945945945946 }, "accuracy": 0.7489177489177489, "ece_hard": 0.2481638809665665, "calibration": 60.095692092187704, "p50_s": 1.4003681987524033, "p95_s": 2.72840940579772, "cost_usd_per_1000": 0.030777515151515146, "axes": { "intelligence": 76.0865031698365, "calibration": 60.095692092187704, "speed": 74.17848222247912, "cost": 55.35299338654697 }, "jevbench_score": 65.825248176186 }, "heldout": { "n": 303, "escalation": 0.46864686468646866, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6513761467889908 }, "accuracy": 0.834983498349835, "ece_hard": 0.17411343292375897, "calibration": 70.81279605889893, "p50_s": 1.8354156464338303, "p95_s": 2.5873304694890975, "cost_usd_per_1000": 0.033586617866281004, "axes": { "intelligence": 83.3403711616606, "calibration": 70.81279605889893, "speed": 73.23413667393994, "cost": 54.215011807314006 }, "jevbench_score": 69.57449681038759 } }, { "threshold": 0.88, "public": { "n": 231, "escalation": 0.4025974025974026, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.5945945945945946 }, "accuracy": 0.7489177489177489, "ece_hard": 0.24817635073079486, "calibration": 60.09444511576486, "p50_s": 1.409024566411972, "p95_s": 2.72840940579772, "cost_usd_per_1000": 0.03095030405175349, "axes": { "intelligence": 76.0865031698365, "calibration": 60.09444511576486, "speed": 74.15171896849931, "cost": 55.280052405115434 }, "jevbench_score": 65.79727474050081 }, "heldout": { "n": 303, "escalation": 0.48184818481848185, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6422018348623854 }, "accuracy": 0.8316831683168316, "ece_hard": 0.1827397649661731, "calibration": 69.9501628546575, "p50_s": 1.884497232735157, "p95_s": 2.5873304694890975, "cost_usd_per_1000": 0.034113538472948415, "axes": { "intelligence": 83.06514180386245, "calibration": 69.9501628546575, "speed": 73.11952592971753, "cost": 54.01219691419592 }, "jevbench_score": 69.21228501179455 } }, { "threshold": 0.89, "public": { "n": 231, "escalation": 0.42857142857142855, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6126126126126126 }, "accuracy": 0.7575757575757576, "ece_hard": 0.22773942855862647, "calibration": 62.1381373329817, "p50_s": 1.4595188051462173, "p95_s": 2.72840940579772, "cost_usd_per_1000": 0.031987037453183514, "axes": { "intelligence": 76.83725392058726, "calibration": 62.1381373329817, "speed": 73.99880766950506, "cost": 54.85077943453307 }, "jevbench_score": 66.34900247038111 }, "heldout": { "n": 303, "escalation": 0.49174917491749176, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6422018348623854 }, "accuracy": 0.8316831683168316, "ece_hard": 0.1827397649661731, "calibration": 69.9501628546575, "p50_s": 1.8997346237301826, "p95_s": 2.5873304694890975, "cost_usd_per_1000": 0.03450872892794897, "axes": { "intelligence": 83.06514180386245, "calibration": 69.9501628546575, "speed": 73.08455158965305, "cost": 53.86213110802222 }, "jevbench_score": 69.15588843905931 } }, { "threshold": 0.9, "public": { "n": 231, "escalation": 0.45454545454545453, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6306306306306306 }, "accuracy": 0.7662337662337663, "ece_hard": 0.2079758506506174, "calibration": 65.18248163391785, "p50_s": 1.5103851854801178, "p95_s": 2.801310855895281, "cost_usd_per_1000": 0.03302377085461355, "axes": { "intelligence": 77.58800467133801, "calibration": 65.18248163391785, "speed": 73.73550978482982, "cost": 54.43520013584778 }, "jevbench_score": 67.12286672640398 }, "heldout": { "n": 303, "escalation": 0.5016501650165016, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6422018348623854 }, "accuracy": 0.8316831683168316, "ece_hard": 0.18123289375147272, "calibration": 70.92209478813926, "p50_s": 1.9051726013422012, "p95_s": 2.5873304694890975, "cost_usd_per_1000": 0.034903919382949525, "axes": { "intelligence": 83.06514180386245, "calibration": 70.92209478813926, "speed": 73.07213769812867, "cost": 53.71377409310265 }, "jevbench_score": 69.34409132539577 } }, { "threshold": 0.91, "public": { "n": 231, "escalation": 0.4588744588744589, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6306306306306306 }, "accuracy": 0.7662337662337663, "ece_hard": 0.20500888838185682, "calibration": 66.05084191995604, "p50_s": 1.5125302523374557, "p95_s": 2.801310855895281, "cost_usd_per_1000": 0.03319655975485189, "axes": { "intelligence": 77.58800467133801, "calibration": 66.05084191995604, "speed": 73.72934625908798, "cost": 54.3672076306553 }, "jevbench_score": 67.32286474637353 }, "heldout": { "n": 303, "escalation": 0.5181518151815182, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6513761467889908 }, "accuracy": 0.834983498349835, "ece_hard": 0.17496041133880202, "calibration": 72.00344628238719, "p50_s": 1.910685919225216, "p95_s": 2.6104362942278376, "cost_usd_per_1000": 0.03556257014128379, "axes": { "intelligence": 83.3403711616606, "calibration": 72.00344628238719, "speed": 73.02097597870777, "cost": 53.47020579058064 }, "jevbench_score": 69.57321146778418 } }, { "threshold": 0.92, "public": { "n": 231, "escalation": 0.47619047619047616, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6306306306306306 }, "accuracy": 0.7662337662337663, "ece_hard": 0.20584125704623318, "calibration": 65.96760505351841, "p50_s": 1.891363449394703, "p95_s": 2.801310855895281, "cost_usd_per_1000": 0.03388771535580524, "axes": { "intelligence": 77.58800467133801, "calibration": 65.96760505351841, "speed": 72.75863703668266, "cost": 54.0987312840783 }, "jevbench_score": 66.99605554354734 }, "heldout": { "n": 303, "escalation": 0.5412541254125413, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6513761467889908 }, "accuracy": 0.834983498349835, "ece_hard": 0.16489277435022337, "calibration": 73.01020998124505, "p50_s": 1.9170156568288803, "p95_s": 2.6104362942278376, "cost_usd_per_1000": 0.03648468120295175, "axes": { "intelligence": 83.3403711616606, "calibration": 73.01020998124505, "speed": 73.00661241057512, "cost": 53.13668332562059 }, "jevbench_score": 69.70259018950385 } }, { "threshold": 0.93, "public": { "n": 231, "escalation": 0.49783549783549785, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6396396396396397 }, "accuracy": 0.7705627705627706, "ece_hard": 0.20081836645697448, "calibration": 66.46989411244428, "p50_s": 1.900885358452797, "p95_s": 2.801310855895281, "cost_usd_per_1000": 0.034751659856996936, "axes": { "intelligence": 77.96338004671338, "calibration": 66.46989411244428, "speed": 72.736827701583, "cost": 53.770733416084745 }, "jevbench_score": 67.09713718480057 }, "heldout": { "n": 303, "escalation": 0.5676567656765676, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6513761467889908 }, "accuracy": 0.834983498349835, "ece_hard": 0.16623675355140893, "calibration": 72.8758120611265, "p50_s": 1.920856922864914, "p95_s": 2.6104362942278376, "cost_usd_per_1000": 0.037538522416286565, "axes": { "intelligence": 83.3403711616606, "calibration": 72.8758120611265, "speed": 72.99791883794916, "cost": 52.765684779689366 }, "jevbench_score": 69.54649121331376 } }, { "threshold": 0.94, "public": { "n": 231, "escalation": 0.5238095238095238, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6576576576576577 }, "accuracy": 0.7792207792207793, "ece_hard": 0.17921616279625435, "calibration": 68.63011447851629, "p50_s": 1.9172174036502838, "p95_s": 2.801310855895281, "cost_usd_per_1000": 0.03578839325842696, "axes": { "intelligence": 78.71413079746414, "calibration": 68.63011447851629, "speed": 72.69967333036678, "cost": 53.38773397257718 }, "jevbench_score": 67.66831209155139 }, "heldout": { "n": 303, "escalation": 0.594059405940594, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6513761467889908 }, "accuracy": 0.834983498349835, "ece_hard": 0.16623675355140893, "calibration": 72.8758120611265, "p50_s": 1.9281360357999802, "p95_s": 2.6104362942278376, "cost_usd_per_1000": 0.03859236362962139, "axes": { "intelligence": 83.3403711616606, "calibration": 72.8758120611265, "speed": 72.9814922952996, "cost": 52.40495865384568 }, "jevbench_score": 69.4234176671553 } }, { "threshold": 0.95, "public": { "n": 231, "escalation": 0.5454545454545454, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6666666666666666 }, "accuracy": 0.7835497835497836, "ece_hard": 0.15929358914390102, "calibration": 71.52112406568239, "p50_s": 1.9277653470635414, "p95_s": 2.801310855895281, "cost_usd_per_1000": 0.03665233775961865, "axes": { "intelligence": 79.0895061728395, "calibration": 71.52112406568239, "speed": 72.67584526450338, "cost": 53.07694959898829 }, "jevbench_score": 68.34587541481137 }, "heldout": { "n": 303, "escalation": 0.6204620462046204, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6513761467889908 }, "accuracy": 0.834983498349835, "ece_hard": 0.16404560607524768, "calibration": 73.09492680874263, "p50_s": 1.940494865179062, "p95_s": 2.610842273384332, "cost_usd_per_1000": 0.039646204842956195, "axes": { "intelligence": 83.3403711616606, "calibration": 73.09492680874263, "speed": 72.9530686617781, "cost": 52.05395138379444 }, "jevbench_score": 69.35215860199163 } }, { "threshold": 0.96, "public": { "n": 231, "escalation": 0.5627705627705628, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6666666666666666 }, "accuracy": 0.7835497835497836, "ece_hard": 0.16236249114508872, "calibration": 71.21423386556361, "p50_s": 1.9360549077391624, "p95_s": 2.801310855895281, "cost_usd_per_1000": 0.03734349336057201, "axes": { "intelligence": 79.0895061728395, "calibration": 71.21423386556361, "speed": 72.65721025694586, "cost": 52.83355173012821 }, "jevbench_score": 68.18966376797756 }, "heldout": { "n": 303, "escalation": 0.6633663366336634, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6697247706422018 }, "accuracy": 0.8415841584158416, "ece_hard": 0.1424608599813228, "calibration": 75.25340141813511, "p50_s": 1.9488137289881706, "p95_s": 2.612931399047375, "cost_usd_per_1000": 0.04135869681462528, "axes": { "intelligence": 83.89082987725693, "calibration": 75.25340141813511, "speed": 72.93101661787097, "cost": 51.50299462097298 }, "jevbench_score": 69.78247096915874 } }, { "threshold": 0.97, "public": { "n": 231, "escalation": 0.6103896103896104, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6756756756756757 }, "accuracy": 0.7878787878787878, "ece_hard": 0.14677099593751908, "calibration": 72.77338338632059, "p50_s": 1.9412290751934052, "p95_s": 2.823735985904932, "cost_usd_per_1000": 0.039244171263193725, "axes": { "intelligence": 79.46488154821488, "calibration": 72.77338338632059, "speed": 72.61099124018477, "cost": 52.18674513026955 }, "jevbench_score": 68.41913761683729 }, "heldout": { "n": 303, "escalation": 0.6963696369636964, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6697247706422018 }, "accuracy": 0.8415841584158416, "ece_hard": 0.1310520802508322, "calibration": 76.39427939118417, "p50_s": 1.9527576640248299, "p95_s": 2.612931399047375, "cost_usd_per_1000": 0.0426759983312938, "axes": { "intelligence": 83.89082987725693, "calibration": 76.39427939118417, "speed": 72.92223641300333, "cost": 51.09448931574614 }, "jevbench_score": 69.90405077949144 } }, { "threshold": 0.98, "public": { "n": 231, "escalation": 0.6796536796536796, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.6846846846846847 }, "accuracy": 0.7922077922077922, "ece_hard": 0.12898968669372407, "calibration": 74.55151431070009, "p50_s": 1.954352654516697, "p95_s": 2.823735985904932, "cost_usd_per_1000": 0.04200879366700715, "axes": { "intelligence": 79.84025692359026, "calibration": 74.55151431070009, "speed": 72.58172978431068, "cost": 51.29979368712684 }, "jevbench_score": 68.61282990575785 }, "heldout": { "n": 303, "escalation": 0.7458745874587459, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6788990825688074 }, "accuracy": 0.8448844884488449, "ece_hard": 0.11724626321260129, "calibration": 78.28237367873987, "p50_s": 1.959138184785843, "p95_s": 2.62162209674716, "cost_usd_per_1000": 0.04465195060629658, "axes": { "intelligence": 84.16605923505509, "calibration": 78.28237367873987, "speed": 72.89364841726497, "cost": 50.50478693033097 }, "jevbench_score": 70.17877813752308 } }, { "threshold": 0.99, "public": { "n": 231, "escalation": 0.7316017316017316, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.7027027027027027 }, "accuracy": 0.8008658008658008, "ece_hard": 0.10777876153483887, "calibration": 76.64312384651612, "p50_s": 1.961857296526432, "p95_s": 2.823735985904932, "cost_usd_per_1000": 0.044082260469867206, "axes": { "intelligence": 80.59100767434103, "calibration": 76.64312384651612, "speed": 72.56508497392635, "cost": 50.67208430915788 }, "jevbench_score": 69.0341632296934 }, "heldout": { "n": 303, "escalation": 0.8547854785478548, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6788990825688074 }, "accuracy": 0.8448844884488449, "ece_hard": 0.1088133754767477, "calibration": 79.12566245232523, "p50_s": 1.9719288051128387, "p95_s": 2.66972187384963, "cost_usd_per_1000": 0.048999045611302694, "axes": { "intelligence": 84.16605923505509, "calibration": 79.12566245232523, "speed": 72.78642749148585, "cost": 49.29437136839852 }, "jevbench_score": 69.91583072560415 } }, { "threshold": 1.0, "public": { "n": 231, "escalation": 1.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.7117117117117117 }, "accuracy": 0.8051948051948052, "ece_hard": 0.09203203203203206, "calibration": 78.2177967967968, "p50_s": 1.9686473831534386, "p95_s": 2.823735985904932, "cost_usd_per_1000": 0.054795172284644186, "axes": { "intelligence": 80.9663830497164, "calibration": 78.2177967967968, "speed": 72.55007977572599, "cost": 47.837731097201925 }, "jevbench_score": 68.47068871996393 }, "heldout": { "n": 303, "escalation": 1.0, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6972477064220184 }, "accuracy": 0.8514851485148515, "ece_hard": 0.08064220183486236, "calibration": 81.94277981651376, "p50_s": 1.9836426004767418, "p95_s": 2.683542966842651, "cost_usd_per_1000": 0.054795172284644186, "axes": { "intelligence": 84.71651795065142, "calibration": 81.94277981651376, "speed": 72.73828020152585, "cost": 47.837731097201925 }, "jevbench_score": 70.10566252087517 } } ] }, { "cheap": "open-alternative-jev", "backend": "jev-1.13.0", "chosen": { "threshold": 1.0, "public": { "n": 231, "escalation": 1.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.7117117117117117 }, "accuracy": 0.8051948051948052, "ece_hard": 0.09203203203203206, "calibration": 78.2177967967968, "p50_s": 1.226769672334194, "p95_s": 2.0335648261010646, "cost_usd_per_1000": 0.06208564044943819, "axes": { "intelligence": 80.9663830497164, "calibration": 78.2177967967968, "speed": 76.02978947569241, "cost": 46.21026503561948 }, "jevbench_score": 68.68045145419504 }, "heldout": { "n": 303, "escalation": 1.0, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6972477064220184 }, "accuracy": 0.8514851485148515, "ece_hard": 0.08064220183486236, "calibration": 81.94277981651376, "p50_s": 1.2545109555125236, "p95_s": 1.8302100747823713, "cost_usd_per_1000": 0.06208564044943819, "axes": { "intelligence": 84.71651795065142, "calibration": 81.94277981651376, "speed": 76.39024599996569, "cost": 46.21026503561948 }, "jevbench_score": 70.3580492090166 } }, "full": { "tiers": { "easy": 1.0, "standard": 0.8229166666666666, "judge": 0.9452054794520548, "hard": 0.7045454545454546 }, "accuracy": 0.8314606741573034, "ece_hard": 0.08502525252525245, "calibration": 80.21147474747477, "p50_s": 1.2439309187233447, "p95_s": 1.8417592387646435, "cost_usd_per_1000": 0.06208564044943819, "axes": { "intelligence": 84.64378372768783, "calibration": 80.21147474747477, "speed": 76.39970880299045, "cost": 46.21026503561948 }, "jevbench_score": 69.97057344425629 }, "full_escalation": 1.0, "backend_heldout_accuracy": 0.8514851485148515, "useful": false, "curve": [ { "threshold": 0.0, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 63.4379581919484, "calibration": 59.97952523584675, "speed": 81.86491236780878, "cost": 59.62620384142349 }, "jevbench_score": 65.64804472134733 } }, { "threshold": 0.01, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 63.4379581919484, "calibration": 59.97952523584675, "speed": 81.86491236780878, "cost": 59.62620384142349 }, "jevbench_score": 65.64804472134733 } }, { "threshold": 0.02, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 63.4379581919484, "calibration": 59.97952523584675, "speed": 81.86491236780878, "cost": 59.62620384142349 }, "jevbench_score": 65.64804472134733 } }, { "threshold": 0.03, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 63.4379581919484, "calibration": 59.97952523584675, "speed": 81.86491236780878, "cost": 59.62620384142349 }, "jevbench_score": 65.64804472134733 } }, { "threshold": 0.04, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 63.4379581919484, "calibration": 59.97952523584675, "speed": 81.86491236780878, "cost": 59.62620384142349 }, "jevbench_score": 65.64804472134733 } }, { "threshold": 0.05, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 63.4379581919484, "calibration": 59.97952523584675, "speed": 81.86491236780878, "cost": 59.62620384142349 }, "jevbench_score": 65.64804472134733 } }, { "threshold": 0.06, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 63.4379581919484, "calibration": 59.97952523584675, "speed": 81.86491236780878, "cost": 59.62620384142349 }, "jevbench_score": 65.64804472134733 } }, { "threshold": 0.07, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 63.4379581919484, "calibration": 59.97952523584675, "speed": 81.86491236780878, "cost": 59.62620384142349 }, "jevbench_score": 65.64804472134733 } }, { "threshold": 0.08, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 63.4379581919484, "calibration": 59.97952523584675, "speed": 81.86491236780878, "cost": 59.62620384142349 }, "jevbench_score": 65.64804472134733 } }, { "threshold": 0.09, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 63.4379581919484, "calibration": 59.97952523584675, "speed": 81.86491236780878, "cost": 59.62620384142349 }, "jevbench_score": 65.64804472134733 } }, { "threshold": 0.1, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 63.4379581919484, "calibration": 59.97952523584675, "speed": 81.86491236780878, "cost": 59.62620384142349 }, "jevbench_score": 65.64804472134733 } }, { "threshold": 0.11, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 63.4379581919484, "calibration": 59.97952523584675, "speed": 81.86491236780878, "cost": 59.62620384142349 }, "jevbench_score": 65.64804472134733 } }, { "threshold": 0.12, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 63.4379581919484, "calibration": 59.97952523584675, "speed": 81.86491236780878, "cost": 59.62620384142349 }, "jevbench_score": 65.64804472134733 } }, { "threshold": 0.13, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 63.4379581919484, "calibration": 59.97952523584675, "speed": 81.86491236780878, "cost": 59.62620384142349 }, "jevbench_score": 65.64804472134733 } }, { "threshold": 0.14, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 63.4379581919484, "calibration": 59.97952523584675, "speed": 81.86491236780878, "cost": 59.62620384142349 }, "jevbench_score": 65.64804472134733 } }, { "threshold": 0.15, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 63.4379581919484, "calibration": 59.97952523584675, "speed": 81.86491236780878, "cost": 59.62620384142349 }, "jevbench_score": 65.64804472134733 } }, { "threshold": 0.16, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 63.4379581919484, "calibration": 59.97952523584675, "speed": 81.86491236780878, "cost": 59.62620384142349 }, "jevbench_score": 65.64804472134733 } }, { "threshold": 0.17, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 63.4379581919484, "calibration": 59.97952523584675, "speed": 81.86491236780878, "cost": 59.62620384142349 }, "jevbench_score": 65.64804472134733 } }, { "threshold": 0.18, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 63.4379581919484, "calibration": 59.97952523584675, "speed": 81.86491236780878, "cost": 59.62620384142349 }, "jevbench_score": 65.64804472134733 } }, { "threshold": 0.19, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 63.4379581919484, "calibration": 59.97952523584675, "speed": 81.86491236780878, "cost": 59.62620384142349 }, "jevbench_score": 65.64804472134733 } }, { "threshold": 0.2, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 63.4379581919484, "calibration": 59.97952523584675, "speed": 81.86491236780878, "cost": 59.62620384142349 }, "jevbench_score": 65.64804472134733 } }, { "threshold": 0.21, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 63.4379581919484, "calibration": 59.97952523584675, "speed": 81.86491236780878, "cost": 59.62620384142349 }, "jevbench_score": 65.64804472134733 } }, { "threshold": 0.22, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 63.4379581919484, "calibration": 59.97952523584675, "speed": 81.86491236780878, "cost": 59.62620384142349 }, "jevbench_score": 65.64804472134733 } }, { "threshold": 0.23, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 63.4379581919484, "calibration": 59.97952523584675, "speed": 81.86491236780878, "cost": 59.62620384142349 }, "jevbench_score": 65.64804472134733 } }, { "threshold": 0.24, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 63.4379581919484, "calibration": 59.97952523584675, "speed": 81.86491236780878, "cost": 59.62620384142349 }, "jevbench_score": 65.64804472134733 } }, { "threshold": 0.25, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 63.4379581919484, "calibration": 59.97952523584675, "speed": 81.86491236780878, "cost": 59.62620384142349 }, "jevbench_score": 65.64804472134733 } }, { "threshold": 0.26, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 63.4379581919484, "calibration": 59.97952523584675, "speed": 81.86491236780878, "cost": 59.62620384142349 }, "jevbench_score": 65.64804472134733 } }, { "threshold": 0.27, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 63.4379581919484, "calibration": 59.97952523584675, "speed": 81.86491236780878, "cost": 59.62620384142349 }, "jevbench_score": 65.64804472134733 } }, { "threshold": 0.28, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 63.4379581919484, "calibration": 59.97952523584675, "speed": 81.86491236780878, "cost": 59.62620384142349 }, "jevbench_score": 65.64804472134733 } }, { "threshold": 0.29, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 63.4379581919484, "calibration": 59.97952523584675, "speed": 81.86491236780878, "cost": 59.62620384142349 }, "jevbench_score": 65.64804472134733 } }, { "threshold": 0.3, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2723344657716148, "calibration": 59.97952523584675, "p50_s": 0.5870145052671433, "p95_s": 1.1088171392679214, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 63.4379581919484, "calibration": 59.97952523584675, "speed": 81.86491236780878, "cost": 59.62620384142349 }, "jevbench_score": 65.64804472134733 } }, { "threshold": 0.31, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.0033003300330033004, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2788851345408365, "calibration": 59.32445835892458, "p50_s": 0.5870145052671433, "p95_s": 1.1089038744568824, "cost_usd_per_1000": 0.022303134646048876, "axes": { "intelligence": 63.4379581919484, "calibration": 59.32445835892458, "speed": 81.86457266219443, "cost": 59.549022811950415 }, "jevbench_score": 65.44679809917865 } }, { "threshold": 0.32, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.006600660066006601, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2862559524401423, "calibration": 58.587376568994, "p50_s": 0.5870145052671433, "p95_s": 1.1089038744568824, "cost_usd_per_1000": 0.02243486479771573, "axes": { "intelligence": 63.4379581919484, "calibration": 58.587376568994, "speed": 81.86457266219443, "cost": 59.47229630113193 }, "jevbench_score": 65.22153092289962 } }, { "threshold": 0.33, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.006600660066006601, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2862559524401423, "calibration": 58.587376568994, "p50_s": 0.5870145052671433, "p95_s": 1.1089038744568824, "cost_usd_per_1000": 0.02243486479771573, "axes": { "intelligence": 63.4379581919484, "calibration": 58.587376568994, "speed": 81.86457266219443, "cost": 59.47229630113193 }, "jevbench_score": 65.22153092289962 } }, { "threshold": 0.34, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.013201320132013201, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5045871559633027 }, "accuracy": 0.5610561056105611, "ece_hard": 0.28425866550464834, "calibration": 58.78710526254339, "p50_s": 0.5870145052671433, "p95_s": 1.1165145859122274, "cost_usd_per_1000": 0.02269832510104943, "axes": { "intelligence": 63.16272883415023, "calibration": 58.78710526254339, "speed": 81.83486766124035, "cost": 59.320185640429074 }, "jevbench_score": 65.15848253531232 } }, { "threshold": 0.35, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.013201320132013201, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5045871559633027 }, "accuracy": 0.5610561056105611, "ece_hard": 0.28425866550464834, "calibration": 58.78710526254339, "p50_s": 0.5870145052671433, "p95_s": 1.1165145859122274, "cost_usd_per_1000": 0.02269832510104943, "axes": { "intelligence": 63.16272883415023, "calibration": 58.78710526254339, "speed": 81.83486766124035, "cost": 59.320185640429074 }, "jevbench_score": 65.15848253531232 } }, { "threshold": 0.36, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.0165016501650165, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5045871559633027 }, "accuracy": 0.5610561056105611, "ece_hard": 0.28027567811472737, "calibration": 59.185404001535495, "p50_s": 0.5870145052671433, "p95_s": 1.120798410475254, "cost_usd_per_1000": 0.022830055252716284, "axes": { "intelligence": 63.16272883415023, "calibration": 59.185404001535495, "speed": 81.81823660742067, "cost": 59.244791123202596 }, "jevbench_score": 65.24450600180543 } }, { "threshold": 0.37, "public": { "n": 231, "escalation": 0.0, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.23568660819842263, "calibration": 57.69125137407893, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022171404494382024, "axes": { "intelligence": 66.27252252252254, "calibration": 57.69125137407893, "speed": 81.73312596625158, "cost": 59.62620384142349 }, "jevbench_score": 65.70065247995181 }, "heldout": { "n": 303, "escalation": 0.019801980198019802, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.27585673808635774, "calibration": 59.631459162103155, "p50_s": 0.5870145052671433, "p95_s": 1.1283639609813687, "cost_usd_per_1000": 0.022961785404383137, "axes": { "intelligence": 63.4379581919484, "calibration": 59.631459162103155, "speed": 81.78901960404914, "cost": 59.169830385706256 }, "jevbench_score": 65.41163277182078 } }, { "threshold": 0.38, "public": { "n": 231, "escalation": 0.004329004329004329, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5495495495495496 }, "accuracy": 0.6666666666666666, "ece_hard": 0.22654418955195743, "calibration": 58.605493238725444, "p50_s": 0.5530259400606156, "p95_s": 1.2132265359163283, "cost_usd_per_1000": 0.022344193394620362, "axes": { "intelligence": 66.6478978978979, "calibration": 58.605493238725444, "speed": 81.73312596625158, "cost": 59.525059550880485 }, "jevbench_score": 66.02458533992528 }, "heldout": { "n": 303, "escalation": 0.019801980198019802, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.27585673808635774, "calibration": 59.631459162103155, "p50_s": 0.5870145052671433, "p95_s": 1.1283639609813687, "cost_usd_per_1000": 0.022961785404383137, "axes": { "intelligence": 63.4379581919484, "calibration": 59.631459162103155, "speed": 81.78901960404914, "cost": 59.169830385706256 }, "jevbench_score": 65.41163277182078 } }, { "threshold": 0.39, "public": { "n": 231, "escalation": 0.008658008658008658, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5495495495495496 }, "accuracy": 0.6666666666666666, "ece_hard": 0.22643991325148444, "calibration": 58.61592086877275, "p50_s": 0.5530259400606156, "p95_s": 1.2245997205376624, "cost_usd_per_1000": 0.0225169822948587, "axes": { "intelligence": 66.6478978978979, "calibration": 58.61592086877275, "speed": 81.69260341346663, "cost": 59.42469440996864 }, "jevbench_score": 65.99149018732358 }, "heldout": { "n": 303, "escalation": 0.019801980198019802, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.27585673808635774, "calibration": 59.631459162103155, "p50_s": 0.5870145052671433, "p95_s": 1.1283639609813687, "cost_usd_per_1000": 0.022961785404383137, "axes": { "intelligence": 63.4379581919484, "calibration": 59.631459162103155, "speed": 81.78901960404914, "cost": 59.169830385706256 }, "jevbench_score": 65.41163277182078 } }, { "threshold": 0.4, "public": { "n": 231, "escalation": 0.017316017316017316, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5585585585585585 }, "accuracy": 0.670995670995671, "ece_hard": 0.22781664987506176, "calibration": 58.47824720641501, "p50_s": 0.5530259400606156, "p95_s": 1.2525466509163379, "cost_usd_per_1000": 0.022862560095335376, "axes": { "intelligence": 67.02327327327328, "calibration": 58.47824720641501, "speed": 81.59460587262502, "cost": 59.22625419850474 }, "jevbench_score": 65.97037063420939 }, "heldout": { "n": 303, "escalation": 0.019801980198019802, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5137614678899083 }, "accuracy": 0.5643564356435643, "ece_hard": 0.27585673808635774, "calibration": 59.631459162103155, "p50_s": 0.5870145052671433, "p95_s": 1.1283639609813687, "cost_usd_per_1000": 0.022961785404383137, "axes": { "intelligence": 63.4379581919484, "calibration": 59.631459162103155, "speed": 81.78901960404914, "cost": 59.169830385706256 }, "jevbench_score": 65.41163277182078 } }, { "threshold": 0.41, "public": { "n": 231, "escalation": 0.017316017316017316, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5585585585585585 }, "accuracy": 0.670995670995671, "ece_hard": 0.22781664987506176, "calibration": 58.47824720641501, "p50_s": 0.5530259400606156, "p95_s": 1.2525466509163379, "cost_usd_per_1000": 0.022862560095335376, "axes": { "intelligence": 67.02327327327328, "calibration": 58.47824720641501, "speed": 81.59460587262502, "cost": 59.22625419850474 }, "jevbench_score": 65.97037063420939 }, "heldout": { "n": 303, "escalation": 0.0231023102310231, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5045871559633027 }, "accuracy": 0.5610561056105611, "ece_hard": 0.2851663429345224, "calibration": 58.70049867728669, "p50_s": 0.5870145052671433, "p95_s": 1.1592439234256735, "cost_usd_per_1000": 0.02309351555604999, "axes": { "intelligence": 63.16272883415023, "calibration": 58.70049867728669, "speed": 81.6717633903008, "cost": 59.095298465007346 }, "jevbench_score": 65.0402024793748 } }, { "threshold": 0.42, "public": { "n": 231, "escalation": 0.017316017316017316, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5585585585585585 }, "accuracy": 0.670995670995671, "ece_hard": 0.22781664987506176, "calibration": 58.47824720641501, "p50_s": 0.5530259400606156, "p95_s": 1.2525466509163379, "cost_usd_per_1000": 0.022862560095335376, "axes": { "intelligence": 67.02327327327328, "calibration": 58.47824720641501, "speed": 81.59460587262502, "cost": 59.22625419850474 }, "jevbench_score": 65.97037063420939 }, "heldout": { "n": 303, "escalation": 0.0231023102310231, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5045871559633027 }, "accuracy": 0.5610561056105611, "ece_hard": 0.2851663429345224, "calibration": 58.70049867728669, "p50_s": 0.5870145052671433, "p95_s": 1.1592439234256735, "cost_usd_per_1000": 0.02309351555604999, "axes": { "intelligence": 63.16272883415023, "calibration": 58.70049867728669, "speed": 81.6717633903008, "cost": 59.095298465007346 }, "jevbench_score": 65.0402024793748 } }, { "threshold": 0.43, "public": { "n": 231, "escalation": 0.030303030303030304, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5585585585585585 }, "accuracy": 0.670995670995671, "ece_hard": 0.23063224476485567, "calibration": 58.19668771743562, "p50_s": 0.5530259400606156, "p95_s": 1.3340997859835624, "cost_usd_per_1000": 0.023380926796050392, "axes": { "intelligence": 67.02327327327328, "calibration": 58.19668771743562, "speed": 81.32066182838275, "cost": 58.93414833530273 }, "jevbench_score": 65.75411765908889 }, "heldout": { "n": 303, "escalation": 0.0231023102310231, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5045871559633027 }, "accuracy": 0.5610561056105611, "ece_hard": 0.2851663429345224, "calibration": 58.70049867728669, "p50_s": 0.5870145052671433, "p95_s": 1.1592439234256735, "cost_usd_per_1000": 0.02309351555604999, "axes": { "intelligence": 63.16272883415023, "calibration": 58.70049867728669, "speed": 81.6717633903008, "cost": 59.095298465007346 }, "jevbench_score": 65.0402024793748 } }, { "threshold": 0.44, "public": { "n": 231, "escalation": 0.030303030303030304, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5585585585585585 }, "accuracy": 0.670995670995671, "ece_hard": 0.23063224476485567, "calibration": 58.19668771743562, "p50_s": 0.5530259400606156, "p95_s": 1.3340997859835624, "cost_usd_per_1000": 0.023380926796050392, "axes": { "intelligence": 67.02327327327328, "calibration": 58.19668771743562, "speed": 81.32066182838275, "cost": 58.93414833530273 }, "jevbench_score": 65.75411765908889 }, "heldout": { "n": 303, "escalation": 0.026402640264026403, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5045871559633027 }, "accuracy": 0.5610561056105611, "ece_hard": 0.29035188461876194, "calibration": 58.18194450886274, "p50_s": 0.5878239214420319, "p95_s": 1.2063746720552433, "cost_usd_per_1000": 0.02322524570771684, "axes": { "intelligence": 63.16272883415023, "calibration": 58.18194450886274, "speed": 81.49270533758894, "cost": 59.02119048286132 }, "jevbench_score": 64.84014154934319 } }, { "threshold": 0.45, "public": { "n": 231, "escalation": 0.03463203463203463, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5495495495495496 }, "accuracy": 0.6666666666666666, "ece_hard": 0.2396919734372042, "calibration": 57.29071485020077, "p50_s": 0.5530259400606156, "p95_s": 1.3340997859835624, "cost_usd_per_1000": 0.02355371569628873, "axes": { "intelligence": 66.6478978978979, "calibration": 57.29071485020077, "speed": 81.32066182838275, "cost": 58.83821714127103 }, "jevbench_score": 65.37817249228333 }, "heldout": { "n": 303, "escalation": 0.0297029702970297, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5068493150684932, "hard": 0.5045871559633027 }, "accuracy": 0.5610561056105611, "ece_hard": 0.2947065171267957, "calibration": 57.74648125805936, "p50_s": 0.5878239214420319, "p95_s": 1.222848697006702, "cost_usd_per_1000": 0.02335697585938369, "axes": { "intelligence": 63.16272883415023, "calibration": 57.74648125805936, "speed": 81.43380019197517, "cost": 58.9475016437958 }, "jevbench_score": 64.68657084204645 } }, { "threshold": 0.46, "public": { "n": 231, "escalation": 0.04329004329004329, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5405405405405406 }, "accuracy": 0.6623376623376623, "ece_hard": 0.2532469326755076, "calibration": 55.93521892637044, "p50_s": 0.5530259400606156, "p95_s": 1.4122533962130546, "cost_usd_per_1000": 0.02389929349676541, "axes": { "intelligence": 66.27252252252254, "calibration": 55.93521892637044, "speed": 81.07341869528295, "cost": 58.64844812008064 }, "jevbench_score": 64.79454787658689 }, "heldout": { "n": 303, "escalation": 0.039603960396039604, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5136986301369864, "hard": 0.5045871559633027 }, "accuracy": 0.5643564356435643, "ece_hard": 0.2949991596128602, "calibration": 57.71721700945291, "p50_s": 0.5895052790641785, "p95_s": 1.2606141895055765, "cost_usd_per_1000": 0.02375216631438425, "axes": { "intelligence": 63.35450965606804, "calibration": 57.71721700945291, "speed": 81.2893011016477, "cost": 58.72890323320096 }, "jevbench_score": 64.63861600597465 } }, { "threshold": 0.47, "public": { "n": 231, "escalation": 0.047619047619047616, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5495495495495496 }, "accuracy": 0.6666666666666666, "ece_hard": 0.24547438872144473, "calibration": 56.71247332177672, "p50_s": 0.5530259400606156, "p95_s": 1.4122533962130546, "cost_usd_per_1000": 0.024072082397003747, "axes": { "intelligence": 66.6478978978979, "calibration": 56.71247332177672, "speed": 81.07341869528295, "cost": 58.554590161201276 }, "jevbench_score": 65.08428214335962 }, "heldout": { "n": 303, "escalation": 0.0462046204620462, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5205479452054794, "hard": 0.5045871559633027 }, "accuracy": 0.5676567656765676, "ece_hard": 0.2949991596128602, "calibration": 57.71721700945291, "p50_s": 0.5900527179241181, "p95_s": 1.2770627483725543, "cost_usd_per_1000": 0.02401562661771795, "axes": { "intelligence": 63.54629047798584, "calibration": 57.71721700945291, "speed": 81.22896948020045, "cost": 58.585182316888194 }, "jevbench_score": 64.6358670249557 } }, { "threshold": 0.48, "public": { "n": 231, "escalation": 0.05194805194805195, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5495495495495496 }, "accuracy": 0.6666666666666666, "ece_hard": 0.2496437802919674, "calibration": 56.295534164724444, "p50_s": 0.5530259400606156, "p95_s": 1.4313000380992889, "cost_usd_per_1000": 0.024244871297242086, "axes": { "intelligence": 66.6478978978979, "calibration": 56.295534164724444, "speed": 81.01523814498383, "cost": 58.46140350922449 }, "jevbench_score": 64.92681331635728 }, "heldout": { "n": 303, "escalation": 0.0462046204620462, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5205479452054794, "hard": 0.5045871559633027 }, "accuracy": 0.5676567656765676, "ece_hard": 0.2949991596128602, "calibration": 57.71721700945291, "p50_s": 0.5900527179241181, "p95_s": 1.2770627483725543, "cost_usd_per_1000": 0.02401562661771795, "axes": { "intelligence": 63.54629047798584, "calibration": 57.71721700945291, "speed": 81.22896948020045, "cost": 58.585182316888194 }, "jevbench_score": 64.6358670249557 } }, { "threshold": 0.49, "public": { "n": 231, "escalation": 0.05627705627705628, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5495495495495496 }, "accuracy": 0.6666666666666666, "ece_hard": 0.243823767898048, "calibration": 56.877535404116394, "p50_s": 0.5530259400606156, "p95_s": 1.4582971796393394, "cost_usd_per_1000": 0.02441766019748042, "axes": { "intelligence": 66.6478978978979, "calibration": 56.877535404116394, "speed": 80.93408461471964, "cost": 58.36887862943428 }, "jevbench_score": 65.05190343138874 }, "heldout": { "n": 303, "escalation": 0.0627062706270627, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5273972602739726, "hard": 0.5229357798165137 }, "accuracy": 0.5775577557755776, "ece_hard": 0.2819208688117406, "calibration": 59.27963238066127, "p50_s": 0.5905896663665772, "p95_s": 1.2855553507804869, "cost_usd_per_1000": 0.02467427737605221, "axes": { "intelligence": 64.28853001549999, "calibration": 59.27963238066127, "speed": 81.1962337588426, "cost": 58.23266672350478 }, "jevbench_score": 65.15314589691974 } }, { "threshold": 0.5, "public": { "n": 231, "escalation": 0.06060606060606061, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5585585585585585 }, "accuracy": 0.670995670995671, "ece_hard": 0.239101070226627, "calibration": 57.501008951695354, "p50_s": 0.5530259400606156, "p95_s": 1.4727609172463416, "cost_usd_per_1000": 0.02459044909771876, "axes": { "intelligence": 67.02327327327328, "calibration": 57.501008951695354, "speed": 80.89122246589763, "cost": 58.27700618881849 }, "jevbench_score": 65.28673240505192 }, "heldout": { "n": 303, "escalation": 0.06930693069306931, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5273972602739726, "hard": 0.5321100917431193 }, "accuracy": 0.5808580858085809, "ece_hard": 0.2768441492617718, "calibration": 60.76350811609501, "p50_s": 0.5923556029796601, "p95_s": 1.2975253440439698, "cost_usd_per_1000": 0.024937737679385917, "axes": { "intelligence": 64.56375937329814, "calibration": 60.76350811609501, "speed": 81.14301649775062, "cost": 58.0942884318075 }, "jevbench_score": 65.57737959626436 } }, { "threshold": 0.51, "public": { "n": 231, "escalation": 0.06060606060606061, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5585585585585585 }, "accuracy": 0.670995670995671, "ece_hard": 0.239101070226627, "calibration": 57.501008951695354, "p50_s": 0.5530259400606156, "p95_s": 1.4727609172463416, "cost_usd_per_1000": 0.02459044909771876, "axes": { "intelligence": 67.02327327327328, "calibration": 57.501008951695354, "speed": 80.89122246589763, "cost": 58.27700618881849 }, "jevbench_score": 65.28673240505192 }, "heldout": { "n": 303, "escalation": 0.0924092409240924, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5616438356164384, "hard": 0.5321100917431193 }, "accuracy": 0.5973597359735974, "ece_hard": 0.2764686322610189, "calibration": 60.8010598161703, "p50_s": 0.5930232793092728, "p95_s": 1.342048987746238, "cost_usd_per_1000": 0.025859848741053883, "axes": { "intelligence": 65.5226634828872, "calibration": 60.8010598161703, "speed": 80.99159889110892, "cost": 57.62122059146196 }, "jevbench_score": 65.66459647053959 } }, { "threshold": 0.52, "public": { "n": 231, "escalation": 0.06060606060606061, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5585585585585585 }, "accuracy": 0.670995670995671, "ece_hard": 0.239101070226627, "calibration": 57.501008951695354, "p50_s": 0.5530259400606156, "p95_s": 1.4727609172463416, "cost_usd_per_1000": 0.02459044909771876, "axes": { "intelligence": 67.02327327327328, "calibration": 57.501008951695354, "speed": 80.89122246589763, "cost": 58.27700618881849 }, "jevbench_score": 65.28673240505192 }, "heldout": { "n": 303, "escalation": 0.0924092409240924, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5616438356164384, "hard": 0.5321100917431193 }, "accuracy": 0.5973597359735974, "ece_hard": 0.2764686322610189, "calibration": 60.8010598161703, "p50_s": 0.5930232793092728, "p95_s": 1.342048987746238, "cost_usd_per_1000": 0.025859848741053883, "axes": { "intelligence": 65.5226634828872, "calibration": 60.8010598161703, "speed": 80.99159889110892, "cost": 57.62122059146196 }, "jevbench_score": 65.66459647053959 } }, { "threshold": 0.53, "public": { "n": 231, "escalation": 0.06926406926406926, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5585585585585585 }, "accuracy": 0.670995670995671, "ece_hard": 0.24145394340559476, "calibration": 57.26572163379858, "p50_s": 0.5530259400606156, "p95_s": 1.5114113330841064, "cost_usd_per_1000": 0.024936026898195438, "axes": { "intelligence": 67.02327327327328, "calibration": 57.26572163379858, "speed": 80.77871822953344, "cost": 58.095182267880254 }, "jevbench_score": 65.1462410695929 }, "heldout": { "n": 303, "escalation": 0.10561056105610561, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5753424657534246, "hard": 0.5412844036697247 }, "accuracy": 0.6072607260726073, "ece_hard": 0.2752506337615269, "calibration": 60.72165124241474, "p50_s": 0.5963243931531906, "p95_s": 1.3474827371537683, "cost_usd_per_1000": 0.02638676934772129, "axes": { "intelligence": 66.18145448452097, "calibration": 60.72165124241474, "speed": 80.94994215039263, "cost": 57.35841337526125 }, "jevbench_score": 65.72390938748919 } }, { "threshold": 0.54, "public": { "n": 231, "escalation": 0.08225108225108226, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5675675675675675 }, "accuracy": 0.6753246753246753, "ece_hard": 0.2355195392284576, "calibration": 58.22086556043122, "p50_s": 0.5530259400606156, "p95_s": 1.5455794125795363, "cost_usd_per_1000": 0.025454393598910455, "axes": { "intelligence": 67.39864864864866, "calibration": 58.22086556043122, "speed": 80.68163173114914, "cost": 57.827117341573555 }, "jevbench_score": 65.41223913380924 }, "heldout": { "n": 303, "escalation": 0.12871287128712872, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5958904109589042, "hard": 0.5412844036697247 }, "accuracy": 0.6171617161716172, "ece_hard": 0.2790010064828686, "calibration": 60.34661397028057, "p50_s": 0.5986908316612244, "p95_s": 1.4190636679530135, "cost_usd_per_1000": 0.027308880409389252, "axes": { "intelligence": 66.7567969502744, "calibration": 60.34661397028057, "speed": 80.70795511554897, "cost": 56.910883132012174 }, "jevbench_score": 65.58658472960575 } }, { "threshold": 0.55, "public": { "n": 231, "escalation": 0.08225108225108226, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5675675675675675 }, "accuracy": 0.6753246753246753, "ece_hard": 0.2355195392284576, "calibration": 58.22086556043122, "p50_s": 0.5530259400606156, "p95_s": 1.5455794125795363, "cost_usd_per_1000": 0.025454393598910455, "axes": { "intelligence": 67.39864864864866, "calibration": 58.22086556043122, "speed": 80.68163173114914, "cost": 57.827117341573555 }, "jevbench_score": 65.41223913380924 }, "heldout": { "n": 303, "escalation": 0.12871287128712872, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5958904109589042, "hard": 0.5412844036697247 }, "accuracy": 0.6171617161716172, "ece_hard": 0.2790010064828686, "calibration": 60.34661397028057, "p50_s": 0.5986908316612244, "p95_s": 1.4190636679530135, "cost_usd_per_1000": 0.027308880409389252, "axes": { "intelligence": 66.7567969502744, "calibration": 60.34661397028057, "speed": 80.70795511554897, "cost": 56.910883132012174 }, "jevbench_score": 65.58658472960575 } }, { "threshold": 0.56, "public": { "n": 231, "escalation": 0.09090909090909091, "tiers": { "easy": 1.0, "standard": 0.625, "hard": 0.5675675675675675 }, "accuracy": 0.6753246753246753, "ece_hard": 0.23297291806259396, "calibration": 58.47552767701758, "p50_s": 0.5530259400606156, "p95_s": 1.6051472067832946, "cost_usd_per_1000": 0.02579997139938713, "axes": { "intelligence": 67.39864864864866, "calibration": 58.47552767701758, "speed": 80.51739629965851, "cost": 57.65142326423709 }, "jevbench_score": 65.40053087283053 }, "heldout": { "n": 303, "escalation": 0.12871287128712872, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.5958904109589042, "hard": 0.5412844036697247 }, "accuracy": 0.6171617161716172, "ece_hard": 0.2790010064828686, "calibration": 60.34661397028057, "p50_s": 0.5986908316612244, "p95_s": 1.4190636679530135, "cost_usd_per_1000": 0.027308880409389252, "axes": { "intelligence": 66.7567969502744, "calibration": 60.34661397028057, "speed": 80.70795511554897, "cost": 56.910883132012174 }, "jevbench_score": 65.58658472960575 } }, { "threshold": 0.57, "public": { "n": 231, "escalation": 0.11255411255411256, "tiers": { "easy": 1.0, "standard": 0.6388888888888888, "hard": 0.5675675675675675 }, "accuracy": 0.6796536796536796, "ece_hard": 0.23205810448455705, "calibration": 59.05989153925026, "p50_s": 0.5537255644798279, "p95_s": 1.696248324215412, "cost_usd_per_1000": 0.026663915900578823, "axes": { "intelligence": 67.93877210543877, "calibration": 59.05989153925026, "speed": 80.2721599384351, "cost": 57.222282074003815 }, "jevbench_score": 65.52169372224925 }, "heldout": { "n": 303, "escalation": 0.13861386138613863, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.6095890410958904, "hard": 0.5504587155963303 }, "accuracy": 0.6270627062706271, "ece_hard": 0.2734764514288704, "calibration": 60.89906947568039, "p50_s": 0.6008343935012818, "p95_s": 1.4190636679530135, "cost_usd_per_1000": 0.02770407086438981, "axes": { "intelligence": 67.41558795190818, "calibration": 60.89906947568039, "speed": 80.69243333993008, "cost": 56.723692317190476 }, "jevbench_score": 65.84034107573699 } }, { "threshold": 0.58, "public": { "n": 231, "escalation": 0.12121212121212122, "tiers": { "easy": 1.0, "standard": 0.6388888888888888, "hard": 0.5675675675675675 }, "accuracy": 0.6796536796536796, "ece_hard": 0.23521134017933984, "calibration": 58.74456796977198, "p50_s": 0.5537255644798279, "p95_s": 1.7532506600022315, "cost_usd_per_1000": 0.0270094937010555, "axes": { "intelligence": 67.93877210543877, "calibration": 58.74456796977198, "speed": 80.12861414397628, "cost": 57.054506700476416 }, "jevbench_score": 65.35679523531465 }, "heldout": { "n": 303, "escalation": 0.1419141914191419, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.6095890410958904, "hard": 0.5596330275229358 }, "accuracy": 0.6303630363036303, "ece_hard": 0.26291749503040013, "calibration": 61.954965115527415, "p50_s": 0.6008917182683945, "p95_s": 1.4190636679530135, "cost_usd_per_1000": 0.027835801016056663, "axes": { "intelligence": 67.69081730970635, "calibration": 61.954965115527415, "speed": 80.69201900541893, "cost": 56.661888301469745 }, "jevbench_score": 66.1731601346602 } }, { "threshold": 0.59, "public": { "n": 231, "escalation": 0.12121212121212122, "tiers": { "easy": 1.0, "standard": 0.6388888888888888, "hard": 0.5675675675675675 }, "accuracy": 0.6796536796536796, "ece_hard": 0.23521134017933984, "calibration": 58.74456796977198, "p50_s": 0.5537255644798279, "p95_s": 1.7532506600022315, "cost_usd_per_1000": 0.0270094937010555, "axes": { "intelligence": 67.93877210543877, "calibration": 58.74456796977198, "speed": 80.12861414397628, "cost": 57.054506700476416 }, "jevbench_score": 65.35679523531465 }, "heldout": { "n": 303, "escalation": 0.1419141914191419, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.6095890410958904, "hard": 0.5596330275229358 }, "accuracy": 0.6303630363036303, "ece_hard": 0.26291749503040013, "calibration": 61.954965115527415, "p50_s": 0.6008917182683945, "p95_s": 1.4190636679530135, "cost_usd_per_1000": 0.027835801016056663, "axes": { "intelligence": 67.69081730970635, "calibration": 61.954965115527415, "speed": 80.69201900541893, "cost": 56.661888301469745 }, "jevbench_score": 66.1731601346602 } }, { "threshold": 0.6, "public": { "n": 231, "escalation": 0.12987012987012986, "tiers": { "easy": 1.0, "standard": 0.6527777777777778, "hard": 0.5675675675675675 }, "accuracy": 0.683982683982684, "ece_hard": 0.2363584879275014, "calibration": 58.62985319495583, "p50_s": 0.5537255644798279, "p95_s": 1.7532506600022315, "cost_usd_per_1000": 0.027355071501532175, "axes": { "intelligence": 68.4788955622289, "calibration": 58.62985319495583, "speed": 80.12861414397628, "cost": 56.88886437138028 }, "jevbench_score": 65.40675640258553 }, "heldout": { "n": 303, "escalation": 0.1617161716171617, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.6301369863013698, "hard": 0.5596330275229358 }, "accuracy": 0.6402640264026402, "ece_hard": 0.264703428976197, "calibration": 62.103705721407024, "p50_s": 0.6067677825689316, "p95_s": 1.4190636679530135, "cost_usd_per_1000": 0.028626181926057773, "axes": { "intelligence": 68.26615977545978, "calibration": 62.103705721407024, "speed": 80.64975605897828, "cost": 56.29709719018032 }, "jevbench_score": 66.2373596207036 } }, { "threshold": 0.61, "public": { "n": 231, "escalation": 0.13852813852813853, "tiers": { "easy": 1.0, "standard": 0.6527777777777778, "hard": 0.5855855855855856 }, "accuracy": 0.6926406926406926, "ece_hard": 0.22162603254831353, "calibration": 60.103098732874614, "p50_s": 0.5537255644798279, "p95_s": 1.7532506600022315, "cost_usd_per_1000": 0.027700649302008853, "axes": { "intelligence": 69.22964631297965, "calibration": 60.103098732874614, "speed": 80.12861414397628, "cost": 56.725301529177074 }, "jevbench_score": 65.94598435766238 }, "heldout": { "n": 303, "escalation": 0.1716171617161716, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.636986301369863, "hard": 0.5596330275229358 }, "accuracy": 0.6435643564356436, "ece_hard": 0.26268635102738525, "calibration": 61.35803774981527, "p50_s": 0.6076879739761353, "p95_s": 1.4190636679530135, "cost_usd_per_1000": 0.02902137238105833, "axes": { "intelligence": 68.45794059737757, "calibration": 61.35803774981527, "speed": 80.64317477143828, "cost": 56.118461626595575 }, "jevbench_score": 66.0301321774486 } }, { "threshold": 0.62, "public": { "n": 231, "escalation": 0.14285714285714285, "tiers": { "easy": 1.0, "standard": 0.6527777777777778, "hard": 0.5855855855855856 }, "accuracy": 0.6926406926406926, "ece_hard": 0.22162603254831353, "calibration": 60.103098732874614, "p50_s": 0.5538872867822647, "p95_s": 1.7532506600022315, "cost_usd_per_1000": 0.02787343820224719, "axes": { "intelligence": 69.22964631297965, "calibration": 60.103098732874614, "speed": 80.12734591918152, "cost": 56.644283726365316 }, "jevbench_score": 65.92216408319494 }, "heldout": { "n": 303, "escalation": 0.1716171617161716, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.636986301369863, "hard": 0.5596330275229358 }, "accuracy": 0.6435643564356436, "ece_hard": 0.26268635102738525, "calibration": 61.35803774981527, "p50_s": 0.6076879739761353, "p95_s": 1.4190636679530135, "cost_usd_per_1000": 0.02902137238105833, "axes": { "intelligence": 68.45794059737757, "calibration": 61.35803774981527, "speed": 80.64317477143828, "cost": 56.118461626595575 }, "jevbench_score": 66.0301321774486 } }, { "threshold": 0.63, "public": { "n": 231, "escalation": 0.16017316017316016, "tiers": { "easy": 1.0, "standard": 0.6666666666666666, "hard": 0.5855855855855856 }, "accuracy": 0.696969696969697, "ece_hard": 0.22563577432846027, "calibration": 60.03982789885066, "p50_s": 0.5563413441181183, "p95_s": 1.7532506600022315, "cost_usd_per_1000": 0.028564593803200544, "axes": { "intelligence": 69.76976976976977, "calibration": 60.03982789885066, "speed": 80.10814653510246, "cost": 56.32515842058628 }, "jevbench_score": 65.93582712303797 }, "heldout": { "n": 303, "escalation": 0.18151815181518152, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.6575342465753424, "hard": 0.5596330275229358 }, "accuracy": 0.6534653465346535, "ece_hard": 0.26268635102738525, "calibration": 61.35803774981527, "p50_s": 0.6084316313266754, "p95_s": 1.4190636679530135, "cost_usd_per_1000": 0.029416562836058886, "axes": { "intelligence": 69.033283063131, "calibration": 61.35803774981527, "speed": 80.63786334774915, "cost": 55.9422422071319 }, "jevbench_score": 66.11533728414715 } }, { "threshold": 0.64, "public": { "n": 231, "escalation": 0.1645021645021645, "tiers": { "easy": 1.0, "standard": 0.6666666666666666, "hard": 0.5945945945945946 }, "accuracy": 0.7012987012987013, "ece_hard": 0.2196654445994337, "calibration": 60.63686087175332, "p50_s": 0.5563413441181183, "p95_s": 1.7532506600022315, "cost_usd_per_1000": 0.028737382703438882, "axes": { "intelligence": 70.14514514514514, "calibration": 60.63686087175332, "speed": 80.10814653510246, "cost": 56.24658365099602 }, "jevbench_score": 66.16476742149023 }, "heldout": { "n": 303, "escalation": 0.1848184818481848, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.6643835616438356, "hard": 0.5596330275229358 }, "accuracy": 0.6567656765676567, "ece_hard": 0.26268635102738525, "calibration": 61.35803774981527, "p50_s": 0.6108522057533264, "p95_s": 1.4190636679530135, "cost_usd_per_1000": 0.02954829298772574, "axes": { "intelligence": 69.22506388504881, "calibration": 61.35803774981527, "speed": 80.62061972545854, "cost": 55.88402810076575 }, "jevbench_score": 66.14045306431717 } }, { "threshold": 0.65, "public": { "n": 231, "escalation": 0.16883116883116883, "tiers": { "easy": 1.0, "standard": 0.6666666666666666, "hard": 0.6036036036036037 }, "accuracy": 0.7056277056277056, "ece_hard": 0.21277674674905372, "calibration": 61.32573065679132, "p50_s": 0.5563413441181183, "p95_s": 1.7532506600022315, "cost_usd_per_1000": 0.02891017160367722, "axes": { "intelligence": 70.52052052052052, "calibration": 61.32573065679132, "speed": 80.10814653510246, "cost": 56.16847991368977 }, "jevbench_score": 66.41740430485812 }, "heldout": { "n": 303, "escalation": 0.19141914191419143, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "judge": 0.6643835616438356, "hard": 0.5596330275229358 }, "accuracy": 0.6567656765676567, "ece_hard": 0.2616309685938325, "calibration": 61.46357599317054, "p50_s": 0.6108522057533264, "p95_s": 1.4444702014327047, "cost_usd_per_1000": 0.029811753291059444, "axes": { "intelligence": 69.22506388504881, "calibration": 61.46357599317054, "speed": 80.54355266536973, "cost": 55.76837444370329 }, "jevbench_score": 66.11880401049667 } }, { "threshold": 0.66, "public": { "n": 231, "escalation": 0.1774891774891775, "tiers": { "easy": 1.0, "standard": 0.6666666666666666, "hard": 0.6126126126126126 }, "accuracy": 0.70995670995671, "ece_hard": 0.20590548792624197, "calibration": 61.656082215742224, "p50_s": 0.5563752442598343, "p95_s": 1.7532506600022315, "cost_usd_per_1000": 0.0292557494041539, "axes": { "intelligence": 70.89589589589589, "calibration": 61.656082215742224, "speed": 80.1078819098665, "cost": 56.01366318065333 }, "jevbench_score": 66.54900417104056 }, "heldout": { "n": 303, "escalation": 0.21782178217821782, "tiers": { "easy": 1.0, "standard": 0.75, "judge": 0.684931506849315, "hard": 0.5596330275229358 }, "accuracy": 0.66996699669967, "ece_hard": 0.2619601869214181, "calibration": 61.430654160411976, "p50_s": 0.6203177213668823, "p95_s": 1.4444702014327047, "cost_usd_per_1000": 0.03086559450439426, "axes": { "intelligence": 70.96707301746889, "calibration": 61.430654160411976, "speed": 80.47677224607965, "cost": 55.31576060918928 }, "jevbench_score": 66.37283495015282 } }, { "threshold": 0.67, "public": { "n": 231, "escalation": 0.18614718614718614, "tiers": { "easy": 1.0, "standard": 0.6666666666666666, "hard": 0.6216216216216216 }, "accuracy": 0.7142857142857143, "ece_hard": 0.20055176605631675, "calibration": 62.19145440273475, "p50_s": 0.5563752442598343, "p95_s": 1.7532506600022315, "cost_usd_per_1000": 0.029601327204630573, "axes": { "intelligence": 71.27127127127126, "calibration": 62.19145440273475, "speed": 80.1078819098665, "cost": 55.860664494523526 }, "jevbench_score": 66.73545737980997 }, "heldout": { "n": 303, "escalation": 0.23102310231023102, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.6917808219178082, "hard": 0.5688073394495413 }, "accuracy": 0.6798679867986799, "ece_hard": 0.2537301364498217, "calibration": 64.39748704914437, "p50_s": 0.6225624173879624, "p95_s": 1.4473231211304665, "cost_usd_per_1000": 0.03139251511106167, "axes": { "intelligence": 72.60074986385155, "calibration": 64.39748704914437, "speed": 80.45251600126451, "cost": 55.09521664072432 }, "jevbench_score": 67.4708044227353 } }, { "threshold": 0.68, "public": { "n": 231, "escalation": 0.19047619047619047, "tiers": { "easy": 1.0, "standard": 0.6805555555555556, "hard": 0.6216216216216216 }, "accuracy": 0.7186147186147186, "ece_hard": 0.20055176605631675, "calibration": 62.19145440273475, "p50_s": 0.5565016955137253, "p95_s": 1.7532506600022315, "cost_usd_per_1000": 0.029774116104868912, "axes": { "intelligence": 71.8113947280614, "calibration": 62.19145440273475, "speed": 80.10689497087859, "cost": 55.78483367266122 }, "jevbench_score": 66.838628376285 }, "heldout": { "n": 303, "escalation": 0.264026402640264, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.7328767123287672, "hard": 0.5596330275229358 }, "accuracy": 0.6963696369636964, "ece_hard": 0.26337069884270803, "calibration": 63.433430809855736, "p50_s": 0.635247477889061, "p95_s": 1.457220613956451, "cost_usd_per_1000": 0.03270981662773019, "axes": { "intelligence": 73.47620543756021, "calibration": 63.433430809855736, "speed": 80.33531744368321, "cost": 54.55965671610516 }, "jevbench_score": 67.22963694703378 } }, { "threshold": 0.69, "public": { "n": 231, "escalation": 0.19047619047619047, "tiers": { "easy": 1.0, "standard": 0.6805555555555556, "hard": 0.6216216216216216 }, "accuracy": 0.7186147186147186, "ece_hard": 0.20055176605631675, "calibration": 62.19145440273475, "p50_s": 0.5565016955137253, "p95_s": 1.7532506600022315, "cost_usd_per_1000": 0.029774116104868912, "axes": { "intelligence": 71.8113947280614, "calibration": 62.19145440273475, "speed": 80.10689497087859, "cost": 55.78483367266122 }, "jevbench_score": 66.838628376285 }, "heldout": { "n": 303, "escalation": 0.26732673267326734, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.7328767123287672, "hard": 0.5596330275229358 }, "accuracy": 0.6963696369636964, "ece_hard": 0.26337069884270803, "calibration": 63.433430809855736, "p50_s": 0.635247477889061, "p95_s": 1.4646646618843078, "cost_usd_per_1000": 0.03284154677939704, "axes": { "intelligence": 73.47620543756021, "calibration": 63.433430809855736, "speed": 80.31318847127302, "cost": 54.50729189681789 }, "jevbench_score": 67.20887076542523 } }, { "threshold": 0.7, "public": { "n": 231, "escalation": 0.19913419913419914, "tiers": { "easy": 1.0, "standard": 0.6805555555555556, "hard": 0.6396396396396397 }, "accuracy": 0.7272727272727273, "ece_hard": 0.1905707262447503, "calibration": 63.189558383891395, "p50_s": 0.5565016955137253, "p95_s": 1.7659048452973365, "cost_usd_per_1000": 0.03011969390534559, "axes": { "intelligence": 72.56214547881216, "calibration": 63.189558383891395, "speed": 80.07566209967192, "cost": 55.63448338040986 }, "jevbench_score": 67.22797165936606 }, "heldout": { "n": 303, "escalation": 0.2739273927392739, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.7397260273972602, "hard": 0.5596330275229358 }, "accuracy": 0.6996699669966997, "ece_hard": 0.2628070433010912, "calibration": 63.489796364017415, "p50_s": 0.6424911558628082, "p95_s": 1.4646646618843078, "cost_usd_per_1000": 0.033105007082730745, "axes": { "intelligence": 73.66798625947804, "calibration": 63.489796364017415, "speed": 80.2639464276768, "cost": 54.40318944599414 }, "jevbench_score": 67.22516871373834 } }, { "threshold": 0.71, "public": { "n": 231, "escalation": 0.22077922077922077, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "hard": 0.6306306306306306 }, "accuracy": 0.7316017316017316, "ece_hard": 0.2025474171537177, "calibration": 61.99188929299465, "p50_s": 0.5587625443935395, "p95_s": 1.7659048452973365, "cost_usd_per_1000": 0.030983638406537284, "axes": { "intelligence": 73.26701701701703, "calibration": 61.99188929299465, "speed": 80.05805415524424, "cost": 55.266027532326426 }, "jevbench_score": 66.95402118925091 }, "heldout": { "n": 303, "escalation": 0.297029702970297, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.773972602739726, "hard": 0.5596330275229358 }, "accuracy": 0.7161716171617162, "ece_hard": 0.2628070433010912, "calibration": 63.489796364017415, "p50_s": 0.6549014985561371, "p95_s": 1.4646646618843078, "cost_usd_per_1000": 0.03402711814439871, "axes": { "intelligence": 74.62689036906707, "calibration": 63.489796364017415, "speed": 80.18085812392485, "cost": 54.04524493028485 }, "jevbench_score": 67.31422765744871 } }, { "threshold": 0.72, "public": { "n": 231, "escalation": 0.22077922077922077, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "hard": 0.6306306306306306 }, "accuracy": 0.7316017316017316, "ece_hard": 0.2025474171537177, "calibration": 61.99188929299465, "p50_s": 0.5587625443935395, "p95_s": 1.7659048452973365, "cost_usd_per_1000": 0.030983638406537284, "axes": { "intelligence": 73.26701701701703, "calibration": 61.99188929299465, "speed": 80.05805415524424, "cost": 55.266027532326426 }, "jevbench_score": 66.95402118925091 }, "heldout": { "n": 303, "escalation": 0.297029702970297, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.773972602739726, "hard": 0.5596330275229358 }, "accuracy": 0.7161716171617162, "ece_hard": 0.2628070433010912, "calibration": 63.489796364017415, "p50_s": 0.6549014985561371, "p95_s": 1.4646646618843078, "cost_usd_per_1000": 0.03402711814439871, "axes": { "intelligence": 74.62689036906707, "calibration": 63.489796364017415, "speed": 80.18085812392485, "cost": 54.04524493028485 }, "jevbench_score": 67.31422765744871 } }, { "threshold": 0.73, "public": { "n": 231, "escalation": 0.22077922077922077, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "hard": 0.6306306306306306 }, "accuracy": 0.7316017316017316, "ece_hard": 0.2025474171537177, "calibration": 61.99188929299465, "p50_s": 0.5587625443935395, "p95_s": 1.7659048452973365, "cost_usd_per_1000": 0.030983638406537284, "axes": { "intelligence": 73.26701701701703, "calibration": 61.99188929299465, "speed": 80.05805415524424, "cost": 55.266027532326426 }, "jevbench_score": 66.95402118925091 }, "heldout": { "n": 303, "escalation": 0.30363036303630364, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.7808219178082192, "hard": 0.5596330275229358 }, "accuracy": 0.7194719471947195, "ece_hard": 0.2628070433010912, "calibration": 63.489796364017415, "p50_s": 0.655505919456482, "p95_s": 1.4646646618843078, "cost_usd_per_1000": 0.03429057844773241, "axes": { "intelligence": 74.81867119098489, "calibration": 63.489796364017415, "speed": 80.17685178701353, "cost": 53.94475566130289 }, "jevbench_score": 67.32525989881995 } }, { "threshold": 0.74, "public": { "n": 231, "escalation": 0.22510822510822512, "tiers": { "easy": 1.0, "standard": 0.7083333333333334, "hard": 0.6306306306306306 }, "accuracy": 0.7316017316017316, "ece_hard": 0.19497031284173572, "calibration": 62.74959972419285, "p50_s": 0.5587625443935395, "p95_s": 1.7856338374316691, "cost_usd_per_1000": 0.03115642730677562, "axes": { "intelligence": 73.26701701701703, "calibration": 62.74959972419285, "speed": 80.00980306568133, "cost": 55.19357045480143 }, "jevbench_score": 67.12553964754281 }, "heldout": { "n": 303, "escalation": 0.31683168316831684, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8013698630136986, "hard": 0.5596330275229358 }, "accuracy": 0.7293729372937293, "ece_hard": 0.26302682148222617, "calibration": 63.467818545903924, "p50_s": 0.6719379365444184, "p95_s": 1.4646646618843078, "cost_usd_per_1000": 0.03481749905439982, "axes": { "intelligence": 75.3940136567383, "calibration": 63.467818545903924, "speed": 80.06932635226191, "cost": 53.74607282586718 }, "jevbench_score": 67.36368502930145 } }, { "threshold": 0.75, "public": { "n": 231, "escalation": 0.23376623376623376, "tiers": { "easy": 1.0, "standard": 0.7222222222222222, "hard": 0.6306306306306306 }, "accuracy": 0.7359307359307359, "ece_hard": 0.19085934698048745, "calibration": 63.160696310317675, "p50_s": 0.5630453020334244, "p95_s": 1.7856338374316691, "cost_usd_per_1000": 0.0315020051072523, "axes": { "intelligence": 73.80714047380715, "calibration": 63.160696310317675, "speed": 79.9766425372939, "cost": 55.04985407269179 }, "jevbench_score": 67.30791805322696 }, "heldout": { "n": 303, "escalation": 0.31683168316831684, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8013698630136986, "hard": 0.5596330275229358 }, "accuracy": 0.7293729372937293, "ece_hard": 0.26302682148222617, "calibration": 63.467818545903924, "p50_s": 0.6719379365444184, "p95_s": 1.4646646618843078, "cost_usd_per_1000": 0.03481749905439982, "axes": { "intelligence": 75.3940136567383, "calibration": 63.467818545903924, "speed": 80.06932635226191, "cost": 53.74607282586718 }, "jevbench_score": 67.36368502930145 } }, { "threshold": 0.76, "public": { "n": 231, "escalation": 0.24675324675324675, "tiers": { "easy": 1.0, "standard": 0.7222222222222222, "hard": 0.6396396396396397 }, "accuracy": 0.7402597402597403, "ece_hard": 0.1827578558983901, "calibration": 65.10552310041518, "p50_s": 0.5630453020334244, "p95_s": 1.7856338374316691, "cost_usd_per_1000": 0.03202037180796731, "axes": { "intelligence": 74.18251584918252, "calibration": 65.10552310041518, "speed": 79.9766425372939, "cost": 54.83720888592034 }, "jevbench_score": 67.84056796655437 }, "heldout": { "n": 303, "escalation": 0.3333333333333333, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.821917808219178, "hard": 0.5688073394495413 }, "accuracy": 0.7425742574257426, "ece_hard": 0.25518356472197273, "calibration": 64.25214422192927, "p50_s": 0.6774831205606461, "p95_s": 1.4705517575144764, "cost_usd_per_1000": 0.03547614981273408, "axes": { "intelligence": 76.2445854802899, "calibration": 64.25214422192927, "speed": 80.0162120619883, "cost": 53.501905593464976 }, "jevbench_score": 67.6723047943949 } }, { "threshold": 0.77, "public": { "n": 231, "escalation": 0.2510822510822511, "tiers": { "easy": 1.0, "standard": 0.7222222222222222, "hard": 0.6396396396396397 }, "accuracy": 0.7402597402597403, "ece_hard": 0.18443893791772234, "calibration": 64.93741489848195, "p50_s": 0.5630453020334244, "p95_s": 1.7856338374316691, "cost_usd_per_1000": 0.03219316070820565, "axes": { "intelligence": 74.18251584918252, "calibration": 64.93741489848195, "speed": 79.9766425372939, "cost": 54.767091472268504 }, "jevbench_score": 67.77505050096494 }, "heldout": { "n": 303, "escalation": 0.3333333333333333, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.821917808219178, "hard": 0.5688073394495413 }, "accuracy": 0.7425742574257426, "ece_hard": 0.25518356472197273, "calibration": 64.25214422192927, "p50_s": 0.6774831205606461, "p95_s": 1.4705517575144764, "cost_usd_per_1000": 0.03547614981273408, "axes": { "intelligence": 76.2445854802899, "calibration": 64.25214422192927, "speed": 80.0162120619883, "cost": 53.501905593464976 }, "jevbench_score": 67.6723047943949 } }, { "threshold": 0.78, "public": { "n": 231, "escalation": 0.2727272727272727, "tiers": { "easy": 1.0, "standard": 0.7222222222222222, "hard": 0.6486486486486487 }, "accuracy": 0.7445887445887446, "ece_hard": 0.18849938273511355, "calibration": 64.53137041674283, "p50_s": 0.5651318520307541, "p95_s": 1.7946988396346568, "cost_usd_per_1000": 0.03305710520939734, "axes": { "intelligence": 74.55789122455789, "calibration": 64.53137041674283, "speed": 79.9385863223493, "cost": 54.42205540009608 }, "jevbench_score": 67.63928014983782 }, "heldout": { "n": 303, "escalation": 0.35313531353135313, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8424657534246576, "hard": 0.5779816513761468 }, "accuracy": 0.7557755775577558, "ece_hard": 0.2448156698267363, "calibration": 65.52543301732636, "p50_s": 0.6951174080371857, "p95_s": 1.4860385894775388, "cost_usd_per_1000": 0.036266530722735196, "axes": { "intelligence": 77.09515730384149, "calibration": 65.52543301732636, "speed": 79.85911747846932, "cost": 53.21481961880312 }, "jevbench_score": 68.06886746626186 } }, { "threshold": 0.79, "public": { "n": 231, "escalation": 0.27705627705627706, "tiers": { "easy": 1.0, "standard": 0.7222222222222222, "hard": 0.6486486486486487 }, "accuracy": 0.7445887445887446, "ece_hard": 0.1931573718352241, "calibration": 64.06557150673177, "p50_s": 0.5651318520307541, "p95_s": 1.8009195752441882, "cost_usd_per_1000": 0.033229894109635685, "axes": { "intelligence": 74.55789122455789, "calibration": 64.06557150673177, "speed": 79.92355895552453, "cost": 54.354131279250026 }, "jevbench_score": 67.49264116616806 }, "heldout": { "n": 303, "escalation": 0.35973597359735976, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8424657534246576, "hard": 0.5779816513761468 }, "accuracy": 0.7557755775577558, "ece_hard": 0.2485913416848789, "calibration": 65.1478658315121, "p50_s": 0.6960485965013504, "p95_s": 1.4930099442601201, "cost_usd_per_1000": 0.036529991026068895, "axes": { "intelligence": 77.09515730384149, "calibration": 65.1478658315121, "speed": 79.832977375231, "cost": 53.12051303429613 }, "jevbench_score": 67.93490460451363 } }, { "threshold": 0.8, "public": { "n": 231, "escalation": 0.2857142857142857, "tiers": { "easy": 1.0, "standard": 0.7222222222222222, "hard": 0.6576576576576577 }, "accuracy": 0.7489177489177489, "ece_hard": 0.1953431651678354, "calibration": 63.846992173470646, "p50_s": 0.5651318520307541, "p95_s": 1.8009195752441882, "cost_usd_per_1000": 0.033575471910112356, "axes": { "intelligence": 74.93326659993329, "calibration": 63.846992173470646, "speed": 79.92355895552453, "cost": 54.21933623556227 }, "jevbench_score": 67.477817875501 }, "heldout": { "n": 303, "escalation": 0.38613861386138615, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8561643835616438, "hard": 0.5779816513761468 }, "accuracy": 0.7623762376237624, "ece_hard": 0.2514142321338542, "calibration": 64.86557678661458, "p50_s": 0.7260566294193268, "p95_s": 1.5101562038063998, "cost_usd_per_1000": 0.03758383223940372, "axes": { "intelligence": 77.4787189476771, "calibration": 64.86557678661458, "speed": 79.60007633706181, "cost": 52.749968172856036 }, "jevbench_score": 67.77711838879776 } }, { "threshold": 0.81, "public": { "n": 231, "escalation": 0.2857142857142857, "tiers": { "easy": 1.0, "standard": 0.7222222222222222, "hard": 0.6576576576576577 }, "accuracy": 0.7489177489177489, "ece_hard": 0.1953431651678354, "calibration": 63.846992173470646, "p50_s": 0.5651318520307541, "p95_s": 1.8009195752441882, "cost_usd_per_1000": 0.033575471910112356, "axes": { "intelligence": 74.93326659993329, "calibration": 63.846992173470646, "speed": 79.92355895552453, "cost": 54.21933623556227 }, "jevbench_score": 67.477817875501 }, "heldout": { "n": 303, "escalation": 0.38943894389438943, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8561643835616438, "hard": 0.5779816513761468 }, "accuracy": 0.7623762376237624, "ece_hard": 0.25152087340376894, "calibration": 64.8549126596231, "p50_s": 0.7260566294193268, "p95_s": 1.517887160927057, "cost_usd_per_1000": 0.037715562391070566, "axes": { "intelligence": 77.4787189476771, "calibration": 64.8549126596231, "speed": 79.57790017306306, "cost": 52.70438235909528 }, "jevbench_score": 67.75496548752936 } }, { "threshold": 0.82, "public": { "n": 231, "escalation": 0.2857142857142857, "tiers": { "easy": 1.0, "standard": 0.7222222222222222, "hard": 0.6576576576576577 }, "accuracy": 0.7489177489177489, "ece_hard": 0.1953431651678354, "calibration": 63.846992173470646, "p50_s": 0.5651318520307541, "p95_s": 1.8009195752441882, "cost_usd_per_1000": 0.033575471910112356, "axes": { "intelligence": 74.93326659993329, "calibration": 63.846992173470646, "speed": 79.92355895552453, "cost": 54.21933623556227 }, "jevbench_score": 67.477817875501 }, "heldout": { "n": 303, "escalation": 0.429042904290429, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8835616438356164, "hard": 0.5871559633027523 }, "accuracy": 0.7788778877887789, "ece_hard": 0.24239436853118357, "calibration": 65.76756314688164, "p50_s": 0.8348341524600983, "p95_s": 1.5970003664493542, "cost_usd_per_1000": 0.03929632421107279, "axes": { "intelligence": 78.5210715931465, "calibration": 65.76756314688164, "speed": 78.75094776778525, "cost": 52.16944215248382 }, "jevbench_score": 67.86838074654271 } }, { "threshold": 0.83, "public": { "n": 231, "escalation": 0.29004329004329005, "tiers": { "easy": 1.0, "standard": 0.7222222222222222, "hard": 0.6576576576576577 }, "accuracy": 0.7489177489177489, "ece_hard": 0.1994949180283458, "calibration": 63.43181688741961, "p50_s": 0.5651318520307541, "p95_s": 1.8009195752441882, "cost_usd_per_1000": 0.0337482608103507, "axes": { "intelligence": 74.93326659993329, "calibration": 63.43181688741961, "speed": 79.92355895552453, "cost": 54.15245809825959 }, "jevbench_score": 67.34706929570372 }, "heldout": { "n": 303, "escalation": 0.43564356435643564, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8835616438356164, "hard": 0.5871559633027523 }, "accuracy": 0.7788778877887789, "ece_hard": 0.24239436853118357, "calibration": 65.76756314688164, "p50_s": 0.9954077243804932, "p95_s": 1.5970003664493542, "cost_usd_per_1000": 0.03955978451440649, "axes": { "intelligence": 78.5210715931465, "calibration": 65.76756314688164, "speed": 77.98693977654364, "cost": 52.082382481249084 }, "jevbench_score": 67.67490775103 } }, { "threshold": 0.84, "public": { "n": 231, "escalation": 0.30735930735930733, "tiers": { "easy": 1.0, "standard": 0.7222222222222222, "hard": 0.6576576576576577 }, "accuracy": 0.7489177489177489, "ece_hard": 0.2035669308486755, "calibration": 63.024615605386636, "p50_s": 0.5724019199609757, "p95_s": 1.8009195752441882, "cost_usd_per_1000": 0.03443941641130405, "axes": { "intelligence": 74.93326659993329, "calibration": 63.024615605386636, "speed": 79.86804599490372, "cost": 53.88832649311746 }, "jevbench_score": 67.1449193886093 }, "heldout": { "n": 303, "escalation": 0.44224422442244227, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8972602739726028, "hard": 0.5871559633027523 }, "accuracy": 0.7854785478547854, "ece_hard": 0.24239436853118357, "calibration": 65.76756314688164, "p50_s": 1.012372413277626, "p95_s": 1.5970003664493542, "cost_usd_per_1000": 0.0398232448177402, "axes": { "intelligence": 78.90463323698211, "calibration": 65.76756314688164, "speed": 77.91354681902322, "cost": 51.99590069011234 }, "jevbench_score": 67.71331629690427 } }, { "threshold": 0.85, "public": { "n": 231, "escalation": 0.31601731601731603, "tiers": { "easy": 1.0, "standard": 0.7222222222222222, "hard": 0.6576576576576577 }, "accuracy": 0.7489177489177489, "ece_hard": 0.20448819477743016, "calibration": 62.93248921251117, "p50_s": 0.5724860668182373, "p95_s": 1.828130406886339, "cost_usd_per_1000": 0.03478499421178073, "axes": { "intelligence": 74.93326659993329, "calibration": 62.93248921251117, "speed": 79.80227906495193, "cost": 53.75824193718641 }, "jevbench_score": 67.0660119306141 }, "heldout": { "n": 303, "escalation": 0.4521452145214521, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.8972602739726028, "hard": 0.5871559633027523 }, "accuracy": 0.7854785478547854, "ece_hard": 0.23716723184762184, "calibration": 66.29027681523782, "p50_s": 1.0215922087430953, "p95_s": 1.5970003664493542, "cost_usd_per_1000": 0.04021843527274076, "axes": { "intelligence": 78.90463323698211, "calibration": 66.29027681523782, "speed": 77.874174121224, "cost": 51.86724489852846 }, "jevbench_score": 67.79688541786219 } }, { "threshold": 0.86, "public": { "n": 231, "escalation": 0.3203463203463203, "tiers": { "easy": 1.0, "standard": 0.7222222222222222, "hard": 0.6576576576576577 }, "accuracy": 0.7489177489177489, "ece_hard": 0.20447060196690484, "calibration": 62.93424849356371, "p50_s": 0.5724860668182373, "p95_s": 1.828130406886339, "cost_usd_per_1000": 0.03495778311201907, "axes": { "intelligence": 74.93326659993329, "calibration": 62.93424849356371, "speed": 79.80227906495193, "cost": 53.69368349343834 }, "jevbench_score": 67.04633647272554 }, "heldout": { "n": 303, "escalation": 0.47194719471947194, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.910958904109589, "hard": 0.6055045871559633 }, "accuracy": 0.7986798679867987, "ece_hard": 0.21951672171976297, "calibration": 68.0553278280237, "p50_s": 1.1063382118940353, "p95_s": 1.5970003664493542, "cost_usd_per_1000": 0.04100881618274187, "axes": { "intelligence": 79.83865359641406, "calibration": 68.0553278280237, "speed": 77.5280707142943, "cost": 51.613683024318725 }, "jevbench_score": 68.28491772026517 } }, { "threshold": 0.87, "public": { "n": 231, "escalation": 0.3333333333333333, "tiers": { "easy": 1.0, "standard": 0.7222222222222222, "hard": 0.6576576576576577 }, "accuracy": 0.7489177489177489, "ece_hard": 0.20161442395066909, "calibration": 63.21986629518727, "p50_s": 0.5749294698238373, "p95_s": 1.828130406886339, "cost_usd_per_1000": 0.03547614981273408, "axes": { "intelligence": 74.93326659993329, "calibration": 63.21986629518727, "speed": 79.78378257414238, "cost": 53.501905593464976 }, "jevbench_score": 67.05837542834598 }, "heldout": { "n": 303, "escalation": 0.5016501650165016, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9178082191780822, "hard": 0.6422018348623854 }, "accuracy": 0.8151815181518152, "ece_hard": 0.1828075632849353, "calibration": 71.72624367150647, "p50_s": 1.1723873555660247, "p95_s": 1.6761263757944103, "cost_usd_per_1000": 0.04219438754774353, "axes": { "intelligence": 81.13135184952453, "calibration": 71.72624367150647, "speed": 77.06622113900418, "cost": 51.242359375755896 }, "jevbench_score": 69.23727957145928 } }, { "threshold": 0.88, "public": { "n": 231, "escalation": 0.3463203463203463, "tiers": { "easy": 1.0, "standard": 0.7222222222222222, "hard": 0.6756756756756757 }, "accuracy": 0.7575757575757576, "ece_hard": 0.18627698347503596, "calibration": 64.75361034275059, "p50_s": 0.5749294698238373, "p95_s": 1.940097665041685, "cost_usd_per_1000": 0.03599451651344909, "axes": { "intelligence": 75.68401735068403, "calibration": 64.75361034275059, "speed": 79.52561836667317, "cost": 53.31290966809725 }, "jevbench_score": 67.51525326850452 }, "heldout": { "n": 303, "escalation": 0.5016501650165016, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9178082191780822, "hard": 0.6422018348623854 }, "accuracy": 0.8151815181518152, "ece_hard": 0.1828075632849353, "calibration": 71.72624367150647, "p50_s": 1.1723873555660247, "p95_s": 1.6761263757944103, "cost_usd_per_1000": 0.04219438754774353, "axes": { "intelligence": 81.13135184952453, "calibration": 71.72624367150647, "speed": 77.06622113900418, "cost": 51.242359375755896 }, "jevbench_score": 69.23727957145928 } }, { "threshold": 0.89, "public": { "n": 231, "escalation": 0.3593073593073593, "tiers": { "easy": 1.0, "standard": 0.7222222222222222, "hard": 0.6756756756756757 }, "accuracy": 0.7575757575757576, "ece_hard": 0.18627698347503596, "calibration": 64.75361034275059, "p50_s": 0.5810501962900162, "p95_s": 1.940097665041685, "cost_usd_per_1000": 0.03651288321416411, "axes": { "intelligence": 75.68401735068403, "calibration": 64.75361034275059, "speed": 79.47962754782083, "cost": 53.12661615812922 }, "jevbench_score": 67.4464406471891 }, "heldout": { "n": 303, "escalation": 0.5247524752475248, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9246575342465754, "hard": 0.6513761467889908 }, "accuracy": 0.8217821782178217, "ece_hard": 0.17126580803093705, "calibration": 72.8804191969063, "p50_s": 1.1888870164752006, "p95_s": 1.6761263757944103, "cost_usd_per_1000": 0.0431164986094115, "axes": { "intelligence": 81.5983620292405, "calibration": 72.8804191969063, "speed": 77.00552655475471, "cost": 50.96069543303019 }, "jevbench_score": 69.50441397501184 } }, { "threshold": 0.9, "public": { "n": 231, "escalation": 0.3722943722943723, "tiers": { "easy": 1.0, "standard": 0.7222222222222222, "hard": 0.6666666666666666 }, "accuracy": 0.7532467532467533, "ece_hard": 0.19318410594301483, "calibration": 64.0628980959527, "p50_s": 0.5851256638765335, "p95_s": 1.940097665041685, "cost_usd_per_1000": 0.037031249914879126, "axes": { "intelligence": 75.30864197530866, "calibration": 64.0628980959527, "speed": 79.44927260136613, "cost": 52.942948869161874 }, "jevbench_score": 67.11774590005872 }, "heldout": { "n": 303, "escalation": 0.5346534653465347, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9246575342465754, "hard": 0.6513761467889908 }, "accuracy": 0.8217821782178217, "ece_hard": 0.17169416210383293, "calibration": 72.8375837896167, "p50_s": 1.191568100452423, "p95_s": 1.6761263757944103, "cost_usd_per_1000": 0.04351168906441206, "axes": { "intelligence": 81.5983620292405, "calibration": 72.8375837896167, "speed": 76.99574371565896, "cost": 50.84182172982169 }, "jevbench_score": 69.45143097681591 } }, { "threshold": 0.91, "public": { "n": 231, "escalation": 0.3939393939393939, "tiers": { "easy": 1.0, "standard": 0.7361111111111112, "hard": 0.6666666666666666 }, "accuracy": 0.7575757575757576, "ece_hard": 0.1976938912266406, "calibration": 63.61191956759013, "p50_s": 0.5868345886468888, "p95_s": 2.0335648261010646, "cost_usd_per_1000": 0.03789519441607082, "axes": { "intelligence": 75.84876543209876, "calibration": 63.61191956759013, "speed": 79.23226274874801, "cost": 52.64247581513339 }, "jevbench_score": 66.97787207609471 }, "heldout": { "n": 303, "escalation": 0.5445544554455446, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9383561643835616, "hard": 0.6513761467889908 }, "accuracy": 0.8283828382838284, "ece_hard": 0.17252166692786117, "calibration": 72.75483330721389, "p50_s": 1.19361043125391, "p95_s": 1.6761263757944103, "cost_usd_per_1000": 0.04390687951941261, "axes": { "intelligence": 81.98192367307612, "calibration": 72.75483330721389, "speed": 76.9883063418751, "cost": 50.724022818767544 }, "jevbench_score": 69.4711682877671 } }, { "threshold": 0.92, "public": { "n": 231, "escalation": 0.4155844155844156, "tiers": { "easy": 1.0, "standard": 0.7361111111111112, "hard": 0.6576576576576577 }, "accuracy": 0.7532467532467533, "ece_hard": 0.1754529502257165, "calibration": 68.2052311482305, "p50_s": 0.6046394348144531, "p95_s": 2.0335648261010646, "cost_usd_per_1000": 0.038759138917262514, "axes": { "intelligence": 75.47339005672339, "calibration": 68.2052311482305, "speed": 79.10245509956073, "cost": 52.348776397272346 }, "jevbench_score": 67.9480519503336 }, "heldout": { "n": 303, "escalation": 0.570957095709571, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9452054794520548, "hard": 0.6605504587155964 }, "accuracy": 0.834983498349835, "ece_hard": 0.1583341130008429, "calibration": 74.17358869991571, "p50_s": 1.2012724593281745, "p95_s": 1.6761263757944103, "cost_usd_per_1000": 0.04496072073274743, "axes": { "intelligence": 82.4489338527921, "calibration": 74.17358869991571, "speed": 76.96051719294101, "cost": 50.4150020656879 }, "jevbench_score": 69.79358804458593 } }, { "threshold": 0.93, "public": { "n": 231, "escalation": 0.43722943722943725, "tiers": { "easy": 1.0, "standard": 0.7361111111111112, "hard": 0.6576576576576577 }, "accuracy": 0.7532467532467533, "ece_hard": 0.17236925550981208, "calibration": 69.73363298736028, "p50_s": 0.635255092382431, "p95_s": 2.0335648261010646, "cost_usd_per_1000": 0.039623083418454205, "axes": { "intelligence": 75.47339005672339, "calibration": 69.73363298736028, "speed": 78.8879382341092, "cost": 52.06155193698216 }, "jevbench_score": 68.1853330795369 }, "heldout": { "n": 303, "escalation": 0.5874587458745875, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9452054794520548, "hard": 0.6605504587155964 }, "accuracy": 0.834983498349835, "ece_hard": 0.15650951136991914, "calibration": 74.35604886300808, "p50_s": 1.2014862462878226, "p95_s": 1.6836474224925038, "cost_usd_per_1000": 0.04561937149108169, "axes": { "intelligence": 82.4489338527921, "calibration": 74.35604886300808, "speed": 76.94030046495251, "cost": 50.22552107189203 }, "jevbench_score": 69.7661760177022 } }, { "threshold": 0.94, "public": { "n": 231, "escalation": 0.4588744588744589, "tiers": { "easy": 1.0, "standard": 0.75, "hard": 0.6666666666666666 }, "accuracy": 0.7619047619047619, "ece_hard": 0.16197735739070235, "calibration": 70.77282279927125, "p50_s": 0.7952923119068146, "p95_s": 2.0335648261010646, "cost_usd_per_1000": 0.040487027919645896, "axes": { "intelligence": 76.3888888888889, "calibration": 70.77282279927125, "speed": 77.91215194426294, "cost": 51.78052308529474 }, "jevbench_score": 68.3387586989018 }, "heldout": { "n": 303, "escalation": 0.6204620462046204, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9383561643835616, "hard": 0.6788990825688074 }, "accuracy": 0.8382838283828383, "ece_hard": 0.13379515545319137, "calibration": 76.62748445468085, "p50_s": 1.208825373649597, "p95_s": 1.6931800834834574, "cost_usd_per_1000": 0.04693667300775021, "axes": { "intelligence": 82.80761174647063, "calibration": 76.62748445468085, "speed": 76.88933281268109, "cost": 49.854630926730664 }, "jevbench_score": 70.22740266574647 } }, { "threshold": 0.95, "public": { "n": 231, "escalation": 0.47186147186147187, "tiers": { "easy": 1.0, "standard": 0.75, "hard": 0.6666666666666666 }, "accuracy": 0.7619047619047619, "ece_hard": 0.15523890570199117, "calibration": 71.44666796814238, "p50_s": 1.006430798768997, "p95_s": 2.0335648261010646, "cost_usd_per_1000": 0.04100539462036091, "axes": { "intelligence": 76.3888888888889, "calibration": 71.44666796814238, "speed": 76.88958060000337, "cost": 51.614770127869086 }, "jevbench_score": 68.22026711380315 }, "heldout": { "n": 303, "escalation": 0.636963696369637, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "judge": 0.9452054794520548, "hard": 0.6880733944954128 }, "accuracy": 0.8448844884488449, "ece_hard": 0.1159123254360167, "calibration": 78.41576745639833, "p50_s": 1.2121592849493026, "p95_s": 1.741218657046555, "cost_usd_per_1000": 0.04759532376608447, "axes": { "intelligence": 83.27462192618658, "calibration": 78.41576745639833, "speed": 76.75586995590467, "cost": 49.67307143667196 }, "jevbench_score": 70.63780084486055 } }, { "threshold": 0.96, "public": { "n": 231, "escalation": 0.5064935064935064, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.6846846846846847 }, "accuracy": 0.7748917748917749, "ece_hard": 0.12706212532914785, "calibration": 74.71478746708522, "p50_s": 1.1265588864684104, "p95_s": 2.0335648261010646, "cost_usd_per_1000": 0.04238770582226761, "axes": { "intelligence": 77.67976309642977, "calibration": 74.71478746708522, "speed": 76.3998808087127, "cost": 51.182802652063216 }, "jevbench_score": 69.02124686044213 }, "heldout": { "n": 303, "escalation": 0.6501650165016502, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6972477064220184 }, "accuracy": 0.8514851485148515, "ece_hard": 0.10513114879707297, "calibration": 79.4938851202927, "p50_s": 1.215047726035118, "p95_s": 1.741218657046555, "cost_usd_per_1000": 0.04812224437275188, "axes": { "intelligence": 84.71651795065142, "calibration": 79.4938851202927, "speed": 76.74553351067064, "cost": 49.52962377414834 }, "jevbench_score": 71.13035719605142 } }, { "threshold": 0.97, "public": { "n": 231, "escalation": 0.5194805194805194, "tiers": { "easy": 1.0, "standard": 0.7638888888888888, "hard": 0.6846846846846847 }, "accuracy": 0.7748917748917749, "ece_hard": 0.12706212532914785, "calibration": 74.71478746708522, "p50_s": 1.151884862780571, "p95_s": 2.0335648261010646, "cost_usd_per_1000": 0.04290607252298263, "axes": { "intelligence": 77.67976309642977, "calibration": 74.71478746708522, "speed": 76.30332907435763, "cost": 51.02443712495685 }, "jevbench_score": 68.94599470548826 }, "heldout": { "n": 303, "escalation": 0.66996699669967, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9383561643835616, "hard": 0.6880733944954128 }, "accuracy": 0.8448844884488449, "ece_hard": 0.11222001820740891, "calibration": 78.7849981792591, "p50_s": 1.217183382809162, "p95_s": 1.7423106856644144, "cost_usd_per_1000": 0.04891262528275299, "axes": { "intelligence": 84.24950777093545, "calibration": 78.7849981792591, "speed": 76.73518385156936, "cost": 49.31737080118342 }, "jevbench_score": 70.79479666889846 } }, { "threshold": 0.98, "public": { "n": 231, "escalation": 0.5670995670995671, "tiers": { "easy": 1.0, "standard": 0.7777777777777778, "hard": 0.7027027027027027 }, "accuracy": 0.7878787878787878, "ece_hard": 0.105515823508132, "calibration": 76.8694176491868, "p50_s": 1.1843002676963805, "p95_s": 2.0335648261010646, "cost_usd_per_1000": 0.04480675042560436, "axes": { "intelligence": 78.97063730397065, "calibration": 76.8694176491868, "speed": 76.18280151126547, "cost": 50.45969655433579 }, "jevbench_score": 69.50326424518421 }, "heldout": { "n": 303, "escalation": 0.6864686468646864, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9383561643835616, "hard": 0.6880733944954128 }, "accuracy": 0.8448844884488449, "ece_hard": 0.10395036046667869, "calibration": 79.61196395333212, "p50_s": 1.2191806599497794, "p95_s": 1.802198813855648, "cost_usd_per_1000": 0.04957127604108725, "axes": { "intelligence": 84.24950777093545, "calibration": 79.61196395333212, "speed": 76.58129235713635, "cost": 49.14309704623859 }, "jevbench_score": 70.88147247466505 } }, { "threshold": 0.99, "public": { "n": 231, "escalation": 0.6320346320346321, "tiers": { "easy": 1.0, "standard": 0.7916666666666666, "hard": 0.7117117117117117 }, "accuracy": 0.7965367965367965, "ece_hard": 0.09306579428755425, "calibration": 78.11442057124458, "p50_s": 1.200592117011547, "p95_s": 2.0335648261010646, "cost_usd_per_1000": 0.047398583929179436, "axes": { "intelligence": 79.88613613613613, "calibration": 78.11442057124458, "speed": 76.12346491085671, "cost": 49.72703899087235 }, "jevbench_score": 69.71535526564571 }, "heldout": { "n": 303, "escalation": 0.7326732673267327, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6788990825688074 }, "accuracy": 0.8448844884488449, "ece_hard": 0.10878229951295179, "calibration": 79.12877004870481, "p50_s": 1.2278619349002837, "p95_s": 1.802198813855648, "cost_usd_per_1000": 0.05141549816442318, "axes": { "intelligence": 84.16605923505509, "calibration": 79.12877004870481, "speed": 76.55047764527006, "cost": 48.66717855888121 }, "jevbench_score": 70.5771099696399 } }, { "threshold": 1.0, "public": { "n": 231, "escalation": 1.0, "tiers": { "easy": 1.0, "standard": 0.8194444444444444, "hard": 0.7117117117117117 }, "accuracy": 0.8051948051948052, "ece_hard": 0.09203203203203206, "calibration": 78.2177967967968, "p50_s": 1.226769672334194, "p95_s": 2.0335648261010646, "cost_usd_per_1000": 0.06208564044943819, "axes": { "intelligence": 80.9663830497164, "calibration": 78.2177967967968, "speed": 76.02978947569241, "cost": 46.21026503561948 }, "jevbench_score": 68.68045145419504 }, "heldout": { "n": 303, "escalation": 1.0, "tiers": { "easy": 1.0, "standard": 0.8333333333333334, "judge": 0.9452054794520548, "hard": 0.6972477064220184 }, "accuracy": 0.8514851485148515, "ece_hard": 0.08064220183486236, "calibration": 81.94277981651376, "p50_s": 1.2545109555125236, "p95_s": 1.8302100747823713, "cost_usd_per_1000": 0.06208564044943819, "axes": { "intelligence": 84.71651795065142, "calibration": 81.94277981651376, "speed": 76.39024599996569, "cost": 46.21026503561948 }, "jevbench_score": 70.3580492090166 } } ] } ], "best_of_n": [ { "system": "openjev-razorback16", "n": 2, "changed_probability_items": 314, "changed_label_items": 5, "sample_hashes": [ [ "52e35047d92056590932f7dfcb892a4eb7007a79f7009861dbbd6cdd0ca3f37e", "2192f674dcb75ac721b6d8eef6df42fe7fb16345aec9e55ac0446312de380e2c", "292745cf9274a9515a6514aee9f0e56eead1a47564e9b53dc3bde79c5a8d4649" ], [ "d89ebe9738c36786b0736715d1dd6329d5be3e33ceee5c2a217a4a04654281bf", "de21565d2b89d4fbb9b882ff3b7780c973a6407d1dfda5ee233a00104d652ae2", "292745cf9274a9515a6514aee9f0e56eead1a47564e9b53dc3bde79c5a8d4649" ] ], "tiers": { "easy": 1.0, "standard": 0.8020833333333334, "judge": 0.9178082191780822, "hard": 0.6227272727272727 }, "accuracy": 0.7865168539325843, "ece_hard": 0.20970901755212665, "calibration": 61.57829793626705, "p50_s": 0.6461449950933457, "p95_s": 1.079701918363571, "cost_usd_per_1000": 0.1312091011235955, "axes": { "intelligence": 80.83878165213783, "calibration": 61.57829793626705, "speed": 81.56366142274575, "cost": 36.461081191797454 }, "jevbench_score": 62.028806363523834 } ] }