{"models":[{"model_id":"gemini-4-argon-high","display_name":"gemini-4-argon","lab":"google","prompt":"bare","pass_rate":0.8860759493670886,"ci_lower":0.8463111555009678,"ci_upper":0.9165664689231221,"total_runs":316,"cost_p50":0.017341000000000002,"cost_p95":0.0794875,"cost_avg":0.02911925316455697,"latency_p50":16434,"latency_p95":91143.25,"latency_avg":34578.56329113924,"ttft_p50":15185.28,"ttft_p95":75163.0575,"ttft_avg":27371.90876582279,"tokens_per_sec_p50":91.66041448241123,"tokens_per_sec_p95":125.61300365039662,"tokens_per_sec_avg":88.6736232379202,"thinking_ratio_p50":0.9801309761141852,"thinking_ratio_p95":0.9963216138320039,"thinking_ratio_avg":0.9560141632319005,"total_params":null,"active_params":null,"precision":"unknown","base_model":"gemini-4-argon","reasoning_effort":"high","provider":"google-ai-studio"},{"model_id":"claude-fable-5-1-max","display_name":"claude-fable-5-1","lab":"anthropic","prompt":"bare","pass_rate":0.8860759493670886,"ci_lower":0.8463111555009678,"ci_upper":0.9165664689231221,"total_runs":316,"cost_p50":0.1441675,"cost_p95":1.4756675000000001,"cost_avg":0.32623536392405056,"latency_p50":40040.5,"latency_p95":503679.5,"latency_avg":143147.3575949367,"ttft_p50":26196.06,"ttft_p95":439101.625,"ttft_avg":99683.69453968259,"tokens_per_sec_p50":72.26093322109634,"tokens_per_sec_p95":95.10952609664959,"tokens_per_sec_avg":61.72261719896774,"thinking_ratio_p50":null,"thinking_ratio_p95":null,"thinking_ratio_avg":null,"total_params":null,"active_params":null,"precision":"unknown","base_model":"claude-fable-5-1","reasoning_effort":"max","provider":"anthropic"},{"model_id":"claude-opus-4.8-max","display_name":"claude-opus-4.8","lab":"anthropic","prompt":"bare","pass_rate":0.8829113924050633,"ci_lower":0.8427759466901593,"ci_upper":0.9138485824517661,"total_runs":316,"cost_p50":0.1427875,"cost_p95":1.02633375,"cost_avg":0.28489047468354434,"latency_p50":56985.5,"latency_p95":460286.5,"latency_avg":144479.49683544305,"ttft_p50":50944.64,"ttft_p95":413012.08900000015,"ttft_avg":110062.80298412697,"tokens_per_sec_p50":93.63449356736453,"tokens_per_sec_p95":109.48086756918082,"tokens_per_sec_avg":85.1058413736514,"thinking_ratio_p50":null,"thinking_ratio_p95":null,"thinking_ratio_avg":null,"total_params":null,"active_params":null,"precision":"unknown","base_model":"claude-opus-4.8","reasoning_effort":"max","provider":"anthropic"},{"model_id":"claude-opus-5-max","display_name":"claude-opus-5","lab":"anthropic","prompt":"bare","pass_rate":0.879746835443038,"ci_lower":0.8392486909659214,"ci_upper":0.9111227428938393,"total_runs":316,"cost_p50":0.0698525,"cost_p95":0.91276875,"cost_avg":0.2070786867088608,"latency_p50":27841,"latency_p95":426988.75,"latency_avg":114115.43670886075,"ttft_p50":21929.99,"ttft_p95":386977.60300000006,"ttft_avg":78560.72939682541,"tokens_per_sec_p50":85.92265353781843,"tokens_per_sec_p95":101.32417765229792,"tokens_per_sec_avg":73.95210978805993,"thinking_ratio_p50":null,"thinking_ratio_p95":null,"thinking_ratio_avg":null,"total_params":null,"active_params":null,"precision":"unknown","base_model":"claude-opus-5","reasoning_effort":"max","provider":"anthropic"},{"model_id":"claude-sonnet-5-5-xhigh","display_name":"claude-sonnet-5-5","lab":"anthropic","prompt":"bare","pass_rate":0.879746835443038,"ci_lower":0.8392486909659214,"ci_upper":0.9111227428938393,"total_runs":316,"cost_p50":0.013018,"cost_p95":0.4266395,"cost_avg":0.06928749683544304,"latency_p50":9321.5,"latency_p95":178826.25,"latency_avg":34485.648734177215,"ttft_p50":7309.555,"ttft_p95":169240.8025,"ttft_avg":31318.339556962015,"tokens_per_sec_p50":108.53510740343734,"tokens_per_sec_p95":155.34776532766455,"tokens_per_sec_avg":103.79352693790861,"thinking_ratio_p50":null,"thinking_ratio_p95":null,"thinking_ratio_avg":null,"total_params":null,"active_params":null,"precision":"unknown","base_model":"claude-sonnet-5-5","reasoning_effort":"xhigh","provider":"anthropic"},{"model_id":"claude-fable-5-1-medium","display_name":"claude-fable-5-1","lab":"anthropic","prompt":"bare","pass_rate":0.8734177215189873,"ci_lower":0.8322170023469961,"ci_upper":0.9056482409484351,"total_runs":316,"cost_p50":0.031265,"cost_p95":0.1937625,"cost_avg":0.05361463607594939,"latency_p50":8402.5,"latency_p95":78510,"latency_avg":18801.21835443038,"ttft_p50":4465.8099999999995,"ttft_p95":71376.4775,"ttft_avg":14174.840727848105,"tokens_per_sec_p50":62.042250852148996,"tokens_per_sec_p95":89.4809212685905,"tokens_per_sec_avg":53.530773032660065,"thinking_ratio_p50":null,"thinking_ratio_p95":null,"thinking_ratio_avg":null,"total_params":null,"active_params":null,"precision":"unknown","base_model":"claude-fable-5-1","reasoning_effort":"medium","provider":"anthropic"},{"model_id":"gpt-6.1-sol-max","display_name":"gpt-6.1-sol","lab":"openai","prompt":"bare","pass_rate":0.8734177215189873,"ci_lower":0.8322170023469961,"ci_upper":0.9056482409484351,"total_runs":316,"cost_p50":0.00683,"cost_p95":0.04600240000000004,"cost_avg":0.013899358730158727,"latency_p50":21539,"latency_p95":153797.75,"latency_avg":58031.303797468354,"ttft_p50":20202.7,"ttft_p95":142552.27300000013,"ttft_avg":45108.126634920634,"tokens_per_sec_p50":26.347305389221557,"tokens_per_sec_p95":30.359539996745763,"tokens_per_sec_avg":24.843805280258405,"thinking_ratio_p50":0.9459459459459459,"thinking_ratio_p95":0.9896013137706177,"thinking_ratio_avg":0.9271146229139846,"total_params":null,"active_params":null,"precision":"unknown","base_model":"gpt-6.1-sol","reasoning_effort":"max","provider":"openai"},{"model_id":"gpt-6.1-terra-xhigh","display_name":"gpt-6.1-terra","lab":"openai","prompt":"bare","pass_rate":0.8734177215189873,"ci_lower":0.8322170023469961,"ci_upper":0.9056482409484351,"total_runs":316,"cost_p50":0.0017404999999999999,"cost_p95":0.00734325,"cost_avg":0.002492487341772152,"latency_p50":4416.5,"latency_p95":17518.5,"latency_avg":6240.284810126583,"ttft_p50":4360.835,"ttft_p95":17474.8975,"ttft_avg":6184.469715189873,"tokens_per_sec_p50":57.20411406465293,"tokens_per_sec_p95":83.3929714030903,"tokens_per_sec_avg":53.95813315350026,"thinking_ratio_p50":0.873490469954896,"thinking_ratio_p95":0.9718437701970128,"thinking_ratio_avg":0.8366735884160863,"total_params":null,"active_params":null,"precision":"unknown","base_model":"gpt-6.1-terra","reasoning_effort":"xhigh","provider":"openai"},{"model_id":"claude-opus-5-5-xhigh","display_name":"claude-opus-5-5","lab":"anthropic","prompt":"bare","pass_rate":0.8734177215189873,"ci_lower":0.8322170023469961,"ci_upper":0.9056482409484351,"total_runs":316,"cost_p50":0.029376,"cost_p95":0.46891675,"cost_avg":0.09986334177215195,"latency_p50":13636.5,"latency_p95":213338.75,"latency_avg":46485.49683544304,"ttft_p50":8798.835,"ttft_p95":204426.4675,"ttft_avg":39704.613196202525,"tokens_per_sec_p50":86.71792230699808,"tokens_per_sec_p95":112.85402120878676,"tokens_per_sec_avg":77.24011258661808,"thinking_ratio_p50":null,"thinking_ratio_p95":null,"thinking_ratio_avg":null,"total_params":null,"active_params":null,"precision":"unknown","base_model":"claude-opus-5-5","reasoning_effort":"xhigh","provider":"anthropic"},{"model_id":"claude-fable-5-high","display_name":"claude-fable-5","lab":"anthropic","prompt":"bare","pass_rate":0.8734177215189873,"ci_lower":0.8322170023469961,"ci_upper":0.9056482409484351,"total_runs":316,"cost_p50":0.044050000000000006,"cost_p95":0.25874125000000003,"cost_avg":0.08737863924050629,"latency_p50":13147,"latency_p95":122666.75,"latency_avg":37254.04113924051,"ttft_p50":6707.27,"ttft_p95":95921.905,"ttft_avg":19859.99835443038,"tokens_per_sec_p50":61.97294114969789,"tokens_per_sec_p95":88.15935848897739,"tokens_per_sec_avg":53.880211315121336,"thinking_ratio_p50":null,"thinking_ratio_p95":null,"thinking_ratio_avg":null,"total_params":null,"active_params":null,"precision":"unknown","base_model":"claude-fable-5","reasoning_effort":"high","provider":"anthropic"},{"model_id":"gpt-6-astra-max","display_name":"gpt-6-astra","lab":"openai","prompt":"bare","pass_rate":0.8706624605678234,"ci_lower":0.8292374941362494,"ci_upper":0.9032111658878872,"total_runs":317,"cost_p50":0.030019999999999998,"cost_p95":0.208835,"cost_avg":0.060339810126582266,"latency_p50":26109,"latency_p95":203613.39999999985,"latency_avg":72658.37223974764,"ttft_p50":25763.35,"ttft_p95":193147.7,"ttft_avg":52781.37515822782,"tokens_per_sec_p50":19.721899363498828,"tokens_per_sec_p95":26.034414535126253,"tokens_per_sec_avg":19.13073997040515,"thinking_ratio_p50":0.9411051981374132,"thinking_ratio_p95":0.9891005383895131,"thinking_ratio_avg":0.9184869132169861,"total_params":null,"active_params":null,"precision":"unknown","base_model":"gpt-6-astra","reasoning_effort":"max","provider":"openai"},{"model_id":"gpt-6-astra-low","display_name":"gpt-6-astra","lab":"openai","prompt":"bare","pass_rate":0.870253164556962,"ci_lower":0.8287120904037939,"ci_upper":0.9029000576094727,"total_runs":316,"cost_p50":0.009524999999999999,"cost_p95":0.03542,"cost_avg":0.013171392405063286,"latency_p50":7171,"latency_p95":26967.75,"latency_avg":9951.541139240506,"ttft_p50":7135.764999999999,"ttft_p95":26753.93,"ttft_avg":9863.593037974686,"tokens_per_sec_p50":14.138556844952447,"tokens_per_sec_p95":23.446243120927015,"tokens_per_sec_avg":13.618905780086719,"thinking_ratio_p50":0.7393762604436762,"thinking_ratio_p95":0.9374022977184626,"thinking_ratio_avg":0.6928313770409517,"total_params":null,"active_params":null,"precision":"unknown","base_model":"gpt-6-astra","reasoning_effort":"low","provider":"openai"},{"model_id":"gpt-6.1-sol-high","display_name":"gpt-6.1-sol","lab":"openai","prompt":"bare","pass_rate":0.870253164556962,"ci_lower":0.8287120904037939,"ci_upper":0.9029000576094727,"total_runs":316,"cost_p50":0.0028840000000000003,"cost_p95":0.014872,"cost_avg":0.004768297468354429,"latency_p50":9496,"latency_p95":45684.25,"latency_avg":21187.54430379747,"ttft_p50":8293.99,"ttft_p95":39791.1,"ttft_avg":13901.923512658224,"tokens_per_sec_p50":21.090425830547314,"tokens_per_sec_p95":28.78761034852841,"tokens_per_sec_avg":20.068743545407738,"thinking_ratio_p50":0.8286867204695525,"thinking_ratio_p95":0.9657813908991454,"thinking_ratio_avg":0.8002986143082539,"total_params":null,"active_params":null,"precision":"unknown","base_model":"gpt-6.1-sol","reasoning_effort":"high","provider":"openai"},{"model_id":"claude-fable-5-max","display_name":"claude-fable-5","lab":"anthropic","prompt":"bare","pass_rate":0.870253164556962,"ci_lower":0.8287120904037939,"ci_upper":0.9029000576094727,"total_runs":316,"cost_p50":0.12594,"cost_p95":1.058885,"cost_avg":0.26909666139240496,"latency_p50":29920,"latency_p95":448637,"latency_avg":123885.73734177215,"ttft_p50":20040.24,"ttft_p95":397103.1540000002,"ttft_avg":84091.07590476192,"tokens_per_sec_p50":77.85711866924109,"tokens_per_sec_p95":97.27187763339361,"tokens_per_sec_avg":70.46772612093349,"thinking_ratio_p50":null,"thinking_ratio_p95":null,"thinking_ratio_avg":null,"total_params":null,"active_params":null,"precision":"unknown","base_model":"claude-fable-5","reasoning_effort":"max","provider":"anthropic"},{"model_id":"claude-fable-5-1-high","display_name":"claude-fable-5-1","lab":"anthropic","prompt":"bare","pass_rate":0.870253164556962,"ci_lower":0.8287120904037939,"ci_upper":0.9029000576094727,"total_runs":316,"cost_p50":0.038435,"cost_p95":0.24248999999999998,"cost_avg":0.06926931962025319,"latency_p50":11939.5,"latency_p95":105599.5,"latency_avg":25392.199367088608,"ttft_p50":7710.49,"ttft_p95":95921.905,"ttft_avg":19880.95155063291,"tokens_per_sec_p50":58.7377820901385,"tokens_per_sec_p95":93.16316719847802,"tokens_per_sec_avg":53.81973549164245,"thinking_ratio_p50":null,"thinking_ratio_p95":null,"thinking_ratio_avg":null,"total_params":null,"active_params":null,"precision":"unknown","base_model":"claude-fable-5-1","reasoning_effort":"high","provider":"anthropic"},{"model_id":"claude-fable-5-1-xhigh","display_name":"claude-fable-5-1","lab":"anthropic","prompt":"bare","pass_rate":0.870253164556962,"ci_lower":0.8287120904037939,"ci_upper":0.9029000576094727,"total_runs":316,"cost_p50":0.07281499999999999,"cost_p95":0.55563875,"cost_avg":0.14582462025316456,"latency_p50":16757,"latency_p95":252176.5,"latency_avg":50480.873417721516,"ttft_p50":12753.7,"ttft_p95":239579.3225,"ttft_avg":45221.12851265823,"tokens_per_sec_p50":79.6765385650706,"tokens_per_sec_p95":101.33704646637976,"tokens_per_sec_avg":68.5633041430362,"thinking_ratio_p50":null,"thinking_ratio_p95":null,"thinking_ratio_avg":null,"total_params":null,"active_params":null,"precision":"unknown","base_model":"claude-fable-5-1","reasoning_effort":"xhigh","provider":"anthropic"},{"model_id":"gpt-6.1-sol-low","display_name":"gpt-6.1-sol","lab":"openai","prompt":"bare","pass_rate":0.870253164556962,"ci_lower":0.8287120904037939,"ci_upper":0.9029000576094727,"total_runs":316,"cost_p50":0.001974,"cost_p95":0.006729499999999999,"cost_avg":0.0026291518987341775,"latency_p50":6311,"latency_p95":22823,"latency_avg":8609.95569620253,"ttft_p50":5293.365,"ttft_p95":18434.300000000003,"ttft_avg":7198.956329113925,"tokens_per_sec_p50":16.6342239084289,"tokens_per_sec_p95":25.427766894676743,"tokens_per_sec_avg":15.782894453215905,"thinking_ratio_p50":0.7492331288343559,"thinking_ratio_p95":0.932353203811662,"thinking_ratio_avg":0.6926462018315688,"total_params":null,"active_params":null,"precision":"unknown","base_model":"gpt-6.1-sol","reasoning_effort":"low","provider":"openai"},{"model_id":"claude-fable-5-xhigh","display_name":"claude-fable-5","lab":"anthropic","prompt":"bare","pass_rate":0.8670886075949367,"ci_lower":0.8252141708685773,"ci_upper":0.9001448818625245,"total_runs":316,"cost_p50":0.069135,"cost_p95":0.4797225,"cost_avg":0.1245584335443038,"latency_p50":17448,"latency_p95":220226.25,"latency_avg":48103.65189873418,"ttft_p50":10099.405,"ttft_p95":184960.7525,"ttft_avg":38309.68756329113,"tokens_per_sec_p50":67.74963482209436,"tokens_per_sec_p95":92.51647552853572,"tokens_per_sec_avg":60.07439370301091,"thinking_ratio_p50":null,"thinking_ratio_p95":null,"thinking_ratio_avg":null,"total_params":null,"active_params":null,"precision":"unknown","base_model":"claude-fable-5","reasoning_effort":"xhigh","provider":"anthropic"},{"model_id":"claude-opus-4.7-max","display_name":"claude-opus-4.7","lab":"anthropic","prompt":"bare","pass_rate":0.8639240506329114,"ci_lower":0.8217230378099494,"ci_upper":0.8973829196389881,"total_runs":316,"cost_p50":0.1209175,"cost_p95":0.944745,"cost_avg":0.23650338607594934,"latency_p50":42682,"latency_p95":447435.25,"latency_avg":107598.15189873418,"ttft_p50":38672.87,"ttft_p95":396773.41200000007,"ttft_avg":93483.13377777777,"tokens_per_sec_p50":88.41827030249405,"tokens_per_sec_p95":110.72863829700543,"tokens_per_sec_avg":76.9268331026949,"thinking_ratio_p50":null,"thinking_ratio_p95":null,"thinking_ratio_avg":null,"total_params":null,"active_params":null,"precision":"unknown","base_model":"claude-opus-4.7","reasoning_effort":"max","provider":"anthropic"},{"model_id":"gpt-6.1-sol-medium","display_name":"gpt-6.1-sol","lab":"openai","prompt":"bare","pass_rate":0.8639240506329114,"ci_lower":0.8217230378099494,"ci_upper":0.8973829196389881,"total_runs":316,"cost_p50":0.002256,"cost_p95":0.011130999999999999,"cost_avg":0.0033616835443037976,"latency_p50":7741,"latency_p95":33329.25,"latency_avg":11057.73417721519,"ttft_p50":6284.1,"ttft_p95":30764.5575,"ttft_avg":9437.646044303798,"tokens_per_sec_p50":19.159454849564526,"tokens_per_sec_p95":27.262071915167684,"tokens_per_sec_avg":17.716510267971703,"thinking_ratio_p50":0.7966224685005607,"thinking_ratio_p95":0.9528359140859142,"thinking_ratio_avg":0.7473195429701468,"total_params":null,"active_params":null,"precision":"unknown","base_model":"gpt-6.1-sol","reasoning_effort":"medium","provider":"openai"},{"model_id":"gpt-6-astra-xhigh","display_name":"gpt-6-astra","lab":"openai","prompt":"bare","pass_rate":0.8639240506329114,"ci_lower":0.8217230378099494,"ci_upper":0.8973829196389881,"total_runs":316,"cost_p50":0.01899,"cost_p95":0.14231749999999999,"cost_avg":0.04122636075949366,"latency_p50":14093.5,"latency_p95":125084.75,"latency_avg":37676.873417721516,"ttft_p50":13950.085,"ttft_p95":124923.8425,"ttft_avg":31700.640094936694,"tokens_per_sec_p50":20.30772156765796,"tokens_per_sec_p95":27.401051454934937,"tokens_per_sec_avg":19.309301857441596,"thinking_ratio_p50":0.8997816593886463,"thinking_ratio_p95":0.9842867951770733,"thinking_ratio_avg":0.862628103182499,"total_params":null,"active_params":null,"precision":"unknown","base_model":"gpt-6-astra","reasoning_effort":"xhigh","provider":"openai"},{"model_id":"claude-sonnet-5-5-high","display_name":"claude-sonnet-5-5","lab":"anthropic","prompt":"bare","pass_rate":0.8607594936708861,"ci_lower":0.8182384968531615,"ci_upper":0.8946143653136112,"total_runs":316,"cost_p50":0.010099,"cost_p95":0.18748725,"cost_avg":0.0355924493670886,"latency_p50":7414,"latency_p95":86494.5,"latency_avg":18576.537974683546,"ttft_p50":5749.995,"ttft_p95":77430.13249999999,"ttft_avg":15826.939968354434,"tokens_per_sec_p50":102.1629791166948,"tokens_per_sec_p95":146.12588421639325,"tokens_per_sec_avg":94.97901019520373,"thinking_ratio_p50":null,"thinking_ratio_p95":null,"thinking_ratio_avg":null,"total_params":null,"active_params":null,"precision":"unknown","base_model":"claude-sonnet-5-5","reasoning_effort":"high","provider":"anthropic"},{"model_id":"claude-opus-5-high","display_name":"claude-opus-5","lab":"anthropic","prompt":"bare","pass_rate":0.8607594936708861,"ci_lower":0.8182384968531615,"ci_upper":0.8946143653136112,"total_runs":316,"cost_p50":0.03044,"cost_p95":0.23009375,"cost_avg":0.058987816455696195,"latency_p50":14488.5,"latency_p95":104942.75,"latency_avg":27953.712025316454,"ttft_p50":10204.654999999999,"ttft_p95":91973.91750000001,"ttft_avg":22426.324873417718,"tokens_per_sec_p50":68.12561665862833,"tokens_per_sec_p95":93.41957398813419,"tokens_per_sec_avg":60.01640534593706,"thinking_ratio_p50":null,"thinking_ratio_p95":null,"thinking_ratio_avg":null,"total_params":null,"active_params":null,"precision":"unknown","base_model":"claude-opus-5","reasoning_effort":"high","provider":"anthropic"},{"model_id":"gpt-6.1-sol-xhigh","display_name":"gpt-6.1-sol","lab":"openai","prompt":"bare","pass_rate":0.8575949367088608,"ci_lower":0.8147603642958167,"ci_upper":0.8918394025887914,"total_runs":316,"cost_p50":0.004444,"cost_p95":0.0305074,"cost_avg":0.009718850793650795,"latency_p50":14485.5,"latency_p95":112917.5,"latency_avg":45008.29430379747,"ttft_p50":13388.21,"ttft_p95":95798.133,"ttft_avg":32105.147015873023,"tokens_per_sec_p50":23.964550551636826,"tokens_per_sec_p95":29.992610426779912,"tokens_per_sec_avg":22.736941162073247,"thinking_ratio_p50":0.9186507936507936,"thinking_ratio_p95":0.9850277243322678,"thinking_ratio_avg":0.8879305273655715,"total_params":null,"active_params":null,"precision":"unknown","base_model":"gpt-6.1-sol","reasoning_effort":"xhigh","provider":"openai"},{"model_id":"claude-opus-4.8-high","display_name":"claude-opus-4.8","lab":"anthropic","prompt":"bare","pass_rate":0.8544303797468354,"ci_lower":0.8112884663092524,"ci_upper":0.8890582052931909,"total_runs":316,"cost_p50":0.03412,"cost_p95":0.2342325,"cost_avg":0.06485419303797467,"latency_p50":14723.5,"latency_p95":101748,"latency_avg":28631.066455696204,"ttft_p50":10727.045,"ttft_p95":95921.905,"ttft_avg":24556.862974683547,"tokens_per_sec_p50":78.68950092845391,"tokens_per_sec_p95":101.16977214248666,"tokens_per_sec_avg":64.74665489963998,"thinking_ratio_p50":null,"thinking_ratio_p95":null,"thinking_ratio_avg":null,"total_params":null,"active_params":null,"precision":"unknown","base_model":"claude-opus-4.8","reasoning_effort":"high","provider":"anthropic"},{"model_id":"claude-opus-5-xhigh","display_name":"claude-opus-5","lab":"anthropic","prompt":"bare","pass_rate":0.8544303797468354,"ci_lower":0.8112884663092524,"ci_upper":0.8890582052931909,"total_runs":316,"cost_p50":0.0411125,"cost_p95":0.45378874999999996,"cost_avg":0.10102520569620253,"latency_p50":19394.5,"latency_p95":197290.5,"latency_avg":46235.03164556962,"ttft_p50":13475.535,"ttft_p95":184960.7525,"ttft_avg":39842.16161392405,"tokens_per_sec_p50":77.38174661656805,"tokens_per_sec_p95":97.55380314316956,"tokens_per_sec_avg":67.31159756007692,"thinking_ratio_p50":null,"thinking_ratio_p95":null,"thinking_ratio_avg":null,"total_params":null,"active_params":null,"precision":"unknown","base_model":"claude-opus-5","reasoning_effort":"xhigh","provider":"anthropic"},{"model_id":"claude-haiku-5-5-max","display_name":"claude-haiku-5-5","lab":"anthropic","prompt":"bare","pass_rate":0.8544303797468354,"ci_lower":0.8112884663092524,"ci_upper":0.8890582052931909,"total_runs":316,"cost_p50":0.00732175,"cost_p95":0.028675475,"cost_avg":0.009160369303797468,"latency_p50":65561,"latency_p95":252956.25,"latency_avg":83646.82278481012,"ttft_p50":64733.130000000005,"ttft_p95":252093.1575,"ttft_avg":82780.06835443035,"tokens_per_sec_p50":218.94910857437318,"tokens_per_sec_p95":254.798030659233,"tokens_per_sec_avg":205.0645240534888,"thinking_ratio_p50":null,"thinking_ratio_p95":null,"thinking_ratio_avg":null,"total_params":null,"active_params":null,"precision":"unknown","base_model":"claude-haiku-5-5","reasoning_effort":"max","provider":"anthropic"},{"model_id":"claude-opus-4.8-xhigh","display_name":"claude-opus-4.8","lab":"anthropic","prompt":"bare","pass_rate":0.8544303797468354,"ci_lower":0.8112884663092524,"ci_upper":0.8890582052931909,"total_runs":316,"cost_p50":0.07509,"cost_p95":0.45232875,"cost_avg":0.12587974683544306,"latency_p50":28193.5,"latency_p95":196712,"latency_avg":54159.41455696202,"ttft_p50":23116.79,"ttft_p95":190919.76499999998,"ttft_avg":49269.24867088605,"tokens_per_sec_p50":89.62918821341204,"tokens_per_sec_p95":106.74839833437082,"tokens_per_sec_avg":75.45897991504066,"thinking_ratio_p50":null,"thinking_ratio_p95":null,"thinking_ratio_avg":null,"total_params":null,"active_params":null,"precision":"unknown","base_model":"claude-opus-4.8","reasoning_effort":"xhigh","provider":"anthropic"},{"model_id":"claude-opus-4.7-xhigh","display_name":"claude-opus-4.7","lab":"anthropic","prompt":"bare","pass_rate":0.8512658227848101,"ci_lower":0.8078226382156595,"ci_upper":0.8862709381046191,"total_runs":316,"cost_p50":0.0404775,"cost_p95":0.26852,"cost_avg":0.07710664556962028,"latency_p50":18082.5,"latency_p95":121082.5,"latency_avg":34560.76898734177,"ttft_p50":12902.619999999999,"ttft_p95":117202.4375,"ttft_avg":30863.837753164546,"tokens_per_sec_p50":75.54879001084159,"tokens_per_sec_p95":101.95014268754967,"tokens_per_sec_avg":62.08812297764994,"thinking_ratio_p50":null,"thinking_ratio_p95":null,"thinking_ratio_avg":null,"total_params":null,"active_params":null,"precision":"unknown","base_model":"claude-opus-4.7","reasoning_effort":"xhigh","provider":"anthropic"},{"model_id":"claude-opus-5-medium","display_name":"claude-opus-5","lab":"anthropic","prompt":"bare","pass_rate":0.8481012658227848,"ci_lower":0.8043627238323244,"ci_upper":0.8834777572057895,"total_runs":316,"cost_p50":0.02001,"cost_p95":0.16536374999999998,"cost_avg":0.04040224683544306,"latency_p50":11578,"latency_p95":79812,"latency_avg":20284.151898734177,"ttft_p50":7077.055,"ttft_p95":69743.2625,"ttft_avg":15344.496867088605,"tokens_per_sec_p50":58.31469427611498,"tokens_per_sec_p95":86.08080045255599,"tokens_per_sec_avg":52.01935802948268,"thinking_ratio_p50":null,"thinking_ratio_p95":null,"thinking_ratio_avg":null,"total_params":null,"active_params":null,"precision":"unknown","base_model":"claude-opus-5","reasoning_effort":"medium","provider":"anthropic"},{"model_id":"gpt-5.6-sol-max","display_name":"gpt-5.6-sol","lab":"openai","prompt":"bare","pass_rate":0.8481012658227848,"ci_lower":0.8043627238323244,"ci_upper":0.8834777572057895,"total_runs":316,"cost_p50":0.029651999999999998,"cost_p95":0.31843499999999997,"cost_avg":0.07434317721518985,"latency_p50":30471.5,"latency_p95":443940.25,"latency_avg":111934.00949367089,"ttft_p50":29582.46,"ttft_p95":391107.29500000004,"ttft_avg":86194.94028481015,"tokens_per_sec_p50":38.54061057345583,"tokens_per_sec_p95":53.1883679822555,"tokens_per_sec_avg":37.76518214504262,"thinking_ratio_p50":0.9801199116440518,"thinking_ratio_p95":0.9978685030332841,"thinking_ratio_avg":0.9560581123659986,"total_params":null,"active_params":null,"precision":"unknown","base_model":"gpt-5.6-sol","reasoning_effort":"max","provider":"openai"},{"model_id":"claude-fable-5-low","display_name":"claude-fable-5","lab":"anthropic","prompt":"bare","pass_rate":0.8449367088607594,"ci_lower":0.8009085748755198,"ci_upper":0.8806788108804293,"total_runs":316,"cost_p50":0.02198,"cost_p95":0.11051875,"cost_avg":0.034307642405063274,"latency_p50":10965,"latency_p95":58558.25,"latency_avg":18286.25,"ttft_p50":4029.07,"ttft_p95":35624.615000000005,"ttft_avg":8892.8882278481,"tokens_per_sec_p50":26.444812684041928,"tokens_per_sec_p95":69.81324487971168,"tokens_per_sec_avg":30.21071305597523,"thinking_ratio_p50":null,"thinking_ratio_p95":null,"thinking_ratio_avg":null,"total_params":null,"active_params":null,"precision":"unknown","base_model":"claude-fable-5","reasoning_effort":"low","provider":"anthropic"},{"model_id":"claude-haiku-5-5-high","display_name":"claude-haiku-5-5","lab":"anthropic","prompt":"bare","pass_rate":0.8449367088607594,"ci_lower":0.8009085748755198,"ci_upper":0.8806788108804293,"total_runs":316,"cost_p50":0.0010193000000000001,"cost_p95":0.0040352,"cost_avg":0.0014021446202531652,"latency_p50":12258,"latency_p95":51410.75,"latency_avg":17358.436708860758,"ttft_p50":11372.535,"ttft_p95":50589.725,"ttft_avg":16412.812594936706,"tokens_per_sec_p50":146.19062183358255,"tokens_per_sec_p95":223.2393497690593,"tokens_per_sec_avg":138.5927419047492,"thinking_ratio_p50":null,"thinking_ratio_p95":null,"thinking_ratio_avg":null,"total_params":null,"active_params":null,"precision":"unknown","base_model":"claude-haiku-5-5","reasoning_effort":"high","provider":"anthropic"},{"model_id":"gpt-6.1-terra-high","display_name":"gpt-6.1-terra","lab":"openai","prompt":"bare","pass_rate":0.8417721518987342,"ci_lower":0.7974600504175338,"ci_upper":0.8778742400562509,"total_runs":316,"cost_p50":0.0012014999999999999,"cost_p95":0.00399325,"cost_avg":0.0016108101265822786,"latency_p50":3428.5,"latency_p95":9264.25,"latency_avg":4272.604430379747,"ttft_p50":3387.6949999999997,"ttft_p95":9196.2325,"ttft_avg":4218.373955696203,"tokens_per_sec_p50":45.748732729896446,"tokens_per_sec_p95":74.5442883114155,"tokens_per_sec_avg":44.41843863782668,"thinking_ratio_p50":0.81789594053745,"thinking_ratio_p95":0.9549011051351145,"thinking_ratio_avg":0.768438397542412,"total_params":null,"active_params":null,"precision":"unknown","base_model":"gpt-6.1-terra","reasoning_effort":"high","provider":"openai"},{"model_id":"claude-haiku-5-5-xhigh","display_name":"claude-haiku-5-5","lab":"anthropic","prompt":"bare","pass_rate":0.8417721518987342,"ci_lower":0.7974600504175338,"ci_upper":0.8778742400562509,"total_runs":316,"cost_p50":0.0014462,"cost_p95":0.0077403,"cost_avg":0.0023429215189873422,"latency_p50":17214.5,"latency_p95":95405.5,"latency_avg":26637.401898734177,"ttft_p50":16484.775,"ttft_p95":94376.5925,"ttft_avg":25704.621772151888,"tokens_per_sec_p50":152.41761428613356,"tokens_per_sec_p95":233.54471141410943,"tokens_per_sec_avg":148.91724116238018,"thinking_ratio_p50":null,"thinking_ratio_p95":null,"thinking_ratio_avg":null,"total_params":null,"active_params":null,"precision":"unknown","base_model":"claude-haiku-5-5","reasoning_effort":"xhigh","provider":"anthropic"},{"model_id":"gpt-5.3-codex-xhigh","display_name":"gpt-5.3-codex","lab":"openai","prompt":"bare","pass_rate":0.8386075949367089,"ci_lower":0.7940170163911512,"ci_upper":0.8750641788004688,"total_runs":316,"cost_p50":0.062729625,"cost_p95":0.3448440625,"cost_avg":0.1026651787974684,"latency_p50":72761,"latency_p95":464561.75,"latency_avg":130754.87341772152,"ttft_p50":72252.85,"ttft_p95":463804.0075,"ttft_avg":130130.24772151897,"tokens_per_sec_p50":56.29647549589571,"tokens_per_sec_p95":65.02746593462106,"tokens_per_sec_avg":55.177167041515105,"thinking_ratio_p50":0.9942136478327301,"thinking_ratio_p95":0.999066176128722,"thinking_ratio_avg":0.9816134664515889,"total_params":null,"active_params":null,"precision":"unknown","base_model":"gpt-5.3-codex","reasoning_effort":"xhigh","provider":"openai"},{"model_id":"gpt-6.1-terra-max","display_name":"gpt-6.1-terra","lab":"openai","prompt":"bare","pass_rate":0.8386075949367089,"ci_lower":0.7940170163911512,"ci_upper":0.8750641788004688,"total_runs":316,"cost_p50":0.0050155,"cost_p95":0.055304,"cost_avg":0.013411300632911393,"latency_p50":11823.5,"latency_p95":127966.5,"latency_avg":30344.21518987342,"ttft_p50":11792.84,"ttft_p95":127907.265,"ttft_avg":30288.297468354438,"tokens_per_sec_p50":76.59054488998815,"tokens_per_sec_p95":100.78662831781199,"tokens_per_sec_avg":72.40110211607147,"thinking_ratio_p50":0.9640579710144928,"thinking_ratio_p95":0.9954805590652739,"thinking_ratio_avg":0.9468470660659077,"total_params":null,"active_params":null,"precision":"unknown","base_model":"gpt-6.1-terra","reasoning_effort":"max","provider":"openai"},{"model_id":"claude-opus-4.8-medium","display_name":"claude-opus-4.8","lab":"anthropic","prompt":"bare","pass_rate":0.8354430379746836,"ci_lower":0.7905793451366101,"ci_upper":0.8722487547728451,"total_runs":316,"cost_p50":0.025154999999999997,"cost_p95":0.1720525,"cost_avg":0.0487165348101266,"latency_p50":12502.5,"latency_p95":75590,"latency_avg":21955.604430379746,"ttft_p50":8891.07,"ttft_p95":71376.4775,"ttft_avg":18236.196044303797,"tokens_per_sec_p50":71.08180018309557,"tokens_per_sec_p95":98.07044284173294,"tokens_per_sec_avg":60.38925973765981,"thinking_ratio_p50":null,"thinking_ratio_p95":null,"thinking_ratio_avg":null,"total_params":null,"active_params":null,"precision":"unknown","base_model":"claude-opus-4.8","reasoning_effort":"medium","provider":"anthropic"},{"model_id":"claude-sonnet-5-5-medium","display_name":"claude-sonnet-5-5","lab":"anthropic","prompt":"bare","pass_rate":0.8354430379746836,"ci_lower":0.7905793451366101,"ci_upper":0.8722487547728451,"total_runs":316,"cost_p50":0.007563,"cost_p95":0.14231249999999998,"cost_avg":0.023528999999999994,"latency_p50":5255,"latency_p95":68435.75,"latency_avg":12607.75,"ttft_p50":0.19,"ttft_p95":52160.695,"ttft_avg":7658.379746835444,"tokens_per_sec_p50":103.05101599985363,"tokens_per_sec_p95":143.18987232703705,"tokens_per_sec_avg":92.96692080715707,"thinking_ratio_p50":null,"thinking_ratio_p95":null,"thinking_ratio_avg":null,"total_params":null,"active_params":null,"precision":"unknown","base_model":"claude-sonnet-5-5","reasoning_effort":"medium","provider":"anthropic"},{"model_id":"claude-sonnet-5-5-low","display_name":"claude-sonnet-5-5","lab":"anthropic","prompt":"bare","pass_rate":0.8291139240506329,"ci_lower":0.7837196098858548,"ci_upper":0.8666022994592711,"total_runs":316,"cost_p50":0.007511,"cost_p95":0.07298450000000001,"cost_avg":0.015242623417721516,"latency_p50":5045.5,"latency_p95":34814.75,"latency_avg":8800.968354430379,"ttft_p50":0.255,"ttft_p95":28686.8675,"ttft_avg":4005.7881962025317,"tokens_per_sec_p50":100.65442439630714,"tokens_per_sec_p95":138.80689524420657,"tokens_per_sec_avg":90.26548563977842,"thinking_ratio_p50":null,"thinking_ratio_p95":null,"thinking_ratio_avg":null,"total_params":null,"active_params":null,"precision":"unknown","base_model":"claude-sonnet-5-5","reasoning_effort":"low","provider":"anthropic"},{"model_id":"gpt-5.6-sol-xhigh","display_name":"gpt-5.6-sol","lab":"openai","prompt":"bare","pass_rate":0.8259493670886076,"ci_lower":0.7802973190407421,"ci_upper":0.8637714950222191,"total_runs":316,"cost_p50":0.021152,"cost_p95":0.14651,"cost_avg":0.04424503797468353,"latency_p50":21321.5,"latency_p95":166222.5,"latency_avg":56024.88291139241,"ttft_p50":20038.155,"ttft_p95":161446.365,"ttft_avg":49740.28556962025,"tokens_per_sec_p50":42.07594660071567,"tokens_per_sec_p95":54.564170788629866,"tokens_per_sec_avg":37.88411061924781,"thinking_ratio_p50":0.9733078880407124,"thinking_ratio_p95":0.9962662880281893,"thinking_ratio_avg":0.9373575903320355,"total_params":null,"active_params":null,"precision":"unknown","base_model":"gpt-5.6-sol","reasoning_effort":"xhigh","provider":"openai"},{"model_id":"gpt-6.1-terra-medium","display_name":"gpt-6.1-terra","lab":"openai","prompt":"bare","pass_rate":0.819620253164557,"ci_lower":0.7734673613468556,"ci_upper":0.8580952621517763,"total_runs":316,"cost_p50":0.000968,"cost_p95":0.00307925,"cost_avg":0.0012706329113924047,"latency_p50":3060.5,"latency_p95":7172.75,"latency_avg":3526.003164556962,"ttft_p50":2998.6800000000003,"ttft_p95":6928.155,"ttft_avg":3467.1807594936677,"tokens_per_sec_p50":36.2187658029733,"tokens_per_sec_p95":69.46458599473368,"tokens_per_sec_avg":37.201888347666085,"thinking_ratio_p50":0.7555555555555555,"thinking_ratio_p95":0.9377363085634266,"thinking_ratio_avg":0.7098310629129956,"total_params":null,"active_params":null,"precision":"unknown","base_model":"gpt-6.1-terra","reasoning_effort":"medium","provider":"openai"},{"model_id":"gemini-3.8-flash-high","display_name":"gemini-3.8-flash","lab":"google","prompt":"bare","pass_rate":0.819620253164557,"ci_lower":0.7734673613468556,"ci_upper":0.8580952621517763,"total_runs":316,"cost_p50":0.052049250000000005,"cost_p95":0.149974875,"cost_avg":0.060704394778481004,"latency_p50":66092.5,"latency_p95":222152,"latency_avg":87069.37025316455,"ttft_p50":65699.32500000001,"ttft_p95":218842.91,"ttft_avg":82451.65582278483,"tokens_per_sec_p50":180.64661164475092,"tokens_per_sec_p95":228.38783177575385,"tokens_per_sec_avg":176.12588620471848,"thinking_ratio_p50":0.9968512842651325,"thinking_ratio_p95":0.9996134825900684,"thinking_ratio_avg":0.9805966586230243,"total_params":null,"active_params":null,"precision":"unknown","base_model":"gemini-3.8-flash","reasoning_effort":"high","provider":"google-vertex"},{"model_id":"gpt-5.4-xhigh","display_name":"gpt-5.4","lab":"openai","prompt":"bare","pass_rate":0.819620253164557,"ci_lower":0.7734673613468556,"ci_upper":0.8580952621517763,"total_runs":316,"cost_p50":0.056749999999999995,"cost_p95":0.30315749999999997,"cost_avg":0.0964075237341772,"latency_p50":81007,"latency_p95":432996.75,"latency_avg":143661.9905063291,"ttft_p50":80168.095,"ttft_p95":432114.9075,"ttft_avg":136535.0014240506,"tokens_per_sec_p50":48.18868738461137,"tokens_per_sec_p95":54.68154049032714,"tokens_per_sec_avg":46.10943833548435,"thinking_ratio_p50":0.9930760569257655,"thinking_ratio_p95":0.9990122102643714,"thinking_ratio_avg":0.9743365846924299,"total_params":null,"active_params":null,"precision":"unknown","base_model":"gpt-5.4","reasoning_effort":"xhigh","provider":"openai"},{"model_id":"claude-opus-4.5-high","display_name":"claude-opus-4.5","lab":"anthropic","prompt":"bare","pass_rate":0.819620253164557,"ci_lower":0.7734673613468556,"ci_upper":0.8580952621517763,"total_runs":316,"cost_p50":0.096355,"cost_p95":0.53322625,"cost_avg":0.15285536392405077,"latency_p50":54941,"latency_p95":445584.5,"latency_avg":122754.05379746835,"ttft_p50":47131.57,"ttft_p95":430963.5625,"ttft_avg":114135.46575949364,"tokens_per_sec_p50":66.99283154908898,"tokens_per_sec_p95":90.95565899393455,"tokens_per_sec_avg":58.8280905842312,"thinking_ratio_p50":null,"thinking_ratio_p95":null,"thinking_ratio_avg":null,"total_params":null,"active_params":null,"precision":"unknown","base_model":"claude-opus-4.5","reasoning_effort":"high","provider":"anthropic"},{"model_id":"gpt-5.5-xhigh","display_name":"gpt-5.5","lab":"openai","prompt":"bare","pass_rate":0.8164556962025317,"ci_lower":0.7700594964466577,"ci_upper":0.8552500317698095,"total_runs":316,"cost_p50":0.115815,"cost_p95":0.67205625,"cost_avg":0.19125555379746834,"latency_p50":67597,"latency_p95":403020.25,"latency_avg":116267.29746835443,"ttft_p50":67044.725,"ttft_p95":401749.24250000005,"ttft_avg":114033.31205696198,"tokens_per_sec_p50":54.32115760050503,"tokens_per_sec_p95":55.38121342053521,"tokens_per_sec_avg":50.79076807007583,"thinking_ratio_p50":0.9915729179436751,"thinking_ratio_p95":0.9985795980689143,"thinking_ratio_avg":0.968841017677361,"total_params":null,"active_params":null,"precision":"unknown","base_model":"gpt-5.5","reasoning_effort":"xhigh","provider":"openai"},{"model_id":"claude-opus-4.8-low","display_name":"claude-opus-4.8","lab":"anthropic","prompt":"bare","pass_rate":0.8164556962025317,"ci_lower":0.7700594964466577,"ci_upper":0.8552500317698095,"total_runs":316,"cost_p50":0.0162525,"cost_p95":0.09582625,"cost_avg":0.029576898734177213,"latency_p50":8543.5,"latency_p95":41460.25,"latency_avg":14108.167721518987,"ttft_p50":6214.49,"ttft_p95":36330.07,"ttft_avg":11410.200949367092,"tokens_per_sec_p50":57.20047971966382,"tokens_per_sec_p95":91.146695142796,"tokens_per_sec_avg":50.24268677847796,"thinking_ratio_p50":null,"thinking_ratio_p95":null,"thinking_ratio_avg":null,"total_params":null,"active_params":null,"precision":"unknown","base_model":"claude-opus-4.8","reasoning_effort":"low","provider":"anthropic"},{"model_id":"gpt-5.3-codex-high","display_name":"gpt-5.3-codex","lab":"openai","prompt":"bare","pass_rate":0.810126582278481,"ci_lower":0.7632575306791116,"ci_upper":0.8495458069730262,"total_runs":316,"cost_p50":0.02683975,"cost_p95":0.1537025,"cost_avg":0.04450723496835444,"latency_p50":33226,"latency_p95":205177,"latency_avg":57260.534810126584,"ttft_p50":32403.595,"ttft_p95":204721.2275,"ttft_avg":56722.6119936709,"tokens_per_sec_p50":53.39459116081878,"tokens_per_sec_p95":63.04358678476777,"tokens_per_sec_avg":52.29729690955285,"thinking_ratio_p50":0.9860525645190469,"thinking_ratio_p95":0.997384455506976,"thinking_ratio_avg":0.9616122065432947,"total_params":null,"active_params":null,"precision":"unknown","base_model":"gpt-5.3-codex","reasoning_effort":"high","provider":"openai"},{"model_id":"grok-4.7-high","display_name":"grok-4.7","lab":"x-ai","prompt":"bare","pass_rate":0.810126582278481,"ci_lower":0.7632575306791116,"ci_upper":0.8495458069730262,"total_runs":316,"cost_p50":0.054117,"cost_p95":0.26959,"cost_avg":0.08032515822784811,"latency_p50":100442.5,"latency_p95":595847.25,"latency_avg":161463.69936708861,"ttft_p50":99798.91,"ttft_p95":595745.1,"ttft_avg":161010.3365822785,"tokens_per_sec_p50":78.01322709648203,"tokens_per_sec_p95":93.01931584060328,"tokens_per_sec_avg":77.60101183074315,"thinking_ratio_p50":0.997171027615531,"thinking_ratio_p95":0.9996166039589476,"thinking_ratio_avg":0.9910811108476457,"total_params":null,"active_params":null,"precision":"unknown","base_model":"grok-4.7","reasoning_effort":"high","provider":"xai"},{"model_id":"gpt-6-sol-xhigh","display_name":"gpt-6-sol","lab":"openai","prompt":"bare","pass_rate":0.8069620253164557,"ci_lower":0.7598632557882227,"ci_upper":0.8466869865817505,"total_runs":316,"cost_p50":0.006253,"cost_p95":0.0376735,"cost_avg":0.011096208860759498,"latency_p50":13043,"latency_p95":74044.25,"latency_avg":25084.145569620254,"ttft_p50":12351.95,"ttft_p95":71011.9925,"ttft_avg":21387.501139240518,"tokens_per_sec_p50":41.016729156463754,"tokens_per_sec_p95":51.98327170937482,"tokens_per_sec_avg":38.2879161049482,"thinking_ratio_p50":0.9541394120041016,"thinking_ratio_p95":0.9909711289616283,"thinking_ratio_avg":0.9173336242471258,"total_params":null,"active_params":null,"precision":"unknown","base_model":"gpt-6-sol","reasoning_effort":"xhigh","provider":"openai"},{"model_id":"kimi-k3.1-pro-max","display_name":"kimi-k3.1-pro","lab":"moonshotai","prompt":"bare","pass_rate":0.8069620253164557,"ci_lower":0.7598632557882227,"ci_upper":0.8466869865817505,"total_runs":316,"cost_p50":0.057245000000000004,"cost_p95":0.371012,"cost_avg":0.09346755063291136,"latency_p50":104351.5,"latency_p95":691529.5,"latency_avg":178167.27215189874,"ttft_p50":102595.13500000001,"ttft_p95":691132.5525,"ttft_avg":177232.25734177214,"tokens_per_sec_p50":52.306193892119,"tokens_per_sec_p95":57.06241608467111,"tokens_per_sec_avg":49.41897262593173,"thinking_ratio_p50":0.9925153137513094,"thinking_ratio_p95":0.9988687428405778,"thinking_ratio_avg":0.9701679806757594,"total_params":2779.9,"active_params":null,"precision":"mxfp4","base_model":"kimi-k3.1-pro","reasoning_effort":"max","provider":"moonshotai"},{"model_id":"grok-4.6-high","display_name":"grok-4.6","lab":"x-ai","prompt":"bare","pass_rate":0.8037974683544303,"ci_lower":0.7564733427820147,"ci_upper":0.8438238043057937,"total_runs":316,"cost_p50":0.022282999999999997,"cost_p95":0.128663,"cost_avg":0.03649877848101267,"latency_p50":56582,"latency_p95":389529.25,"latency_avg":103062.49050632911,"ttft_p50":56108.33,"ttft_p95":388961.8470000002,"ttft_avg":102886.70663492067,"tokens_per_sec_p50":57.177444373202164,"tokens_per_sec_p95":69.04764792384975,"tokens_per_sec_avg":57.6658860504577,"thinking_ratio_p50":0.9938537433012755,"thinking_ratio_p95":0.9991217105615188,"thinking_ratio_avg":0.9864475537238935,"total_params":null,"active_params":null,"precision":"unknown","base_model":"grok-4.6","reasoning_effort":"high","provider":"xai"},{"model_id":"gpt-6.1-terra-low","display_name":"gpt-6.1-terra","lab":"openai","prompt":"bare","pass_rate":0.8037974683544303,"ci_lower":0.7564733427820147,"ci_upper":0.8438238043057937,"total_runs":316,"cost_p50":0.0008535,"cost_p95":0.0027949999999999997,"cost_avg":0.0011252088607594931,"latency_p50":2678.5,"latency_p95":6779,"latency_avg":3214.6645569620255,"ttft_p50":2567.455,"ttft_p95":6713.4425,"ttft_avg":3152.6711392405077,"tokens_per_sec_p50":29.294173365931677,"tokens_per_sec_p95":65.49372054849631,"tokens_per_sec_avg":32.03310856178017,"thinking_ratio_p50":0.7608695652173914,"thinking_ratio_p95":0.9274809160305344,"thinking_ratio_avg":0.7216927549081804,"total_params":null,"active_params":null,"precision":"unknown","base_model":"gpt-6.1-terra","reasoning_effort":"low","provider":"openai"},{"model_id":"gpt-5.5-high","display_name":"gpt-5.5","lab":"openai","prompt":"bare","pass_rate":0.8037974683544303,"ci_lower":0.7564733427820147,"ci_upper":0.8438238043057937,"total_runs":316,"cost_p50":0.07352500000000001,"cost_p95":0.34583625,"cost_avg":0.1069929588607595,"latency_p50":43281,"latency_p95":226389,"latency_avg":67516.12025316455,"ttft_p50":39019.979999999996,"ttft_p95":203280.33250000002,"ttft_avg":63328.837056961995,"tokens_per_sec_p50":53.60900845746097,"tokens_per_sec_p95":55.22655386936261,"tokens_per_sec_avg":49.32423923858137,"thinking_ratio_p50":0.9862250485740695,"thinking_ratio_p95":0.9975955976693521,"thinking_ratio_avg":0.9537837687548153,"total_params":null,"active_params":null,"precision":"unknown","base_model":"gpt-5.5","reasoning_effort":"high","provider":"openai"},{"model_id":"claude-opus-4.5-16k","display_name":"claude-opus-4.5","lab":"anthropic","prompt":"bare","pass_rate":0.8006329113924051,"ci_lower":0.7530877132181288,"ci_upper":0.8409563385875151,"total_runs":316,"cost_p50":0.101215,"cost_p95":0.292195,"cost_avg":0.11545087025316454,"latency_p50":54171,"latency_p95":167975.5,"latency_avg":67724.65189873418,"ttft_p50":45512.18,"ttft_p95":155244.77000000002,"ttft_avg":55883.523037974686,"tokens_per_sec_p50":69.5531251883246,"tokens_per_sec_p95":88.97160039700321,"tokens_per_sec_avg":65.45357117775471,"thinking_ratio_p50":null,"thinking_ratio_p95":null,"thinking_ratio_avg":null,"total_params":null,"active_params":null,"precision":"unknown","base_model":"claude-opus-4.5","reasoning_effort":"medium","provider":"anthropic"},{"model_id":"gemini-3.1-pro-preview-high","display_name":"gemini-3.1-pro-preview","lab":"google","prompt":"bare","pass_rate":0.7974683544303798,"ci_lower":0.7497062917548237,"ci_upper":0.8380846647686555,"total_runs":316,"cost_p50":0.05,"cost_p95":0.305881,"cost_avg":0.09277807594936707,"latency_p50":76124,"latency_p95":336085.25,"latency_avg":116082.1835443038,"ttft_p50":49691.28,"ttft_p95":304786.1525,"ttft_avg":90567.77892405062,"tokens_per_sec_p50":72.75022153622349,"tokens_per_sec_p95":106.37571093242988,"tokens_per_sec_avg":64.3775366133794,"thinking_ratio_p50":0.9775689525235984,"thinking_ratio_p95":0.9990857498668076,"thinking_ratio_avg":0.9544023313719778,"total_params":null,"active_params":null,"precision":"unknown","base_model":"gemini-3.1-pro-preview","reasoning_effort":"high","provider":"google-vertex"},{"model_id":"muse-wonder-high","display_name":"muse-wonder","lab":"meta","prompt":"bare","pass_rate":0.7943037974683544,"ci_lower":0.7463290059885863,"ci_upper":0.8352088552527281,"total_runs":316,"cost_p50":0.07258,"cost_p95":0.34634299999999996,"cost_avg":0.10696168354430383,"latency_p50":30882.5,"latency_p95":164180.25,"latency_avg":48290.53164556962,"ttft_p50":30587.58,"ttft_p95":164127.145,"ttft_avg":47888.10670886079,"tokens_per_sec_p50":109.56384964216699,"tokens_per_sec_p95":152.97089894485245,"tokens_per_sec_avg":110.50861031525866,"thinking_ratio_p50":0.9650344441084858,"thinking_ratio_p95":0.9845745463468604,"thinking_ratio_avg":0.9523616973594929,"total_params":null,"active_params":null,"precision":"unknown","base_model":"muse-wonder","reasoning_effort":"high","provider":"meta"},{"model_id":"claude-sonnet-4.6-high","display_name":"claude-sonnet-4.6","lab":"anthropic","prompt":"bare","pass_rate":0.7943037974683544,"ci_lower":0.7463290059885863,"ci_upper":0.8352088552527281,"total_runs":316,"cost_p50":0.057929999999999995,"cost_p95":1.0154144999999999,"cost_avg":0.18013818987341776,"latency_p50":48224.5,"latency_p95":1043931.25,"latency_avg":216554.8987341772,"ttft_p50":40242.73,"ttft_p95":651115.8049999999,"ttft_avg":130441.55270096459,"tokens_per_sec_p50":71.00856828567518,"tokens_per_sec_p95":91.48506459315615,"tokens_per_sec_avg":65.25373512972077,"thinking_ratio_p50":null,"thinking_ratio_p95":null,"thinking_ratio_avg":null,"total_params":null,"active_params":null,"precision":"unknown","base_model":"claude-sonnet-4.6","reasoning_effort":"high","provider":"anthropic"},{"model_id":"gemini-3.7-flash-high","display_name":"gemini-3.7-flash","lab":"google","prompt":"bare","pass_rate":0.7911392405063291,"ci_lower":0.7429557863027209,"ci_upper":0.832328979656429,"total_runs":316,"cost_p50":0.0062113125,"cost_p95":0.026149125,"cost_avg":0.008607739319620254,"latency_p50":15149,"latency_p95":63238.75,"latency_avg":21770.70569620253,"ttft_p50":14800.455,"ttft_p95":62820.884999999995,"ttft_avg":21245.586424050627,"tokens_per_sec_p50":197.24863321832754,"tokens_per_sec_p95":230.41243895006318,"tokens_per_sec_avg":182.6622985367434,"thinking_ratio_p50":0.9906818325334681,"thinking_ratio_p95":0.9986976118663906,"thinking_ratio_avg":0.973420386913029,"total_params":null,"active_params":null,"precision":"unknown","base_model":"gemini-3.7-flash","reasoning_effort":"high","provider":"google-vertex"},{"model_id":"gpt-5.4-high","display_name":"gpt-5.4","lab":"openai","prompt":"bare","pass_rate":0.7879746835443038,"ci_lower":0.7395865657260222,"ci_upper":0.8294451049509629,"total_runs":316,"cost_p50":0.032218750000000004,"cost_p95":0.163835,"cost_avg":0.04838613132911392,"latency_p50":41138.5,"latency_p95":269590.25,"latency_avg":72818.81962025317,"ttft_p50":40499.740000000005,"ttft_p95":265950.615,"ttft_avg":70697.50974683544,"tokens_per_sec_p50":45.417427257837744,"tokens_per_sec_p95":54.52279284624223,"tokens_per_sec_avg":42.48200511412073,"thinking_ratio_p50":0.9853474907956539,"thinking_ratio_p95":0.9977999868311312,"thinking_ratio_avg":0.9562302369472259,"total_params":null,"active_params":null,"precision":"unknown","base_model":"gpt-5.4","reasoning_effort":"high","provider":"openai"},{"model_id":"gpt-5.4-medium","display_name":"gpt-5.4","lab":"openai","prompt":"bare","pass_rate":0.7848101265822784,"ci_lower":0.7362212798007275,"ci_upper":0.826557295594093,"total_runs":316,"cost_p50":0.0132825,"cost_p95":0.063553125,"cost_avg":0.018515672468354435,"latency_p50":18448.5,"latency_p95":89856.25,"latency_avg":26936.886075949365,"ttft_p50":16957.92,"ttft_p95":89276.1125,"ttft_avg":26169.89047468354,"tokens_per_sec_p50":41.231199003264365,"tokens_per_sec_p95":53.22017826249505,"tokens_per_sec_avg":40.08174692673262,"thinking_ratio_p50":0.9606145305056093,"thinking_ratio_p95":0.9938381055091027,"thinking_ratio_avg":0.9116410757870893,"total_params":null,"active_params":null,"precision":"unknown","base_model":"gpt-5.4","reasoning_effort":"medium","provider":"openai"},{"model_id":"gpt-5.5-medium","display_name":"gpt-5.5","lab":"openai","prompt":"bare","pass_rate":0.7848101265822784,"ci_lower":0.7362212798007275,"ci_upper":0.826557295594093,"total_runs":316,"cost_p50":0.033562499999999995,"cost_p95":0.12560375,"cost_avg":0.042679889240506315,"latency_p50":20294.5,"latency_p95":75212.75,"latency_avg":25512.708860759492,"ttft_p50":19658.08,"ttft_p95":74342.9225,"ttft_avg":24686.529335443032,"tokens_per_sec_p50":51.8759779583001,"tokens_per_sec_p95":54.81837822977063,"tokens_per_sec_avg":47.43692608020297,"thinking_ratio_p50":0.9723479351367215,"thinking_ratio_p95":0.9934011630131954,"thinking_ratio_avg":0.9328368626526304,"total_params":null,"active_params":null,"precision":"unknown","base_model":"gpt-5.5","reasoning_effort":"medium","provider":"openai"},{"model_id":"gpt-5.3-codex-medium","display_name":"gpt-5.3-codex","lab":"openai","prompt":"bare","pass_rate":0.7784810126582279,"ci_lower":0.7295022659073515,"ci_upper":0.8207701189231398,"total_runs":316,"cost_p50":0.014091875,"cost_p95":0.07160606250000001,"cost_avg":0.020726064082278475,"latency_p50":18126,"latency_p95":95961,"latency_avg":27785.933544303796,"ttft_p50":17306.699999999997,"ttft_p95":95421.2775,"ttft_avg":27239.263607594934,"tokens_per_sec_p50":48.57312778874538,"tokens_per_sec_p95":60.67927802485384,"tokens_per_sec_avg":47.6858655005911,"thinking_ratio_p50":0.9684816470269538,"thinking_ratio_p95":0.9956081457450642,"thinking_ratio_avg":0.9356857147437955,"total_params":null,"active_params":null,"precision":"unknown","base_model":"gpt-5.3-codex","reasoning_effort":"medium","provider":"openai"},{"model_id":"gpt-5.6-sol-low","display_name":"gpt-5.6-sol","lab":"openai","prompt":"bare","pass_rate":0.7626582278481012,"ci_lower":0.7127695083234125,"ci_upper":0.8062374000962552,"total_runs":316,"cost_p50":0.009815999999999998,"cost_p95":0.033804,"cost_avg":0.012130582278481015,"latency_p50":10914,"latency_p95":35634.75,"latency_avg":15352.446202531646,"ttft_p50":10067.745,"ttft_p95":34651.427500000005,"ttft_avg":14207.765664556968,"tokens_per_sec_p50":33.27451050344604,"tokens_per_sec_p95":48.297206406218464,"tokens_per_sec_avg":31.322558524728933,"thinking_ratio_p50":0.9360464529159304,"thinking_ratio_p95":0.9815546822828376,"thinking_ratio_avg":0.8803394630393191,"total_params":null,"active_params":null,"precision":"unknown","base_model":"gpt-5.6-sol","reasoning_effort":"low","provider":"openai"},{"model_id":"grok-4.5-high","display_name":"grok-4.5","lab":"x-ai","prompt":"bare","pass_rate":0.7626582278481012,"ci_lower":0.7127695083234125,"ci_upper":0.8062374000962552,"total_runs":316,"cost_p50":0.025816,"cost_p95":0.160673,"cost_avg":0.044783734177215186,"latency_p50":35701,"latency_p95":273537.5,"latency_avg":73866.02531645569,"ttft_p50":34078.96000000001,"ttft_p95":270188.07,"ttft_avg":68099.71889240509,"tokens_per_sec_p50":104.07636644889263,"tokens_per_sec_p95":134.02215157852078,"tokens_per_sec_avg":104.23813253400085,"thinking_ratio_p50":0.9873462020428873,"thinking_ratio_p95":0.9993794539073132,"thinking_ratio_avg":0.9656541630287182,"total_params":null,"active_params":null,"precision":"unknown","base_model":"grok-4.5","reasoning_effort":"high","provider":"xai"},{"model_id":"gpt-6-sol-low","display_name":"gpt-6-sol","lab":"openai","prompt":"bare","pass_rate":0.7563291139240507,"ci_lower":0.7061012280066065,"ci_upper":0.800399489848732,"total_runs":316,"cost_p50":0.002319,"cost_p95":0.008367,"cost_avg":0.0031093417721518996,"latency_p50":5470.5,"latency_p95":18582.75,"latency_avg":7273.262658227848,"ttft_p50":4660.79,"ttft_p95":17579.6725,"ttft_avg":6483.392594936708,"tokens_per_sec_p50":25.780983416307745,"tokens_per_sec_p95":39.980961713132615,"tokens_per_sec_avg":23.84596262923768,"thinking_ratio_p50":0.8326455416359617,"thinking_ratio_p95":0.9645061555849933,"thinking_ratio_avg":0.7805823179835807,"total_params":null,"active_params":null,"precision":"unknown","base_model":"gpt-6-sol","reasoning_effort":"low","provider":"openai"},{"model_id":"kimi-k3-high","display_name":"kimi-k3","lab":"moonshotai","prompt":"bare","pass_rate":0.7536231884057971,"ci_lower":0.705525206569594,"ci_upper":0.7961351458654367,"total_runs":345,"cost_p50":0.05127,"cost_p95":0.47093760000000007,"cost_avg":0.11645816521739126,"latency_p50":197598,"latency_p95":1425951.2000000011,"latency_avg":407120.28695652174,"ttft_p50":117477.46,"ttft_p95":1103527.1700000002,"ttft_avg":257796.01455072453,"tokens_per_sec_p50":24.679342703765933,"tokens_per_sec_p95":35.14344803522167,"tokens_per_sec_avg":21.264635565933943,"thinking_ratio_p50":0.9876768598813327,"thinking_ratio_p95":0.9987437002573604,"thinking_ratio_avg":0.9512490885114757,"total_params":2779.9,"active_params":null,"precision":"int4","base_model":"kimi-k3","reasoning_effort":"high","provider":"moonshotai"},{"model_id":"gpt-5.5-low","display_name":"gpt-5.5","lab":"openai","prompt":"bare","pass_rate":0.75,"ci_lower":0.6994465676307645,"ci_upper":0.7945479596602446,"total_runs":316,"cost_p50":0.02154,"cost_p95":0.0603,"cost_avg":0.024593022151898725,"latency_p50":11782,"latency_p95":35416,"latency_avg":14568.832278481013,"ttft_p50":10688.08,"ttft_p95":34613.13,"ttft_avg":13737.883512658234,"tokens_per_sec_p50":50.45206472447066,"tokens_per_sec_p95":53.756302566878546,"tokens_per_sec_avg":45.53246884717724,"thinking_ratio_p50":0.9579646017699115,"thinking_ratio_p95":0.9876178607349942,"thinking_ratio_avg":0.9188122226619635,"total_params":null,"active_params":null,"precision":"unknown","base_model":"gpt-5.5","reasoning_effort":"low","provider":"openai"},{"model_id":"kimi-k3-max","display_name":"kimi-k3","lab":"moonshotai","prompt":"bare","pass_rate":0.7436708860759493,"ci_lower":0.6928051849287795,"ci_upper":0.7886831517979002,"total_runs":316,"cost_p50":0.053883,"cost_p95":0.43665420000000005,"cost_avg":0.11048109523809523,"latency_p50":132635.5,"latency_p95":1040460.75,"latency_avg":278656.7563291139,"ttft_p50":132060.54499999998,"ttft_p95":1028045.31,"ttft_avg":264417.8617088607,"tokens_per_sec_p50":25.442583049687194,"tokens_per_sec_p95":31.82252371340545,"tokens_per_sec_avg":23.710800567275736,"thinking_ratio_p50":0.9864902122966639,"thinking_ratio_p95":0.9986907022891919,"thinking_ratio_avg":0.9532078014568953,"total_params":2779.9,"active_params":null,"precision":"int4","base_model":"kimi-k3","reasoning_effort":"max","provider":"moonshotai"},{"model_id":"gpt-5.6-terra-xhigh","display_name":"gpt-5.6-terra","lab":"openai","prompt":"bare","pass_rate":0.740506329113924,"ci_lower":0.689489371850028,"ci_upper":0.7857458695944869,"total_runs":316,"cost_p50":0.013548000000000001,"cost_p95":0.1305755,"cost_avg":0.030311259493670896,"latency_p50":21308,"latency_p95":195701.75,"latency_avg":46794.284810126584,"ttft_p50":19926.260000000002,"ttft_p95":194153.525,"ttft_avg":45018.27734177216,"tokens_per_sec_p50":50.64485734767743,"tokens_per_sec_p95":55.044603890910736,"tokens_per_sec_avg":44.39158550104689,"thinking_ratio_p50":0.969281045751634,"thinking_ratio_p95":0.9968581758062423,"thinking_ratio_avg":0.9298562099650649,"total_params":null,"active_params":null,"precision":"unknown","base_model":"gpt-5.6-terra","reasoning_effort":"xhigh","provider":"openai"},{"model_id":"gpt-5.4-low","display_name":"gpt-5.4","lab":"openai","prompt":"bare","pass_rate":0.7373417721518988,"ci_lower":0.6861767597737298,"ci_upper":0.7828053863886207,"total_runs":316,"cost_p50":0.00804125,"cost_p95":0.03643625,"cost_avg":0.012421590189873417,"latency_p50":11614.5,"latency_p95":53166.75,"latency_avg":18280.49683544304,"ttft_p50":10689.965,"ttft_p95":50847.479999999996,"ttft_avg":17400.03183544303,"tokens_per_sec_p50":39.56122071918995,"tokens_per_sec_p95":52.890134447084385,"tokens_per_sec_avg":38.08696774772651,"thinking_ratio_p50":0.9357541899441341,"thinking_ratio_p95":0.9901280121817034,"thinking_ratio_avg":0.8863290900816454,"total_params":null,"active_params":null,"precision":"unknown","base_model":"gpt-5.4","reasoning_effort":"low","provider":"openai"},{"model_id":"muse-spark-1.2-high","display_name":"muse-spark-1.2","lab":"meta","prompt":"bare","pass_rate":0.7373417721518988,"ci_lower":0.6861767597737298,"ci_upper":0.7828053863886207,"total_runs":316,"cost_p50":0.026311,"cost_p95":0.1440019375,"cost_avg":0.03941281724683544,"latency_p50":48718,"latency_p95":198929.75,"latency_avg":64600.129746835446,"ttft_p50":40438.94,"ttft_p95":190692.27000000002,"ttft_avg":55212.95446202533,"tokens_per_sec_p50":143.14502013984907,"tokens_per_sec_p95":193.2987675963628,"tokens_per_sec_avg":136.1438526221662,"thinking_ratio_p50":0.9766171259130687,"thinking_ratio_p95":0.9985623396870025,"thinking_ratio_avg":0.9562714697153969,"total_params":null,"active_params":null,"precision":"unknown","base_model":"muse-spark-1.2","reasoning_effort":"high","provider":"meta"},{"model_id":"deepseek-v4.1-flash-1017-high","display_name":"deepseek-v4.1-flash-1017","lab":"deepseek","prompt":"bare","pass_rate":0.7373417721518988,"ci_lower":0.6861767597737298,"ci_upper":0.7828053863886207,"total_runs":316,"cost_p50":0.0016551219999999998,"cost_p95":0.009565535,"cost_avg":0.0027421407911392406,"latency_p50":43093.5,"latency_p95":246022.5,"latency_avg":72008.71835443038,"ttft_p50":42575.485,"ttft_p95":236955.24,"ttft_avg":69806.13696202531,"tokens_per_sec_p50":117.95697760166979,"tokens_per_sec_p95":158.39811741898646,"tokens_per_sec_avg":112.35531381005225,"thinking_ratio_p50":0.9941437965654123,"thinking_ratio_p95":0.9993652251126839,"thinking_ratio_avg":0.9694028178577421,"total_params":763.2,"active_params":null,"precision":"fp8","base_model":"deepseek-v4.1-flash-1017","reasoning_effort":"high","provider":"deepinfra"},{"model_id":"mimo-v2.6-pro-high","display_name":"mimo-v2.6-pro","lab":"xiaomi","prompt":"bare","pass_rate":0.7341772151898734,"ci_lower":0.6828673118569675,"ci_upper":0.7798617390232183,"total_runs":316,"cost_p50":0.0074443725,"cost_p95":0.09863407499999999,"cost_avg":0.02413705974683545,"latency_p50":148171.5,"latency_p95":3268078.75,"latency_avg":727605.417721519,"ttft_p50":113136.14,"ttft_p95":1743562.062,"ttft_avg":425583.086416938,"tokens_per_sec_p50":48.13956740385491,"tokens_per_sec_p95":84.09631523105735,"tokens_per_sec_avg":47.91142977127336,"thinking_ratio_p50":0.9259414715658718,"thinking_ratio_p95":0.9995015309818348,"thinking_ratio_avg":0.8489058221147474,"total_params":1024.2,"active_params":null,"precision":"fp8","base_model":"mimo-v2.6-pro","reasoning_effort":"high","provider":"xiaomi"},{"model_id":"muse-spark-open-high","display_name":"muse-spark-open","lab":"meta","prompt":"bare","pass_rate":0.7278481012658228,"ci_lower":0.6762577669633894,"ci_upper":0.773965093352467,"total_runs":316,"cost_p50":0.031413250000000004,"cost_p95":0.1680104375,"cost_avg":0.048358867088607586,"latency_p50":45404.5,"latency_p95":209469.75,"latency_avg":63342.0917721519,"ttft_p50":44514.915,"ttft_p95":209378.255,"ttft_avg":62077.87990506327,"tokens_per_sec_p50":158.5019804360285,"tokens_per_sec_p95":207.78280899642903,"tokens_per_sec_avg":160.8311630060852,"thinking_ratio_p50":0.9811385892548633,"thinking_ratio_p95":0.9990668036096573,"thinking_ratio_avg":0.9676074001497141,"total_params":822,"active_params":null,"precision":"fp4","base_model":"muse-spark-open","reasoning_effort":"high","provider":"baseten"},{"model_id":"gemini-3.5-flash-high","display_name":"gemini-3.5-flash","lab":"google","prompt":"bare","pass_rate":0.7278481012658228,"ci_lower":0.6762577669633894,"ci_upper":0.773965093352467,"total_runs":316,"cost_p50":0.06858225,"cost_p95":0.2289525,"cost_avg":0.0849459018987342,"latency_p50":32219.5,"latency_p95":103241.25,"latency_avg":39545.08544303798,"ttft_p50":30980.17,"ttft_p95":102342.6825,"ttft_avg":38354.922468354445,"tokens_per_sec_p50":227.77075732422816,"tokens_per_sec_p95":258.7431469102557,"tokens_per_sec_avg":212.44319318787262,"thinking_ratio_p50":0.9904183103911365,"thinking_ratio_p95":0.9989318847363895,"thinking_ratio_avg":0.9647081916274748,"total_params":null,"active_params":null,"precision":"unknown","base_model":"gemini-3.5-flash","reasoning_effort":"high","provider":"google-vertex"},{"model_id":"qwen3.7-max-high","display_name":"qwen3.7-max","lab":"qwen","prompt":"bare","pass_rate":0.7246835443037974,"ci_lower":0.6729576020011337,"ci_upper":0.7710121630325579,"total_runs":316,"cost_p50":0.0155339625,"cost_p95":0.068491625,"cost_avg":0.023169071281645578,"latency_p50":28158,"latency_p95":118397.5,"latency_avg":40659.946202531646,"ttft_p50":26884.83,"ttft_p95":117645.28100000005,"ttft_avg":39597.349015873006,"tokens_per_sec_p50":116.58990721484604,"tokens_per_sec_p95":152.3719928063846,"tokens_per_sec_avg":115.7622874419247,"thinking_ratio_p50":0.985823016284874,"thinking_ratio_p95":0.9977184363778732,"thinking_ratio_avg":0.9681478052234872,"total_params":null,"active_params":null,"precision":"unknown","base_model":"qwen3.7-max","reasoning_effort":"high","provider":"alibaba"},{"model_id":"gemini-3-flash-high","display_name":"gemini-3-flash","lab":"google","prompt":"bare","pass_rate":0.7183544303797469,"ci_lower":0.6663663250011351,"ci_upper":0.7650972494682274,"total_runs":316,"cost_p50":0.01066575,"cost_p95":0.076029875,"cost_avg":0.025647835443037968,"latency_p50":29754.5,"latency_p95":163833.75,"latency_avg":59036.82278481012,"ttft_p50":20053.825,"ttft_p95":142276.085,"ttft_avg":48626.412689873454,"tokens_per_sec_p50":142.98488965244633,"tokens_per_sec_p95":197.93246219863022,"tokens_per_sec_avg":130.8642417502783,"thinking_ratio_p50":0.9801192057168153,"thinking_ratio_p95":0.9991285516029635,"thinking_ratio_avg":0.9327059298728027,"total_params":null,"active_params":null,"precision":"bf16","base_model":"gemini-3-flash","reasoning_effort":"high","provider":"google-vertex"},{"model_id":"gpt-6-luna-max","display_name":"gpt-6-luna","lab":"openai","prompt":"bare","pass_rate":0.7120253164556962,"ci_lower":0.659786914114778,"ci_upper":0.7591704697902552,"total_runs":316,"cost_p50":0.00119155,"cost_p95":0.005868175,"cost_avg":0.0018485129746835436,"latency_p50":42935,"latency_p95":223537,"latency_avg":70020.58227848102,"ttft_p50":41918.515,"ttft_p95":222287.92250000002,"ttft_avg":68846.28069620254,"tokens_per_sec_p50":51.350839522379175,"tokens_per_sec_p95":55.10875939341332,"tokens_per_sec_avg":45.92564585660005,"thinking_ratio_p50":0.9815441382056649,"thinking_ratio_p95":0.9970490131071691,"thinking_ratio_avg":0.9556569634687465,"total_params":null,"active_params":null,"precision":"unknown","base_model":"gpt-6-luna","reasoning_effort":"max","provider":"openai"},{"model_id":"deepseek-v4.1-flash-high","display_name":"deepseek-v4.1-flash","lab":"deepseek","prompt":"bare","pass_rate":0.7056962025316456,"ci_lower":0.6532191370456515,"ci_upper":0.7532320562950522,"total_runs":316,"cost_p50":0.007375680000000001,"cost_p95":0.03230345625,"cost_avg":0.010354115553797467,"latency_p50":37374,"latency_p95":162880.25,"latency_avg":60965.45886075949,"ttft_p50":37187.74,"ttft_p95":159325.06,"ttft_avg":50682.71414556962,"tokens_per_sec_p50":241.90843698477673,"tokens_per_sec_p95":280.04269779848124,"tokens_per_sec_avg":214.2538505148671,"thinking_ratio_p50":0.9973748719469426,"thinking_ratio_p95":0.9995316505039884,"thinking_ratio_avg":0.9823774249912833,"total_params":763.2,"active_params":null,"precision":"fp8","base_model":"deepseek-v4.1-flash","reasoning_effort":"high","provider":"novita"},{"model_id":"qwen3.8-2.4t-a95b-high","display_name":"qwen3.8-2.4t-a95b","lab":"qwen","prompt":"bare","pass_rate":0.7025316455696202,"ci_lower":0.6499395424517582,"ci_upper":0.7502585556067808,"total_runs":316,"cost_p50":0.018291,"cost_p95":0.103578,"cost_avg":0.02885308227848101,"latency_p50":83184.5,"latency_p95":298460.25,"latency_avg":115889.31962025317,"ttft_p50":68521.3,"ttft_p95":288895.0675,"ttft_avg":105266.7817721519,"tokens_per_sec_p50":39.12185682501777,"tokens_per_sec_p95":65.53677481424079,"tokens_per_sec_avg":37.69845331769118,"thinking_ratio_p50":0.6625376704958899,"thinking_ratio_p95":1.064795008912656,"thinking_ratio_avg":0.6901779520365139,"total_params":2446.2,"active_params":95,"precision":"unknown","base_model":"qwen3.8-2.4t-a95b","reasoning_effort":"high","provider":"fireworks"},{"model_id":"claude-sonnet-4.5-high","display_name":"claude-sonnet-4.5","lab":"anthropic","prompt":"bare","pass_rate":0.7025316455696202,"ci_lower":0.6499395424517582,"ci_upper":0.7502585556067808,"total_runs":316,"cost_p50":0.065715,"cost_p95":0.25546199999999997,"cost_avg":0.08555206329113926,"latency_p50":60743,"latency_p95":233876,"latency_avg":81909.76265822785,"ttft_p50":53260.455,"ttft_p95":221347.24,"ttft_avg":76054.71414556965,"tokens_per_sec_p50":65.6981721423805,"tokens_per_sec_p95":82.58706821597431,"tokens_per_sec_avg":61.65861771310015,"thinking_ratio_p50":null,"thinking_ratio_p95":null,"thinking_ratio_avg":null,"total_params":null,"active_params":null,"precision":"unknown","base_model":"claude-sonnet-4.5","reasoning_effort":"high","provider":"anthropic"},{"model_id":"gpt-5.4-mini-xhigh","display_name":"gpt-5.4-mini","lab":"openai","prompt":"bare","pass_rate":0.6993670886075949,"ci_lower":0.6466627754885508,"ci_upper":0.7472822272878236,"total_runs":316,"cost_p50":0.113797125,"cost_p95":0.3045105,"cost_avg":0.1199921843354431,"latency_p50":154820,"latency_p95":434192.25,"latency_avg":167087.05063291139,"ttft_p50":153048.07,"ttft_p95":416970.3975,"ttft_avg":161414.98664556965,"tokens_per_sec_p50":165.25722820548606,"tokens_per_sec_p95":166.23839267861865,"tokens_per_sec_avg":151.316469386707,"thinking_ratio_p50":0.998874139278165,"thinking_ratio_p95":0.9996221827247681,"thinking_ratio_avg":0.9893114590787517,"total_params":null,"active_params":null,"precision":"unknown","base_model":"gpt-5.4-mini","reasoning_effort":"xhigh","provider":"openai"},{"model_id":"deepseek-v4-pro-0813-high","display_name":"deepseek-v4-pro-0813","lab":"deepseek","prompt":"bare","pass_rate":0.689873417721519,"ci_lower":0.6368491916221196,"ci_upper":0.7383365253077607,"total_runs":316,"cost_p50":0.08082755999999999,"cost_p95":0.19555239,"cost_avg":0.07906700999999998,"latency_p50":462933,"latency_p95":1194765.75,"latency_avg":473341.003164557,"ttft_p50":462343.06,"ttft_p95":1121247.375,"ttft_avg":451728.85686708847,"tokens_per_sec_p50":42.67304336449919,"tokens_per_sec_p95":49.20620990427234,"tokens_per_sec_avg":40.11314446889114,"thinking_ratio_p50":0.8812817211337542,"thinking_ratio_p95":1.0328866195236106,"thinking_ratio_avg":0.8831505407517879,"total_params":1650.5,"active_params":null,"precision":"unknown","base_model":"deepseek-v4-pro-0813","reasoning_effort":"high","provider":"cloudflare"},{"model_id":"kimi-k2.6-high","display_name":"kimi-k2.6","lab":"moonshotai","prompt":"bare","pass_rate":0.689873417721519,"ci_lower":0.6368491916221196,"ci_upper":0.7383365253077607,"total_runs":316,"cost_p50":0.055232149999999994,"cost_p95":0.2337858375,"cost_avg":0.07423945205696197,"latency_p50":170416.5,"latency_p95":921190.75,"latency_avg":272046.11392405065,"ttft_p50":170239.51,"ttft_p95":920335.2025,"ttft_avg":271079.8064873417,"tokens_per_sec_p50":66.66183304893708,"tokens_per_sec_p95":110.57898886156974,"tokens_per_sec_avg":72.656111535823,"thinking_ratio_p50":0.9792872080259007,"thinking_ratio_p95":1.1592594748326301,"thinking_ratio_avg":0.9736030949607115,"total_params":1026.9,"active_params":32,"precision":"int4","base_model":"kimi-k2.6","reasoning_effort":"high","provider":"moonshotai"},{"model_id":"claude-sonnet-5-low","display_name":"claude-sonnet-5","lab":"anthropic","prompt":"bare","pass_rate":0.6835443037974683,"ci_lower":0.6303204973952294,"ci_upper":0.7323590289703215,"total_runs":316,"cost_p50":0.005269,"cost_p95":0.0968485,"cost_avg":0.019711234177215188,"latency_p50":6890.5,"latency_p95":87190.75,"latency_avg":18497.23417721519,"ttft_p50":4622.815,"ttft_p95":84143.99,"ttft_avg":16299.796139240503,"tokens_per_sec_p50":56.72888962389739,"tokens_per_sec_p95":112.31915788396094,"tokens_per_sec_avg":55.610826425671185,"thinking_ratio_p50":null,"thinking_ratio_p95":null,"thinking_ratio_avg":null,"total_params":null,"active_params":null,"precision":"unknown","base_model":"claude-sonnet-5","reasoning_effort":"low","provider":"anthropic"},{"model_id":"gpt-5.6-terra-low","display_name":"gpt-5.6-terra","lab":"openai","prompt":"bare","pass_rate":0.680379746835443,"ci_lower":0.6270601917203724,"ci_upper":0.7293662393630138,"total_runs":316,"cost_p50":0.006507,"cost_p95":0.033365,"cost_avg":0.010762272151898731,"latency_p50":9941.5,"latency_p95":51707.25,"latency_avg":16573.48734177215,"ttft_p50":9205.085,"ttft_p95":47971.2125,"ttft_avg":15536.147151898736,"tokens_per_sec_p50":47.3039899176445,"tokens_per_sec_p95":54.21452452218432,"tokens_per_sec_avg":42.34131908971324,"thinking_ratio_p50":0.945054945054945,"thinking_ratio_p95":0.989234194598349,"thinking_ratio_avg":0.893856567867262,"total_params":null,"active_params":null,"precision":"unknown","base_model":"gpt-5.6-terra","reasoning_effort":"low","provider":"openai"},{"model_id":"claude-opus-4.1-32k","display_name":"claude-opus-4.1","lab":"anthropic","prompt":"bare","pass_rate":0.6708860759493671,"ci_lower":0.6172951899864231,"ci_upper":0.7203719552504693,"total_runs":316,"cost_p50":0.13368750000000001,"cost_p95":1.08144375,"cost_avg":0.24962530063291155,"latency_p50":35785,"latency_p95":300315.75,"latency_avg":73408.76898734177,"ttft_p50":28494.35,"ttft_p95":270686.315,"ttft_avg":59061.12794303795,"tokens_per_sec_p50":45.75755691190537,"tokens_per_sec_p95":58.512598060715064,"tokens_per_sec_avg":44.16025273385055,"thinking_ratio_p50":null,"thinking_ratio_p95":null,"thinking_ratio_avg":null,"total_params":null,"active_params":null,"precision":"unknown","base_model":"claude-opus-4.1","reasoning_effort":"high","provider":"anthropic"},{"model_id":"claude-opus-4.1-16k","display_name":"claude-opus-4.1","lab":"anthropic","prompt":"bare","pass_rate":0.6550632911392406,"ci_lower":0.6010720858189762,"ci_upper":0.7053295830070928,"total_runs":316,"cost_p50":0.12729000000000001,"cost_p95":0.67681875,"cost_avg":0.19794920886075965,"latency_p50":33242,"latency_p95":193391.5,"latency_avg":52782.01898734177,"ttft_p50":25391.045,"ttft_p95":182067.14750000002,"ttft_avg":46357.72876582278,"tokens_per_sec_p50":44.2290004896642,"tokens_per_sec_p95":58.53061260990294,"tokens_per_sec_avg":43.358117880376824,"thinking_ratio_p50":null,"thinking_ratio_p95":null,"thinking_ratio_avg":null,"total_params":null,"active_params":null,"precision":"unknown","base_model":"claude-opus-4.1","reasoning_effort":"medium","provider":"anthropic"},{"model_id":"kimi-k2.7-code-high","display_name":"kimi-k2.7-code","lab":"moonshotai","prompt":"bare","pass_rate":0.6550632911392406,"ci_lower":0.6010720858189762,"ci_upper":0.7053295830070928,"total_runs":316,"cost_p50":0.022659725,"cost_p95":0.1668853125,"cost_avg":0.04440692041139239,"latency_p50":120743,"latency_p95":915230.75,"latency_avg":240857.93354430379,"ttft_p50":119178.205,"ttft_p95":914728.9650000001,"ttft_avg":239787.48462025318,"tokens_per_sec_p50":44.52779899392587,"tokens_per_sec_p95":53.12899489844036,"tokens_per_sec_avg":43.223139152190825,"thinking_ratio_p50":0.9705235952085814,"thinking_ratio_p95":1.1563550971567638,"thinking_ratio_avg":0.9695193560588219,"total_params":1026.9,"active_params":32,"precision":"int4","base_model":"kimi-k2.7-code","reasoning_effort":"high","provider":"moonshotai"},{"model_id":"kimi-k2.5-high","display_name":"kimi-k2.5","lab":"moonshotai","prompt":"bare","pass_rate":0.6550632911392406,"ci_lower":0.6010720858189762,"ci_upper":0.7053295830070928,"total_runs":316,"cost_p50":0.012142000000000002,"cost_p95":0.038552675,"cost_avg":0.013982657436708854,"latency_p50":124604.5,"latency_p95":388916,"latency_avg":147961.1234177215,"ttft_p50":122663.36499999999,"ttft_p95":378663.4275,"ttft_avg":142732.7935443038,"tokens_per_sec_p50":42.00588253678683,"tokens_per_sec_p95":53.06765310037275,"tokens_per_sec_avg":41.63080214580112,"thinking_ratio_p50":0.9318918189771233,"thinking_ratio_p95":1.1689885977975343,"thinking_ratio_avg":0.946946675191826,"total_params":1026.9,"active_params":32,"precision":"int4","base_model":"kimi-k2.5","reasoning_effort":"high","provider":"moonshotai"},{"model_id":"deepseek-v4-flash-0731","display_name":"deepseek-v4-flash-0731","lab":"deepseek","prompt":"bare","pass_rate":0.6494252873563219,"ci_lower":0.5979088044549136,"ci_upper":0.6976787554015673,"total_runs":348,"cost_p50":0,"cost_p95":0,"cost_avg":0,"latency_p50":550968,"latency_p95":1566844.7999999998,"latency_avg":644758.1810344828,"ttft_p50":535572.35,"ttft_p95":1436842.7974999996,"ttft_avg":592053.7375574713,"tokens_per_sec_p50":36.11205673109487,"tokens_per_sec_p95":44.03524247068201,"tokens_per_sec_avg":36.0642866954212,"thinking_ratio_p50":null,"thinking_ratio_p95":null,"thinking_ratio_avg":null,"total_params":304,"active_params":13,"precision":"fp8-fp4experts","base_model":"deepseek-v4-flash-0731","reasoning_effort":"max","provider":"starflinger"},{"model_id":"gpt-5.4-mini-high","display_name":"gpt-5.4-mini","lab":"openai","prompt":"bare","pass_rate":0.6455696202531646,"ci_lower":0.5913685914921556,"ci_upper":0.6962737914874193,"total_runs":316,"cost_p50":0.020625749999999998,"cost_p95":0.12324824999999999,"cost_avg":0.03358379193037974,"latency_p50":28068.5,"latency_p95":165108,"latency_avg":45546.943037974685,"ttft_p50":27528.239999999998,"ttft_p95":164892.18,"ttft_avg":45279.767531645586,"tokens_per_sec_p50":160.68597963682348,"tokens_per_sec_p95":165.64421894023437,"tokens_per_sec_avg":143.49747857068485,"thinking_ratio_p50":0.9941089798559866,"thinking_ratio_p95":0.9989011008489267,"thinking_ratio_avg":0.9705416201040902,"total_params":null,"active_params":null,"precision":"unknown","base_model":"gpt-5.4-mini","reasoning_effort":"high","provider":"openai"},{"model_id":"deepseek-v4-flash-0731-high","display_name":"deepseek-v4-flash-0731","lab":"deepseek","prompt":"bare","pass_rate":0.6392405063291139,"ci_lower":0.5849119847104416,"ci_upper":0.6902242077048039,"total_runs":316,"cost_p50":0,"cost_p95":0,"cost_avg":0,"latency_p50":229553,"latency_p95":676989.75,"latency_avg":255963.67088607594,"ttft_p50":228635.03,"ttft_p95":604998.4550000001,"ttft_avg":244530.5570886076,"tokens_per_sec_p50":91.82965034778394,"tokens_per_sec_p95":121.36975291352236,"tokens_per_sec_avg":71.18761432888336,"thinking_ratio_p50":null,"thinking_ratio_p95":null,"thinking_ratio_avg":null,"total_params":304,"active_params":13,"precision":"fp8-fp4experts","base_model":"deepseek-v4-flash-0731","reasoning_effort":"high","provider":"starflinger"},{"model_id":"gpt-5.1-high","display_name":"gpt-5.1","lab":"openai","prompt":"bare","pass_rate":0.6392405063291139,"ci_lower":0.5849119847104416,"ci_upper":0.6902242077048039,"total_runs":316,"cost_p50":0.0979425,"cost_p95":0.3402190625,"cost_avg":0.12397265427215193,"latency_p50":68961,"latency_p95":235348,"latency_avg":87064.5,"ttft_p50":68438.23,"ttft_p95":233662.39,"ttft_avg":86534.98515822782,"tokens_per_sec_p50":138.69215286792405,"tokens_per_sec_p95":163.13937497904752,"tokens_per_sec_avg":134.22817737623737,"thinking_ratio_p50":0.997065964011015,"thinking_ratio_p95":0.9992528076370059,"thinking_ratio_avg":0.9794770570849652,"total_params":null,"active_params":null,"precision":"unknown","base_model":"gpt-5.1","reasoning_effort":"high","provider":"openai"},{"model_id":"qwen3.8-flash-high","display_name":"qwen3.8-flash","lab":"qwen","prompt":"bare","pass_rate":0.6360759493670886,"ci_lower":0.58168735461107,"ci_upper":0.6871957425220108,"total_runs":316,"cost_p50":0.01527332,"cost_p95":0.0472828225,"cost_avg":0.018776142468354438,"latency_p50":1701820.5,"latency_p95":6361150,"latency_avg":2264766.009493671,"ttft_p50":409087.09,"ttft_p95":1247085.5450000004,"ttft_avg":486127.57688888896,"tokens_per_sec_p50":20.88861275307491,"tokens_per_sec_p95":89.23504870759359,"tokens_per_sec_avg":29.939605086671264,"thinking_ratio_p50":0.9987153705068514,"thinking_ratio_p95":0.9997962780996703,"thinking_ratio_avg":0.9868159391639036,"total_params":180,"active_params":6,"precision":"unknown","base_model":"qwen3.8-flash","reasoning_effort":"high","provider":"alibaba"},{"model_id":"claude-sonnet-4-5-16k","display_name":"claude-sonnet-4.5","lab":"anthropic","prompt":"bare","pass_rate":0.629746835443038,"ci_lower":0.5752453729437816,"ci_upper":0.6811315336249699,"total_runs":316,"cost_p50":0.044772,"cost_p95":0.1226775,"cost_avg":0.049982079113924036,"latency_p50":41219,"latency_p95":120420.75,"latency_avg":48581.38924050633,"ttft_p50":34834.94,"ttft_p95":111516.68,"ttft_avg":42877.24224683547,"tokens_per_sec_p50":63.45164705743355,"tokens_per_sec_p95":80.75567273607513,"tokens_per_sec_avg":60.091337723810284,"thinking_ratio_p50":null,"thinking_ratio_p95":null,"thinking_ratio_avg":null,"total_params":null,"active_params":null,"precision":"unknown","base_model":"claude-sonnet-4.5","reasoning_effort":"medium","provider":"anthropic"},{"model_id":"gpt-5.3-chat","display_name":"gpt-5.3-chat","lab":"openai","prompt":"bare","pass_rate":0.629746835443038,"ci_lower":0.5752453729437816,"ci_upper":0.6811315336249699,"total_runs":316,"cost_p50":0.011126500000000001,"cost_p95":0.038302249999999996,"cost_avg":0.014777083069620247,"latency_p50":10268.5,"latency_p95":32575,"latency_avg":13065.715189873417,"ttft_p50":8861.154999999999,"ttft_p95":31937.8075,"ttft_avg":11953.27186708861,"tokens_per_sec_p50":71.66607886201018,"tokens_per_sec_p95":89.95974077508455,"tokens_per_sec_avg":70.61165309562033,"thinking_ratio_p50":0.8235927300829389,"thinking_ratio_p95":0.9835021930283221,"thinking_ratio_avg":0.7956320386929653,"total_params":null,"active_params":null,"precision":"unknown","base_model":"gpt-5.3-chat","reasoning_effort":null,"provider":"openai"},{"model_id":"kimi-k2.5","display_name":"kimi-k2.5","lab":"moonshotai","prompt":"bare","pass_rate":0.6265822784810127,"ci_lower":0.5720279954525296,"ci_upper":0.6780958158340573,"total_runs":316,"cost_p50":0.012104375,"cost_p95":0.03619660000000001,"cost_avg":0.013481342879746835,"latency_p50":165593.5,"latency_p95":857915.25,"latency_avg":246004.80379746837,"ttft_p50":131442.64,"ttft_p95":406969.72250000003,"ttft_avg":149270.94981012653,"tokens_per_sec_p50":37.07349243678054,"tokens_per_sec_p95":43.89407783875036,"tokens_per_sec_avg":32.22559271894478,"thinking_ratio_p50":0.9442937408881682,"thinking_ratio_p95":1.179339793809478,"thinking_ratio_avg":0.9498160378642155,"total_params":1026.9,"active_params":32,"precision":"int4","base_model":"kimi-k2.5","reasoning_effort":null,"provider":"moonshotai"},{"model_id":"gpt-5-high","display_name":"gpt-5","lab":"openai","prompt":"bare","pass_rate":0.6234177215189873,"ci_lower":0.5688130102718261,"ci_upper":0.6750577057325962,"total_runs":316,"cost_p50":0.078971875,"cost_p95":0.23852374999999998,"cost_avg":0.09742186313291139,"latency_p50":52402.5,"latency_p95":158299.75,"latency_avg":63184.09493670886,"ttft_p50":52037.229999999996,"ttft_p95":157956.03499999997,"ttft_avg":62778.13927215189,"tokens_per_sec_p50":148.6969740698092,"tokens_per_sec_p95":172.7653214676271,"tokens_per_sec_avg":148.30231242242624,"thinking_ratio_p50":0.9930708320613628,"thinking_ratio_p95":0.9981599688548344,"thinking_ratio_avg":0.9865736324863373,"total_params":null,"active_params":null,"precision":"unknown","base_model":"gpt-5","reasoning_effort":"high","provider":"openai"},{"model_id":"deepseek-v4-pro-high","display_name":"deepseek-v4-pro","lab":"deepseek","prompt":"bare","pass_rate":0.6139240506329114,"ci_lower":0.5591822912825375,"ci_upper":0.6659291388753908,"total_runs":316,"cost_p50":0.012406200000000001,"cost_p95":0.097623135,"cost_avg":0.028542756265822783,"latency_p50":84974.5,"latency_p95":688296,"latency_avg":201679.05696202532,"ttft_p50":83029.225,"ttft_p95":687594.405,"ttft_avg":200698.3699050632,"tokens_per_sec_p50":37.119696357579414,"tokens_per_sec_p95":48.37673009125413,"tokens_per_sec_avg":37.86438046618708,"thinking_ratio_p50":0.9932567305066099,"thinking_ratio_p95":0.9992870230660428,"thinking_ratio_avg":0.9760697276521344,"total_params":1598.8,"active_params":49,"precision":"unknown","base_model":"deepseek-v4-pro","reasoning_effort":"high","provider":"deepseek"},{"model_id":"glm-5.2-high","display_name":"glm-5.2","lab":"z-ai","prompt":"bare","pass_rate":0.6107594936708861,"ci_lower":0.5559767599860462,"ci_upper":0.6628815748897173,"total_runs":316,"cost_p50":0.0155381,"cost_p95":0.12056595,"cost_avg":0.04062713987341773,"latency_p50":41583.5,"latency_p95":475726.25,"latency_avg":135000.8987341772,"ttft_p50":38078.25,"ttft_p95":310735.8920000002,"ttft_avg":100667.94517460317,"tokens_per_sec_p50":72.61019462317543,"tokens_per_sec_p95":95.73504610589046,"tokens_per_sec_avg":67.87117108203928,"thinking_ratio_p50":0.9547913108220558,"thinking_ratio_p95":0.9983379331019663,"thinking_ratio_avg":0.92251036248776,"total_params":753.3,"active_params":null,"precision":"fp8","base_model":"glm-5.2","reasoning_effort":"high","provider":"z-ai"},{"model_id":"gpt-5.4-mini-medium","display_name":"gpt-5.4-mini","lab":"openai","prompt":"bare","pass_rate":0.6075949367088608,"ci_lower":0.55277356524569,"ci_upper":0.6598316743479089,"total_runs":316,"cost_p50":0.009996,"cost_p95":0.0572754375,"cost_avg":0.016298081487341767,"latency_p50":14171.5,"latency_p95":81685.5,"latency_avg":23146.129746835442,"ttft_p50":13739.565,"ttft_p95":81414.32500000001,"ttft_avg":22858.052246835447,"tokens_per_sec_p50":152.69343501930211,"tokens_per_sec_p95":164.69819294164057,"tokens_per_sec_avg":133.20111034474323,"thinking_ratio_p50":0.9853727508533169,"thinking_ratio_p95":0.9981804162757338,"thinking_ratio_avg":0.9534876575338171,"total_params":null,"active_params":null,"precision":"unknown","base_model":"gpt-5.4-mini","reasoning_effort":"medium","provider":"openai"},{"model_id":"gpt-5.2-codex-low","display_name":"gpt-5.2-codex","lab":"openai","prompt":"bare","pass_rate":0.6075949367088608,"ci_lower":0.55277356524569,"ci_upper":0.6598316743479089,"total_runs":316,"cost_p50":0.00736575,"cost_p95":0.0814016875,"cost_avg":0.0202677579113924,"latency_p50":9884.5,"latency_p95":103822.5,"latency_avg":28045.40506329114,"ttft_p50":9287.54,"ttft_p95":102378.49,"ttft_avg":27411.538892405053,"tokens_per_sec_p50":48.384387226832466,"tokens_per_sec_p95":54.883301756499165,"tokens_per_sec_avg":44.649362456373844,"thinking_ratio_p50":0.9061946902654867,"thinking_ratio_p95":0.9905437695832866,"thinking_ratio_avg":0.858920933768603,"total_params":null,"active_params":null,"precision":"unknown","base_model":"gpt-5.2-codex","reasoning_effort":"low","provider":"openai"},{"model_id":"claude-sonnet-4-high","display_name":"claude-sonnet-4","lab":"anthropic","prompt":"bare","pass_rate":0.6044303797468354,"ci_lower":0.549572697038391,"ci_upper":0.6567794472730433,"total_runs":316,"cost_p50":0.042996000000000006,"cost_p95":0.22508475,"cost_avg":0.0712362246835443,"latency_p50":36366,"latency_p95":203465.5,"latency_avg":65344.29746835443,"ttft_p50":30279,"ttft_p95":197124.55,"ttft_avg":60428.364398734186,"tokens_per_sec_p50":65.8094208290889,"tokens_per_sec_p95":85.93257427109371,"tokens_per_sec_avg":64.68333013265611,"thinking_ratio_p50":null,"thinking_ratio_p95":null,"thinking_ratio_avg":null,"total_params":null,"active_params":null,"precision":"unknown","base_model":"claude-sonnet-4","reasoning_effort":"high","provider":"anthropic"},{"model_id":"glm-5.3-flash-fp8-baseten-high-131k","display_name":"glm-5.3-flash","lab":"z-ai","prompt":"bare","pass_rate":0.5949367088607594,"ci_lower":0.5399839567280832,"ci_upper":0.6476089017368569,"total_runs":316,"cost_p50":0.000999975,"cost_p95":0.026909074999999998,"cost_avg":0.005107151424050632,"latency_p50":22167.5,"latency_p95":1214227,"latency_avg":198240.81645569621,"ttft_p50":17938.375,"ttft_p95":676700.86,"ttft_avg":117546.2704113924,"tokens_per_sec_p50":70.87667264603209,"tokens_per_sec_p95":109.6899645679619,"tokens_per_sec_avg":65.19183184004604,"thinking_ratio_p50":0.8567990373044525,"thinking_ratio_p95":0.990936802944615,"thinking_ratio_avg":0.7977466249584084,"total_params":321.3,"active_params":18,"precision":"fp8","base_model":"glm-5.3-flash","reasoning_effort":"high","provider":"baseten"},{"model_id":"glm-5.3-flash-fp4-local-qad-high","display_name":"glm-5.3-flash","lab":"z-ai","prompt":"bare","pass_rate":0.5886075949367089,"ci_lower":0.5336029279493076,"ci_upper":0.6414837399513033,"total_runs":316,"cost_p50":0,"cost_p95":0,"cost_avg":0,"latency_p50":10546,"latency_p95":565746,"latency_avg":90677.28481012658,"ttft_p50":8174.64,"ttft_p95":365899.33499999996,"ttft_avg":68355.36012658228,"tokens_per_sec_p50":127.93904582460793,"tokens_per_sec_p95":148.18933563147158,"tokens_per_sec_avg":113.9183138986381,"thinking_ratio_p50":0.8374094202898551,"thinking_ratio_p95":0.9911062402387757,"thinking_ratio_avg":0.7859314099946407,"total_params":320,"active_params":18,"precision":"nvidia-fp4-local-qad","base_model":"glm-5.3-flash","reasoning_effort":"high","provider":"starflinger"},{"model_id":"gpt-5.6-luna-low","display_name":"gpt-5.6-luna","lab":"openai","prompt":"bare","pass_rate":0.5791139240506329,"ci_lower":0.5240484193005199,"ci_upper":0.6322789627535967,"total_runs":316,"cost_p50":0.0009705,"cost_p95":0.0025373999999999996,"cost_avg":0.0010927006329113915,"latency_p50":6883,"latency_p95":15866.75,"latency_avg":7526.085443037975,"ttft_p50":6818.745000000001,"ttft_p95":15725.904999999999,"ttft_avg":7414.336613924051,"tokens_per_sec_p50":105.03189801852446,"tokens_per_sec_p95":138.18164948088972,"tokens_per_sec_avg":94.71355800496288,"thinking_ratio_p50":0.9618604651162791,"thinking_ratio_p95":0.988176122520014,"thinking_ratio_avg":0.9249868311952778,"total_params":null,"active_params":null,"precision":"unknown","base_model":"gpt-5.6-luna","reasoning_effort":"low","provider":"openai"},{"model_id":"claude-sonnet-4-16k","display_name":"claude-sonnet-4","lab":"anthropic","prompt":"bare","pass_rate":0.5791139240506329,"ci_lower":0.5240484193005199,"ci_upper":0.6322789627535967,"total_runs":316,"cost_p50":0.034505999999999995,"cost_p95":0.152016,"cost_avg":0.05096568037974684,"latency_p50":32882,"latency_p95":136576,"latency_avg":46652.623417721516,"ttft_p50":28223.239999999998,"ttft_p95":132120.875,"ttft_avg":41939.6681329114,"tokens_per_sec_p50":66.49478707803814,"tokens_per_sec_p95":84.53954194519616,"tokens_per_sec_avg":64.34566838126554,"thinking_ratio_p50":null,"thinking_ratio_p95":null,"thinking_ratio_avg":null,"total_params":null,"active_params":null,"precision":"unknown","base_model":"claude-sonnet-4","reasoning_effort":"medium","provider":"anthropic"},{"model_id":"qwen3.8-flash-xhigh","display_name":"qwen3.8-flash","lab":"qwen","prompt":"bare","pass_rate":0.5759493670886076,"ci_lower":0.5208680960090742,"ci_upper":0.6292061907628779,"total_runs":316,"cost_p50":0.016192825,"cost_p95":0.050719787499999995,"cost_avg":0.018156554556962027,"latency_p50":3381911,"latency_p95":11669388,"latency_avg":5308219.028481012,"ttft_p50":392647.745,"ttft_p95":1155483.53,"ttft_avg":426089.25028481014,"tokens_per_sec_p50":6.731379061596312,"tokens_per_sec_p95":86.84743192534955,"tokens_per_sec_avg":20.61594344123458,"thinking_ratio_p50":0.9987178758271349,"thinking_ratio_p95":0.9997599144821592,"thinking_ratio_avg":0.9862039476931296,"total_params":180,"active_params":6,"precision":"unknown","base_model":"qwen3.8-flash","reasoning_effort":"xhigh","provider":"alibaba"},{"model_id":"hunter-alpha-high","display_name":"hunter-alpha","lab":"openrouter","prompt":"bare","pass_rate":0.5759493670886076,"ci_lower":0.5208680960090742,"ci_upper":0.6292061907628779,"total_runs":316,"cost_p50":0,"cost_p95":0,"cost_avg":0,"latency_p50":127865,"latency_p95":353246.75,"latency_avg":148525.00316455695,"ttft_p50":126924.495,"ttft_p95":318320.915,"ttft_avg":134076.4568670886,"tokens_per_sec_p50":54.58751305480043,"tokens_per_sec_p95":76.76960383457231,"tokens_per_sec_avg":48.74078033236604,"thinking_ratio_p50":0.9203750088824716,"thinking_ratio_p95":1.1645032999419387,"thinking_ratio_avg":0.9010362135334203,"total_params":1,"active_params":null,"precision":"unknown","base_model":"hunter-alpha","reasoning_effort":"high","provider":"stealth"},{"model_id":"qwen3.6-plus-preview-high","display_name":"qwen3.6-plus-preview","lab":"qwen","prompt":"bare","pass_rate":0.569620253164557,"ci_lower":0.5145141781447586,"ci_upper":0.6230539180628643,"total_runs":316,"cost_p50":0,"cost_p95":0,"cost_avg":0,"latency_p50":57114.5,"latency_p95":137891.25,"latency_avg":63350.49367088608,"ttft_p50":47056.119999999995,"ttft_p95":120674.905,"ttft_avg":51926.85287974684,"tokens_per_sec_p50":107.73153884760791,"tokens_per_sec_p95":126.59523575204976,"tokens_per_sec_avg":97.02227596102509,"thinking_ratio_p50":0.993732150082694,"thinking_ratio_p95":0.9986605895951033,"thinking_ratio_avg":0.9820502550034028,"total_params":null,"active_params":null,"precision":"unknown","base_model":"qwen3.6-plus-preview","reasoning_effort":"high","provider":"alibaba"},{"model_id":"ox-alpha-high","display_name":"ox-alpha","lab":"stealth","prompt":"bare","pass_rate":0.5664556962025317,"ci_lower":0.511340571082102,"ci_upper":0.6199744298433563,"total_runs":316,"cost_p50":0,"cost_p95":0,"cost_avg":0,"latency_p50":25477,"latency_p95":290627.75,"latency_avg":76797.37341772152,"ttft_p50":9206.045,"ttft_p95":265071.8725,"ttft_avg":52555.704082278484,"tokens_per_sec_p50":49.92511303493626,"tokens_per_sec_p95":75.87124375305945,"tokens_per_sec_avg":43.71234930003081,"thinking_ratio_p50":null,"thinking_ratio_p95":null,"thinking_ratio_avg":null,"total_params":null,"active_params":null,"precision":"unknown","base_model":"ox-alpha","reasoning_effort":"high","provider":"stealth"},{"model_id":"grok-4.3-high","display_name":"grok-4.3","lab":"x-ai","prompt":"bare","pass_rate":0.5632911392405063,"ci_lower":0.5081691907696114,"ci_upper":0.616892714873682,"total_runs":316,"cost_p50":0.020640625000000003,"cost_p95":0.0712665625,"cost_avg":0.04075189082278481,"latency_p50":104145,"latency_p95":369198.75,"latency_avg":131352.58544303797,"ttft_p50":103439.36499999999,"ttft_p95":337787.915,"ttft_avg":124496.53417721517,"tokens_per_sec_p50":82.07419866145543,"tokens_per_sec_p95":114.72216109067244,"tokens_per_sec_avg":96.1815676987436,"thinking_ratio_p50":0.9969604231386633,"thinking_ratio_p95":0.9995591933836812,"thinking_ratio_avg":0.9858556695380374,"total_params":null,"active_params":null,"precision":"unknown","base_model":"grok-4.3","reasoning_effort":"high","provider":"xai"},{"model_id":"grok-4.20-beta-high","display_name":"grok-4.20-beta","lab":"x-ai","prompt":"bare","pass_rate":0.5632911392405063,"ci_lower":0.5081691907696114,"ci_upper":0.616892714873682,"total_runs":316,"cost_p50":0.013052500000000002,"cost_p95":0.0497675,"cost_avg":0.016590727848101262,"latency_p50":25072.5,"latency_p95":111847,"latency_avg":33065.08544303798,"ttft_p50":24849.565000000002,"ttft_p95":110801.70000000001,"ttft_avg":32589.249841772154,"tokens_per_sec_p50":199.94918989130917,"tokens_per_sec_p95":237.23297478634274,"tokens_per_sec_avg":191.72661777113856,"thinking_ratio_p50":0.9949485767795279,"thinking_ratio_p95":0.9993663767902609,"thinking_ratio_avg":0.9696676500621543,"total_params":null,"active_params":null,"precision":"unknown","base_model":"grok-4.20-beta","reasoning_effort":"high","provider":"xai"},{"model_id":"glm-5-high","display_name":"glm-5","lab":"z-ai","prompt":"bare","pass_rate":0.560126582278481,"ci_lower":0.5050000317023345,"ci_upper":0.6138087786587942,"total_runs":316,"cost_p50":0.0237693,"cost_p95":0.07372235,"cost_avg":0.02785657658227849,"latency_p50":200202,"latency_p95":668962.25,"latency_avg":242473.71518987342,"ttft_p50":188867.97,"ttft_p95":639802.15,"ttft_avg":229880.85797468355,"tokens_per_sec_p50":34.35535830574456,"tokens_per_sec_p95":46.46238202082351,"tokens_per_sec_avg":35.135742387126086,"thinking_ratio_p50":0.9083518429183743,"thinking_ratio_p95":1.1250453906234525,"thinking_ratio_avg":0.9069406408669302,"total_params":753.9,"active_params":null,"precision":"unknown","base_model":"glm-5","reasoning_effort":"high","provider":"z-ai"},{"model_id":"mistral-large-4-0-high","display_name":"mistral-large-4-0","lab":"mistral","prompt":"bare","pass_rate":0.560126582278481,"ci_lower":0.5050000317023345,"ci_upper":0.6138087786587942,"total_runs":316,"cost_p50":0.017342865,"cost_p95":0.05823100999999999,"cost_avg":0.022758730854430375,"latency_p50":102837,"latency_p95":523499.5,"latency_avg":180561.8575949367,"ttft_p50":101635.51000000001,"ttft_p95":397941.9625,"ttft_avg":142755.00575949365,"tokens_per_sec_p50":76.9064424921204,"tokens_per_sec_p95":108.11539644761024,"tokens_per_sec_avg":76.58239625091485,"thinking_ratio_p50":0.9150274130256673,"thinking_ratio_p95":1.0734119980780483,"thinking_ratio_avg":0.8927094468088597,"total_params":null,"active_params":null,"precision":"unknown","base_model":"mistral-large-4-0","reasoning_effort":"high","provider":"mistral"},{"model_id":"glm-5.3-high","display_name":"glm-5.3","lab":"z-ai","prompt":"bare","pass_rate":0.5569620253164557,"ci_lower":0.5018330886648706,"ci_upper":0.6107226264140934,"total_runs":316,"cost_p50":0.006520600000000001,"cost_p95":0.19494715,"cost_avg":0.03669625506329114,"latency_p50":23254,"latency_p95":938040,"latency_avg":176220.03797468354,"ttft_p50":19086.915,"ttft_p95":759036.615,"ttft_avg":128612.95518987344,"tokens_per_sec_p50":48.66447710619904,"tokens_per_sec_p95":88.05507879514852,"tokens_per_sec_avg":46.95134990894264,"thinking_ratio_p50":0.7933419801124081,"thinking_ratio_p95":0.9893116061064021,"thinking_ratio_avg":0.7478821476712558,"total_params":753.3,"active_params":null,"precision":"fp8","base_model":"glm-5.3","reasoning_effort":"high","provider":"z-ai"},{"model_id":"mimo-v2.5-pro-high","display_name":"mimo-v2.5-pro","lab":"xiaomi","prompt":"bare","pass_rate":0.5506329113924051,"ci_lower":0.4955058312461932,"ci_upper":0.6045436932684417,"total_runs":316,"cost_p50":0.0040470225,"cost_p95":0.022291466249999996,"cost_avg":0.006564050886075948,"latency_p50":70366.5,"latency_p95":497330,"latency_avg":150275.88924050634,"ttft_p50":69968.54000000001,"ttft_p95":423686.145,"ttft_avg":115680.91854430381,"tokens_per_sec_p50":57.53821640849424,"tokens_per_sec_p95":68.75974713640619,"tokens_per_sec_avg":55.366198149268264,"thinking_ratio_p50":0.9883695469305731,"thinking_ratio_p95":0.9988926640844762,"thinking_ratio_avg":0.9498388106130405,"total_params":1023.2,"active_params":null,"precision":"fp8","base_model":"mimo-v2.5-pro","reasoning_effort":"high","provider":"xiaomi"},{"model_id":"deepseek-v4-flash","display_name":"deepseek-v4-flash","lab":"deepseek","prompt":"bare","pass_rate":0.5474683544303798,"ci_lower":0.49234550785359626,"ci_upper":0.6014509213788739,"total_runs":316,"cost_p50":0,"cost_p95":0,"cost_avg":0,"latency_p50":116297,"latency_p95":634391.25,"latency_avg":211753.19620253163,"ttft_p50":113707.41500000001,"ttft_p95":517444.3719999997,"ttft_avg":178754.2261464968,"tokens_per_sec_p50":86.4043804284972,"tokens_per_sec_p95":93.38332679912622,"tokens_per_sec_avg":72.0993976983024,"thinking_ratio_p50":null,"thinking_ratio_p95":null,"thinking_ratio_avg":null,"total_params":null,"active_params":null,"precision":"unknown","base_model":"deepseek-v4-flash","reasoning_effort":"max","provider":"starflinger"},{"model_id":"minimax-m3-high","display_name":"minimax-m3","lab":"minimax","prompt":"bare","pass_rate":0.5443037974683544,"ci_lower":0.4891873824624862,"ci_upper":0.5983559514878193,"total_runs":316,"cost_p50":0.008330849999999999,"cost_p95":0.05910015,"cost_avg":0.016495246518987337,"latency_p50":144474.5,"latency_p95":1331942.75,"latency_avg":343505.85126582277,"ttft_p50":129813.63500000001,"ttft_p95":814835.9969999995,"ttft_avg":230948.52573248392,"tokens_per_sec_p50":47.4631159972227,"tokens_per_sec_p95":60.95101210259081,"tokens_per_sec_avg":43.954137680156315,"thinking_ratio_p50":0.8826967083259989,"thinking_ratio_p95":1.130212945393211,"thinking_ratio_avg":0.8691493621533707,"total_params":427,"active_params":null,"precision":"fp8","base_model":"minimax-m3","reasoning_effort":"high","provider":"minimax"},{"model_id":"glm-5.3-flash-nvfp4-sglang-h-high-65k","display_name":"glm-5.3-flash","lab":"z-ai","prompt":"bare","pass_rate":0.5443037974683544,"ci_lower":0.4891873824624862,"ci_upper":0.5983559514878193,"total_runs":316,"cost_p50":0,"cost_p95":0,"cost_avg":0,"latency_p50":11558.5,"latency_p95":510881.5,"latency_avg":96061.71518987342,"ttft_p50":8583.08,"ttft_p95":289756.8999999996,"ttft_avg":48350.547028753994,"tokens_per_sec_p50":131.75471638440322,"tokens_per_sec_p95":157.7832274939489,"tokens_per_sec_avg":120.83975202702982,"thinking_ratio_p50":null,"thinking_ratio_p95":null,"thinking_ratio_avg":null,"total_params":320,"active_params":18,"precision":"nvfp4-w4a16-k32","base_model":"glm-5.3-flash","reasoning_effort":"high","provider":"starflinger"},{"model_id":"claude-3.7-sonnet-thinking","display_name":"claude-3.7-sonnet","lab":"anthropic","prompt":"bare","pass_rate":0.5411392405063291,"ci_lower":0.48603145126019415,"ci_upper":0.5952587874079466,"total_runs":316,"cost_p50":0.12642150000000002,"cost_p95":0.33938399999999996,"cost_avg":0.1407052215189874,"latency_p50":106627,"latency_p95":288130.25,"latency_avg":121906.71518987342,"ttft_p50":102473.04000000001,"ttft_p95":281265.6225,"ttft_avg":117295.13107594938,"tokens_per_sec_p50":73.57839352818203,"tokens_per_sec_p95":84.65426316728691,"tokens_per_sec_avg":69.26215638873491,"thinking_ratio_p50":0.844275333451175,"thinking_ratio_p95":1.05046303774296,"thinking_ratio_avg":0.8434155188181026,"total_params":null,"active_params":null,"precision":"unknown","base_model":"claude-3.7-sonnet","reasoning_effort":"high","provider":"google-vertex"},{"model_id":"deepseek-v4-flash-0731-low","display_name":"deepseek-v4-flash-0731","lab":"deepseek","prompt":"bare","pass_rate":0.5348101265822784,"ci_lower":0.4797261575334397,"ci_upper":0.5890578905703715,"total_runs":316,"cost_p50":0,"cost_p95":0,"cost_avg":0,"latency_p50":221774.5,"latency_p95":378253.25,"latency_avg":212296.47151898735,"ttft_p50":221555.075,"ttft_p95":365696.2025,"ttft_avg":208395.05193037968,"tokens_per_sec_p50":21.20678793373154,"tokens_per_sec_p95":112.27720803748403,"tokens_per_sec_avg":28.741631062079612,"thinking_ratio_p50":null,"thinking_ratio_p95":null,"thinking_ratio_avg":null,"total_params":304,"active_params":13,"precision":"fp8-fp4experts","base_model":"deepseek-v4-flash-0731","reasoning_effort":"low","provider":"starflinger"},{"model_id":"qwen3.5-397b-a17b","display_name":"qwen3.5-397b-a17b","lab":"qwen","prompt":"bare","pass_rate":0.5284810126582279,"ci_lower":0.47342960159124187,"ci_upper":0.5828482559482402,"total_runs":316,"cost_p50":0.015568799999999999,"cost_p95":0.03533439,"cost_avg":0.016885327689873414,"latency_p50":84260.5,"latency_p95":229714.25,"latency_avg":95172.68987341772,"ttft_p50":82675.91500000001,"ttft_p95":195490.345,"ttft_avg":90615.55477848096,"tokens_per_sec_p50":78.3116766000439,"tokens_per_sec_p95":90.83605521428474,"tokens_per_sec_avg":76.25879904059806,"thinking_ratio_p50":0.9951178819746367,"thinking_ratio_p95":0.998831169149558,"thinking_ratio_avg":0.9837325211720405,"total_params":403.4,"active_params":17,"precision":"unknown","base_model":"qwen3.5-397b-a17b","reasoning_effort":null,"provider":"alibaba"},{"model_id":"glm-5.3-flash-nvfp4-qad-kraken-high-131k","display_name":"glm-5.3-flash","lab":"z-ai","prompt":"bare","pass_rate":0.5221518987341772,"ci_lower":0.4671417626231299,"ci_upper":0.5766299043520229,"total_runs":316,"cost_p50":0,"cost_p95":0,"cost_avg":0,"latency_p50":9809,"latency_p95":200640.5,"latency_avg":53128.95253164557,"ttft_p50":7397.915000000001,"ttft_p95":191940.77000000002,"ttft_avg":40384.65363924051,"tokens_per_sec_p50":130.11203885094326,"tokens_per_sec_p95":152.9142547874464,"tokens_per_sec_avg":116.73695090378045,"thinking_ratio_p50":0.8075539568345323,"thinking_ratio_p95":0.9815487738512818,"thinking_ratio_avg":0.7572464888080247,"total_params":320,"active_params":18,"precision":"nvfp4-csf-mxfp8-qad","base_model":"glm-5.3-flash","reasoning_effort":"high","provider":"starflinger"},{"model_id":"deepseek-v4-flash-high","display_name":"deepseek-v4-flash","lab":"deepseek","prompt":"bare","pass_rate":0.5158227848101266,"ci_lower":0.4608626240401377,"ci_upper":0.5704028523706857,"total_runs":316,"cost_p50":0.00097503,"cost_p95":0.004824365000000001,"cost_avg":0.0014183820886075954,"latency_p50":42112,"latency_p95":216494,"latency_avg":63266.63291139241,"ttft_p50":40690.61,"ttft_p95":216116.58750000002,"ttft_avg":62683.16003164557,"tokens_per_sec_p50":73.27080060293042,"tokens_per_sec_p95":81.91077939990377,"tokens_per_sec_avg":70.79512744739523,"thinking_ratio_p50":0.9928560998292433,"thinking_ratio_p95":0.9989993661505217,"thinking_ratio_avg":0.9658225611950161,"total_params":290.9,"active_params":13,"precision":"unknown","base_model":"deepseek-v4-flash","reasoning_effort":"high","provider":"deepseek"},{"model_id":"gemini-3-flash","display_name":"gemini-3-flash","lab":"google","prompt":"bare","pass_rate":0.5158227848101266,"ci_lower":0.4608626240401377,"ci_upper":0.5704028523706857,"total_runs":316,"cost_p50":0.00129425,"cost_p95":0.00405,"cost_avg":0.001535639240506328,"latency_p50":4596,"latency_p95":8715.75,"latency_avg":5371.655063291139,"ttft_p50":2281.71,"ttft_p95":2992.9275000000002,"ttft_avg":2300.81411392405,"tokens_per_sec_p50":79.79102054534933,"tokens_per_sec_p95":133.96697243845253,"tokens_per_sec_avg":64.82688404493655,"thinking_ratio_p50":null,"thinking_ratio_p95":null,"thinking_ratio_avg":null,"total_params":null,"active_params":null,"precision":"bf16","base_model":"gemini-3-flash","reasoning_effort":null,"provider":"google-vertex"},{"model_id":"gpt-5.4-mini-low","display_name":"gpt-5.4-mini","lab":"openai","prompt":"bare","pass_rate":0.5126582278481012,"ci_lower":0.45772631335333386,"ci_upper":0.5672860677753249,"total_runs":316,"cost_p50":0.003463125,"cost_p95":0.0129178125,"cost_avg":0.004448143196202532,"latency_p50":5367.5,"latency_p95":17186,"latency_avg":6687.224683544304,"ttft_p50":5135.495,"ttft_p95":16882.6,"ttft_avg":6400.098037974685,"tokens_per_sec_p50":130.6525286443964,"tokens_per_sec_p95":156.2859006574242,"tokens_per_sec_avg":115.377328642887,"thinking_ratio_p50":0.9593523028283346,"thinking_ratio_p95":0.9921815957375038,"thinking_ratio_avg":0.9052806822183259,"total_params":null,"active_params":null,"precision":"unknown","base_model":"gpt-5.4-mini","reasoning_effort":"low","provider":"openai"},{"model_id":"deepseek-r1","display_name":"deepseek-r1","lab":"deepseek","prompt":"bare","pass_rate":0.5063291139240507,"ci_lower":0.45146020338289966,"ci_upper":0.5610459871814297,"total_runs":316,"cost_p50":0.008407149999999999,"cost_p95":0.048846725,"cost_avg":0.013855443354430385,"latency_p50":112822,"latency_p95":689587.5,"latency_avg":193896.48734177215,"ttft_p50":109547.23999999999,"ttft_p95":676958.0625,"ttft_avg":186444.2133227848,"tokens_per_sec_p50":27.768632648551566,"tokens_per_sec_p95":29.24372486810254,"tokens_per_sec_avg":26.937030837685793,"thinking_ratio_p50":0.9890430153426132,"thinking_ratio_p95":0.998768955240118,"thinking_ratio_avg":0.9416172374821761,"total_params":684.5,"active_params":37,"precision":"fp8","base_model":"deepseek-r1","reasoning_effort":"high","provider":"novita"},{"model_id":"inkling-high","display_name":"inkling","lab":"thinkingmachines","prompt":"bare","pass_rate":0.5,"ci_lower":0.4452027705582164,"ci_upper":0.5547972294417836,"total_runs":316,"cost_p50":0.055659475,"cost_p95":0.1322298125,"cost_avg":0.057780453481012635,"latency_p50":371855.5,"latency_p95":860846.25,"latency_avg":400273.7594936709,"ttft_p50":213342.82,"ttft_p95":470669.8825,"ttft_avg":219933.025443038,"tokens_per_sec_p50":36.30847652482271,"tokens_per_sec_p95":74.35225190686552,"tokens_per_sec_avg":39.97317744986921,"thinking_ratio_p50":0.9981384837543964,"thinking_ratio_p95":0.9993961714911586,"thinking_ratio_avg":0.9823842347098551,"total_params":952.4,"active_params":41,"precision":"unknown","base_model":"inkling","reasoning_effort":"high","provider":"together"},{"model_id":"gemini-3.1-flash-lite-preview-high","display_name":"gemini-3.1-flash-lite-preview","lab":"google","prompt":"bare","pass_rate":0.4936708860759494,"ci_lower":0.4389540128185704,"ci_upper":0.5485397966171005,"total_runs":316,"cost_p50":0.0064531250000000005,"cost_p95":0.02827475,"cost_avg":0.009212235759493675,"latency_p50":13888,"latency_p95":45084,"latency_avg":17673.816455696204,"ttft_p50":12532.075,"ttft_p95":43311.134999999995,"ttft_avg":16293.462310126575,"tokens_per_sec_p50":306.1663079103884,"tokens_per_sec_p95":419.299508277183,"tokens_per_sec_avg":282.92323120037673,"thinking_ratio_p50":0.9703235439638112,"thinking_ratio_p95":0.9988314162868641,"thinking_ratio_avg":0.9482750432582767,"total_params":null,"active_params":null,"precision":"unknown","base_model":"gemini-3.1-flash-lite-preview","reasoning_effort":"high","provider":"google-vertex"},{"model_id":"qwen3-235b-a22b-thinking-2507","display_name":"qwen3-235b-a22b-thinking-2507","lab":"qwen","prompt":"bare","pass_rate":0.49050632911392406,"ci_lower":0.4358328875885491,"ci_upper":0.545407826564957,"total_runs":316,"cost_p50":0.018933945,"cost_p95":0.054513795,"cost_avg":0.021272523860759493,"latency_p50":378629,"latency_p95":3143365,"latency_avg":710701.7183544304,"ttft_p50":273422.45999999996,"ttft_p95":817862.305,"ttft_avg":319995.2509810126,"tokens_per_sec_p50":27.749265630675815,"tokens_per_sec_p95":34.907710485640756,"tokens_per_sec_avg":23.907121739836818,"thinking_ratio_p50":0.8895056444650576,"thinking_ratio_p95":1.1471094046833443,"thinking_ratio_avg":0.886944214053273,"total_params":235.1,"active_params":22,"precision":"fp8","base_model":"qwen3-235b-a22b-thinking-2507","reasoning_effort":"high","provider":"deepinfra"},{"model_id":"gpt-5.3-codex-spark-xhigh","display_name":"gpt-5.3-codex-spark","lab":"openai","prompt":"bare","pass_rate":0.4873417721518987,"ci_lower":0.43271393222467525,"ci_upper":0.5422736866466662,"total_runs":316,"cost_p50":0.223524875,"cost_p95":0.8528305625,"cost_avg":0.2952708433544303,"latency_p50":17917,"latency_p95":76334.5,"latency_avg":24769.098101265823,"ttft_p50":17800.13,"ttft_p95":76191.0675,"ttft_avg":24593.966392405073,"tokens_per_sec_p50":896.041670860159,"tokens_per_sec_p95":1064.4231690809802,"tokens_per_sec_avg":808.2348354333743,"thinking_ratio_p50":0.9980936963634566,"thinking_ratio_p95":0.9996106877865343,"thinking_ratio_avg":0.9932833158441519,"total_params":null,"active_params":null,"precision":"unknown","base_model":"gpt-5.3-codex-spark","reasoning_effort":"xhigh","provider":"openai"},{"model_id":"glm-4.7-high","display_name":"glm-4.7","lab":"z-ai","prompt":"bare","pass_rate":0.46835443037974683,"ci_lower":0.4140458359177566,"ci_upper":0.5234232112605968,"total_runs":316,"cost_p50":0.015419700000000001,"cost_p95":0.0480073,"cost_avg":0.01858274556962026,"latency_p50":216454,"latency_p95":813854.25,"latency_avg":297822.88924050634,"ttft_p50":95603.875,"ttft_p95":344830.08999999997,"ttft_avg":128345.04661392413,"tokens_per_sec_p50":41.004927960946105,"tokens_per_sec_p95":83.16798284656521,"tokens_per_sec_avg":42.55117282357176,"thinking_ratio_p50":0.9951682737286567,"thinking_ratio_p95":0.9993773721545248,"thinking_ratio_avg":0.9773882144984998,"total_params":null,"active_params":null,"precision":"unknown","base_model":"glm-4.7","reasoning_effort":"high","provider":"z-ai"},{"model_id":"minimax-m3.1-flash-xhigh","display_name":"minimax-m3.1-flash","lab":"minimax","prompt":"bare","pass_rate":0.4620253164556962,"ci_lower":0.40784056732091606,"ci_upper":0.517122289293108,"total_runs":316,"cost_p50":0,"cost_p95":0,"cost_avg":0,"latency_p50":49595,"latency_p95":382653.25,"latency_avg":113940.0506329114,"ttft_p50":49029.03999999999,"ttft_p95":373342.6725,"ttft_avg":109561.6301265823,"tokens_per_sec_p50":91.34087879154447,"tokens_per_sec_p95":128.79613481464278,"tokens_per_sec_avg":88.69839127872477,"thinking_ratio_p50":0.995895428391722,"thinking_ratio_p95":0.9994398061272697,"thinking_ratio_avg":0.9816734082930036,"total_params":null,"active_params":null,"precision":"unknown","base_model":"minimax-m3.1-flash","reasoning_effort":"xhigh","provider":"minimax"},{"model_id":"deepseek-r1-0528","display_name":"deepseek-r1-0528","lab":"deepseek","prompt":"bare","pass_rate":0.4620253164556962,"ci_lower":0.40784056732091606,"ci_upper":0.517122289293108,"total_runs":316,"cost_p50":0.009087580000000001,"cost_p95":0.027905025000000003,"cost_avg":0.010704775063291132,"latency_p50":237751.5,"latency_p95":735170.25,"latency_avg":287805.11392405065,"ttft_p50":218686.83500000002,"ttft_p95":706695.8275,"ttft_avg":267326.9250316456,"tokens_per_sec_p50":16.632077033872854,"tokens_per_sec_p95":18.470710048877883,"tokens_per_sec_avg":16.00747294572446,"thinking_ratio_p50":0.9330311204797821,"thinking_ratio_p95":0.9817419685500586,"thinking_ratio_avg":0.8994077986741563,"total_params":684.5,"active_params":37,"precision":"fp8","base_model":"deepseek-r1-0528","reasoning_effort":"high","provider":"siliconflow"},{"model_id":"qwen3-max-thinking","display_name":"qwen3-max-thinking","lab":"qwen","prompt":"bare","pass_rate":0.4588607594936709,"ci_lower":0.4047412125920535,"ci_upper":0.5139685487398059,"total_runs":316,"cost_p50":0.02914626,"cost_p95":0.095667975,"cost_avg":0.03662561335443037,"latency_p50":192630.5,"latency_p95":588943.25,"latency_avg":233138.3829113924,"ttft_p50":185070.685,"ttft_p95":588677.6925,"ttft_avg":228186.92639240503,"tokens_per_sec_p50":38.04055435136256,"tokens_per_sec_p95":45.18720741338798,"tokens_per_sec_avg":37.791221428881,"thinking_ratio_p50":0.9860503016151003,"thinking_ratio_p95":0.9991235180433082,"thinking_ratio_avg":0.968841540957317,"total_params":null,"active_params":null,"precision":"unknown","base_model":"qwen3-max-thinking","reasoning_effort":"high","provider":"alibaba"},{"model_id":"healer-alpha-high","display_name":"healer-alpha","lab":"openrouter","prompt":"bare","pass_rate":0.45569620253164556,"ci_lower":0.40164404851218083,"ci_upper":0.5108126175375138,"total_runs":316,"cost_p50":0,"cost_p95":0,"cost_avg":0,"latency_p50":49565.5,"latency_p95":185344.75,"latency_avg":75460.31329113925,"ttft_p50":49088.28,"ttft_p95":180203.17,"ttft_avg":65082.21047468352,"tokens_per_sec_p50":92.97198060253338,"tokens_per_sec_p95":111.19481745010307,"tokens_per_sec_avg":86.33150818464624,"thinking_ratio_p50":0.9296760447770218,"thinking_ratio_p95":1.1443209251897362,"thinking_ratio_avg":0.8970936132219671,"total_params":null,"active_params":null,"precision":"unknown","base_model":"healer-alpha","reasoning_effort":"high","provider":"stealth"},{"model_id":"o4-mini-high","display_name":"o4-mini","lab":"openai","prompt":"bare","pass_rate":0.4462025316455696,"ci_lower":0.3923657369313149,"ci_upper":0.5013316432718857,"total_runs":316,"cost_p50":0.037473700000000006,"cost_p95":0.13852740000000002,"cost_avg":0.04990317436708862,"latency_p50":76725,"latency_p95":298390.75,"latency_avg":106182.46518987342,"ttft_p50":75718.125,"ttft_p95":298253.9625,"ttft_avg":105606.62088607594,"tokens_per_sec_p50":107.94678266016996,"tokens_per_sec_p95":126.83766503387488,"tokens_per_sec_avg":105.70877533004732,"thinking_ratio_p50":0.992482969270204,"thinking_ratio_p95":0.9986557374831586,"thinking_ratio_avg":0.9807645380138578,"total_params":null,"active_params":null,"precision":"unknown","base_model":"o4-mini","reasoning_effort":"high","provider":"openai"},{"model_id":"grok-code-fast-1-high","display_name":"grok-code-fast-1","lab":"x-ai","prompt":"bare","pass_rate":0.439873417721519,"ci_lower":0.3861912213412059,"ci_upper":0.49499996829766546,"total_runs":316,"cost_p50":0.00366525,"cost_p95":0.016793574999999998,"cost_avg":0.005564460126582276,"latency_p50":27766.5,"latency_p95":122240,"latency_avg":49650.40189873418,"ttft_p50":27479.35,"ttft_p95":119398.84,"ttft_avg":40413.84889240507,"tokens_per_sec_p50":94.91555346216795,"tokens_per_sec_p95":136.11000420602954,"tokens_per_sec_avg":88.85798609707284,"thinking_ratio_p50":0.992750721512096,"thinking_ratio_p95":0.9984408109546132,"thinking_ratio_avg":0.9769305777359014,"total_params":null,"active_params":null,"precision":"unknown","base_model":"grok-code-fast-1","reasoning_effort":"high","provider":"xai"},{"model_id":"claude-haiku-4.5-high","display_name":"claude-haiku-4.5","lab":"anthropic","prompt":"bare","pass_rate":0.43670886075949367,"ci_lower":0.38310728512631803,"ci_upper":0.49183080923038863,"total_runs":316,"cost_p50":0.051000000000000004,"cost_p95":0.15724,"cost_avg":0.06014621835443035,"latency_p50":69200.5,"latency_p95":206171.25,"latency_avg":79583.37974683545,"ttft_p50":67133.785,"ttft_p95":202750.9325,"ttft_avg":77600.58012658227,"tokens_per_sec_p50":144.08034849605542,"tokens_per_sec_p95":176.56828509031564,"tokens_per_sec_avg":135.11927652247044,"thinking_ratio_p50":0.3347193752274027,"thinking_ratio_p95":0.9502892662353435,"thinking_ratio_avg":0.4258317331665193,"total_params":null,"active_params":null,"precision":"unknown","base_model":"claude-haiku-4.5","reasoning_effort":"high","provider":"anthropic"},{"model_id":"qwen3.6-27b-high","display_name":"qwen3.6-27b","lab":"qwen","prompt":"bare","pass_rate":0.43670886075949367,"ci_lower":0.38310728512631803,"ci_upper":0.49183080923038863,"total_runs":316,"cost_p50":0.0159336,"cost_p95":0.05676446250000001,"cost_avg":0.02054672515822785,"latency_p50":1088864.5,"latency_p95":3096074.25,"latency_avg":1275540.6424050634,"ttft_p50":91598.805,"ttft_p95":330321.02,"ttft_avg":117269.1098101266,"tokens_per_sec_p50":5.581735511150283,"tokens_per_sec_p95":63.09774975339118,"tokens_per_sec_avg":12.360556378046523,"thinking_ratio_p50":0.9933146296105081,"thinking_ratio_p95":0.9987011471221015,"thinking_ratio_avg":0.9816132881272482,"total_params":27.8,"active_params":27,"precision":"unknown","base_model":"qwen3.6-27b","reasoning_effort":"high","provider":"alibaba"},{"model_id":"qwen3.8-27b","display_name":"qwen3.8-27b","lab":"qwen","prompt":"bare","pass_rate":0.43354430379746833,"ci_lower":0.3800255701566438,"ci_upper":0.48865942891789815,"total_runs":316,"cost_p50":0,"cost_p95":0,"cost_avg":0,"latency_p50":1056437,"latency_p95":2943196.75,"latency_avg":1123958.3670886075,"ttft_p50":1028932.995,"ttft_p95":2847429.205,"ttft_avg":1088762.655537975,"tokens_per_sec_p50":28.182554841248766,"tokens_per_sec_p95":55.114095186037545,"tokens_per_sec_avg":30.754976910680817,"thinking_ratio_p50":null,"thinking_ratio_p95":null,"thinking_ratio_avg":null,"total_params":27,"active_params":27,"precision":"bf16","base_model":"qwen3.8-27b","reasoning_effort":"xhigh","provider":"starflinger"},{"model_id":"minimax-m2.7-high","display_name":"minimax-m2.7","lab":"minimax","prompt":"bare","pass_rate":0.43037974683544306,"ci_lower":0.37694608193713586,"ci_upper":0.4854858218552415,"total_runs":316,"cost_p50":0.008147999999999999,"cost_p95":0.058851074999999996,"cost_avg":0.015772394620253167,"latency_p50":160848,"latency_p95":1216808.5,"latency_avg":331082.67088607594,"ttft_p50":156076.31,"ttft_p95":1111459.74,"ttft_avg":304718.2219620253,"tokens_per_sec_p50":39.43561236084955,"tokens_per_sec_p95":53.18223279191949,"tokens_per_sec_avg":40.2934354518175,"thinking_ratio_p50":0.996765458457209,"thinking_ratio_p95":0.9997091427131576,"thinking_ratio_avg":0.9664132846614477,"total_params":228.7,"active_params":null,"precision":"fp8","base_model":"minimax-m2.7","reasoning_effort":"high","provider":"minimax"},{"model_id":"qwen3.5-122b-a10b-sf","display_name":"qwen3.5-122b-a10b","lab":"qwen","prompt":"bare","pass_rate":0.4272151898734177,"ci_lower":0.37386882626575346,"ci_upper":0.4823099822444592,"total_runs":316,"cost_p50":0,"cost_p95":0,"cost_avg":0,"latency_p50":104286,"latency_p95":288092,"latency_avg":123074.19303797468,"ttft_p50":102344.79000000001,"ttft_p95":286047.905,"ttft_avg":121448.3243037975,"tokens_per_sec_p50":57.62394630797488,"tokens_per_sec_p95":86.7379564676923,"tokens_per_sec_avg":56.7267376642029,"thinking_ratio_p50":null,"thinking_ratio_p95":null,"thinking_ratio_avg":null,"total_params":122,"active_params":10,"precision":"nvfp4","base_model":"qwen3.5-122b-a10b","reasoning_effort":null,"provider":"starflinger"},{"model_id":"qwen3-max","display_name":"qwen3-max","lab":"qwen","prompt":"bare","pass_rate":0.4272151898734177,"ci_lower":0.37386882626575346,"ci_upper":0.4823099822444592,"total_runs":316,"cost_p50":0.00059787,"cost_p95":0.015092024999999998,"cost_avg":0.002786505569620253,"latency_p50":119030.5,"latency_p95":343260,"latency_avg":151143.32911392406,"ttft_p50":1191.865,"ttft_p95":1602.835,"ttft_avg":1252.5605063291143,"tokens_per_sec_p50":0.1931477953096734,"tokens_per_sec_p95":19.09820720786726,"tokens_per_sec_avg":3.433140120299284,"thinking_ratio_p50":null,"thinking_ratio_p95":null,"thinking_ratio_avg":null,"total_params":null,"active_params":null,"precision":"unknown","base_model":"qwen3-max","reasoning_effort":null,"provider":"alibaba"},{"model_id":"qwen3.8-27b-nvfp4-xhigh","display_name":"qwen3.8-27b","lab":"qwen","prompt":"bare","pass_rate":0.4272151898734177,"ci_lower":0.37386882626575346,"ci_upper":0.4823099822444592,"total_runs":316,"cost_p50":0,"cost_p95":0,"cost_avg":0,"latency_p50":414204,"latency_p95":1046616,"latency_avg":436686.2278481013,"ttft_p50":412175.06,"ttft_p95":984838.958,"ttft_avg":401295.5812698412,"tokens_per_sec_p50":80.28563813230394,"tokens_per_sec_p95":101.87347771220296,"tokens_per_sec_avg":80.17443755564797,"thinking_ratio_p50":null,"thinking_ratio_p95":null,"thinking_ratio_avg":null,"total_params":27,"active_params":27,"precision":"nvfp4","base_model":"qwen3.8-27b","reasoning_effort":"xhigh","provider":"starflinger"},{"model_id":"claude-haiku-4.5","display_name":"claude-haiku-4.5","lab":"anthropic","prompt":"bare","pass_rate":0.4208860759493671,"ci_lower":0.3677210372464032,"ci_upper":0.4759515806994801,"total_runs":316,"cost_p50":0.002903,"cost_p95":0.0061785,"cost_avg":0.0030555537974683556,"latency_p50":4788,"latency_p95":7954.25,"latency_avg":4683.360759493671,"ttft_p50":950.48,"ttft_p95":1532.475,"ttft_avg":1040.053069620253,"tokens_per_sec_p50":101.03516895352737,"tokens_per_sec_p95":151.65565859264666,"tokens_per_sec_avg":91.963966041227,"thinking_ratio_p50":null,"thinking_ratio_p95":null,"thinking_ratio_avg":null,"total_params":null,"active_params":null,"precision":"unknown","base_model":"claude-haiku-4.5","reasoning_effort":null,"provider":"anthropic"},{"model_id":"minimax-m2.5","display_name":"minimax-m2.5","lab":"minimax","prompt":"bare","pass_rate":0.40625,"ci_lower":0.35622199632092405,"ci_upper":0.4583022189478922,"total_runs":352,"cost_p50":0.0067326,"cost_p95":0.035751314999999985,"cost_avg":0.010344905113636366,"latency_p50":140121.5,"latency_p95":679476.0499999996,"latency_avg":212618.08238636365,"ttft_p50":110001.43,"ttft_p95":678648.2769999997,"ttft_avg":183004.7435511364,"tokens_per_sec_p50":39.50853382347152,"tokens_per_sec_p95":54.275502677426296,"tokens_per_sec_avg":37.495485864121356,"thinking_ratio_p50":0.995492625522785,"thinking_ratio_p95":0.9995873285358512,"thinking_ratio_avg":0.9604458097879548,"total_params":228.7,"active_params":10,"precision":"fp8","base_model":"minimax-m2.5","reasoning_effort":null,"provider":"minimax"},{"model_id":"gpt-oss-120b-low-ef-ecodev","display_name":"gpt-oss-120b","lab":"openai","prompt":"bare","pass_rate":0.4050632911392405,"ci_lower":0.352391098263143,"ci_upper":0.4600160432719169,"total_runs":316,"cost_p50":0,"cost_p95":0,"cost_avg":0,"latency_p50":22865.5,"latency_p95":113791.75,"latency_avg":37110.648734177215,"ttft_p50":21671.665,"ttft_p95":113349.4725,"ttft_avg":36067.75727848102,"tokens_per_sec_p50":57.52001039890227,"tokens_per_sec_p95":67.09494837987673,"tokens_per_sec_avg":57.46041251667971,"thinking_ratio_p50":null,"thinking_ratio_p95":null,"thinking_ratio_avg":null,"total_params":117,"active_params":5.1,"precision":"unknown","base_model":"gpt-oss-120b","reasoning_effort":"low","provider":"ef-inference"},{"model_id":"deepseek-v3.1","display_name":"deepseek-v3.1","lab":"deepseek","prompt":"bare","pass_rate":0.39556962025316456,"ci_lower":0.3432205527269568,"ci_upper":0.45042730296160916,"total_runs":316,"cost_p50":0.00049084,"cost_p95":0.0030385425,"cost_avg":0.0009654027531645566,"latency_p50":20423,"latency_p95":171547.5,"latency_avg":43970.784810126584,"ttft_p50":1919.975,"ttft_p95":2414.5275,"ttft_avg":1956.55908227848,"tokens_per_sec_p50":23.720069754187605,"tokens_per_sec_p95":33.76220745273081,"tokens_per_sec_avg":18.68323122590931,"thinking_ratio_p50":null,"thinking_ratio_p95":null,"thinking_ratio_avg":null,"total_params":684.5,"active_params":37,"precision":"fp8","base_model":"deepseek-v3.1","reasoning_effort":null,"provider":"novita"},{"model_id":"gpt-oss-120b-medium-ef-ecodev","display_name":"gpt-oss-120b","lab":"openai","prompt":"bare","pass_rate":0.39556962025316456,"ci_lower":0.3432205527269568,"ci_upper":0.45042730296160916,"total_runs":316,"cost_p50":0,"cost_p95":0,"cost_avg":0,"latency_p50":17588,"latency_p95":78306.75,"latency_avg":27823.518987341773,"ttft_p50":16324.81,"ttft_p95":77941.72750000001,"ttft_avg":27052.595221518994,"tokens_per_sec_p50":83.14080946388916,"tokens_per_sec_p95":99.29187940849334,"tokens_per_sec_avg":82.27194231357767,"thinking_ratio_p50":null,"thinking_ratio_p95":null,"thinking_ratio_avg":null,"total_params":117,"active_params":5.1,"precision":"unknown","base_model":"gpt-oss-120b","reasoning_effort":"medium","provider":"ef-inference"},{"model_id":"qwen3.5-27b","display_name":"qwen3.5-27b","lab":"qwen","prompt":"bare","pass_rate":0.39556962025316456,"ci_lower":0.3432205527269568,"ci_upper":0.45042730296160916,"total_runs":316,"cost_p50":0.013264095,"cost_p95":0.030192045,"cost_avg":0.014739138560126583,"latency_p50":99722.5,"latency_p95":219377.25,"latency_avg":108217.78164556962,"ttft_p50":97749.76999999999,"ttft_p95":208087.745,"ttft_avg":105849.37386075949,"tokens_per_sec_p50":84.25171768118895,"tokens_per_sec_p95":102.28563097889301,"tokens_per_sec_avg":84.783539848676,"thinking_ratio_p50":0.9955845387616342,"thinking_ratio_p95":0.9990287106323289,"thinking_ratio_avg":0.9778394518418412,"total_params":27.8,"active_params":27,"precision":"unknown","base_model":"qwen3.5-27b","reasoning_effort":null,"provider":"alibaba"},{"model_id":"step-3.7-flash-high","display_name":"step-3.7-flash","lab":"stepfun","prompt":"bare","pass_rate":0.38924050632911394,"ci_lower":0.3371184251102828,"ci_upper":0.4440232400139538,"total_runs":316,"cost_p50":0.030271975,"cost_p95":0.0867421875,"cost_avg":0.03404386250000001,"latency_p50":167546,"latency_p95":445656.75,"latency_avg":182603.65506329114,"ttft_p50":150458.635,"ttft_p95":437477.96249999997,"ttft_avg":171193.72322784812,"tokens_per_sec_p50":159.79085271052443,"tokens_per_sec_p95":211.45040789530898,"tokens_per_sec_avg":154.61182066752335,"thinking_ratio_p50":null,"thinking_ratio_p95":null,"thinking_ratio_avg":null,"total_params":201.4,"active_params":11,"precision":"fp8","base_model":"step-3.7-flash","reasoning_effort":"high","provider":"stepfun"},{"model_id":"gpt-oss-120b-medium","display_name":"gpt-oss-120b","lab":"openai","prompt":"bare","pass_rate":0.3860759493670886,"ci_lower":0.33407086112460926,"ci_upper":0.4408177087174627,"total_runs":316,"cost_p50":0.00022661000000000003,"cost_p95":0.0011258780000000003,"cost_avg":0.00037176407911392414,"latency_p50":7670.5,"latency_p95":51632.25,"latency_avg":13530.79430379747,"ttft_p50":6536,"ttft_p95":34796.4625,"ttft_avg":10989.17101265823,"tokens_per_sec_p50":166.3128262264093,"tokens_per_sec_p95":241.63365931468041,"tokens_per_sec_avg":161.90137629340722,"thinking_ratio_p50":0.8955728753721184,"thinking_ratio_p95":1.1336336376554033,"thinking_ratio_avg":0.8985276170560076,"total_params":116.8,"active_params":5.1,"precision":"bf16","base_model":"gpt-oss-120b","reasoning_effort":"medium","provider":"deepinfra"},{"model_id":"llama-4-maverick","display_name":"llama-4-maverick","lab":"meta","prompt":"bare","pass_rate":0.3829113924050633,"ci_lower":0.33102564408402607,"ci_upper":0.43760983047588115,"total_runs":316,"cost_p50":0.00044235000000000005,"cost_p95":0.00095925,"cost_avg":0.00047375079113924054,"latency_p50":26417,"latency_p95":119772.5,"latency_avg":41995.987341772154,"ttft_p50":736.56,"ttft_p95":1157.3725,"ttft_avg":778.7437658227855,"tokens_per_sec_p50":22.400157054235272,"tokens_per_sec_p95":39.53743210030169,"tokens_per_sec_avg":22.072566356864385,"thinking_ratio_p50":null,"thinking_ratio_p95":null,"thinking_ratio_avg":null,"total_params":401.6,"active_params":17,"precision":"fp8","base_model":"llama-4-maverick","reasoning_effort":null,"provider":"deepinfra"},{"model_id":"deepseek-v3.2","display_name":"deepseek-v3.2","lab":"deepseek","prompt":"bare","pass_rate":0.3829113924050633,"ci_lower":0.33102564408402607,"ci_upper":0.43760983047588115,"total_runs":316,"cost_p50":0.00037219,"cost_p95":0.00168417,"cost_avg":0.0005485099367088607,"latency_p50":19005.5,"latency_p95":110039,"latency_avg":32984.15189873418,"ttft_p50":2451.545,"ttft_p95":3987.325,"ttft_avg":2588.261202531645,"tokens_per_sec_p50":30.016595301317277,"tokens_per_sec_p95":39.478858959649756,"tokens_per_sec_avg":23.50194408130272,"thinking_ratio_p50":null,"thinking_ratio_p95":null,"thinking_ratio_avg":null,"total_params":685.4,"active_params":37,"precision":"fp8","base_model":"deepseek-v3.2","reasoning_effort":null,"provider":"atlas-cloud"},{"model_id":"deepseek-v3","display_name":"deepseek-v3","lab":"deepseek","prompt":"bare","pass_rate":0.37658227848101267,"ci_lower":0.3249422942674039,"ci_upper":0.43118698972817404,"total_runs":316,"cost_p50":0.00050965,"cost_p95":0.001134525,"cost_avg":0.0005480357594936704,"latency_p50":9041,"latency_p95":17947.5,"latency_avg":9511.89240506329,"ttft_p50":1519.3899999999999,"ttft_p95":2049.9725,"ttft_avg":1553.1236392405053,"tokens_per_sec_p50":27.7432959137495,"tokens_per_sec_p95":32.15411738516899,"tokens_per_sec_avg":23.73685974173843,"thinking_ratio_p50":null,"thinking_ratio_p95":null,"thinking_ratio_avg":null,"total_params":684.5,"active_params":37,"precision":"fp8","base_model":"deepseek-v3","reasoning_effort":null,"provider":"novita"},{"model_id":"qwen3.8-27b-nvfp4-medium","display_name":"qwen3.8-27b","lab":"qwen","prompt":"bare","pass_rate":0.37658227848101267,"ci_lower":0.3249422942674039,"ci_upper":0.43118698972817404,"total_runs":316,"cost_p50":0,"cost_p95":0,"cost_avg":0,"latency_p50":28836.5,"latency_p95":241362.25,"latency_avg":56882.6582278481,"ttft_p50":26582.175,"ttft_p95":137653.77500000002,"ttft_avg":40833.576044303816,"tokens_per_sec_p50":125.61149718266873,"tokens_per_sec_p95":143.10885007224337,"tokens_per_sec_avg":120.97376076600571,"thinking_ratio_p50":null,"thinking_ratio_p95":null,"thinking_ratio_avg":null,"total_params":27,"active_params":27,"precision":"nvfp4","base_model":"qwen3.8-27b","reasoning_effort":"medium","provider":"starflinger"},{"model_id":"muse-glimmer-30b-high","display_name":"muse-glimmer-30b","lab":"meta","prompt":"bare","pass_rate":0.37658227848101267,"ci_lower":0.3249422942674039,"ci_upper":0.43118698972817404,"total_runs":316,"cost_p50":0.013087874999999999,"cost_p95":0.03652175,"cost_avg":0.014457886867088607,"latency_p50":111556.5,"latency_p95":335393.75,"latency_avg":133242.75316455695,"ttft_p50":108767.565,"ttft_p95":300114.09750000003,"ttft_avg":118995.93136075944,"tokens_per_sec_p50":78.26596173790162,"tokens_per_sec_p95":83.41019429655707,"tokens_per_sec_avg":75.83281594734817,"thinking_ratio_p50":0.9459104014060347,"thinking_ratio_p95":1.1342877443044301,"thinking_ratio_avg":0.9576595113659134,"total_params":29.8,"active_params":30,"precision":"unknown","base_model":"muse-glimmer-30b","reasoning_effort":"high","provider":"together"},{"model_id":"gpt-oss-120b-high-ef-ecodev","display_name":"gpt-oss-120b","lab":"openai","prompt":"bare","pass_rate":0.37341772151898733,"ci_lower":0.32190418416594274,"ci_upper":0.4279720045474704,"total_runs":316,"cost_p50":0,"cost_p95":0,"cost_avg":0,"latency_p50":16271,"latency_p95":83749.75,"latency_avg":27183.645569620254,"ttft_p50":15015.295,"ttft_p95":83607.5475,"ttft_avg":26438.09056962025,"tokens_per_sec_p50":82.59348993411402,"tokens_per_sec_p95":98.3963804353838,"tokens_per_sec_avg":82.02727925634484,"thinking_ratio_p50":null,"thinking_ratio_p95":null,"thinking_ratio_avg":null,"total_params":117,"active_params":5.1,"precision":"unknown","base_model":"gpt-oss-120b","reasoning_effort":"high","provider":"ef-inference"},{"model_id":"qwen3.6-35b-a3b-high","display_name":"qwen3.6-35b-a3b","lab":"qwen","prompt":"bare","pass_rate":0.37341772151898733,"ci_lower":0.32190418416594274,"ci_upper":0.4279720045474704,"total_runs":316,"cost_p50":0.0064525,"cost_p95":0.01879275,"cost_avg":0.008082024525316452,"latency_p50":42081,"latency_p95":153030.25,"latency_avg":59437.876582278484,"ttft_p50":28724.525,"ttft_p95":76120.94,"ttft_avg":33988.971550632894,"tokens_per_sec_p50":156.17207743228604,"tokens_per_sec_p95":198.7316074724639,"tokens_per_sec_avg":133.81819254427302,"thinking_ratio_p50":0.9939742987330198,"thinking_ratio_p95":0.998748691787131,"thinking_ratio_avg":0.9844579799732402,"total_params":36,"active_params":3,"precision":"fp8","base_model":"qwen3.6-35b-a3b","reasoning_effort":"high","provider":"atlas-cloud"},{"model_id":"gemini-2.5-flash-high","display_name":"gemini-2.5-flash","lab":"google","prompt":"bare","pass_rate":0.37341772151898733,"ci_lower":0.32190418416594274,"ci_upper":0.4279720045474704,"total_runs":316,"cost_p50":0.009420749999999999,"cost_p95":0.0431958,"cost_avg":0.014642784810126585,"latency_p50":43965,"latency_p95":139501.75,"latency_avg":54941.42721518988,"ttft_p50":25430.82,"ttft_p95":111019.5125,"ttft_avg":37462.48354430381,"tokens_per_sec_p50":127.31928300520704,"tokens_per_sec_p95":221.61896921938114,"tokens_per_sec_avg":119.97051164344694,"thinking_ratio_p50":0.9294509673366065,"thinking_ratio_p95":0.998518997584226,"thinking_ratio_avg":0.9012102126081826,"total_params":null,"active_params":null,"precision":"unknown","base_model":"gemini-2.5-flash","reasoning_effort":"high","provider":"google-vertex"},{"model_id":"grok-4.1-fast-high","display_name":"grok-4.1-fast","lab":"x-ai","prompt":"bare","pass_rate":0.3670886075949367,"ci_lower":0.31583515322922534,"ci_upper":0.42153484491985854,"total_runs":316,"cost_p50":0.00350345,"cost_p95":0.011093475,"cost_avg":0.004251693987341773,"latency_p50":63215.5,"latency_p95":237145,"latency_avg":82385.82594936709,"ttft_p50":62902.03999999999,"ttft_p95":236707.8675,"ttft_avg":82010.48275316454,"tokens_per_sec_p50":96.29516450951485,"tokens_per_sec_p95":142.05041238735316,"tokens_per_sec_avg":97.23892282173595,"thinking_ratio_p50":0.997281170240508,"thinking_ratio_p95":0.9995640320713177,"thinking_ratio_avg":0.9861815384015071,"total_params":null,"active_params":null,"precision":"unknown","base_model":"grok-4.1-fast","reasoning_effort":"high","provider":"xai"},{"model_id":"owl-alpha","display_name":"owl-alpha","lab":"openrouter","prompt":"bare","pass_rate":0.3639240506329114,"ci_lower":0.31280425747798923,"ci_upper":0.41831264538893,"total_runs":316,"cost_p50":0,"cost_p95":0,"cost_avg":0,"latency_p50":30458,"latency_p95":275565,"latency_avg":102578.25949367089,"ttft_p50":12117.505000000001,"ttft_p95":24224.865,"ttft_avg":12157.290537974683,"tokens_per_sec_p50":9.588637392283285,"tokens_per_sec_p95":28.673681612388886,"tokens_per_sec_avg":12.366352509538883,"thinking_ratio_p50":null,"thinking_ratio_p95":null,"thinking_ratio_avg":null,"total_params":null,"active_params":null,"precision":"int8","base_model":"owl-alpha","reasoning_effort":null,"provider":"stealth"},{"model_id":"grok-3","display_name":"grok-3","lab":"x-ai","prompt":"bare","pass_rate":0.3639240506329114,"ci_lower":0.31280425747798923,"ci_upper":0.41831264538893,"total_runs":316,"cost_p50":0.002214,"cost_p95":0.015917999999999998,"cost_avg":0.0050728291139240525,"latency_p50":1360,"latency_p95":12916,"latency_avg":3932.0981012658226,"ttft_p50":673.8050000000001,"ttft_p95":2502.0675,"ttft_avg":929.5772151898727,"tokens_per_sec_p50":21.12073972539089,"tokens_per_sec_p95":87.23593159513057,"tokens_per_sec_avg":35.26369448131183,"thinking_ratio_p50":null,"thinking_ratio_p95":null,"thinking_ratio_avg":null,"total_params":null,"active_params":null,"precision":"unknown","base_model":"grok-3","reasoning_effort":null,"provider":"xai"},{"model_id":"mistral-small-2603-high","display_name":"mistral-small-2603","lab":"mistral","prompt":"bare","pass_rate":0.36075949367088606,"ci_lower":0.30977579229519603,"ci_upper":0.41508801528955847,"total_runs":316,"cost_p50":0.00154905,"cost_p95":0.008890725,"cost_avg":0.0027705783227848103,"latency_p50":15144,"latency_p95":89291.75,"latency_avg":27498.87658227848,"ttft_p50":14626.64,"ttft_p95":86596.0125,"ttft_avg":26146.41212025317,"tokens_per_sec_p50":157.8176437581323,"tokens_per_sec_p95":189.69394861189818,"tokens_per_sec_avg":148.56367823749517,"thinking_ratio_p50":0.8441240114718153,"thinking_ratio_p95":1.1082206592556256,"thinking_ratio_avg":0.8476667953450299,"total_params":119.4,"active_params":null,"precision":"unknown","base_model":"mistral-small-2603","reasoning_effort":"high","provider":"mistral"},{"model_id":"claude-3.5-haiku","display_name":"claude-3.5-haiku","lab":"anthropic","prompt":"bare","pass_rate":0.3575949367088608,"ci_lower":0.3067497712890721,"ci_upper":0.4118609410135178,"total_runs":316,"cost_p50":0.001148,"cost_p95":0.0024394,"cost_avg":0.0011824683544303792,"latency_p50":4194.5,"latency_p95":6690,"latency_avg":3976.772151898734,"ttft_p50":800.315,"ttft_p95":1424.8899999999999,"ttft_avg":892.6605696202525,"tokens_per_sec_p50":44.42285969195466,"tokens_per_sec_p95":63.8078993463752,"tokens_per_sec_avg":40.15120905970286,"thinking_ratio_p50":null,"thinking_ratio_p95":null,"thinking_ratio_avg":null,"total_params":null,"active_params":null,"precision":"unknown","base_model":"claude-3.5-haiku","reasoning_effort":null,"provider":"amazon-bedrock"},{"model_id":"qwen3.5-35b-a3b","display_name":"qwen3.5-35b-a3b","lab":"qwen","prompt":"bare","pass_rate":0.3575949367088608,"ci_lower":0.3067497712890721,"ci_upper":0.4118609410135178,"total_runs":316,"cost_p50":0.012591718750000001,"cost_p95":0.033021259375,"cost_avg":0.013927728441455695,"latency_p50":79751,"latency_p95":210334.5,"latency_avg":88453.36392405063,"ttft_p50":78438.01500000001,"ttft_p95":206569.67250000002,"ttft_avg":86183.62186708863,"tokens_per_sec_p50":122.90329425454675,"tokens_per_sec_p95":152.17206892981363,"tokens_per_sec_avg":119.3292914297918,"thinking_ratio_p50":0.9963194152041707,"thinking_ratio_p95":0.9992416329050792,"thinking_ratio_avg":0.961937312653924,"total_params":36,"active_params":3,"precision":"unknown","base_model":"qwen3.5-35b-a3b","reasoning_effort":null,"provider":"alibaba"},{"model_id":"grok-3-mini-high","display_name":"grok-3-mini","lab":"x-ai","prompt":"bare","pass_rate":0.3512658227848101,"ci_lower":0.300705118474274,"ci_upper":0.4053994032639865,"total_runs":316,"cost_p50":0.003352,"cost_p95":0.011428375000000001,"cost_avg":0.004092142088607597,"latency_p50":56696.5,"latency_p95":201808.25,"latency_avg":74854.03797468354,"ttft_p50":55708.705,"ttft_p95":200876.1025,"ttft_avg":70824.11408227848,"tokens_per_sec_p50":109.26577740081669,"tokens_per_sec_p95":122.26359590642491,"tokens_per_sec_avg":108.02137075349314,"thinking_ratio_p50":0.9965840361465308,"thinking_ratio_p95":0.9995965118616552,"thinking_ratio_avg":0.9842059609813917,"total_params":null,"active_params":null,"precision":"unknown","base_model":"grok-3-mini","reasoning_effort":"high","provider":"xai"},{"model_id":"minimax-m2.5-high","display_name":"minimax-m2.5","lab":"minimax","prompt":"bare","pass_rate":0.3449367088607595,"ci_lower":0.29467041699290725,"ci_upper":0.3989279141810239,"total_runs":316,"cost_p50":0.0065255999999999995,"cost_p95":0.039124425,"cost_avg":0.011085236392405065,"latency_p50":118952.5,"latency_p95":597545.25,"latency_avg":200989.88607594935,"ttft_p50":113463.76500000001,"ttft_p95":597157.465,"ttft_avg":198560.4826265823,"tokens_per_sec_p50":41.13403633354157,"tokens_per_sec_p95":59.42140816204318,"tokens_per_sec_avg":40.78777460060987,"thinking_ratio_p50":0.9957450385364761,"thinking_ratio_p95":0.9995781200296023,"thinking_ratio_avg":0.9596568517449825,"total_params":228.7,"active_params":10,"precision":"fp8","base_model":"minimax-m2.5","reasoning_effort":"high","provider":"minimax"},{"model_id":"mistral-large-2512","display_name":"mistral-large-2512","lab":"mistral","prompt":"bare","pass_rate":0.34177215189873417,"ci_lower":0.291656836949517,"ci_upper":0.3956883989422495,"total_runs":316,"cost_p50":0.00065075,"cost_p95":0.001984,"cost_avg":0.0008088338607594935,"latency_p50":5388,"latency_p95":17267.5,"latency_avg":6418.705696202532,"ttft_p50":830.855,"ttft_p95":1681.7975000000001,"ttft_avg":985.9035759493673,"tokens_per_sec_p50":39.35324597499563,"tokens_per_sec_p95":67.43594518901752,"tokens_per_sec_avg":36.71507787876335,"thinking_ratio_p50":null,"thinking_ratio_p95":null,"thinking_ratio_avg":null,"total_params":675,"active_params":41,"precision":"unknown","base_model":"mistral-large-2512","reasoning_effort":null,"provider":"mistral"},{"model_id":"qwen3.6-35b-a3b-q8","display_name":"qwen3.6-35b-a3b","lab":"qwen","prompt":"bare","pass_rate":0.33860759493670883,"ci_lower":0.2886457924689756,"ci_upper":0.3924463481406262,"total_runs":316,"cost_p50":0,"cost_p95":0,"cost_avg":0,"latency_p50":293617.5,"latency_p95":897652.75,"latency_avg":349035.1740506329,"ttft_p50":287621.92000000004,"ttft_p95":896844.23,"ttft_avg":344970.5052215189,"tokens_per_sec_p50":20.02190847127556,"tokens_per_sec_p95":25.769701756787438,"tokens_per_sec_avg":19.634909595636834,"thinking_ratio_p50":0.9951515151515151,"thinking_ratio_p95":0.9989021809295943,"thinking_ratio_avg":0.9846189102961143,"total_params":35,"active_params":3,"precision":"q8_0","base_model":"qwen3.6-35b-a3b","reasoning_effort":null,"provider":"selfhosted"},{"model_id":"qwen3.5-27b-uncensored","display_name":"qwen3.5-27b-uncensored-hauhaucs-aggressive","lab":"qwen","prompt":"bare","pass_rate":0.33860759493670883,"ci_lower":0.2886457924689756,"ci_upper":0.3924463481406262,"total_runs":316,"cost_p50":0,"cost_p95":0,"cost_avg":0,"latency_p50":267908,"latency_p95":790189.75,"latency_avg":313755.35126582277,"ttft_p50":1370.6599999999999,"ttft_p95":7767.705,"ttft_avg":2993.6343670886076,"tokens_per_sec_p50":31.582451329019495,"tokens_per_sec_p95":44.30158048599586,"tokens_per_sec_avg":31.7107590583383,"thinking_ratio_p50":null,"thinking_ratio_p95":null,"thinking_ratio_avg":null,"total_params":27,"active_params":27,"precision":"q4_k_m","base_model":"qwen3.5-27b-uncensored-hauhaucs-aggressive","reasoning_effort":null,"provider":"selfhosted"},{"model_id":"mistral-medium-3.1","display_name":"mistral-medium-3.1","lab":"mistral","prompt":"bare","pass_rate":0.3322784810126582,"ci_lower":0.2826313785967871,"ci_upper":0.38595457144848533,"total_runs":316,"cost_p50":0.00046100000000000004,"cost_p95":0.0019662,"cost_avg":0.0006718582278481012,"latency_p50":2839,"latency_p95":34307.75,"latency_avg":9393.971518987342,"ttft_p50":769.765,"ttft_p95":2252.065,"ttft_avg":1065.3396202531646,"tokens_per_sec_p50":14.620166593762262,"tokens_per_sec_p95":42.457088953753185,"tokens_per_sec_avg":18.350669029996396,"thinking_ratio_p50":null,"thinking_ratio_p95":null,"thinking_ratio_avg":null,"total_params":null,"active_params":null,"precision":"unknown","base_model":"mistral-medium-3.1","reasoning_effort":null,"provider":"mistral"},{"model_id":"grok-4-fast-high","display_name":"grok-4-fast","lab":"x-ai","prompt":"bare","pass_rate":0.3291139240506329,"ci_lower":0.2796280447495307,"ci_upper":0.3827048100135771,"total_runs":316,"cost_p50":0.002771,"cost_p95":0.010128475,"cost_avg":0.0035565468354430375,"latency_p50":32218,"latency_p95":107050.75,"latency_avg":39862.73417721519,"ttft_p50":31809.370000000003,"ttft_p95":106852.29000000001,"ttft_avg":39490.33791139241,"tokens_per_sec_p50":162.5,"tokens_per_sec_p95":196.50559722694396,"tokens_per_sec_avg":157.9907061318699,"thinking_ratio_p50":0.9959757085020242,"thinking_ratio_p95":0.9992978812304033,"thinking_ratio_avg":0.9776252259232519,"total_params":null,"active_params":null,"precision":"unknown","base_model":"grok-4-fast","reasoning_effort":"high","provider":"xai"},{"model_id":"laguna-s-2.1-fp8","display_name":"laguna-s-2.1","lab":"starflinger","prompt":"bare","pass_rate":0.3259493670886076,"ci_lower":0.27662731758415726,"ci_upper":0.37945244189678595,"total_runs":316,"cost_p50":0,"cost_p95":0,"cost_avg":0,"latency_p50":2938,"latency_p95":122652.75,"latency_avg":22064.3417721519,"ttft_p50":157.835,"ttft_p95":2748.055,"ttft_avg":445.2265189873418,"tokens_per_sec_p50":88.61178743898913,"tokens_per_sec_p95":161.3838039225999,"tokens_per_sec_avg":88.53379472258393,"thinking_ratio_p50":null,"thinking_ratio_p95":null,"thinking_ratio_avg":null,"total_params":118,"active_params":8,"precision":"fp8","base_model":"laguna-s-2.1","reasoning_effort":null,"provider":"starflinger"},{"model_id":"gpt-oss-120b-low","display_name":"gpt-oss-120b","lab":"openai","prompt":"bare","pass_rate":0.3227848101265823,"ci_lower":0.273629216286367,"ci_upper":0.3761974479124115,"total_runs":316,"cost_p50":0.00005504650000000001,"cost_p95":0.00021315549999999998,"cost_avg":0.00007930436392405067,"latency_p50":1657,"latency_p95":5720.25,"latency_avg":2236.8987341772154,"ttft_p50":1367.0300000000002,"ttft_p95":4418.065,"ttft_avg":1847.0016139240504,"tokens_per_sec_p50":130.1146873843877,"tokens_per_sec_p95":210.4226136238733,"tokens_per_sec_avg":129.1628377156649,"thinking_ratio_p50":0.7723279310105657,"thinking_ratio_p95":0.9643253968253969,"thinking_ratio_avg":0.7324685203675739,"total_params":116.8,"active_params":5.1,"precision":"bf16","base_model":"gpt-oss-120b","reasoning_effort":"low","provider":"deepinfra"},{"model_id":"gpt-5-nano-high","display_name":"gpt-5-nano","lab":"openai","prompt":"bare","pass_rate":0.31962025316455694,"ci_lower":0.2706337606369862,"ci_upper":0.3729398082796276,"total_runs":316,"cost_p50":0.0038233,"cost_p95":0.010790737500000001,"cost_avg":0.004733908702531643,"latency_p50":99039.5,"latency_p95":280502,"latency_avg":119990.53797468354,"ttft_p50":98840.685,"ttft_p95":280258.0575,"ttft_avg":119745.29753164558,"tokens_per_sec_p50":93.32163817712237,"tokens_per_sec_p95":147.710233784841,"tokens_per_sec_avg":97.83353260352024,"thinking_ratio_p50":0.9938289981087122,"thinking_ratio_p95":0.9983974856239957,"thinking_ratio_avg":0.9901721272491595,"total_params":null,"active_params":null,"precision":"unknown","base_model":"gpt-5-nano","reasoning_effort":"high","provider":"openai"},{"model_id":"gpt-4o-2024-05-13","display_name":"gpt-4o-2024-05-13","lab":"openai","prompt":"bare","pass_rate":0.31962025316455694,"ci_lower":0.2706337606369862,"ci_upper":0.3729398082796276,"total_runs":316,"cost_p50":0.0038075,"cost_p95":0.01839375,"cost_avg":0.006075348101265821,"latency_p50":1296.5,"latency_p95":8676.5,"latency_avg":2881.5727848101264,"ttft_p50":646.51,"ttft_p95":1322.575,"ttft_avg":769.2228164556961,"tokens_per_sec_p50":32.043187673783834,"tokens_per_sec_p95":125.08240759403614,"tokens_per_sec_avg":49.38290447884982,"thinking_ratio_p50":null,"thinking_ratio_p95":null,"thinking_ratio_avg":null,"total_params":null,"active_params":null,"precision":"unknown","base_model":"gpt-4o-2024-05-13","reasoning_effort":null,"provider":"openai"},{"model_id":"gpt-4o","display_name":"gpt-4o","lab":"openai","prompt":"bare","pass_rate":0.31645569620253167,"ci_lower":0.26764097102967865,"ci_upper":0.3696795026047705,"total_runs":316,"cost_p50":0.0022387500000000003,"cost_p95":0.01095875,"cost_avg":0.003397365506329115,"latency_p50":1567.5,"latency_p95":8870.75,"latency_avg":3081.8132911392404,"ttft_p50":698.3,"ttft_p95":1320.915,"ttft_avg":803.587753164557,"tokens_per_sec_p50":33.58821250313376,"tokens_per_sec_p95":104.98202225569551,"tokens_per_sec_avg":44.27718787740005,"thinking_ratio_p50":null,"thinking_ratio_p95":null,"thinking_ratio_avg":null,"total_params":null,"active_params":null,"precision":"unknown","base_model":"gpt-4o","reasoning_effort":null,"provider":"openai"},{"model_id":"flint-v1.0","display_name":"flint-v1.0","lab":"recursive","prompt":"bare","pass_rate":0.31329113924050633,"ci_lower":0.26465086848948566,"ci_upper":0.36641650986279883,"total_runs":316,"cost_p50":0,"cost_p95":0,"cost_avg":0,"latency_p50":125251,"latency_p95":233204.75,"latency_avg":132543.07594936708,"ttft_p50":119072.84,"ttft_p95":222614.275,"ttft_avg":113467.18338607597,"tokens_per_sec_p50":138.73369722139256,"tokens_per_sec_p95":154.83896223198118,"tokens_per_sec_avg":131.02104847833647,"thinking_ratio_p50":0.9808764644597845,"thinking_ratio_p95":1.1683482455363023,"thinking_ratio_avg":0.9559838640452267,"total_params":39.8,"active_params":null,"precision":"bf16","base_model":"flint-v1.0","reasoning_effort":null,"provider":"recursive"},{"model_id":"codestral-2508","display_name":"codestral-2508","lab":"mistral","prompt":"bare","pass_rate":0.310126582278481,"ci_lower":0.2616634746922393,"ci_upper":0.3631508083778805,"total_runs":316,"cost_p50":0.00029145,"cost_p95":0.001208925,"cost_avg":0.00039757879746835426,"latency_p50":1000.5,"latency_p95":5309.25,"latency_avg":1824.8291139240507,"ttft_p50":539.89,"ttft_p95":959.15,"ttft_avg":617.2047784810132,"tokens_per_sec_p50":73.49167803396992,"tokens_per_sec_p95":186.52900693535568,"tokens_per_sec_avg":76.90481163524747,"thinking_ratio_p50":null,"thinking_ratio_p95":null,"thinking_ratio_avg":null,"total_params":null,"active_params":null,"precision":"unknown","base_model":"codestral-2508","reasoning_effort":null,"provider":"mistral"},{"model_id":"gpt-4.1-mini","display_name":"gpt-4.1-mini","lab":"openai","prompt":"bare","pass_rate":0.310126582278481,"ci_lower":0.2616634746922393,"ci_upper":0.3631508083778805,"total_runs":316,"cost_p50":0.0011589999999999999,"cost_p95":0.004769700000000001,"cost_avg":0.0016058037974683546,"latency_p50":9524.5,"latency_p95":45826.75,"latency_avg":14858.12658227848,"ttft_p50":904.88,"ttft_p95":1825.3175,"ttft_avg":1142.7623101265824,"tokens_per_sec_p50":56.933425135023256,"tokens_per_sec_p95":75.48404661695186,"tokens_per_sec_avg":47.92833869193058,"thinking_ratio_p50":null,"thinking_ratio_p95":null,"thinking_ratio_avg":null,"total_params":null,"active_params":null,"precision":"unknown","base_model":"gpt-4.1-mini","reasoning_effort":null,"provider":"openai"},{"model_id":"mixtral-8x22b-instruct","display_name":"mixtral-8x22b-instruct","lab":"mistral","prompt":"bare","pass_rate":0.3069620253164557,"ci_lower":0.2586788119848943,"ci_upper":0.35988237580306087,"total_runs":316,"cost_p50":0.001271,"cost_p95":0.0056945,"cost_avg":0.001898550632911392,"latency_p50":1077,"latency_p95":4609.5,"latency_avg":1688.129746835443,"ttft_p50":796.235,"ttft_p95":2606.1,"ttft_avg":1115.7381012658236,"tokens_per_sec_p50":19.130063520001393,"tokens_per_sec_p95":126.87437757860292,"tokens_per_sec_avg":36.41373485288135,"thinking_ratio_p50":null,"thinking_ratio_p95":null,"thinking_ratio_avg":null,"total_params":140.6,"active_params":39,"precision":"unknown","base_model":"mixtral-8x22b-instruct","reasoning_effort":null,"provider":"mistral"},{"model_id":"laguna-s-2.1-nvfp4-thinking","display_name":"laguna-s-2.1","lab":"starflinger","prompt":"bare","pass_rate":0.3069620253164557,"ci_lower":0.2586788119848943,"ci_upper":0.35988237580306087,"total_runs":316,"cost_p50":0,"cost_p95":0,"cost_avg":0,"latency_p50":2195,"latency_p95":322214,"latency_avg":36780.148734177215,"ttft_p50":157.34,"ttft_p95":344.69,"ttft_avg":353.6393987341772,"tokens_per_sec_p50":106.33654242549167,"tokens_per_sec_p95":198.49030301249223,"tokens_per_sec_avg":109.28881864395608,"thinking_ratio_p50":null,"thinking_ratio_p95":null,"thinking_ratio_avg":null,"total_params":118,"active_params":8,"precision":"nvfp4","base_model":"laguna-s-2.1","reasoning_effort":"high","provider":"starflinger"},{"model_id":"laguna-s-2.1-fp8-thinking","display_name":"laguna-s-2.1","lab":"starflinger","prompt":"bare","pass_rate":0.29430379746835444,"ci_lower":0.24676794370494787,"ci_upper":0.34678086295434857,"total_runs":316,"cost_p50":0,"cost_p95":0,"cost_avg":0,"latency_p50":1241,"latency_p95":85495.5,"latency_avg":24310.740506329115,"ttft_p50":194.195,"ttft_p95":315.0525,"ttft_avg":620.4966139240506,"tokens_per_sec_p50":87.52760582429691,"tokens_per_sec_p95":149.36691108706975,"tokens_per_sec_avg":86.5052804361714,"thinking_ratio_p50":null,"thinking_ratio_p95":null,"thinking_ratio_avg":null,"total_params":118,"active_params":8,"precision":"fp8","base_model":"laguna-s-2.1","reasoning_effort":"high","provider":"starflinger"},{"model_id":"gemma-4-31b-it-high","display_name":"gemma-4-31b-it","lab":"google","prompt":"bare","pass_rate":0.2911392405063291,"ci_lower":0.24379729669081032,"ci_upper":0.3434984146863215,"total_runs":316,"cost_p50":0.00128188,"cost_p95":0.004824115,"cost_avg":0.0018820065822784812,"latency_p50":92587.5,"latency_p95":347959.5,"latency_avg":135574.77215189874,"ttft_p50":80849.525,"ttft_p95":330873.07,"ttft_avg":120928.52626582276,"tokens_per_sec_p50":33.73603112314682,"tokens_per_sec_p95":48.93897657335982,"tokens_per_sec_avg":34.09435591295593,"thinking_ratio_p50":0.6579711859788018,"thinking_ratio_p95":0.9210284490314272,"thinking_ratio_avg":0.6616338507586965,"total_params":31.3,"active_params":31,"precision":"bf16","base_model":"gemma-4-31b-it","reasoning_effort":"high","provider":"parasail"},{"model_id":"gpt-oss-20b-high","display_name":"gpt-oss-20b","lab":"openai","prompt":"bare","pass_rate":0.2879746835443038,"ci_lower":0.24082953020974499,"ci_upper":0.34021308588522203,"total_runs":316,"cost_p50":0.0022532200000000002,"cost_p95":0.0078257675,"cost_avg":0.0028686059177215196,"latency_p50":161619.5,"latency_p95":728220.25,"latency_avg":238201.75632911394,"ttft_p50":161304.89,"ttft_p95":727349.475,"ttft_avg":237313.83740506315,"tokens_per_sec_p50":116.85164179038446,"tokens_per_sec_p95":189.62497463011465,"tokens_per_sec_avg":108.83174918515284,"thinking_ratio_p50":1.0110384262907046,"thinking_ratio_p95":1.1995908106940698,"thinking_ratio_avg":0.9964402612410485,"total_params":20.9,"active_params":3.6,"precision":"bf16","base_model":"gpt-oss-20b","reasoning_effort":"high","provider":"deepinfra"},{"model_id":"devstral-medium","display_name":"devstral-medium","lab":"mistral","prompt":"bare","pass_rate":0.2879746835443038,"ci_lower":0.24082953020974499,"ci_upper":0.34021308588522203,"total_runs":316,"cost_p50":0.0004354,"cost_p95":0.0035199000000000003,"cost_avg":0.0009818139240506327,"latency_p50":1632.5,"latency_p95":14588.5,"latency_avg":4301.693037974684,"ttft_p50":644.4100000000001,"ttft_p95":1175.1375,"ttft_avg":731.9926582278482,"tokens_per_sec_p50":40.63101865287709,"tokens_per_sec_p95":110.50730078590131,"tokens_per_sec_avg":46.56492872519127,"thinking_ratio_p50":null,"thinking_ratio_p95":null,"thinking_ratio_avg":null,"total_params":null,"active_params":null,"precision":"unknown","base_model":"devstral-medium","reasoning_effort":null,"provider":"mistral"},{"model_id":"gpt-oss-20b-medium","display_name":"gpt-oss-20b","lab":"openai","prompt":"bare","pass_rate":0.2848101265822785,"ci_lower":0.23786467196456465,"ci_upper":0.33692484884823776,"total_runs":316,"cost_p50":0.000368115,"cost_p95":0.0016641600000000002,"cost_avg":0.0005621599683544303,"latency_p50":36608.5,"latency_p95":213443,"latency_avg":66556.68987341772,"ttft_p50":32114.715,"ttft_p95":138753.3125,"ttft_avg":46168.51901898734,"tokens_per_sec_p50":57.943397148472215,"tokens_per_sec_p95":254.78730331422614,"tokens_per_sec_avg":107.0242870507282,"thinking_ratio_p50":0.9447590139715929,"thinking_ratio_p95":1.1632894956299284,"thinking_ratio_avg":0.9395861631485042,"total_params":20.9,"active_params":3.6,"precision":"bf16","base_model":"gpt-oss-20b","reasoning_effort":"medium","provider":"deepinfra"},{"model_id":"laguna-s-2.1-nvfp4","display_name":"laguna-s-2.1","lab":"starflinger","prompt":"bare","pass_rate":0.28164556962025317,"ci_lower":0.23490275053177267,"ci_upper":0.33363367499886504,"total_runs":316,"cost_p50":0,"cost_p95":0,"cost_avg":0,"latency_p50":1986.5,"latency_p95":350997,"latency_avg":34826.92405063291,"ttft_p50":163.28,"ttft_p95":458.9750000000001,"ttft_avg":521.510857142857,"tokens_per_sec_p50":97.19251708752728,"tokens_per_sec_p95":180.17656211452973,"tokens_per_sec_avg":95.7892277681382,"thinking_ratio_p50":null,"thinking_ratio_p95":null,"thinking_ratio_avg":null,"total_params":118,"active_params":8,"precision":"nvfp4","base_model":"laguna-s-2.1","reasoning_effort":null,"provider":"starflinger"},{"model_id":"qwq-32b","display_name":"qwq-32b","lab":"qwen","prompt":"bare","pass_rate":0.27531645569620256,"ci_lower":0.22898783696744218,"ci_upper":0.32704239799886625,"total_runs":316,"cost_p50":0.00312891,"cost_p95":0.011898734999999999,"cost_avg":0.004423888987341771,"latency_p50":131239.5,"latency_p95":556359.75,"latency_avg":204384.7658227848,"ttft_p50":130597.53,"ttft_p95":550614.8225,"ttft_avg":200540.99575949367,"tokens_per_sec_p50":37.11298227698515,"tokens_per_sec_p95":44.86290917328109,"tokens_per_sec_avg":36.59855437764916,"thinking_ratio_p50":0.9902723383131682,"thinking_ratio_p95":0.999240551439244,"thinking_ratio_avg":0.9686892078718328,"total_params":32,"active_params":32,"precision":"fp8","base_model":"qwq-32b","reasoning_effort":"high","provider":"siliconflow"},{"model_id":"mistral-large-2411","display_name":"mistral-large-2411","lab":"mistral","prompt":"bare","pass_rate":0.2721518987341772,"ci_lower":0.22603490664753303,"ci_upper":0.3237422330366107,"total_runs":316,"cost_p50":0.003268,"cost_p95":0.012220499999999999,"cost_avg":0.0046334367088607595,"latency_p50":8972,"latency_p95":35215.5,"latency_avg":12239.70253164557,"ttft_p50":648.61,"ttft_p95":1321.5125,"ttft_avg":753.1148734177215,"tokens_per_sec_p50":39.77928909109043,"tokens_per_sec_p95":51.758981894332216,"tokens_per_sec_avg":32.553391433519174,"thinking_ratio_p50":null,"thinking_ratio_p95":null,"thinking_ratio_avg":null,"total_params":123,"active_params":123,"precision":"unknown","base_model":"mistral-large-2411","reasoning_effort":null,"provider":"mistral"},{"model_id":"gemini-2.0-flash","display_name":"gemini-2.0-flash","lab":"google","prompt":"bare","pass_rate":0.2689873417721519,"ci_lower":0.22308503683440079,"ci_upper":0.3204390075675782,"total_runs":316,"cost_p50":0.00014470000000000002,"cost_p95":0.00069315,"cost_avg":0.0002479810126582279,"latency_p50":2500.5,"latency_p95":9577.25,"latency_avg":3752.25,"ttft_p50":622.1800000000001,"ttft_p95":1576.11,"ttft_avg":741.9283227848101,"tokens_per_sec_p50":95.99784579953803,"tokens_per_sec_p95":156.05606702163115,"tokens_per_sec_avg":79.13022951996845,"thinking_ratio_p50":null,"thinking_ratio_p95":null,"thinking_ratio_avg":null,"total_params":null,"active_params":null,"precision":"unknown","base_model":"gemini-2.0-flash","reasoning_effort":null,"provider":"google-vertex"},{"model_id":"gpt-4-turbo","display_name":"gpt-4-turbo","lab":"openai","prompt":"bare","pass_rate":0.2626582278481013,"ci_lower":0.2171946136113794,"ci_upper":0.31382324022627034,"total_runs":316,"cost_p50":0.01593,"cost_p95":0.0314325,"cost_avg":0.01500594936708861,"latency_p50":17998.5,"latency_p95":33313.5,"latency_avg":15856.575949367088,"ttft_p50":1331.515,"ttft_p95":1819.71,"ttft_avg":1350.7854746835442,"tokens_per_sec_p50":20.9046429059435,"tokens_per_sec_p95":23.328752192321573,"tokens_per_sec_avg":15.838520147271595,"thinking_ratio_p50":null,"thinking_ratio_p95":null,"thinking_ratio_avg":null,"total_params":null,"active_params":null,"precision":"unknown","base_model":"gpt-4-turbo","reasoning_effort":null,"provider":"openai"},{"model_id":"qwen-2.5-72b-instruct","display_name":"qwen-2.5-72b-instruct","lab":"qwen","prompt":"bare","pass_rate":0.2626582278481013,"ci_lower":0.2171946136113794,"ci_upper":0.31382324022627034,"total_runs":316,"cost_p50":0.00026873999999999995,"cost_p95":0.00095926,"cost_avg":0.00038048455696202527,"latency_p50":2286.5,"latency_p95":34172.75,"latency_avg":11553.174050632912,"ttft_p50":570.7049999999999,"ttft_p95":1514.2425,"ttft_avg":714.5879430379746,"tokens_per_sec_p50":21.46594837043338,"tokens_per_sec_p95":43.17740907882941,"tokens_per_sec_avg":22.129015799399586,"thinking_ratio_p50":null,"thinking_ratio_p95":null,"thinking_ratio_avg":null,"total_params":72.7,"active_params":72,"precision":"fp8","base_model":"qwen-2.5-72b-instruct","reasoning_effort":null,"provider":"deepinfra"},{"model_id":"llama-4-scout","display_name":"llama-4-scout","lab":"meta","prompt":"bare","pass_rate":0.25316455696202533,"ci_lower":0.20838280506712323,"ci_upper":0.3038757629240325,"total_runs":316,"cost_p50":0.0002321,"cost_p95":0.00039487499999999994,"cost_avg":0.0002168053797468355,"latency_p50":10931,"latency_p95":19121.75,"latency_avg":10550.958860759494,"ttft_p50":614.015,"ttft_p95":1242.3799999999999,"ttft_avg":722.0376898734178,"tokens_per_sec_p50":52.32426823831638,"tokens_per_sec_p95":63.1344597753714,"tokens_per_sec_avg":44.98143743041465,"thinking_ratio_p50":null,"thinking_ratio_p95":null,"thinking_ratio_avg":null,"total_params":108.6,"active_params":17,"precision":"fp8","base_model":"llama-4-scout","reasoning_effort":null,"provider":"deepinfra"},{"model_id":"gemma-4-26b-a4b-it-high","display_name":"gemma-4-26b-a4b-it","lab":"google","prompt":"bare","pass_rate":0.24367088607594936,"ci_lower":0.199600510151268,"ci_upper":0.29389877199339365,"total_runs":316,"cost_p50":0.0023806700000000005,"cost_p95":0.010200195000000002,"cost_avg":0.0036450802531645585,"latency_p50":217952.5,"latency_p95":784478.25,"latency_avg":286628.9588607595,"ttft_p50":184948.17,"ttft_p95":737556.36,"ttft_avg":257839.531107595,"tokens_per_sec_p50":28.459091092295708,"tokens_per_sec_p95":61.5442226454649,"tokens_per_sec_avg":32.135705647838904,"thinking_ratio_p50":0.7089508365230026,"thinking_ratio_p95":0.9420321026285023,"thinking_ratio_avg":0.7067842160831926,"total_params":25.8,"active_params":4,"precision":"bf16","base_model":"gemma-4-26b-a4b-it","reasoning_effort":"high","provider":"parasail"},{"model_id":"hermes-4-405b","display_name":"hermes-4-405b","lab":"nousresearch","prompt":"bare","pass_rate":0.24367088607594936,"ci_lower":0.199600510151268,"ci_upper":0.29389877199339365,"total_runs":316,"cost_p50":0.0006725,"cost_p95":0.0021715000000000003,"cost_avg":0.000858835443037975,"latency_p50":2599,"latency_p95":22285.5,"latency_avg":6753.1740506329115,"ttft_p50":693.6800000000001,"ttft_p95":1866.41,"ttft_avg":872.2877531645571,"tokens_per_sec_p50":13.644225200321618,"tokens_per_sec_p95":22.33961609966189,"tokens_per_sec_avg":13.527312009136114,"thinking_ratio_p50":null,"thinking_ratio_p95":null,"thinking_ratio_avg":null,"total_params":405.9,"active_params":405,"precision":"fp8","base_model":"hermes-4-405b","reasoning_effort":null,"provider":"nebius"},{"model_id":"claude-3-haiku","display_name":"claude-3-haiku","lab":"anthropic","prompt":"bare","pass_rate":0.23734177215189872,"ci_lower":0.20582067540011031,"ci_upper":0.2720367029154953,"total_runs":632,"cost_p50":0.00032162499999999997,"cost_p95":0.0009159874999999998,"cost_avg":0.0003655181962025317,"latency_p50":1985.5,"latency_p95":5464.7,"latency_avg":2315.9731012658226,"ttft_p50":259.64,"ttft_p95":1246.7254999999998,"ttft_avg":426.0841613924048,"tokens_per_sec_p50":63.433611880278335,"tokens_per_sec_p95":121.95939192720502,"tokens_per_sec_avg":60.55904263871156,"thinking_ratio_p50":null,"thinking_ratio_p95":null,"thinking_ratio_avg":null,"total_params":null,"active_params":null,"precision":"unknown","base_model":"claude-3-haiku","reasoning_effort":null,"provider":"amazon-bedrock"},{"model_id":"command-a","display_name":"command-a","lab":"cohere","prompt":"bare","pass_rate":0.23734177215189872,"ci_lower":0.19376259990374478,"ci_upper":0.28723049167658754,"total_runs":316,"cost_p50":0.00322125,"cost_p95":0.0114475,"cost_avg":0.004271368670886078,"latency_p50":5349.5,"latency_p95":30716.25,"latency_avg":9500.354430379746,"ttft_p50":1450.52,"ttft_p95":4333.62,"ttft_avg":1866.1313607594948,"tokens_per_sec_p50":21.54270128691635,"tokens_per_sec_p95":55.02580419039833,"tokens_per_sec_avg":23.03192975448347,"thinking_ratio_p50":null,"thinking_ratio_p95":null,"thinking_ratio_avg":null,"total_params":111.1,"active_params":111,"precision":"unknown","base_model":"command-a","reasoning_effort":null,"provider":"cohere"},{"model_id":"glm-4.7-flash-high","display_name":"glm-4.7-flash","lab":"z-ai","prompt":"bare","pass_rate":0.23734177215189872,"ci_lower":0.19376259990374478,"ci_upper":0.28723049167658754,"total_runs":316,"cost_p50":0.0026603,"cost_p95":0.008222485,"cost_avg":0.003164062215189875,"latency_p50":130177,"latency_p95":410145.75,"latency_avg":160724.39556962025,"ttft_p50":127021.66,"ttft_p95":409560.86,"ttft_avg":157248.75825949374,"tokens_per_sec_p50":48.757750067353854,"tokens_per_sec_p95":66.26741639208814,"tokens_per_sec_avg":47.18157137238911,"thinking_ratio_p50":0.9949291663847892,"thinking_ratio_p95":0.999330441684413,"thinking_ratio_avg":0.9791565209589325,"total_params":31.2,"active_params":3,"precision":"unknown","base_model":"glm-4.7-flash","reasoning_effort":"high","provider":"z-ai"},{"model_id":"llama-3.3-70b-instruct","display_name":"llama-3.3-70b-instruct","lab":"meta","prompt":"bare","pass_rate":0.2310126582278481,"ci_lower":0.18793867595940303,"ci_upper":0.2805482250566,"total_runs":316,"cost_p50":0.00022927,"cost_p95":0.0005517125000000001,"cost_avg":0.0002519334651898734,"latency_p50":10187,"latency_p95":23948.75,"latency_avg":10708.541139240506,"ttft_p50":911.745,"ttft_p95":1252.815,"ttft_avg":964.6442721518981,"tokens_per_sec_p50":42.2111146734239,"tokens_per_sec_p95":47.08099375358607,"tokens_per_sec_avg":32.247676241371984,"thinking_ratio_p50":null,"thinking_ratio_p95":null,"thinking_ratio_avg":null,"total_params":70.6,"active_params":70,"precision":"bf16","base_model":"llama-3.3-70b-instruct","reasoning_effort":null,"provider":"novita"},{"model_id":"gemini-2.5-flash-lite","display_name":"gemini-2.5-flash-lite","lab":"google","prompt":"bare","pass_rate":0.2120253164556962,"ci_lower":0.1705548950490372,"ci_upper":0.26041343427397773,"total_runs":316,"cost_p50":0.00013930000000000002,"cost_p95":0.002013725,"cost_avg":0.00044735569620253163,"latency_p50":1801,"latency_p95":13628,"latency_avg":3788.1645569620255,"ttft_p50":599.775,"ttft_p95":1642.0825,"ttft_avg":820.1384177215186,"tokens_per_sec_p50":102.47834696569018,"tokens_per_sec_p95":369.9182613164799,"tokens_per_sec_avg":146.99204918347212,"thinking_ratio_p50":null,"thinking_ratio_p95":null,"thinking_ratio_avg":null,"total_params":null,"active_params":null,"precision":"unknown","base_model":"gemini-2.5-flash-lite","reasoning_effort":null,"provider":"google-vertex"},{"model_id":"gpt-4.1-nano","display_name":"gpt-4.1-nano","lab":"openai","prompt":"bare","pass_rate":0.20253164556962025,"ci_lower":0.16191533523134458,"ci_upper":0.2502937082451763,"total_runs":316,"cost_p50":0.0001284,"cost_p95":0.00047055,"cost_avg":0.00017584651898734188,"latency_p50":2321.5,"latency_p95":6563.5,"latency_avg":2932.1139240506327,"ttft_p50":742.5799999999999,"ttft_p95":1469.6100000000001,"ttft_avg":850.7941772151901,"tokens_per_sec_p50":92.26873465359833,"tokens_per_sec_p95":170.4258967883498,"tokens_per_sec_avg":79.56085506884497,"thinking_ratio_p50":null,"thinking_ratio_p95":null,"thinking_ratio_avg":null,"total_params":null,"active_params":null,"precision":"unknown","base_model":"gpt-4.1-nano","reasoning_effort":null,"provider":"openai"},{"model_id":"llama-3.1-instruct-70b","display_name":"llama-3.1-instruct-70b","lab":"meta","prompt":"bare","pass_rate":0.19936708860759494,"ci_lower":0.1590436614124851,"ci_upper":0.24691228678187116,"total_runs":316,"cost_p50":0.0002848,"cost_p95":0.0006569,"cost_avg":0.00031996455696202533,"latency_p50":15445.5,"latency_p95":47070.25,"latency_avg":17873.028481012658,"ttft_p50":558.0899999999999,"ttft_p95":1422.9225,"ttft_avg":667.8685126582284,"tokens_per_sec_p50":16.033920361515666,"tokens_per_sec_p95":20.693359566450585,"tokens_per_sec_avg":13.90691378960291,"thinking_ratio_p50":null,"thinking_ratio_p95":null,"thinking_ratio_avg":null,"total_params":70.6,"active_params":70,"precision":"bf16","base_model":"llama-3.1-instruct-70b","reasoning_effort":null,"provider":"deepinfra"},{"model_id":"gpt-4o-mini","display_name":"gpt-4o-mini","lab":"openai","prompt":"bare","pass_rate":0.18354430379746836,"ci_lower":0.14474996823019048,"ci_upper":0.22994050355334242,"total_runs":316,"cost_p50":0.00012007499999999998,"cost_p95":0.0005061,"cost_avg":0.00019342357594936705,"latency_p50":1526.5,"latency_p95":10199.5,"latency_avg":3909.3670886075947,"ttft_p50":715.315,"ttft_p95":1369.74,"ttft_avg":820.3509493670884,"tokens_per_sec_p50":22.421335513384705,"tokens_per_sec_p95":66.61253268893377,"tokens_per_sec_avg":31.25365806975717,"thinking_ratio_p50":null,"thinking_ratio_p95":null,"thinking_ratio_avg":null,"total_params":null,"active_params":null,"precision":"unknown","base_model":"gpt-4o-mini","reasoning_effort":null,"provider":"openai"},{"model_id":"command-r-plus","display_name":"command-r-plus","lab":"cohere","prompt":"bare","pass_rate":0.18037974683544303,"ci_lower":0.14190473784822366,"ci_upper":0.22653263865314452,"total_runs":316,"cost_p50":0.001555,"cost_p95":0.004894374999999999,"cost_avg":0.002114691455696201,"latency_p50":1851.5,"latency_p95":19631,"latency_avg":4227.579113924051,"ttft_p50":996.615,"ttft_p95":7657.31,"ttft_avg":1647.801518987343,"tokens_per_sec_p50":9.389682383174792,"tokens_per_sec_p95":28.12274847480871,"tokens_per_sec_avg":12.220591132401632,"thinking_ratio_p50":null,"thinking_ratio_p95":null,"thinking_ratio_avg":null,"total_params":104,"active_params":104,"precision":"unknown","base_model":"command-r-plus","reasoning_effort":null,"provider":"cohere"},{"model_id":"gpt-3.5-turbo","display_name":"gpt-3.5-turbo","lab":"openai","prompt":"bare","pass_rate":0.17405063291139242,"ci_lower":0.13622850497778094,"ci_upper":0.21970268095925796,"total_runs":316,"cost_p50":0.0002245,"cost_p95":0.0006745,"cost_avg":0.00029404113924050626,"latency_p50":887,"latency_p95":2076,"latency_avg":1089.5,"ttft_p50":674.355,"ttft_p95":1050.1125000000002,"ttft_avg":709.7320569620257,"tokens_per_sec_p50":18.164695530831864,"tokens_per_sec_p95":56.6478015121955,"tokens_per_sec_avg":22.963943563941104,"thinking_ratio_p50":null,"thinking_ratio_p95":null,"thinking_ratio_avg":null,"total_params":null,"active_params":null,"precision":"unknown","base_model":"gpt-3.5-turbo","reasoning_effort":null,"provider":"openai"},{"model_id":"command-r","display_name":"command-r","lab":"cohere","prompt":"bare","pass_rate":0.17088607594936708,"ci_lower":0.13339770054072886,"ci_upper":0.21628039011414535,"total_runs":316,"cost_p50":0.00010125,"cost_p95":0.00029917500000000005,"cost_avg":0.00012854810126582282,"latency_p50":2468,"latency_p95":23120.75,"latency_avg":7111.914556962025,"ttft_p50":932.03,"ttft_p95":1832.32,"ttft_avg":1086.8673417721523,"tokens_per_sec_p50":8.428569757023842,"tokens_per_sec_p95":31.497039703759288,"tokens_per_sec_avg":11.078059085779225,"thinking_ratio_p50":null,"thinking_ratio_p95":null,"thinking_ratio_avg":null,"total_params":35,"active_params":35,"precision":"unknown","base_model":"command-r","reasoning_effort":null,"provider":"cohere"},{"model_id":"hermes-4-70b","display_name":"hermes-4-70b","lab":"nousresearch","prompt":"bare","pass_rate":0.16455696202531644,"ci_lower":0.12775124522715484,"ci_upper":0.20942065486338998,"total_runs":316,"cost_p50":0.00006262000000000001,"cost_p95":0.000188825,"cost_avg":0.00008327015822784808,"latency_p50":2497.5,"latency_p95":26490.5,"latency_avg":6673.189873417721,"ttft_p50":825.6500000000001,"ttft_p95":1703.2175000000002,"ttft_avg":937.626107594937,"tokens_per_sec_p50":6.332725450140426,"tokens_per_sec_p95":13.663565384075927,"tokens_per_sec_avg":7.149522587808703,"thinking_ratio_p50":null,"thinking_ratio_p95":null,"thinking_ratio_avg":null,"total_params":70,"active_params":70,"precision":"fp8","base_model":"hermes-4-70b","reasoning_effort":null,"provider":"nebius"},{"model_id":"phi-4","display_name":"phi-4","lab":"microsoft","prompt":"bare","pass_rate":0.1550632911392405,"ci_lower":0.11932118911957067,"ci_upper":0.19909142512448016,"total_runs":316,"cost_p50":0.000092015,"cost_p95":0.00021119000000000004,"cost_avg":0.00010118743670886081,"latency_p50":7646.5,"latency_p95":15407.25,"latency_avg":7757.107594936709,"ttft_p50":378.45500000000004,"ttft_p95":774.005,"ttft_avg":466.1387341772151,"tokens_per_sec_p50":61.249287230849646,"tokens_per_sec_p95":66.4657129190764,"tokens_per_sec_avg":52.18657161519274,"thinking_ratio_p50":null,"thinking_ratio_p95":null,"thinking_ratio_avg":null,"total_params":14.7,"active_params":14,"precision":"bf16","base_model":"phi-4","reasoning_effort":null,"provider":"deepinfra"},{"model_id":"llama-3-instruct-8b","display_name":"llama-3-instruct-8b","lab":"meta","prompt":"bare","pass_rate":0.10759493670886076,"ci_lower":0.0780227604632617,"ci_upper":0.14659342454831908,"total_runs":316,"cost_p50":0.000017295,"cost_p95":0.000044605,"cost_avg":0.000019689335443037968,"latency_p50":1438.5,"latency_p95":11561,"latency_avg":3715.9556962025317,"ttft_p50":469.62,"ttft_p95":917.9525,"ttft_avg":546.4986708860761,"tokens_per_sec_p50":17.673393603914697,"tokens_per_sec_p95":39.91467168106132,"tokens_per_sec_avg":21.88533489952136,"thinking_ratio_p50":null,"thinking_ratio_p95":null,"thinking_ratio_avg":null,"total_params":8,"active_params":8,"precision":"bf16","base_model":"llama-3-instruct-8b","reasoning_effort":null,"provider":"deepinfra"},{"model_id":"llama-3.1-instruct-8b","display_name":"llama-3.1-instruct-8b","lab":"meta","prompt":"bare","pass_rate":0.0949367088607595,"ci_lower":0.06731043634030419,"ci_upper":0.1322933675426179,"total_runs":316,"cost_p50":0.000014565,"cost_p95":0.0000554675,"cost_avg":0.00002010436708860759,"latency_p50":1468.5,"latency_p95":16510.25,"latency_avg":4824.816455696203,"ttft_p50":651.265,"ttft_p95":1095.9325000000001,"ttft_avg":789.9098417721516,"tokens_per_sec_p50":17.257517257517257,"tokens_per_sec_p95":77.45610239014255,"tokens_per_sec_avg":28.58962798247965,"thinking_ratio_p50":null,"thinking_ratio_p95":null,"thinking_ratio_avg":null,"total_params":8,"active_params":8,"precision":"bf16","base_model":"llama-3.1-instruct-8b","reasoning_effort":null,"provider":"deepinfra"},{"model_id":"llama-3.2-instruct-3b","display_name":"llama-3.2-instruct-3b","lab":"meta","prompt":"bare","pass_rate":0.06962025316455696,"ci_lower":0.04642211732326561,"ci_upper":0.10315692430233908,"total_runs":316,"cost_p50":0.000051600999999999996,"cost_p95":0.000358784325,"cost_avg":0.00013941542721518987,"latency_p50":1254,"latency_p95":19322.5,"latency_avg":8238.360759493671,"ttft_p50":626.43,"ttft_p95":1499.465,"ttft_avg":731.3735759493669,"tokens_per_sec_p50":22.497216317441037,"tokens_per_sec_p95":114.1385999930985,"tokens_per_sec_avg":39.58190166358696,"thinking_ratio_p50":null,"thinking_ratio_p95":null,"thinking_ratio_avg":null,"total_params":3.2,"active_params":3,"precision":"unknown","base_model":"llama-3.2-instruct-3b","reasoning_effort":null,"provider":"cloudflare"},{"model_id":"mixtral-8x7b-instruct","display_name":"mixtral-8x7b-instruct","lab":"mistral","prompt":"bare","pass_rate":0.060126582278481014,"ci_lower":0.03882664502120181,"ci_upper":0.09199311075790886,"total_runs":316,"cost_p50":0.00040581000000000005,"cost_p95":0.0012984300000000002,"cost_avg":0.0005237931645569616,"latency_p50":3147,"latency_p95":17468,"latency_avg":5755.427215189873,"ttft_p50":401.63,"ttft_p95":861.4100000000001,"ttft_avg":495.7054430379748,"tokens_per_sec_p50":40.91092878381812,"tokens_per_sec_p95":48.518795553517776,"tokens_per_sec_avg":37.64982822314441,"thinking_ratio_p50":null,"thinking_ratio_p95":null,"thinking_ratio_avg":null,"total_params":47,"active_params":13,"precision":"fp8","base_model":"mixtral-8x7b-instruct","reasoning_effort":null,"provider":"deepinfra"},{"model_id":"llama-3.2-instruct-1b","display_name":"llama-3.2-instruct-1b","lab":"meta","prompt":"bare","pass_rate":0.006329113924050633,"ci_lower":0.0017373543361146123,"ci_upper":0.022779781646196633,"total_runs":316,"cost_p50":0.00003258,"cost_p95":0.00235610025,"cost_avg":0.000266152756329114,"latency_p50":2083,"latency_p95":177091,"latency_avg":20299.651898734177,"ttft_p50":597.29,"ttft_p95":2206.92,"ttft_avg":863.2800949367087,"tokens_per_sec_p50":30.99315311563995,"tokens_per_sec_p95":134.60582530841438,"tokens_per_sec_avg":45.51895483039501,"thinking_ratio_p50":null,"thinking_ratio_p95":null,"thinking_ratio_avg":null,"total_params":1.2,"active_params":1,"precision":"unknown","base_model":"llama-3.2-instruct-1b","reasoning_effort":null,"provider":"cloudflare"}]}
