{"model_scores":[{"model_id":"claude-fable-5-1-max","display_name":"claude-fable-5-1","lab":"anthropic","prompt":"bare","release_date":"2026-09-01","precision":"unknown","base_model":"claude-fable-5-1","reasoning_effort":"max","provider":"anthropic","notes":null,"pass_rate":0.8860759493670886,"ci_lower":0.8463111555009678,"ci_upper":0.9165664689231221,"total_questions":316,"total_runs":316,"cost_avg":0.32623536392405067,"cost_total":103.09037500000001,"latency_p50":40040.5,"category_pass_rates":{"calculations":1,"conceptual":1,"consensus-epoch-processing":0.9038461538461539,"consensus-fork-choice":0.9333333333333333,"consensus-shuffling":0,"consensus-state-transition":1,"constants":1,"cross-fork":1,"eip-interactions":1,"evm-execution":0.7833333333333333,"trick":0.8333333333333334}},{"model_id":"gemini-4-argon-high","display_name":"gemini-4-argon","lab":"google","prompt":"bare","release_date":"2026-10-13","precision":"unknown","base_model":"gemini-4-argon","reasoning_effort":"high","provider":"google-ai-studio","notes":null,"pass_rate":0.8860759493670886,"ci_lower":0.8463111555009678,"ci_upper":0.9165664689231221,"total_questions":316,"total_runs":316,"cost_avg":0.029119253164556956,"cost_total":9.201683999999998,"latency_p50":16434,"category_pass_rates":{"calculations":1,"conceptual":1,"consensus-epoch-processing":0.9230769230769231,"consensus-fork-choice":0.9333333333333333,"consensus-shuffling":0,"consensus-state-transition":1,"constants":1,"cross-fork":1,"eip-interactions":1,"evm-execution":0.7833333333333333,"trick":0.6666666666666666}},{"model_id":"claude-opus-4.8-max","display_name":"claude-opus-4.8","lab":"anthropic","prompt":"bare","release_date":"2026-05-28","precision":"unknown","base_model":"claude-opus-4.8","reasoning_effort":"max","provider":"anthropic","notes":null,"pass_rate":0.8829113924050633,"ci_lower":0.8427759466901593,"ci_upper":0.9138485824517661,"total_questions":316,"total_runs":316,"cost_avg":0.2848904746835444,"cost_total":90.02539000000003,"latency_p50":56985.5,"category_pass_rates":{"calculations":1,"conceptual":1,"consensus-epoch-processing":0.9230769230769231,"consensus-fork-choice":0.9333333333333333,"consensus-shuffling":0,"consensus-state-transition":1,"constants":1,"cross-fork":1,"eip-interactions":1,"evm-execution":0.7666666666666667,"trick":0.8333333333333334}},{"model_id":"claude-opus-5-max","display_name":"claude-opus-5","lab":"anthropic","prompt":"bare","release_date":"2026-07-24","precision":"unknown","base_model":"claude-opus-5","reasoning_effort":"max","provider":"anthropic","notes":null,"pass_rate":0.879746835443038,"ci_lower":0.8392486909659214,"ci_upper":0.9111227428938393,"total_questions":316,"total_runs":316,"cost_avg":0.20707868670886054,"cost_total":65.43686499999993,"latency_p50":27841,"category_pass_rates":{"calculations":1,"conceptual":1,"consensus-epoch-processing":0.9038461538461539,"consensus-fork-choice":0.9333333333333333,"consensus-shuffling":0,"consensus-state-transition":0.98,"constants":1,"cross-fork":1,"eip-interactions":1,"evm-execution":0.775,"trick":0.8333333333333334}},{"model_id":"claude-sonnet-5-5-xhigh","display_name":"claude-sonnet-5-5","lab":"anthropic","prompt":"bare","release_date":"2026-09-28","precision":"unknown","base_model":"claude-sonnet-5-5","reasoning_effort":"xhigh","provider":"anthropic","notes":null,"pass_rate":0.879746835443038,"ci_lower":0.8392486909659214,"ci_upper":0.9111227428938393,"total_questions":316,"total_runs":316,"cost_avg":0.06928749683544302,"cost_total":21.894848999999994,"latency_p50":9321.5,"category_pass_rates":{"calculations":1,"conceptual":0.8333333333333334,"consensus-epoch-processing":0.9230769230769231,"consensus-fork-choice":0.8666666666666667,"consensus-shuffling":0,"consensus-state-transition":1,"constants":1,"cross-fork":1,"eip-interactions":1,"evm-execution":0.7666666666666667,"trick":1}},{"model_id":"gpt-6.1-terra-xhigh","display_name":"gpt-6.1-terra","lab":"openai","prompt":"bare","release_date":"2026-10-26","precision":"unknown","base_model":"gpt-6.1-terra","reasoning_effort":"xhigh","provider":"openai","notes":null,"pass_rate":0.8734177215189873,"ci_lower":0.8322170023469961,"ci_upper":0.9056482409484351,"total_questions":316,"total_runs":316,"cost_avg":0.002492487341772152,"cost_total":0.787626,"latency_p50":4416.5,"category_pass_rates":{"calculations":1,"conceptual":0.8333333333333334,"consensus-epoch-processing":0.8846153846153846,"consensus-fork-choice":0.8666666666666667,"consensus-shuffling":0,"consensus-state-transition":0.98,"constants":1,"cross-fork":1,"eip-interactions":1,"evm-execution":0.7833333333333333,"trick":0.8333333333333334}},{"model_id":"claude-fable-5-high","display_name":"claude-fable-5","lab":"anthropic","prompt":"bare","release_date":"2026-06-07","precision":"unknown","base_model":"claude-fable-5","reasoning_effort":"high","provider":"anthropic","notes":null,"pass_rate":0.8734177215189873,"ci_lower":0.8322170023469961,"ci_upper":0.9056482409484351,"total_questions":316,"total_runs":316,"cost_avg":0.08737863924050626,"cost_total":27.61164999999998,"latency_p50":13147,"category_pass_rates":{"calculations":1,"conceptual":0.8333333333333334,"consensus-epoch-processing":0.9230769230769231,"consensus-fork-choice":0.8666666666666667,"consensus-shuffling":0,"consensus-state-transition":1,"constants":1,"cross-fork":1,"eip-interactions":1,"evm-execution":0.7666666666666667,"trick":0.6666666666666666}},{"model_id":"claude-fable-5-1-medium","display_name":"claude-fable-5-1","lab":"anthropic","prompt":"bare","release_date":"2026-09-01","precision":"unknown","base_model":"claude-fable-5-1","reasoning_effort":"medium","provider":"anthropic","notes":null,"pass_rate":0.8734177215189873,"ci_lower":0.8322170023469961,"ci_upper":0.9056482409484351,"total_questions":316,"total_runs":316,"cost_avg":0.053614636075949344,"cost_total":16.942224999999993,"latency_p50":8402.5,"category_pass_rates":{"calculations":1,"conceptual":0.8333333333333334,"consensus-epoch-processing":0.9038461538461539,"consensus-fork-choice":0.9333333333333333,"consensus-shuffling":0,"consensus-state-transition":1,"constants":0.9824561403508771,"cross-fork":1,"eip-interactions":1,"evm-execution":0.7583333333333333,"trick":1}},{"model_id":"gpt-6.1-sol-max","display_name":"gpt-6.1-sol","lab":"openai","prompt":"bare","release_date":"2026-09-29","precision":"unknown","base_model":"gpt-6.1-sol","reasoning_effort":"max","provider":"openai","notes":null,"pass_rate":0.8734177215189873,"ci_lower":0.8322170023469961,"ci_upper":0.9056482409484351,"total_questions":316,"total_runs":316,"cost_avg":0.013899358730158731,"cost_total":4.378298,"latency_p50":21539,"category_pass_rates":{"calculations":1,"conceptual":1,"consensus-epoch-processing":0.8461538461538461,"consensus-fork-choice":0.9333333333333333,"consensus-shuffling":0,"consensus-state-transition":1,"constants":1,"cross-fork":1,"eip-interactions":0,"evm-execution":0.775,"trick":1}},{"model_id":"claude-opus-5-5-xhigh","display_name":"claude-opus-5-5","lab":"anthropic","prompt":"bare","release_date":"2026-09-22","precision":"unknown","base_model":"claude-opus-5-5","reasoning_effort":"xhigh","provider":"anthropic","notes":null,"pass_rate":0.8734177215189873,"ci_lower":0.8322170023469961,"ci_upper":0.9056482409484351,"total_questions":316,"total_runs":316,"cost_avg":0.0998633417721519,"cost_total":31.556816,"latency_p50":13636.5,"category_pass_rates":{"calculations":1,"conceptual":1,"consensus-epoch-processing":0.9038461538461539,"consensus-fork-choice":0.8666666666666667,"consensus-shuffling":0,"consensus-state-transition":1,"constants":1,"cross-fork":1,"eip-interactions":1,"evm-execution":0.75,"trick":1}},{"model_id":"gpt-6-astra-max","display_name":"gpt-6-astra","lab":"openai","prompt":"bare","release_date":"2026-09-04","precision":"unknown","base_model":"gpt-6-astra","reasoning_effort":"max","provider":"openai","notes":null,"pass_rate":0.8706624605678234,"ci_lower":0.8292374941362494,"ci_upper":0.9032111658878872,"total_questions":316,"total_runs":317,"cost_avg":0.06033981012658228,"cost_total":19.06738,"latency_p50":26109,"category_pass_rates":{"calculations":1,"conceptual":1,"consensus-epoch-processing":0.8679245283018868,"consensus-fork-choice":0.9333333333333333,"consensus-shuffling":0,"consensus-state-transition":1,"constants":1,"cross-fork":1,"eip-interactions":1,"evm-execution":0.7583333333333333,"trick":0.8333333333333334}},{"model_id":"gpt-6.1-sol-low","display_name":"gpt-6.1-sol","lab":"openai","prompt":"bare","release_date":"2026-09-29","precision":"unknown","base_model":"gpt-6.1-sol","reasoning_effort":"low","provider":"openai","notes":null,"pass_rate":0.870253164556962,"ci_lower":0.8287120904037939,"ci_upper":0.9029000576094727,"total_questions":316,"total_runs":316,"cost_avg":0.002629151898734179,"cost_total":0.8308120000000004,"latency_p50":6311,"category_pass_rates":{"calculations":1,"conceptual":0.8333333333333334,"consensus-epoch-processing":0.9038461538461539,"consensus-fork-choice":0.8666666666666667,"consensus-shuffling":0,"consensus-state-transition":0.98,"constants":1,"cross-fork":1,"eip-interactions":1,"evm-execution":0.7666666666666667,"trick":0.8333333333333334}},{"model_id":"claude-fable-5-1-high","display_name":"claude-fable-5-1","lab":"anthropic","prompt":"bare","release_date":"2026-09-01","precision":"unknown","base_model":"claude-fable-5-1","reasoning_effort":"high","provider":"anthropic","notes":null,"pass_rate":0.870253164556962,"ci_lower":0.8287120904037939,"ci_upper":0.9029000576094727,"total_questions":316,"total_runs":316,"cost_avg":0.06926931962025311,"cost_total":21.889104999999983,"latency_p50":11939.5,"category_pass_rates":{"calculations":1,"conceptual":0.8333333333333334,"consensus-epoch-processing":0.8653846153846154,"consensus-fork-choice":0.8666666666666667,"consensus-shuffling":0,"consensus-state-transition":1,"constants":0.9824561403508771,"cross-fork":1,"eip-interactions":1,"evm-execution":0.7833333333333333,"trick":0.8333333333333334}},{"model_id":"claude-fable-5-max","display_name":"claude-fable-5","lab":"anthropic","prompt":"bare","release_date":"2026-06-07","precision":"unknown","base_model":"claude-fable-5","reasoning_effort":"max","provider":"anthropic","notes":null,"pass_rate":0.870253164556962,"ci_lower":0.8287120904037939,"ci_upper":0.9029000576094727,"total_questions":316,"total_runs":316,"cost_avg":0.2690966613924053,"cost_total":85.03454500000007,"latency_p50":29920,"category_pass_rates":{"calculations":1,"conceptual":0.8333333333333334,"consensus-epoch-processing":0.9038461538461539,"consensus-fork-choice":0.8666666666666667,"consensus-shuffling":0,"consensus-state-transition":1,"constants":0.9649122807017544,"cross-fork":1,"eip-interactions":1,"evm-execution":0.775,"trick":0.8333333333333334}},{"model_id":"claude-fable-5-1-xhigh","display_name":"claude-fable-5-1","lab":"anthropic","prompt":"bare","release_date":"2026-09-01","precision":"unknown","base_model":"claude-fable-5-1","reasoning_effort":"xhigh","provider":"anthropic","notes":null,"pass_rate":0.870253164556962,"ci_lower":0.8287120904037939,"ci_upper":0.9029000576094727,"total_questions":316,"total_runs":316,"cost_avg":0.14582462025316448,"cost_total":46.080579999999976,"latency_p50":16757,"category_pass_rates":{"calculations":1,"conceptual":1,"consensus-epoch-processing":0.9038461538461539,"consensus-fork-choice":0.9333333333333333,"consensus-shuffling":0,"consensus-state-transition":1,"constants":0.9824561403508771,"cross-fork":1,"eip-interactions":1,"evm-execution":0.7416666666666667,"trick":1}},{"model_id":"gpt-6-astra-low","display_name":"gpt-6-astra","lab":"openai","prompt":"bare","release_date":"2026-09-04","precision":"unknown","base_model":"gpt-6-astra","reasoning_effort":"low","provider":"openai","notes":null,"pass_rate":0.870253164556962,"ci_lower":0.8287120904037939,"ci_upper":0.9029000576094727,"total_questions":316,"total_runs":316,"cost_avg":0.013171392405063284,"cost_total":4.162159999999997,"latency_p50":7171,"category_pass_rates":{"calculations":1,"conceptual":0.8333333333333334,"consensus-epoch-processing":0.8653846153846154,"consensus-fork-choice":0.9333333333333333,"consensus-shuffling":0,"consensus-state-transition":1,"constants":1,"cross-fork":1,"eip-interactions":1,"evm-execution":0.7583333333333333,"trick":1}},{"model_id":"gpt-6.1-sol-high","display_name":"gpt-6.1-sol","lab":"openai","prompt":"bare","release_date":"2026-09-29","precision":"unknown","base_model":"gpt-6.1-sol","reasoning_effort":"high","provider":"openai","notes":null,"pass_rate":0.870253164556962,"ci_lower":0.8287120904037939,"ci_upper":0.9029000576094727,"total_questions":316,"total_runs":316,"cost_avg":0.004768297468354431,"cost_total":1.5067820000000003,"latency_p50":9496,"category_pass_rates":{"calculations":1,"conceptual":1,"consensus-epoch-processing":0.8461538461538461,"consensus-fork-choice":0.8666666666666667,"consensus-shuffling":0,"consensus-state-transition":0.98,"constants":1,"cross-fork":1,"eip-interactions":1,"evm-execution":0.775,"trick":1}},{"model_id":"claude-fable-5-xhigh","display_name":"claude-fable-5","lab":"anthropic","prompt":"bare","release_date":"2026-06-07","precision":"unknown","base_model":"claude-fable-5","reasoning_effort":"xhigh","provider":"anthropic","notes":null,"pass_rate":0.8670886075949367,"ci_lower":0.8252141708685773,"ci_upper":0.9001448818625245,"total_questions":316,"total_runs":316,"cost_avg":0.12455843354430383,"cost_total":39.36046500000001,"latency_p50":17448,"category_pass_rates":{"calculations":1,"conceptual":0.8333333333333334,"consensus-epoch-processing":0.9038461538461539,"consensus-fork-choice":0.9333333333333333,"consensus-shuffling":0,"consensus-state-transition":1,"constants":1,"cross-fork":1,"eip-interactions":1,"evm-execution":0.7416666666666667,"trick":0.8333333333333334}},{"model_id":"gpt-6.1-sol-medium","display_name":"gpt-6.1-sol","lab":"openai","prompt":"bare","release_date":"2026-09-29","precision":"unknown","base_model":"gpt-6.1-sol","reasoning_effort":"medium","provider":"openai","notes":null,"pass_rate":0.8639240506329114,"ci_lower":0.8217230378099494,"ci_upper":0.8973829196389881,"total_questions":316,"total_runs":316,"cost_avg":0.003361683544303801,"cost_total":1.0622920000000011,"latency_p50":7741,"category_pass_rates":{"calculations":1,"conceptual":0.8333333333333334,"consensus-epoch-processing":0.8846153846153846,"consensus-fork-choice":0.9333333333333333,"consensus-shuffling":0,"consensus-state-transition":1,"constants":1,"cross-fork":1,"eip-interactions":0,"evm-execution":0.75,"trick":0.8333333333333334}},{"model_id":"gpt-6-astra-xhigh","display_name":"gpt-6-astra","lab":"openai","prompt":"bare","release_date":"2026-09-04","precision":"unknown","base_model":"gpt-6-astra","reasoning_effort":"xhigh","provider":"openai","notes":null,"pass_rate":0.8639240506329114,"ci_lower":0.8217230378099494,"ci_upper":0.8973829196389881,"total_questions":316,"total_runs":316,"cost_avg":0.04122636075949369,"cost_total":13.027530000000006,"latency_p50":14093.5,"category_pass_rates":{"calculations":1,"conceptual":1,"consensus-epoch-processing":0.8461538461538461,"consensus-fork-choice":0.9333333333333333,"consensus-shuffling":0,"consensus-state-transition":0.98,"constants":1,"cross-fork":1,"eip-interactions":0,"evm-execution":0.7583333333333333,"trick":1}},{"model_id":"claude-opus-4.7-max","display_name":"claude-opus-4.7","lab":"anthropic","prompt":"bare","release_date":"2026-04-14","precision":"unknown","base_model":"claude-opus-4.7","reasoning_effort":"max","provider":"anthropic","notes":null,"pass_rate":0.8639240506329114,"ci_lower":0.8217230378099494,"ci_upper":0.8973829196389881,"total_questions":316,"total_runs":316,"cost_avg":0.23650338607594912,"cost_total":74.73506999999992,"latency_p50":42682,"category_pass_rates":{"calculations":1,"conceptual":1,"consensus-epoch-processing":0.8653846153846154,"consensus-fork-choice":0.8666666666666667,"consensus-shuffling":0,"consensus-state-transition":1,"constants":0.9824561403508771,"cross-fork":1,"eip-interactions":1,"evm-execution":0.7583333333333333,"trick":0.8333333333333334}},{"model_id":"claude-sonnet-5-5-high","display_name":"claude-sonnet-5-5","lab":"anthropic","prompt":"bare","release_date":"2026-09-28","precision":"unknown","base_model":"claude-sonnet-5-5","reasoning_effort":"high","provider":"anthropic","notes":null,"pass_rate":0.8607594936708861,"ci_lower":0.8182384968531615,"ci_upper":0.8946143653136112,"total_questions":316,"total_runs":316,"cost_avg":0.035592449367088615,"cost_total":11.247214000000001,"latency_p50":7414,"category_pass_rates":{"calculations":1,"conceptual":0.8333333333333334,"consensus-epoch-processing":0.8461538461538461,"consensus-fork-choice":0.8666666666666667,"consensus-shuffling":0,"consensus-state-transition":1,"constants":0.9824561403508771,"cross-fork":1,"eip-interactions":1,"evm-execution":0.7583333333333333,"trick":1}},{"model_id":"claude-opus-5-high","display_name":"claude-opus-5","lab":"anthropic","prompt":"bare","release_date":"2026-07-24","precision":"unknown","base_model":"claude-opus-5","reasoning_effort":"high","provider":"anthropic","notes":null,"pass_rate":0.8607594936708861,"ci_lower":0.8182384968531615,"ci_upper":0.8946143653136112,"total_questions":316,"total_runs":316,"cost_avg":0.058987816455696195,"cost_total":18.64015,"latency_p50":14488.5,"category_pass_rates":{"calculations":1,"conceptual":1,"consensus-epoch-processing":0.8846153846153846,"consensus-fork-choice":0.8666666666666667,"consensus-shuffling":0,"consensus-state-transition":0.94,"constants":1,"cross-fork":1,"eip-interactions":1,"evm-execution":0.7583333333333333,"trick":0.8333333333333334}},{"model_id":"gpt-6.1-sol-xhigh","display_name":"gpt-6.1-sol","lab":"openai","prompt":"bare","release_date":"2026-09-29","precision":"unknown","base_model":"gpt-6.1-sol","reasoning_effort":"xhigh","provider":"openai","notes":null,"pass_rate":0.8575949367088608,"ci_lower":0.8147603642958167,"ci_upper":0.8918394025887914,"total_questions":316,"total_runs":316,"cost_avg":0.009718850793650795,"cost_total":3.0614380000000008,"latency_p50":14485.5,"category_pass_rates":{"calculations":1,"conceptual":1,"consensus-epoch-processing":0.7884615384615384,"consensus-fork-choice":0.9333333333333333,"consensus-shuffling":0,"consensus-state-transition":1,"constants":1,"cross-fork":1,"eip-interactions":1,"evm-execution":0.75,"trick":1}},{"model_id":"claude-haiku-5-5-max","display_name":"claude-haiku-5-5","lab":"anthropic","prompt":"bare","release_date":"2026-10-07","precision":"unknown","base_model":"claude-haiku-5-5","reasoning_effort":"max","provider":"anthropic","notes":null,"pass_rate":0.8544303797468354,"ci_lower":0.8112884663092524,"ci_upper":0.8890582052931909,"total_questions":316,"total_runs":316,"cost_avg":0.009160369303797464,"cost_total":2.894676699999999,"latency_p50":65561,"category_pass_rates":{"calculations":1,"conceptual":1,"consensus-epoch-processing":0.8076923076923077,"consensus-fork-choice":0.8666666666666667,"consensus-shuffling":0,"consensus-state-transition":1,"constants":0.9824561403508771,"cross-fork":1,"eip-interactions":1,"evm-execution":0.75,"trick":1}},{"model_id":"claude-opus-4.8-high","display_name":"claude-opus-4.8","lab":"anthropic","prompt":"bare","release_date":"2026-05-28","precision":"unknown","base_model":"claude-opus-4.8","reasoning_effort":"high","provider":"anthropic","notes":null,"pass_rate":0.8544303797468354,"ci_lower":0.8112884663092524,"ci_upper":0.8890582052931909,"total_questions":316,"total_runs":316,"cost_avg":0.06485419303797467,"cost_total":20.493924999999997,"latency_p50":14723.5,"category_pass_rates":{"calculations":1,"conceptual":0.8333333333333334,"consensus-epoch-processing":0.8269230769230769,"consensus-fork-choice":0.8666666666666667,"consensus-shuffling":0,"consensus-state-transition":1,"constants":0.9824561403508771,"cross-fork":1,"eip-interactions":1,"evm-execution":0.7583333333333333,"trick":0.8333333333333334}},{"model_id":"claude-opus-5-xhigh","display_name":"claude-opus-5","lab":"anthropic","prompt":"bare","release_date":"2026-07-24","precision":"unknown","base_model":"claude-opus-5","reasoning_effort":"xhigh","provider":"anthropic","notes":null,"pass_rate":0.8544303797468354,"ci_lower":0.8112884663092524,"ci_upper":0.8890582052931909,"total_questions":316,"total_runs":316,"cost_avg":0.1010252056962026,"cost_total":31.923965000000024,"latency_p50":19394.5,"category_pass_rates":{"calculations":1,"conceptual":1,"consensus-epoch-processing":0.8846153846153846,"consensus-fork-choice":0.8666666666666667,"consensus-shuffling":0,"consensus-state-transition":0.96,"constants":1,"cross-fork":1,"eip-interactions":1,"evm-execution":0.7333333333333333,"trick":0.8333333333333334}},{"model_id":"claude-opus-4.8-xhigh","display_name":"claude-opus-4.8","lab":"anthropic","prompt":"bare","release_date":"2026-05-28","precision":"unknown","base_model":"claude-opus-4.8","reasoning_effort":"xhigh","provider":"anthropic","notes":null,"pass_rate":0.8544303797468354,"ci_lower":0.8112884663092524,"ci_upper":0.8890582052931909,"total_questions":316,"total_runs":316,"cost_avg":0.12587974683544303,"cost_total":39.778,"latency_p50":28193.5,"category_pass_rates":{"calculations":1,"conceptual":0.8333333333333334,"consensus-epoch-processing":0.8846153846153846,"consensus-fork-choice":0.8666666666666667,"consensus-shuffling":0,"consensus-state-transition":1,"constants":0.9824561403508771,"cross-fork":1,"eip-interactions":1,"evm-execution":0.7333333333333333,"trick":0.8333333333333334}},{"model_id":"claude-opus-4.7-xhigh","display_name":"claude-opus-4.7","lab":"anthropic","prompt":"bare","release_date":"2026-04-14","precision":"unknown","base_model":"claude-opus-4.7","reasoning_effort":"xhigh","provider":"anthropic","notes":null,"pass_rate":0.8512658227848101,"ci_lower":0.8078226382156595,"ci_upper":0.8862709381046191,"total_questions":316,"total_runs":316,"cost_avg":0.07710664556962027,"cost_total":24.365700000000004,"latency_p50":18082.5,"category_pass_rates":{"calculations":1,"conceptual":1,"consensus-epoch-processing":0.8269230769230769,"consensus-fork-choice":0.8666666666666667,"consensus-shuffling":0,"consensus-state-transition":0.98,"constants":0.9824561403508771,"cross-fork":1,"eip-interactions":1,"evm-execution":0.75,"trick":0.8333333333333334}},{"model_id":"gpt-5.6-sol-max","display_name":"gpt-5.6-sol","lab":"openai","prompt":"bare","release_date":"2026-07-09","precision":"unknown","base_model":"gpt-5.6-sol","reasoning_effort":"max","provider":"openai","notes":null,"pass_rate":0.8481012658227848,"ci_lower":0.8043627238323244,"ci_upper":0.8834777572057895,"total_questions":316,"total_runs":316,"cost_avg":0.07434317721518989,"cost_total":23.492444000000006,"latency_p50":30471.5,"category_pass_rates":{"calculations":1,"conceptual":1,"consensus-epoch-processing":0.7115384615384616,"consensus-fork-choice":0.9333333333333333,"consensus-shuffling":0,"consensus-state-transition":1,"constants":1,"cross-fork":1,"eip-interactions":1,"evm-execution":0.7666666666666667,"trick":0.8333333333333334}},{"model_id":"claude-opus-5-medium","display_name":"claude-opus-5","lab":"anthropic","prompt":"bare","release_date":"2026-07-24","precision":"unknown","base_model":"claude-opus-5","reasoning_effort":"medium","provider":"anthropic","notes":null,"pass_rate":0.8481012658227848,"ci_lower":0.8043627238323244,"ci_upper":0.8834777572057895,"total_questions":316,"total_runs":316,"cost_avg":0.040402246835443016,"cost_total":12.767109999999994,"latency_p50":11578,"category_pass_rates":{"calculations":1,"conceptual":1,"consensus-epoch-processing":0.8653846153846154,"consensus-fork-choice":0.8,"consensus-shuffling":0,"consensus-state-transition":1,"constants":1,"cross-fork":1,"eip-interactions":1,"evm-execution":0.7166666666666667,"trick":0.8333333333333334}},{"model_id":"claude-fable-5-low","display_name":"claude-fable-5","lab":"anthropic","prompt":"bare","release_date":"2026-06-07","precision":"unknown","base_model":"claude-fable-5","reasoning_effort":"low","provider":"anthropic","notes":null,"pass_rate":0.8449367088607594,"ci_lower":0.8009085748755198,"ci_upper":0.8806788108804293,"total_questions":316,"total_runs":316,"cost_avg":0.034307642405063295,"cost_total":10.841215,"latency_p50":10965,"category_pass_rates":{"calculations":1,"conceptual":1,"consensus-epoch-processing":0.8076923076923077,"consensus-fork-choice":0.8666666666666667,"consensus-shuffling":0,"consensus-state-transition":1,"constants":0.9824561403508771,"cross-fork":1,"eip-interactions":1,"evm-execution":0.7333333333333333,"trick":0.8333333333333334}},{"model_id":"claude-haiku-5-5-high","display_name":"claude-haiku-5-5","lab":"anthropic","prompt":"bare","release_date":"2026-10-07","precision":"unknown","base_model":"claude-haiku-5-5","reasoning_effort":"high","provider":"anthropic","notes":null,"pass_rate":0.8449367088607594,"ci_lower":0.8009085748755198,"ci_upper":0.8806788108804293,"total_questions":316,"total_runs":316,"cost_avg":0.0014021446202531656,"cost_total":0.44307770000000035,"latency_p50":12258,"category_pass_rates":{"calculations":1,"conceptual":0.6666666666666666,"consensus-epoch-processing":0.75,"consensus-fork-choice":0.8666666666666667,"consensus-shuffling":0,"consensus-state-transition":0.98,"constants":1,"cross-fork":1,"eip-interactions":1,"evm-execution":0.775,"trick":0.8333333333333334}},{"model_id":"gpt-6.1-terra-high","display_name":"gpt-6.1-terra","lab":"openai","prompt":"bare","release_date":"2026-10-26","precision":"unknown","base_model":"gpt-6.1-terra","reasoning_effort":"high","provider":"openai","notes":null,"pass_rate":0.8417721518987342,"ci_lower":0.7974600504175338,"ci_upper":0.8778742400562509,"total_questions":316,"total_runs":316,"cost_avg":0.0016108101265822781,"cost_total":0.5090159999999999,"latency_p50":3428.5,"category_pass_rates":{"calculations":1,"conceptual":1,"consensus-epoch-processing":0.8653846153846154,"consensus-fork-choice":0.8,"consensus-shuffling":0,"consensus-state-transition":0.94,"constants":0.9824561403508771,"cross-fork":1,"eip-interactions":0,"evm-execution":0.7416666666666667,"trick":0.8333333333333334}},{"model_id":"claude-haiku-5-5-xhigh","display_name":"claude-haiku-5-5","lab":"anthropic","prompt":"bare","release_date":"2026-10-07","precision":"unknown","base_model":"claude-haiku-5-5","reasoning_effort":"xhigh","provider":"anthropic","notes":null,"pass_rate":0.8417721518987342,"ci_lower":0.7974600504175338,"ci_upper":0.8778742400562509,"total_questions":316,"total_runs":316,"cost_avg":0.002342921518987341,"cost_total":0.7403631999999997,"latency_p50":17214.5,"category_pass_rates":{"calculations":1,"conceptual":0.8333333333333334,"consensus-epoch-processing":0.7307692307692307,"consensus-fork-choice":0.8666666666666667,"consensus-shuffling":0,"consensus-state-transition":0.96,"constants":0.9824561403508771,"cross-fork":1,"eip-interactions":1,"evm-execution":0.775,"trick":1}},{"model_id":"gpt-5.3-codex-xhigh","display_name":"gpt-5.3-codex","lab":"openai","prompt":"bare","release_date":"2026-02-05","precision":"unknown","base_model":"gpt-5.3-codex","reasoning_effort":"xhigh","provider":"openai","notes":null,"pass_rate":0.8386075949367089,"ci_lower":0.7940170163911512,"ci_upper":0.8750641788004688,"total_questions":316,"total_runs":316,"cost_avg":0.10266517879746838,"cost_total":32.44219650000001,"latency_p50":72761,"category_pass_rates":{"calculations":1,"conceptual":1,"consensus-epoch-processing":0.6538461538461539,"consensus-fork-choice":0.9333333333333333,"consensus-shuffling":0,"consensus-state-transition":1,"constants":1,"cross-fork":1,"eip-interactions":0,"evm-execution":0.7833333333333333,"trick":0.6666666666666666}},{"model_id":"gpt-6.1-terra-max","display_name":"gpt-6.1-terra","lab":"openai","prompt":"bare","release_date":"2026-10-26","precision":"unknown","base_model":"gpt-6.1-terra","reasoning_effort":"max","provider":"openai","notes":null,"pass_rate":0.8386075949367089,"ci_lower":0.7940170163911512,"ci_upper":0.8750641788004688,"total_questions":316,"total_runs":316,"cost_avg":0.013411300632911386,"cost_total":4.237970999999998,"latency_p50":11823.5,"category_pass_rates":{"calculations":1,"conceptual":1,"consensus-epoch-processing":0.7692307692307693,"consensus-fork-choice":0.8,"consensus-shuffling":0,"consensus-state-transition":0.9,"constants":1,"cross-fork":1,"eip-interactions":1,"evm-execution":0.775,"trick":0.8333333333333334}},{"model_id":"claude-opus-4.8-medium","display_name":"claude-opus-4.8","lab":"anthropic","prompt":"bare","release_date":"2026-05-28","precision":"unknown","base_model":"claude-opus-4.8","reasoning_effort":"medium","provider":"anthropic","notes":null,"pass_rate":0.8354430379746836,"ci_lower":0.7905793451366101,"ci_upper":0.8722487547728451,"total_questions":316,"total_runs":316,"cost_avg":0.04871653481012665,"cost_total":15.394425000000021,"latency_p50":12502.5,"category_pass_rates":{"calculations":1,"conceptual":0.6666666666666666,"consensus-epoch-processing":0.8076923076923077,"consensus-fork-choice":0.8666666666666667,"consensus-shuffling":0,"consensus-state-transition":1,"constants":1,"cross-fork":1,"eip-interactions":1,"evm-execution":0.7166666666666667,"trick":0.8333333333333334}},{"model_id":"claude-sonnet-5-5-medium","display_name":"claude-sonnet-5-5","lab":"anthropic","prompt":"bare","release_date":"2026-09-28","precision":"unknown","base_model":"claude-sonnet-5-5","reasoning_effort":"medium","provider":"anthropic","notes":null,"pass_rate":0.8354430379746836,"ci_lower":0.7905793451366101,"ci_upper":0.8722487547728451,"total_questions":316,"total_runs":316,"cost_avg":0.02352900000000001,"cost_total":7.435164000000004,"latency_p50":5255,"category_pass_rates":{"calculations":1,"conceptual":0.8333333333333334,"consensus-epoch-processing":0.8269230769230769,"consensus-fork-choice":0.8666666666666667,"consensus-shuffling":0,"consensus-state-transition":0.98,"constants":0.9473684210526315,"cross-fork":1,"eip-interactions":1,"evm-execution":0.725,"trick":1}},{"model_id":"claude-sonnet-5-5-low","display_name":"claude-sonnet-5-5","lab":"anthropic","prompt":"bare","release_date":"2026-09-28","precision":"unknown","base_model":"claude-sonnet-5-5","reasoning_effort":"low","provider":"anthropic","notes":null,"pass_rate":0.8291139240506329,"ci_lower":0.7837196098858548,"ci_upper":0.8666022994592711,"total_questions":316,"total_runs":316,"cost_avg":0.015242623417721536,"cost_total":4.816669000000005,"latency_p50":5045.5,"category_pass_rates":{"calculations":1,"conceptual":0.8333333333333334,"consensus-epoch-processing":0.8269230769230769,"consensus-fork-choice":0.8666666666666667,"consensus-shuffling":0,"consensus-state-transition":1,"constants":0.9824561403508771,"cross-fork":1,"eip-interactions":1,"evm-execution":0.6833333333333333,"trick":1}},{"model_id":"gpt-5.6-sol-xhigh","display_name":"gpt-5.6-sol","lab":"openai","prompt":"bare","release_date":"2026-07-09","precision":"unknown","base_model":"gpt-5.6-sol","reasoning_effort":"xhigh","provider":"openai","notes":null,"pass_rate":0.8259493670886076,"ci_lower":0.7802973190407421,"ci_upper":0.8637714950222191,"total_questions":316,"total_runs":316,"cost_avg":0.044245037974683536,"cost_total":13.981431999999998,"latency_p50":21321.5,"category_pass_rates":{"calculations":1,"conceptual":1,"consensus-epoch-processing":0.6538461538461539,"consensus-fork-choice":0.9333333333333333,"consensus-shuffling":0,"consensus-state-transition":0.98,"constants":0.9824561403508771,"cross-fork":1,"eip-interactions":0,"evm-execution":0.7583333333333333,"trick":0.8333333333333334}},{"model_id":"claude-opus-4.5-high","display_name":"claude-opus-4.5","lab":"anthropic","prompt":"bare","release_date":"2025-11-24","precision":"unknown","base_model":"claude-opus-4.5","reasoning_effort":"high","provider":"anthropic","notes":null,"pass_rate":0.819620253164557,"ci_lower":0.7734673613468556,"ci_upper":0.8580952621517763,"total_questions":316,"total_runs":316,"cost_avg":0.15285536392405044,"cost_total":48.30229499999994,"latency_p50":54941,"category_pass_rates":{"calculations":1,"conceptual":0.6666666666666666,"consensus-epoch-processing":0.6730769230769231,"consensus-fork-choice":0.8666666666666667,"consensus-shuffling":0,"consensus-state-transition":0.96,"constants":0.9649122807017544,"cross-fork":0.75,"eip-interactions":1,"evm-execution":0.7916666666666666,"trick":0.5}},{"model_id":"gpt-5.4-xhigh","display_name":"gpt-5.4","lab":"openai","prompt":"bare","release_date":"2026-03-05","precision":"unknown","base_model":"gpt-5.4","reasoning_effort":"xhigh","provider":"openai","notes":null,"pass_rate":0.819620253164557,"ci_lower":0.7734673613468556,"ci_upper":0.8580952621517763,"total_questions":316,"total_runs":316,"cost_avg":0.09640752373417724,"cost_total":30.464777500000007,"latency_p50":81007,"category_pass_rates":{"calculations":1,"conceptual":1,"consensus-epoch-processing":0.6538461538461539,"consensus-fork-choice":0.9333333333333333,"consensus-shuffling":0,"consensus-state-transition":1,"constants":1,"cross-fork":0.5,"eip-interactions":0,"evm-execution":0.75,"trick":0.6666666666666666}},{"model_id":"gpt-6.1-terra-medium","display_name":"gpt-6.1-terra","lab":"openai","prompt":"bare","release_date":"2026-10-26","precision":"unknown","base_model":"gpt-6.1-terra","reasoning_effort":"medium","provider":"openai","notes":null,"pass_rate":0.819620253164557,"ci_lower":0.7734673613468556,"ci_upper":0.8580952621517763,"total_questions":316,"total_runs":316,"cost_avg":0.001270632911392405,"cost_total":0.40152,"latency_p50":3060.5,"category_pass_rates":{"calculations":1,"conceptual":0.6666666666666666,"consensus-epoch-processing":0.8076923076923077,"consensus-fork-choice":0.8,"consensus-shuffling":0,"consensus-state-transition":0.94,"constants":1,"cross-fork":1,"eip-interactions":0,"evm-execution":0.7166666666666667,"trick":0.8333333333333334}},{"model_id":"gemini-3.8-flash-high","display_name":"gemini-3.8-flash","lab":"google","prompt":"bare","release_date":"2026-09-02","precision":"unknown","base_model":"gemini-3.8-flash","reasoning_effort":"high","provider":"google-vertex","notes":null,"pass_rate":0.819620253164557,"ci_lower":0.7734673613468556,"ci_upper":0.8580952621517763,"total_questions":316,"total_runs":316,"cost_avg":0.060704394778481024,"cost_total":19.182588750000004,"latency_p50":66092.5,"category_pass_rates":{"calculations":1,"conceptual":0.8333333333333334,"consensus-epoch-processing":0.6153846153846154,"consensus-fork-choice":0.8666666666666667,"consensus-shuffling":0,"consensus-state-transition":0.94,"constants":0.9824561403508771,"cross-fork":1,"eip-interactions":1,"evm-execution":0.7916666666666666,"trick":0.6666666666666666}},{"model_id":"claude-opus-4.8-low","display_name":"claude-opus-4.8","lab":"anthropic","prompt":"bare","release_date":"2026-05-28","precision":"unknown","base_model":"claude-opus-4.8","reasoning_effort":"low","provider":"anthropic","notes":null,"pass_rate":0.8164556962025317,"ci_lower":0.7700594964466577,"ci_upper":0.8552500317698095,"total_questions":316,"total_runs":316,"cost_avg":0.029576898734177258,"cost_total":9.346300000000014,"latency_p50":8543.5,"category_pass_rates":{"calculations":1,"conceptual":0.6666666666666666,"consensus-epoch-processing":0.8076923076923077,"consensus-fork-choice":0.8666666666666667,"consensus-shuffling":0,"consensus-state-transition":1,"constants":1,"cross-fork":1,"eip-interactions":1,"evm-execution":0.6666666666666666,"trick":0.8333333333333334}},{"model_id":"gpt-5.5-xhigh","display_name":"gpt-5.5","lab":"openai","prompt":"bare","release_date":"2026-04-23","precision":"unknown","base_model":"gpt-5.5","reasoning_effort":"xhigh","provider":"openai","notes":null,"pass_rate":0.8164556962025317,"ci_lower":0.7700594964466577,"ci_upper":0.8552500317698095,"total_questions":316,"total_runs":316,"cost_avg":0.19125555379746825,"cost_total":60.43675499999997,"latency_p50":67597,"category_pass_rates":{"calculations":1,"conceptual":1,"consensus-epoch-processing":0.7115384615384616,"consensus-fork-choice":0.8666666666666667,"consensus-shuffling":0,"consensus-state-transition":1,"constants":1,"cross-fork":1,"eip-interactions":0,"evm-execution":0.7,"trick":0.8333333333333334}},{"model_id":"gpt-5.3-codex-high","display_name":"gpt-5.3-codex","lab":"openai","prompt":"bare","release_date":"2026-02-05","precision":"unknown","base_model":"gpt-5.3-codex","reasoning_effort":"high","provider":"openai","notes":null,"pass_rate":0.810126582278481,"ci_lower":0.7632575306791116,"ci_upper":0.8495458069730262,"total_questions":316,"total_runs":316,"cost_avg":0.04450723496835442,"cost_total":14.064286249999995,"latency_p50":33226,"category_pass_rates":{"calculations":1,"conceptual":0.8333333333333334,"consensus-epoch-processing":0.6346153846153846,"consensus-fork-choice":0.9333333333333333,"consensus-shuffling":0,"consensus-state-transition":1,"constants":1,"cross-fork":0.75,"eip-interactions":0,"evm-execution":0.7333333333333333,"trick":0.6666666666666666}},{"model_id":"grok-4.7-high","display_name":"grok-4.7","lab":"x-ai","prompt":"bare","release_date":"2026-09-21","precision":"unknown","base_model":"grok-4.7","reasoning_effort":"high","provider":"xai","notes":null,"pass_rate":0.810126582278481,"ci_lower":0.7632575306791116,"ci_upper":0.8495458069730262,"total_questions":316,"total_runs":316,"cost_avg":0.08032515822784811,"cost_total":25.38275,"latency_p50":100442.5,"category_pass_rates":{"calculations":1,"conceptual":1,"consensus-epoch-processing":0.7884615384615384,"consensus-fork-choice":0.8666666666666667,"consensus-shuffling":0,"consensus-state-transition":0.98,"constants":0.9473684210526315,"cross-fork":0.75,"eip-interactions":0,"evm-execution":0.7,"trick":0.6666666666666666}},{"model_id":"gpt-6-sol-xhigh","display_name":"gpt-6-sol","lab":"openai","prompt":"bare","release_date":"2026-09-22","precision":"unknown","base_model":"gpt-6-sol","reasoning_effort":"xhigh","provider":"openai","notes":null,"pass_rate":0.8069620253164557,"ci_lower":0.7598632557882227,"ci_upper":0.8466869865817505,"total_questions":316,"total_runs":316,"cost_avg":0.011096208860759495,"cost_total":3.5064020000000005,"latency_p50":13043,"category_pass_rates":{"calculations":1,"conceptual":0.8333333333333334,"consensus-epoch-processing":0.6346153846153846,"consensus-fork-choice":0.8,"consensus-shuffling":0,"consensus-state-transition":0.96,"constants":1,"cross-fork":0.75,"eip-interactions":0,"evm-execution":0.7416666666666667,"trick":1}},{"model_id":"kimi-k3.1-pro-max","display_name":"kimi-k3.1-pro","lab":"moonshotai","prompt":"bare","release_date":"2026-10-23","precision":"mxfp4","base_model":"kimi-k3.1-pro","reasoning_effort":"max","provider":"moonshotai","notes":null,"pass_rate":0.8069620253164557,"ci_lower":0.7598632557882227,"ci_upper":0.8466869865817505,"total_questions":316,"total_runs":316,"cost_avg":0.09346755063291132,"cost_total":29.535745999999975,"latency_p50":104351.5,"category_pass_rates":{"calculations":1,"conceptual":0.8333333333333334,"consensus-epoch-processing":0.6153846153846154,"consensus-fork-choice":0.9333333333333333,"consensus-shuffling":0,"consensus-state-transition":0.94,"constants":0.9824561403508771,"cross-fork":1,"eip-interactions":0,"evm-execution":0.75,"trick":0.8333333333333334}},{"model_id":"gpt-5.5-high","display_name":"gpt-5.5","lab":"openai","prompt":"bare","release_date":"2026-04-23","precision":"unknown","base_model":"gpt-5.5","reasoning_effort":"high","provider":"openai","notes":null,"pass_rate":0.8037974683544303,"ci_lower":0.7564733427820147,"ci_upper":0.8438238043057937,"total_questions":316,"total_runs":316,"cost_avg":0.10699295886075955,"cost_total":33.809775000000016,"latency_p50":43281,"category_pass_rates":{"calculations":1,"conceptual":0.8333333333333334,"consensus-epoch-processing":0.6538461538461539,"consensus-fork-choice":0.8666666666666667,"consensus-shuffling":0,"consensus-state-transition":1,"constants":0.9824561403508771,"cross-fork":1,"eip-interactions":0,"evm-execution":0.7166666666666667,"trick":0.6666666666666666}},{"model_id":"gpt-6.1-terra-low","display_name":"gpt-6.1-terra","lab":"openai","prompt":"bare","release_date":"2026-10-26","precision":"unknown","base_model":"gpt-6.1-terra","reasoning_effort":"low","provider":"openai","notes":null,"pass_rate":0.8037974683544303,"ci_lower":0.7564733427820147,"ci_upper":0.8438238043057937,"total_questions":316,"total_runs":316,"cost_avg":0.0011252088607594936,"cost_total":0.35556599999999994,"latency_p50":2678.5,"category_pass_rates":{"calculations":1,"conceptual":0.6666666666666666,"consensus-epoch-processing":0.8076923076923077,"consensus-fork-choice":0.8,"consensus-shuffling":0,"consensus-state-transition":0.92,"constants":0.9824561403508771,"cross-fork":0.75,"eip-interactions":0,"evm-execution":0.7,"trick":0.8333333333333334}},{"model_id":"grok-4.6-high","display_name":"grok-4.6","lab":"x-ai","prompt":"bare","release_date":"2026-08-12","precision":"unknown","base_model":"grok-4.6","reasoning_effort":"high","provider":"xai","notes":null,"pass_rate":0.8037974683544303,"ci_lower":0.7564733427820147,"ci_upper":0.8438238043057937,"total_questions":316,"total_runs":316,"cost_avg":0.03649877848101266,"cost_total":11.533614000000002,"latency_p50":56582,"category_pass_rates":{"calculations":1,"conceptual":0.8333333333333334,"consensus-epoch-processing":0.6730769230769231,"consensus-fork-choice":0.8666666666666667,"consensus-shuffling":0,"consensus-state-transition":1,"constants":0.9824561403508771,"cross-fork":1,"eip-interactions":0,"evm-execution":0.7166666666666667,"trick":0.5}},{"model_id":"claude-opus-4.5-16k","display_name":"claude-opus-4.5","lab":"anthropic","prompt":"bare","release_date":"2025-11-24","precision":"unknown","base_model":"claude-opus-4.5","reasoning_effort":"medium","provider":"anthropic","notes":null,"pass_rate":0.8006329113924051,"ci_lower":0.7530877132181288,"ci_upper":0.8409563385875151,"total_questions":316,"total_runs":316,"cost_avg":0.11545087025316456,"cost_total":36.482475,"latency_p50":54171,"category_pass_rates":{"calculations":1,"conceptual":0.6666666666666666,"consensus-epoch-processing":0.5961538461538461,"consensus-fork-choice":0.8666666666666667,"consensus-shuffling":0,"consensus-state-transition":0.98,"constants":0.9824561403508771,"cross-fork":0.75,"eip-interactions":1,"evm-execution":0.75,"trick":0.6666666666666666}},{"model_id":"gemini-3.1-pro-preview-high","display_name":"gemini-3.1-pro-preview","lab":"google","prompt":"bare","release_date":"2026-02-19","precision":"unknown","base_model":"gemini-3.1-pro-preview","reasoning_effort":"high","provider":"google-vertex","notes":null,"pass_rate":0.7974683544303798,"ci_lower":0.7497062917548237,"ci_upper":0.8380846647686555,"total_questions":316,"total_runs":316,"cost_avg":0.09277807594936707,"cost_total":29.317871999999994,"latency_p50":76124,"category_pass_rates":{"calculations":1,"conceptual":0.8333333333333334,"consensus-epoch-processing":0.5769230769230769,"consensus-fork-choice":0.7333333333333333,"consensus-shuffling":0,"consensus-state-transition":0.96,"constants":0.9824561403508771,"cross-fork":0.75,"eip-interactions":0,"evm-execution":0.7833333333333333,"trick":0.5}},{"model_id":"claude-sonnet-4.6-high","display_name":"claude-sonnet-4.6","lab":"anthropic","prompt":"bare","release_date":"2026-02-17","precision":"unknown","base_model":"claude-sonnet-4.6","reasoning_effort":"high","provider":"anthropic","notes":null,"pass_rate":0.7943037974683544,"ci_lower":0.7463290059885863,"ci_upper":0.8352088552527281,"total_questions":316,"total_runs":316,"cost_avg":0.18013818987341776,"cost_total":56.92366800000001,"latency_p50":48224.5,"category_pass_rates":{"calculations":1,"conceptual":0.6666666666666666,"consensus-epoch-processing":0.5384615384615384,"consensus-fork-choice":0.8666666666666667,"consensus-shuffling":0,"consensus-state-transition":0.96,"constants":0.9649122807017544,"cross-fork":0.75,"eip-interactions":1,"evm-execution":0.7833333333333333,"trick":0.5}},{"model_id":"muse-wonder-high","display_name":"muse-wonder","lab":"meta","prompt":"bare","release_date":"2026-10-12","precision":"unknown","base_model":"muse-wonder","reasoning_effort":"high","provider":"meta","notes":null,"pass_rate":0.7943037974683544,"ci_lower":0.7463290059885863,"ci_upper":0.8352088552527281,"total_questions":316,"total_runs":316,"cost_avg":0.10696168354430391,"cost_total":33.799892000000035,"latency_p50":30882.5,"category_pass_rates":{"calculations":1,"conceptual":1,"consensus-epoch-processing":0.6923076923076923,"consensus-fork-choice":0.8666666666666667,"consensus-shuffling":0,"consensus-state-transition":0.86,"constants":1,"cross-fork":1,"eip-interactions":1,"evm-execution":0.7166666666666667,"trick":0.5}},{"model_id":"gemini-3.7-flash-high","display_name":"gemini-3.7-flash","lab":"google","prompt":"bare","release_date":"2026-08-13","precision":"unknown","base_model":"gemini-3.7-flash","reasoning_effort":"high","provider":"google-vertex","notes":null,"pass_rate":0.7911392405063291,"ci_lower":0.7429557863027209,"ci_upper":0.832328979656429,"total_questions":316,"total_runs":316,"cost_avg":0.008607739319620257,"cost_total":2.7200456250000014,"latency_p50":15149,"category_pass_rates":{"calculations":1,"conceptual":0.5,"consensus-epoch-processing":0.5769230769230769,"consensus-fork-choice":0.8666666666666667,"consensus-shuffling":0,"consensus-state-transition":0.88,"constants":0.9824561403508771,"cross-fork":1,"eip-interactions":1,"evm-execution":0.775,"trick":0.6666666666666666}},{"model_id":"gpt-5.4-high","display_name":"gpt-5.4","lab":"openai","prompt":"bare","release_date":"2026-03-05","precision":"unknown","base_model":"gpt-5.4","reasoning_effort":"high","provider":"openai","notes":null,"pass_rate":0.7879746835443038,"ci_lower":0.7395865657260222,"ci_upper":0.8294451049509629,"total_questions":316,"total_runs":316,"cost_avg":0.04838613132911389,"cost_total":15.290017499999989,"latency_p50":41138.5,"category_pass_rates":{"calculations":1,"conceptual":0.6666666666666666,"consensus-epoch-processing":0.5576923076923077,"consensus-fork-choice":0.8666666666666667,"consensus-shuffling":0,"consensus-state-transition":0.96,"constants":1,"cross-fork":0.75,"eip-interactions":0,"evm-execution":0.7416666666666667,"trick":0.6666666666666666}},{"model_id":"gpt-5.5-medium","display_name":"gpt-5.5","lab":"openai","prompt":"bare","release_date":"2026-04-23","precision":"unknown","base_model":"gpt-5.5","reasoning_effort":"medium","provider":"openai","notes":null,"pass_rate":0.7848101265822784,"ci_lower":0.7362212798007275,"ci_upper":0.826557295594093,"total_questions":316,"total_runs":316,"cost_avg":0.04267988924050637,"cost_total":13.486845000000013,"latency_p50":20294.5,"category_pass_rates":{"calculations":1,"conceptual":1,"consensus-epoch-processing":0.6730769230769231,"consensus-fork-choice":0.8666666666666667,"consensus-shuffling":0,"consensus-state-transition":1,"constants":1,"cross-fork":1,"eip-interactions":0,"evm-execution":0.6333333333333333,"trick":0.8333333333333334}},{"model_id":"gpt-5.4-medium","display_name":"gpt-5.4","lab":"openai","prompt":"bare","release_date":"2026-03-05","precision":"unknown","base_model":"gpt-5.4","reasoning_effort":"medium","provider":"openai","notes":null,"pass_rate":0.7848101265822784,"ci_lower":0.7362212798007275,"ci_upper":0.826557295594093,"total_questions":316,"total_runs":316,"cost_avg":0.018515672468354435,"cost_total":5.850952500000002,"latency_p50":18448.5,"category_pass_rates":{"calculations":1,"conceptual":0.8333333333333334,"consensus-epoch-processing":0.5961538461538461,"consensus-fork-choice":0.8666666666666667,"consensus-shuffling":0,"consensus-state-transition":0.96,"constants":0.9824561403508771,"cross-fork":0.75,"eip-interactions":1,"evm-execution":0.7166666666666667,"trick":0.5}},{"model_id":"gpt-5.3-codex-medium","display_name":"gpt-5.3-codex","lab":"openai","prompt":"bare","release_date":"2026-02-05","precision":"unknown","base_model":"gpt-5.3-codex","reasoning_effort":"medium","provider":"openai","notes":null,"pass_rate":0.7784810126582279,"ci_lower":0.7295022659073515,"ci_upper":0.8207701189231398,"total_questions":316,"total_runs":316,"cost_avg":0.020726064082278492,"cost_total":6.549436250000003,"latency_p50":18126,"category_pass_rates":{"calculations":1,"conceptual":0.8333333333333334,"consensus-epoch-processing":0.6153846153846154,"consensus-fork-choice":0.8666666666666667,"consensus-shuffling":0,"consensus-state-transition":0.94,"constants":0.9824561403508771,"cross-fork":0.5,"eip-interactions":0,"evm-execution":0.7166666666666667,"trick":0.5}},{"model_id":"gpt-5.6-sol-low","display_name":"gpt-5.6-sol","lab":"openai","prompt":"bare","release_date":"2026-07-09","precision":"unknown","base_model":"gpt-5.6-sol","reasoning_effort":"low","provider":"openai","notes":null,"pass_rate":0.7626582278481012,"ci_lower":0.7127695083234125,"ci_upper":0.8062374000962552,"total_questions":316,"total_runs":316,"cost_avg":0.012130582278481015,"cost_total":3.8332640000000007,"latency_p50":10914,"category_pass_rates":{"calculations":1,"conceptual":0.8333333333333334,"consensus-epoch-processing":0.5,"consensus-fork-choice":0.8666666666666667,"consensus-shuffling":0,"consensus-state-transition":1,"constants":0.9298245614035088,"cross-fork":1,"eip-interactions":0,"evm-execution":0.6916666666666667,"trick":0.8333333333333334}},{"model_id":"grok-4.5-high","display_name":"grok-4.5","lab":"x-ai","prompt":"bare","release_date":"2026-07-08","precision":"unknown","base_model":"grok-4.5","reasoning_effort":"high","provider":"xai","notes":null,"pass_rate":0.7626582278481012,"ci_lower":0.7127695083234125,"ci_upper":0.8062374000962552,"total_questions":316,"total_runs":316,"cost_avg":0.044783734177215186,"cost_total":14.15166,"latency_p50":35701,"category_pass_rates":{"calculations":1,"conceptual":0.6666666666666666,"consensus-epoch-processing":0.6153846153846154,"consensus-fork-choice":0.7333333333333333,"consensus-shuffling":0,"consensus-state-transition":1,"constants":0.9649122807017544,"cross-fork":1,"eip-interactions":0,"evm-execution":0.6583333333333333,"trick":0.6666666666666666}},{"model_id":"gpt-6-sol-low","display_name":"gpt-6-sol","lab":"openai","prompt":"bare","release_date":"2026-09-22","precision":"unknown","base_model":"gpt-6-sol","reasoning_effort":"low","provider":"openai","notes":null,"pass_rate":0.7563291139240507,"ci_lower":0.7061012280066065,"ci_upper":0.800399489848732,"total_questions":316,"total_runs":316,"cost_avg":0.003109341772151898,"cost_total":0.9825519999999998,"latency_p50":5470.5,"category_pass_rates":{"calculations":1,"conceptual":0.6666666666666666,"consensus-epoch-processing":0.5961538461538461,"consensus-fork-choice":0.7333333333333333,"consensus-shuffling":0,"consensus-state-transition":0.92,"constants":0.9473684210526315,"cross-fork":0.75,"eip-interactions":0,"evm-execution":0.6916666666666667,"trick":0.8333333333333334}},{"model_id":"kimi-k3-high","display_name":"kimi-k3","lab":"moonshotai","prompt":"bare","release_date":"2026-07-16","precision":"int4","base_model":"kimi-k3","reasoning_effort":"high","provider":"moonshotai","notes":null,"pass_rate":0.7536231884057971,"ci_lower":0.705525206569594,"ci_upper":0.7961351458654367,"total_questions":316,"total_runs":345,"cost_avg":0.11645816521739126,"cost_total":40.178066999999984,"latency_p50":197598,"category_pass_rates":{"calculations":1,"conceptual":1,"consensus-epoch-processing":0.5178571428571429,"consensus-fork-choice":0.8666666666666667,"consensus-shuffling":0,"consensus-state-transition":0.8727272727272727,"constants":0.9545454545454546,"cross-fork":1,"eip-interactions":1,"evm-execution":0.6793893129770993,"trick":0.8333333333333334}},{"model_id":"gpt-5.5-low","display_name":"gpt-5.5","lab":"openai","prompt":"bare","release_date":"2026-04-23","precision":"unknown","base_model":"gpt-5.5","reasoning_effort":"low","provider":"openai","notes":null,"pass_rate":0.75,"ci_lower":0.6994465676307645,"ci_upper":0.7945479596602446,"total_questions":316,"total_runs":316,"cost_avg":0.024593022151898736,"cost_total":7.771395,"latency_p50":11782,"category_pass_rates":{"calculations":1,"conceptual":0.8333333333333334,"consensus-epoch-processing":0.6346153846153846,"consensus-fork-choice":0.8666666666666667,"consensus-shuffling":0,"consensus-state-transition":0.94,"constants":0.9824561403508771,"cross-fork":1,"eip-interactions":0,"evm-execution":0.6083333333333333,"trick":0.6666666666666666}},{"model_id":"kimi-k3-max","display_name":"kimi-k3","lab":"moonshotai","prompt":"bare","release_date":"2026-07-16","precision":"int4","base_model":"kimi-k3","reasoning_effort":"max","provider":"moonshotai","notes":null,"pass_rate":0.7436708860759493,"ci_lower":0.6928051849287795,"ci_upper":0.7886831517979002,"total_questions":316,"total_runs":316,"cost_avg":0.11048109523809525,"cost_total":34.801545000000004,"latency_p50":132635.5,"category_pass_rates":{"calculations":1,"conceptual":0.8333333333333334,"consensus-epoch-processing":0.4423076923076923,"consensus-fork-choice":0.8,"consensus-shuffling":0,"consensus-state-transition":0.88,"constants":0.9649122807017544,"cross-fork":1,"eip-interactions":1,"evm-execution":0.7,"trick":0.8333333333333334}},{"model_id":"gpt-5.6-terra-xhigh","display_name":"gpt-5.6-terra","lab":"openai","prompt":"bare","release_date":"2026-07-09","precision":"unknown","base_model":"gpt-5.6-terra","reasoning_effort":"xhigh","provider":"openai","notes":null,"pass_rate":0.740506329113924,"ci_lower":0.689489371850028,"ci_upper":0.7857458695944869,"total_questions":316,"total_runs":316,"cost_avg":0.030311259493670886,"cost_total":9.578358,"latency_p50":21308,"category_pass_rates":{"calculations":1,"conceptual":0.6666666666666666,"consensus-epoch-processing":0.5961538461538461,"consensus-fork-choice":0.8666666666666667,"consensus-shuffling":0,"consensus-state-transition":0.88,"constants":0.9649122807017544,"cross-fork":0.75,"eip-interactions":1,"evm-execution":0.6333333333333333,"trick":0.8333333333333334}},{"model_id":"deepseek-v4.1-flash-1017-high","display_name":"deepseek-v4.1-flash-1017","lab":"deepseek","prompt":"bare","release_date":"2026-10-17","precision":"fp8","base_model":"deepseek-v4.1-flash-1017","reasoning_effort":"high","provider":"deepinfra","notes":null,"pass_rate":0.7373417721518988,"ci_lower":0.6861767597737298,"ci_upper":0.7828053863886207,"total_questions":316,"total_runs":316,"cost_avg":0.002742140791139241,"cost_total":0.8665164900000002,"latency_p50":43093.5,"category_pass_rates":{"calculations":1,"conceptual":0.8333333333333334,"consensus-epoch-processing":0.5,"consensus-fork-choice":0.7333333333333333,"consensus-shuffling":0,"consensus-state-transition":0.74,"constants":0.9649122807017544,"cross-fork":1,"eip-interactions":1,"evm-execution":0.7333333333333333,"trick":0.6666666666666666}},{"model_id":"gpt-5.4-low","display_name":"gpt-5.4","lab":"openai","prompt":"bare","release_date":"2026-03-05","precision":"unknown","base_model":"gpt-5.4","reasoning_effort":"low","provider":"openai","notes":null,"pass_rate":0.7373417721518988,"ci_lower":0.6861767597737298,"ci_upper":0.7828053863886207,"total_questions":316,"total_runs":316,"cost_avg":0.012421590189873419,"cost_total":3.9252225000000003,"latency_p50":11614.5,"category_pass_rates":{"calculations":1,"conceptual":0.6666666666666666,"consensus-epoch-processing":0.5576923076923077,"consensus-fork-choice":0.8666666666666667,"consensus-shuffling":0,"consensus-state-transition":0.92,"constants":1,"cross-fork":0.75,"eip-interactions":1,"evm-execution":0.625,"trick":0.5}},{"model_id":"muse-spark-1.2-high","display_name":"muse-spark-1.2","lab":"meta","prompt":"bare","release_date":"2026-08-05","precision":"unknown","base_model":"muse-spark-1.2","reasoning_effort":"high","provider":"meta","notes":null,"pass_rate":0.7373417721518988,"ci_lower":0.6861767597737298,"ci_upper":0.7828053863886207,"total_questions":316,"total_runs":316,"cost_avg":0.03941281724683544,"cost_total":12.454450249999999,"latency_p50":48718,"category_pass_rates":{"calculations":1,"conceptual":0.8333333333333334,"consensus-epoch-processing":0.6538461538461539,"consensus-fork-choice":0.8666666666666667,"consensus-shuffling":0,"consensus-state-transition":0.7,"constants":0.9824561403508771,"cross-fork":1,"eip-interactions":1,"evm-execution":0.6583333333333333,"trick":0.6666666666666666}},{"model_id":"mimo-v2.6-pro-high","display_name":"mimo-v2.6-pro","lab":"xiaomi","prompt":"bare","release_date":"2026-09-22","precision":"fp8","base_model":"mimo-v2.6-pro","reasoning_effort":"high","provider":"xiaomi","notes":null,"pass_rate":0.7341772151898734,"ci_lower":0.6828673118569675,"ci_upper":0.7798617390232183,"total_questions":316,"total_runs":316,"cost_avg":0.024137059746835447,"cost_total":7.627310880000001,"latency_p50":148171.5,"category_pass_rates":{"calculations":1,"conceptual":1,"consensus-epoch-processing":0.6346153846153846,"consensus-fork-choice":0.8666666666666667,"consensus-shuffling":0,"consensus-state-transition":0.82,"constants":0.8771929824561403,"cross-fork":0.75,"eip-interactions":1,"evm-execution":0.6666666666666666,"trick":0.5}},{"model_id":"gemini-3.5-flash-high","display_name":"gemini-3.5-flash","lab":"google","prompt":"bare","release_date":"2026-05-19","precision":"unknown","base_model":"gemini-3.5-flash","reasoning_effort":"high","provider":"google-vertex","notes":null,"pass_rate":0.7278481012658228,"ci_lower":0.6762577669633894,"ci_upper":0.773965093352467,"total_questions":316,"total_runs":316,"cost_avg":0.08494590189873419,"cost_total":26.842905000000005,"latency_p50":32219.5,"category_pass_rates":{"calculations":1,"conceptual":0.5,"consensus-epoch-processing":0.4423076923076923,"consensus-fork-choice":0.8,"consensus-shuffling":0,"consensus-state-transition":0.84,"constants":0.8947368421052632,"cross-fork":0.75,"eip-interactions":1,"evm-execution":0.7583333333333333,"trick":0.3333333333333333}},{"model_id":"muse-spark-open-high","display_name":"muse-spark-open","lab":"meta","prompt":"bare","release_date":"2026-10-15","precision":"fp4","base_model":"muse-spark-open","reasoning_effort":"high","provider":"baseten","notes":null,"pass_rate":0.7278481012658228,"ci_lower":0.6762577669633894,"ci_upper":0.773965093352467,"total_questions":316,"total_runs":316,"cost_avg":0.04835886708860766,"cost_total":15.281402000000021,"latency_p50":45404.5,"category_pass_rates":{"calculations":1,"conceptual":1,"consensus-epoch-processing":0.6923076923076923,"consensus-fork-choice":0.7333333333333333,"consensus-shuffling":0,"consensus-state-transition":0.7,"constants":0.9649122807017544,"cross-fork":1,"eip-interactions":1,"evm-execution":0.625,"trick":0.8333333333333334}},{"model_id":"qwen3.7-max-high","display_name":"qwen3.7-max","lab":"qwen","prompt":"bare","release_date":"2026-05-21","precision":"unknown","base_model":"qwen3.7-max","reasoning_effort":"high","provider":"alibaba","notes":null,"pass_rate":0.7246835443037974,"ci_lower":0.6729576020011337,"ci_upper":0.7710121630325579,"total_questions":316,"total_runs":316,"cost_avg":0.023169071281645567,"cost_total":7.321426524999999,"latency_p50":28158,"category_pass_rates":{"calculations":1,"conceptual":0.6666666666666666,"consensus-epoch-processing":0.5576923076923077,"consensus-fork-choice":0.7333333333333333,"consensus-shuffling":0,"consensus-state-transition":0.92,"constants":0.8596491228070176,"cross-fork":0.75,"eip-interactions":1,"evm-execution":0.6833333333333333,"trick":0.3333333333333333}},{"model_id":"gemini-3-flash-high","display_name":"gemini-3-flash","lab":"google","prompt":"bare","release_date":"2025-12-17","precision":"bf16","base_model":"gemini-3-flash","reasoning_effort":"high","provider":"google-vertex","notes":null,"pass_rate":0.7183544303797469,"ci_lower":0.6663663250011351,"ci_upper":0.7650972494682274,"total_questions":316,"total_runs":316,"cost_avg":0.025647835443037968,"cost_total":8.104715999999998,"latency_p50":29754.5,"category_pass_rates":{"calculations":1,"conceptual":0.5,"consensus-epoch-processing":0.5,"consensus-fork-choice":0.8,"consensus-shuffling":0,"consensus-state-transition":0.78,"constants":0.8947368421052632,"cross-fork":0.75,"eip-interactions":1,"evm-execution":0.7333333333333333,"trick":0.3333333333333333}},{"model_id":"gpt-6-luna-max","display_name":"gpt-6-luna","lab":"openai","prompt":"bare","release_date":"2026-09-22","precision":"unknown","base_model":"gpt-6-luna","reasoning_effort":"max","provider":"openai","notes":null,"pass_rate":0.7120253164556962,"ci_lower":0.659786914114778,"ci_upper":0.7591704697902552,"total_questions":316,"total_runs":316,"cost_avg":0.0018485129746835431,"cost_total":0.5841300999999997,"latency_p50":42935,"category_pass_rates":{"calculations":1,"conceptual":0.6666666666666666,"consensus-epoch-processing":0.40384615384615385,"consensus-fork-choice":0.7333333333333333,"consensus-shuffling":0,"consensus-state-transition":0.84,"constants":0.9298245614035088,"cross-fork":0.75,"eip-interactions":1,"evm-execution":0.7,"trick":0.6666666666666666}},{"model_id":"deepseek-v4.1-flash-high","display_name":"deepseek-v4.1-flash","lab":"deepseek","prompt":"bare","release_date":"2026-09-10","precision":"fp8","base_model":"deepseek-v4.1-flash","reasoning_effort":"high","provider":"novita","notes":null,"pass_rate":0.7056962025316456,"ci_lower":0.6532191370456515,"ci_upper":0.7532320562950522,"total_questions":316,"total_runs":316,"cost_avg":0.010354115553797463,"cost_total":3.2719005149999982,"latency_p50":37374,"category_pass_rates":{"calculations":1,"conceptual":0.6666666666666666,"consensus-epoch-processing":0.40384615384615385,"consensus-fork-choice":0.8666666666666667,"consensus-shuffling":0,"consensus-state-transition":0.84,"constants":0.9473684210526315,"cross-fork":1,"eip-interactions":1,"evm-execution":0.6416666666666667,"trick":0.8333333333333334}},{"model_id":"qwen3.8-2.4t-a95b-high","display_name":"qwen3.8-2.4t-a95b","lab":"qwen","prompt":"bare","release_date":"2026-08-12","precision":"unknown","base_model":"qwen3.8-2.4t-a95b","reasoning_effort":"high","provider":"fireworks","notes":null,"pass_rate":0.7025316455696202,"ci_lower":0.6499395424517582,"ci_upper":0.7502585556067808,"total_questions":316,"total_runs":316,"cost_avg":0.02885308227848102,"cost_total":9.117574000000003,"latency_p50":83184.5,"category_pass_rates":{"calculations":1,"conceptual":0.6666666666666666,"consensus-epoch-processing":0.4807692307692308,"consensus-fork-choice":0.8666666666666667,"consensus-shuffling":0,"consensus-state-transition":0.7,"constants":0.9473684210526315,"cross-fork":0.75,"eip-interactions":1,"evm-execution":0.6666666666666666,"trick":0.8333333333333334}},{"model_id":"claude-sonnet-4.5-high","display_name":"claude-sonnet-4.5","lab":"anthropic","prompt":"bare","release_date":"2025-09-29","precision":"unknown","base_model":"claude-sonnet-4.5","reasoning_effort":"high","provider":"anthropic","notes":null,"pass_rate":0.7025316455696202,"ci_lower":0.6499395424517582,"ci_upper":0.7502585556067808,"total_questions":316,"total_runs":316,"cost_avg":0.08555206329113926,"cost_total":27.034452000000005,"latency_p50":60743,"category_pass_rates":{"calculations":1,"conceptual":0.5,"consensus-epoch-processing":0.4230769230769231,"consensus-fork-choice":0.8666666666666667,"consensus-shuffling":0,"consensus-state-transition":0.9,"constants":0.9298245614035088,"cross-fork":0.75,"eip-interactions":1,"evm-execution":0.6333333333333333,"trick":0.6666666666666666}},{"model_id":"gpt-5.4-mini-xhigh","display_name":"gpt-5.4-mini","lab":"openai","prompt":"bare","release_date":"2026-03-17","precision":"unknown","base_model":"gpt-5.4-mini","reasoning_effort":"xhigh","provider":"openai","notes":null,"pass_rate":0.6993670886075949,"ci_lower":0.6466627754885508,"ci_upper":0.7472822272878236,"total_questions":316,"total_runs":316,"cost_avg":0.11999218433544304,"cost_total":37.91753025,"latency_p50":154820,"category_pass_rates":{"calculations":1,"conceptual":0.6666666666666666,"consensus-epoch-processing":0.4230769230769231,"consensus-fork-choice":0.8666666666666667,"consensus-shuffling":0,"consensus-state-transition":0.8,"constants":0.8771929824561403,"cross-fork":0.5,"eip-interactions":1,"evm-execution":0.6916666666666667,"trick":0.6666666666666666}},{"model_id":"deepseek-v4-pro-0813-high","display_name":"deepseek-v4-pro-0813","lab":"deepseek","prompt":"bare","release_date":"2026-08-12","precision":"unknown","base_model":"deepseek-v4-pro-0813","reasoning_effort":"high","provider":"cloudflare","notes":null,"pass_rate":0.689873417721519,"ci_lower":0.6368491916221196,"ci_upper":0.7383365253077607,"total_questions":316,"total_runs":316,"cost_avg":0.07906701000000001,"cost_total":24.985175160000004,"latency_p50":462933,"category_pass_rates":{"calculations":1,"conceptual":0.6666666666666666,"consensus-epoch-processing":0.4807692307692308,"consensus-fork-choice":0.6666666666666666,"consensus-shuffling":0,"consensus-state-transition":0.78,"constants":0.8947368421052632,"cross-fork":0.5,"eip-interactions":1,"evm-execution":0.6666666666666666,"trick":0.6666666666666666}},{"model_id":"kimi-k2.6-high","display_name":"kimi-k2.6","lab":"moonshotai","prompt":"bare","release_date":"2026-04-21","precision":"int4","base_model":"kimi-k2.6","reasoning_effort":"high","provider":"moonshotai","notes":null,"pass_rate":0.689873417721519,"ci_lower":0.6368491916221196,"ci_upper":0.7383365253077607,"total_questions":316,"total_runs":316,"cost_avg":0.074239452056962,"cost_total":23.459666849999994,"latency_p50":170416.5,"category_pass_rates":{"calculations":1,"conceptual":0.8333333333333334,"consensus-epoch-processing":0.5576923076923077,"consensus-fork-choice":0.6666666666666666,"consensus-shuffling":0,"consensus-state-transition":0.84,"constants":0.9298245614035088,"cross-fork":0.5,"eip-interactions":0,"evm-execution":0.5916666666666667,"trick":0.6666666666666666}},{"model_id":"claude-sonnet-5-low","display_name":"claude-sonnet-5","lab":"anthropic","prompt":"bare","release_date":"2026-06-29","precision":"unknown","base_model":"claude-sonnet-5","reasoning_effort":"low","provider":"anthropic","notes":null,"pass_rate":0.6835443037974683,"ci_lower":0.6303204973952294,"ci_upper":0.7323590289703215,"total_questions":316,"total_runs":316,"cost_avg":0.019711234177215164,"cost_total":6.228749999999992,"latency_p50":6890.5,"category_pass_rates":{"calculations":1,"conceptual":0.6666666666666666,"consensus-epoch-processing":0.5192307692307693,"consensus-fork-choice":0.7333333333333333,"consensus-shuffling":0,"consensus-state-transition":0.86,"constants":0.9649122807017544,"cross-fork":1,"eip-interactions":0,"evm-execution":0.55,"trick":0.6666666666666666}},{"model_id":"gpt-5.6-terra-low","display_name":"gpt-5.6-terra","lab":"openai","prompt":"bare","release_date":"2026-07-09","precision":"unknown","base_model":"gpt-5.6-terra","reasoning_effort":"low","provider":"openai","notes":null,"pass_rate":0.680379746835443,"ci_lower":0.6270601917203724,"ci_upper":0.7293662393630138,"total_questions":316,"total_runs":316,"cost_avg":0.010762272151898735,"cost_total":3.400878,"latency_p50":9941.5,"category_pass_rates":{"calculations":1,"conceptual":0.8333333333333334,"consensus-epoch-processing":0.38461538461538464,"consensus-fork-choice":0.8,"consensus-shuffling":0,"consensus-state-transition":0.74,"constants":0.9824561403508771,"cross-fork":0.75,"eip-interactions":1,"evm-execution":0.6166666666666667,"trick":0.8333333333333334}},{"model_id":"claude-opus-4.1-32k","display_name":"claude-opus-4.1","lab":"anthropic","prompt":"bare","release_date":"2025-08-05","precision":"unknown","base_model":"claude-opus-4.1","reasoning_effort":"high","provider":"anthropic","notes":null,"pass_rate":0.6708860759493671,"ci_lower":0.6172951899864231,"ci_upper":0.7203719552504693,"total_questions":316,"total_runs":316,"cost_avg":0.24962530063291127,"cost_total":78.88159499999996,"latency_p50":35785,"category_pass_rates":{"calculations":1,"conceptual":0.6666666666666666,"consensus-epoch-processing":0.40384615384615385,"consensus-fork-choice":0.7333333333333333,"consensus-shuffling":0,"consensus-state-transition":0.84,"constants":0.8947368421052632,"cross-fork":0.75,"eip-interactions":1,"evm-execution":0.6166666666666667,"trick":0.5}},{"model_id":"kimi-k2.5-high","display_name":"kimi-k2.5","lab":"moonshotai","prompt":"bare","release_date":"2026-01-27","precision":"int4","base_model":"kimi-k2.5","reasoning_effort":"high","provider":"moonshotai","notes":null,"pass_rate":0.6550632911392406,"ci_lower":0.6010720858189762,"ci_upper":0.7053295830070928,"total_questions":316,"total_runs":316,"cost_avg":0.013982657436708845,"cost_total":4.418519749999995,"latency_p50":124604.5,"category_pass_rates":{"calculations":1,"conceptual":0.5,"consensus-epoch-processing":0.5,"consensus-fork-choice":0.7333333333333333,"consensus-shuffling":0,"consensus-state-transition":0.68,"constants":0.9298245614035088,"cross-fork":0.75,"eip-interactions":1,"evm-execution":0.5916666666666667,"trick":0.5}},{"model_id":"claude-opus-4.1-16k","display_name":"claude-opus-4.1","lab":"anthropic","prompt":"bare","release_date":"2025-08-05","precision":"unknown","base_model":"claude-opus-4.1","reasoning_effort":"medium","provider":"anthropic","notes":null,"pass_rate":0.6550632911392406,"ci_lower":0.6010720858189762,"ci_upper":0.7053295830070928,"total_questions":316,"total_runs":316,"cost_avg":0.1979492088607596,"cost_total":62.551950000000026,"latency_p50":33242,"category_pass_rates":{"calculations":1,"conceptual":0.5,"consensus-epoch-processing":0.40384615384615385,"consensus-fork-choice":0.8,"consensus-shuffling":0,"consensus-state-transition":0.86,"constants":0.8771929824561403,"cross-fork":0.5,"eip-interactions":1,"evm-execution":0.5833333333333334,"trick":0.5}},{"model_id":"kimi-k2.7-code-high","display_name":"kimi-k2.7-code","lab":"moonshotai","prompt":"bare","release_date":"2026-06-12","precision":"int4","base_model":"kimi-k2.7-code","reasoning_effort":"high","provider":"moonshotai","notes":null,"pass_rate":0.6550632911392406,"ci_lower":0.6010720858189762,"ci_upper":0.7053295830070928,"total_questions":316,"total_runs":316,"cost_avg":0.04440692041139238,"cost_total":14.032586849999992,"latency_p50":120743,"category_pass_rates":{"calculations":1,"conceptual":0.6666666666666666,"consensus-epoch-processing":0.5961538461538461,"consensus-fork-choice":0.6666666666666666,"consensus-shuffling":0,"consensus-state-transition":0.72,"constants":0.8771929824561403,"cross-fork":0.75,"eip-interactions":1,"evm-execution":0.5583333333333333,"trick":0.5}},{"model_id":"deepseek-v4-flash-0731","display_name":"deepseek-v4-flash-0731","lab":"deepseek","prompt":"bare","release_date":"2026-07-31","precision":"fp8-fp4experts","base_model":"deepseek-v4-flash-0731","reasoning_effort":"max","provider":"starflinger","notes":null,"pass_rate":0.6494252873563219,"ci_lower":0.5979088044549136,"ci_upper":0.6976787554015673,"total_questions":316,"total_runs":348,"cost_avg":0,"cost_total":0,"latency_p50":550968,"category_pass_rates":{"calculations":1,"conceptual":0.7142857142857143,"consensus-epoch-processing":0.288135593220339,"consensus-fork-choice":0.75,"consensus-shuffling":0,"consensus-state-transition":0.6964285714285714,"constants":0.7288135593220338,"cross-fork":0.6,"eip-interactions":1,"evm-execution":0.7557251908396947,"trick":0.5555555555555556}},{"model_id":"gpt-5.4-mini-high","display_name":"gpt-5.4-mini","lab":"openai","prompt":"bare","release_date":"2026-03-17","precision":"unknown","base_model":"gpt-5.4-mini","reasoning_effort":"high","provider":"openai","notes":null,"pass_rate":0.6455696202531646,"ci_lower":0.5913685914921556,"ci_upper":0.6962737914874193,"total_questions":316,"total_runs":316,"cost_avg":0.03358379193037976,"cost_total":10.612478250000004,"latency_p50":28068.5,"category_pass_rates":{"calculations":1,"conceptual":0.5,"consensus-epoch-processing":0.3269230769230769,"consensus-fork-choice":0.8,"consensus-shuffling":0,"consensus-state-transition":0.76,"constants":0.8596491228070176,"cross-fork":0.5,"eip-interactions":0,"evm-execution":0.65,"trick":0.5}},{"model_id":"deepseek-v4-flash-0731-high","display_name":"deepseek-v4-flash-0731","lab":"deepseek","prompt":"bare","release_date":"2026-07-31","precision":"fp8-fp4experts","base_model":"deepseek-v4-flash-0731","reasoning_effort":"high","provider":"starflinger","notes":null,"pass_rate":0.6392405063291139,"ci_lower":0.5849119847104416,"ci_upper":0.6902242077048039,"total_questions":316,"total_runs":316,"cost_avg":0,"cost_total":0,"latency_p50":229553,"category_pass_rates":{"calculations":1,"conceptual":0.8333333333333334,"consensus-epoch-processing":0.3076923076923077,"consensus-fork-choice":0.8666666666666667,"consensus-shuffling":0,"consensus-state-transition":0.62,"constants":0.7719298245614035,"cross-fork":0.75,"eip-interactions":0,"evm-execution":0.7083333333333334,"trick":0.5}},{"model_id":"gpt-5.1-high","display_name":"gpt-5.1","lab":"openai","prompt":"bare","release_date":"2025-11-13","precision":"unknown","base_model":"gpt-5.1","reasoning_effort":"high","provider":"openai","notes":null,"pass_rate":0.6392405063291139,"ci_lower":0.5849119847104416,"ci_upper":0.6902242077048039,"total_questions":316,"total_runs":316,"cost_avg":0.12397265427215194,"cost_total":39.175358750000015,"latency_p50":68961,"category_pass_rates":{"calculations":1,"conceptual":0.8333333333333334,"consensus-epoch-processing":0.38461538461538464,"consensus-fork-choice":0.7333333333333333,"consensus-shuffling":0,"consensus-state-transition":0.84,"constants":0.7719298245614035,"cross-fork":0.5,"eip-interactions":0,"evm-execution":0.6083333333333333,"trick":0.5}},{"model_id":"qwen3.8-flash-high","display_name":"qwen3.8-flash","lab":"qwen","prompt":"bare","release_date":"2026-08-26","precision":"unknown","base_model":"qwen3.8-flash","reasoning_effort":"high","provider":"alibaba","notes":null,"pass_rate":0.6360759493670886,"ci_lower":0.58168735461107,"ci_upper":0.6871957425220108,"total_questions":316,"total_runs":316,"cost_avg":0.01877614246835442,"cost_total":5.933261019999997,"latency_p50":1701820.5,"category_pass_rates":{"calculations":1,"conceptual":0.6666666666666666,"consensus-epoch-processing":0.28846153846153844,"consensus-fork-choice":0.5333333333333333,"consensus-shuffling":0,"consensus-state-transition":0.6,"constants":0.8596491228070176,"cross-fork":0.75,"eip-interactions":1,"evm-execution":0.7,"trick":0.8333333333333334}},{"model_id":"gpt-5.3-chat","display_name":"gpt-5.3-chat","lab":"openai","prompt":"bare","release_date":"2026-03-03","precision":"unknown","base_model":"gpt-5.3-chat","reasoning_effort":null,"provider":"openai","notes":null,"pass_rate":0.629746835443038,"ci_lower":0.5752453729437816,"ci_upper":0.6811315336249699,"total_questions":316,"total_runs":316,"cost_avg":0.014777083069620242,"cost_total":4.669558249999996,"latency_p50":10268.5,"category_pass_rates":{"calculations":1,"conceptual":0.6666666666666666,"consensus-epoch-processing":0.36538461538461536,"consensus-fork-choice":0.7333333333333333,"consensus-shuffling":0,"consensus-state-transition":0.76,"constants":0.9298245614035088,"cross-fork":0.75,"eip-interactions":1,"evm-execution":0.5416666666666666,"trick":0.5}},{"model_id":"claude-sonnet-4-5-16k","display_name":"claude-sonnet-4.5","lab":"anthropic","prompt":"bare","release_date":"2025-09-29","precision":"unknown","base_model":"claude-sonnet-4.5","reasoning_effort":"medium","provider":"anthropic","notes":null,"pass_rate":0.629746835443038,"ci_lower":0.5752453729437816,"ci_upper":0.6811315336249699,"total_questions":316,"total_runs":316,"cost_avg":0.049982079113924084,"cost_total":15.794337000000011,"latency_p50":41219,"category_pass_rates":{"calculations":1,"conceptual":0.3333333333333333,"consensus-epoch-processing":0.36538461538461536,"consensus-fork-choice":0.6,"consensus-shuffling":0,"consensus-state-transition":0.86,"constants":0.9473684210526315,"cross-fork":0.75,"eip-interactions":0,"evm-execution":0.525,"trick":0.6666666666666666}},{"model_id":"kimi-k2.5","display_name":"kimi-k2.5","lab":"moonshotai","prompt":"bare","release_date":"2026-01-27","precision":"int4","base_model":"kimi-k2.5","reasoning_effort":null,"provider":"moonshotai","notes":null,"pass_rate":0.6265822784810127,"ci_lower":0.5720279954525296,"ci_upper":0.6780958158340573,"total_questions":316,"total_runs":316,"cost_avg":0.013481342879746849,"cost_total":4.260104350000004,"latency_p50":165593.5,"category_pass_rates":{"calculations":1,"conceptual":0.6666666666666666,"consensus-epoch-processing":0.4230769230769231,"consensus-fork-choice":0.7333333333333333,"consensus-shuffling":0,"consensus-state-transition":0.66,"constants":0.9473684210526315,"cross-fork":0.75,"eip-interactions":0,"evm-execution":0.55,"trick":0.5}},{"model_id":"gpt-5-high","display_name":"gpt-5","lab":"openai","prompt":"bare","release_date":"2025-08-07","precision":"unknown","base_model":"gpt-5","reasoning_effort":"high","provider":"openai","notes":null,"pass_rate":0.6234177215189873,"ci_lower":0.5688130102718261,"ci_upper":0.6750577057325962,"total_questions":316,"total_runs":316,"cost_avg":0.09742186313291144,"cost_total":30.785308750000016,"latency_p50":52402.5,"category_pass_rates":{"calculations":1,"conceptual":0.8333333333333334,"consensus-epoch-processing":0.36538461538461536,"consensus-fork-choice":0.6666666666666666,"consensus-shuffling":0,"consensus-state-transition":0.68,"constants":0.7894736842105263,"cross-fork":0.75,"eip-interactions":1,"evm-execution":0.6166666666666667,"trick":0.6666666666666666}},{"model_id":"deepseek-v4-pro-high","display_name":"deepseek-v4-pro","lab":"deepseek","prompt":"bare","release_date":"2026-04-24","precision":"unknown","base_model":"deepseek-v4-pro","reasoning_effort":"high","provider":"deepseek","notes":null,"pass_rate":0.6139240506329114,"ci_lower":0.5591822912825375,"ci_upper":0.6659291388753908,"total_questions":316,"total_runs":316,"cost_avg":0.028542756265822797,"cost_total":9.019510980000003,"latency_p50":84974.5,"category_pass_rates":{"calculations":1,"conceptual":0.5,"consensus-epoch-processing":0.40384615384615385,"consensus-fork-choice":0.9333333333333333,"consensus-shuffling":0,"consensus-state-transition":0.64,"constants":0.8421052631578947,"cross-fork":0.5,"eip-interactions":0,"evm-execution":0.575,"trick":0.5}},{"model_id":"glm-5.2-high","display_name":"glm-5.2","lab":"z-ai","prompt":"bare","release_date":"2026-06-13","precision":"fp8","base_model":"glm-5.2","reasoning_effort":"high","provider":"z-ai","notes":null,"pass_rate":0.6107594936708861,"ci_lower":0.5559767599860462,"ci_upper":0.6628815748897173,"total_questions":316,"total_runs":316,"cost_avg":0.040627139873417734,"cost_total":12.838176200000005,"latency_p50":41583.5,"category_pass_rates":{"calculations":1,"conceptual":0.6666666666666666,"consensus-epoch-processing":0.4230769230769231,"consensus-fork-choice":0.8,"consensus-shuffling":0,"consensus-state-transition":0.52,"constants":0.8245614035087719,"cross-fork":0.75,"eip-interactions":0,"evm-execution":0.6083333333333333,"trick":0.6666666666666666}},{"model_id":"gpt-5.4-mini-medium","display_name":"gpt-5.4-mini","lab":"openai","prompt":"bare","release_date":"2026-03-17","precision":"unknown","base_model":"gpt-5.4-mini","reasoning_effort":"medium","provider":"openai","notes":null,"pass_rate":0.6075949367088608,"ci_lower":0.55277356524569,"ci_upper":0.6598316743479089,"total_questions":316,"total_runs":316,"cost_avg":0.016298081487341767,"cost_total":5.150193749999998,"latency_p50":14171.5,"category_pass_rates":{"calculations":1,"conceptual":0.6666666666666666,"consensus-epoch-processing":0.36538461538461536,"consensus-fork-choice":0.8,"consensus-shuffling":0,"consensus-state-transition":0.66,"constants":0.8596491228070176,"cross-fork":0.5,"eip-interactions":1,"evm-execution":0.55,"trick":0.6666666666666666}},{"model_id":"gpt-5.2-codex-low","display_name":"gpt-5.2-codex","lab":"openai","prompt":"bare","release_date":"2025-12-11","precision":"unknown","base_model":"gpt-5.2-codex","reasoning_effort":"low","provider":"openai","notes":null,"pass_rate":0.6075949367088608,"ci_lower":0.55277356524569,"ci_upper":0.6598316743479089,"total_questions":316,"total_runs":316,"cost_avg":0.0202677579113924,"cost_total":6.404611499999998,"latency_p50":9884.5,"category_pass_rates":{"calculations":1,"conceptual":0.6666666666666666,"consensus-epoch-processing":0.34615384615384615,"consensus-fork-choice":0.6666666666666666,"consensus-shuffling":0,"consensus-state-transition":0.76,"constants":0.8421052631578947,"cross-fork":0.5,"eip-interactions":0,"evm-execution":0.5583333333333333,"trick":0.5}},{"model_id":"claude-sonnet-4-high","display_name":"claude-sonnet-4","lab":"anthropic","prompt":"bare","release_date":"2025-05-22","precision":"unknown","base_model":"claude-sonnet-4","reasoning_effort":"high","provider":"anthropic","notes":null,"pass_rate":0.6044303797468354,"ci_lower":0.549572697038391,"ci_upper":0.6567794472730433,"total_questions":316,"total_runs":316,"cost_avg":0.07123622468354436,"cost_total":22.51064700000002,"latency_p50":36366,"category_pass_rates":{"calculations":1,"conceptual":0.5,"consensus-epoch-processing":0.36538461538461536,"consensus-fork-choice":0.7333333333333333,"consensus-shuffling":0,"consensus-state-transition":0.7,"constants":0.8596491228070176,"cross-fork":0.5,"eip-interactions":1,"evm-execution":0.5416666666666666,"trick":0.6666666666666666}},{"model_id":"glm-5.3-flash-fp8-baseten-high-131k","display_name":"glm-5.3-flash","lab":"z-ai","prompt":"bare","release_date":"2026-08-26","precision":"fp8","base_model":"glm-5.3-flash","reasoning_effort":"high","provider":"baseten","notes":"OpenRouter, baseten/fp8 only; fallbacks disabled. FP8, not BF16. High reasoning, 131072 output tokens per attempt, concurrency 30. Upstream backfill replaces only unresolved infrastructure rows; original scored answers, including wrong and no-answer results, retained. Harness 40b7728a28330b0b908809d012dfa300fa185d93. Recorded usage excludes discarded retry attempts.","pass_rate":0.5949367088607594,"ci_lower":0.5399839567280832,"ci_upper":0.6476089017368569,"total_questions":316,"total_runs":316,"cost_avg":0.005107151424050635,"cost_total":1.6138598500000005,"latency_p50":22167.5,"category_pass_rates":{"calculations":1,"conceptual":0.8333333333333334,"consensus-epoch-processing":0.38461538461538464,"consensus-fork-choice":0.7333333333333333,"consensus-shuffling":0,"consensus-state-transition":0.64,"constants":0.8421052631578947,"cross-fork":1,"eip-interactions":1,"evm-execution":0.5083333333333333,"trick":0.6666666666666666}},{"model_id":"glm-5.3-flash-fp4-local-qad-high","display_name":"glm-5.3-flash","lab":"z-ai","prompt":"bare","release_date":"2026-09-07","precision":"nvidia-fp4-local-qad","base_model":"glm-5.3-flash","reasoning_effort":"high","provider":"starflinger","notes":"Starflinger; NVIDIA GLM-5.3-Flash-FP4-Local (QAD; NVFP4 routed experts with csf_32x2_v1 scales, MXFP8/FP8-block attention and shared experts), vLLM local-inference-lab release-10152026, TP2/DCP2, MTP3, FP8 KV. High reasoning, 131072 output tokens per attempt (no fusaka answer exceeded 117089 tokens), concurrency 2. Harness 40b7728a28330b0b908809d012dfa300fa185d93. Recorded usage excludes discarded retry attempts.","pass_rate":0.5886075949367089,"ci_lower":0.5336029279493076,"ci_upper":0.6414837399513033,"total_questions":316,"total_runs":316,"cost_avg":0,"cost_total":0,"latency_p50":10546,"category_pass_rates":{"calculations":1,"conceptual":0.6666666666666666,"consensus-epoch-processing":0.34615384615384615,"consensus-fork-choice":0.6,"consensus-shuffling":0,"consensus-state-transition":0.68,"constants":0.8070175438596491,"cross-fork":1,"eip-interactions":0,"evm-execution":0.5333333333333333,"trick":0.8333333333333334}},{"model_id":"claude-sonnet-4-16k","display_name":"claude-sonnet-4","lab":"anthropic","prompt":"bare","release_date":"2025-05-22","precision":"unknown","base_model":"claude-sonnet-4","reasoning_effort":"medium","provider":"anthropic","notes":null,"pass_rate":0.5791139240506329,"ci_lower":0.5240484193005199,"ci_upper":0.6322789627535967,"total_questions":316,"total_runs":316,"cost_avg":0.05096568037974686,"cost_total":16.105155000000007,"latency_p50":32882,"category_pass_rates":{"calculations":1,"conceptual":0.5,"consensus-epoch-processing":0.3269230769230769,"consensus-fork-choice":0.6,"consensus-shuffling":0,"consensus-state-transition":0.72,"constants":0.9122807017543859,"cross-fork":0.25,"eip-interactions":1,"evm-execution":0.48333333333333334,"trick":0.6666666666666666}},{"model_id":"gpt-5.6-luna-low","display_name":"gpt-5.6-luna","lab":"openai","prompt":"bare","release_date":"2026-07-09","precision":"unknown","base_model":"gpt-5.6-luna","reasoning_effort":"low","provider":"openai","notes":null,"pass_rate":0.5791139240506329,"ci_lower":0.5240484193005199,"ci_upper":0.6322789627535967,"total_questions":316,"total_runs":316,"cost_avg":0.0010927006329113924,"cost_total":0.3452934,"latency_p50":6883,"category_pass_rates":{"calculations":0.5,"conceptual":0.5,"consensus-epoch-processing":0.38461538461538464,"consensus-fork-choice":0.8,"consensus-shuffling":0,"consensus-state-transition":0.64,"constants":0.9473684210526315,"cross-fork":0.75,"eip-interactions":0,"evm-execution":0.44166666666666665,"trick":0.8333333333333334}},{"model_id":"hunter-alpha-high","display_name":"hunter-alpha","lab":"openrouter","prompt":"bare","release_date":"2026-03-11","precision":"unknown","base_model":"hunter-alpha","reasoning_effort":"high","provider":"stealth","notes":null,"pass_rate":0.5759493670886076,"ci_lower":0.5208680960090742,"ci_upper":0.6292061907628779,"total_questions":316,"total_runs":316,"cost_avg":0,"cost_total":0,"latency_p50":127865,"category_pass_rates":{"calculations":1,"conceptual":0.6666666666666666,"consensus-epoch-processing":0.36538461538461536,"consensus-fork-choice":0.7333333333333333,"consensus-shuffling":0,"consensus-state-transition":0.76,"constants":0.7017543859649122,"cross-fork":0.75,"eip-interactions":0,"evm-execution":0.5166666666666667,"trick":0.5}},{"model_id":"qwen3.8-flash-xhigh","display_name":"qwen3.8-flash","lab":"qwen","prompt":"bare","release_date":"2026-08-26","precision":"unknown","base_model":"qwen3.8-flash","reasoning_effort":"xhigh","provider":"alibaba","notes":null,"pass_rate":0.5759493670886076,"ci_lower":0.5208680960090742,"ci_upper":0.6292061907628779,"total_questions":316,"total_runs":316,"cost_avg":0.01815655455696202,"cost_total":5.737471239999999,"latency_p50":3381911,"category_pass_rates":{"calculations":1,"conceptual":0.6666666666666666,"consensus-epoch-processing":0.3269230769230769,"consensus-fork-choice":0.6,"consensus-shuffling":0,"consensus-state-transition":0.48,"constants":0.7543859649122807,"cross-fork":0.5,"eip-interactions":1,"evm-execution":0.65,"trick":0.3333333333333333}},{"model_id":"qwen3.6-plus-preview-high","display_name":"qwen3.6-plus-preview","lab":"qwen","prompt":"bare","release_date":"2026-03-30","precision":"unknown","base_model":"qwen3.6-plus-preview","reasoning_effort":"high","provider":"alibaba","notes":null,"pass_rate":0.569620253164557,"ci_lower":0.5145141781447586,"ci_upper":0.6230539180628643,"total_questions":316,"total_runs":316,"cost_avg":0,"cost_total":0,"latency_p50":57114.5,"category_pass_rates":{"calculations":1,"conceptual":0.5,"consensus-epoch-processing":0.40384615384615385,"consensus-fork-choice":0.6,"consensus-shuffling":0,"consensus-state-transition":0.56,"constants":0.8596491228070176,"cross-fork":0.75,"eip-interactions":0,"evm-execution":0.5083333333333333,"trick":0.6666666666666666}},{"model_id":"ox-alpha-high","display_name":"ox-alpha","lab":"stealth","prompt":"bare","release_date":"2026-08-20","precision":"unknown","base_model":"ox-alpha","reasoning_effort":"high","provider":"stealth","notes":null,"pass_rate":0.5664556962025317,"ci_lower":0.511340571082102,"ci_upper":0.6199744298433563,"total_questions":316,"total_runs":316,"cost_avg":0,"cost_total":0,"latency_p50":25477,"category_pass_rates":{"calculations":1,"conceptual":0.6666666666666666,"consensus-epoch-processing":0.38461538461538464,"consensus-fork-choice":0.8,"consensus-shuffling":0,"consensus-state-transition":0.54,"constants":0.8771929824561403,"cross-fork":1,"eip-interactions":0,"evm-execution":0.4583333333333333,"trick":0.8333333333333334}},{"model_id":"grok-4.3-high","display_name":"grok-4.3","lab":"x-ai","prompt":"bare","release_date":"2026-04-17","precision":"unknown","base_model":"grok-4.3","reasoning_effort":"high","provider":"xai","notes":null,"pass_rate":0.5632911392405063,"ci_lower":0.5081691907696114,"ci_upper":0.616892714873682,"total_questions":316,"total_runs":316,"cost_avg":0.04075189082278481,"cost_total":12.8775975,"latency_p50":104145,"category_pass_rates":{"calculations":1,"conceptual":0.8333333333333334,"consensus-epoch-processing":0.34615384615384615,"consensus-fork-choice":0.7333333333333333,"consensus-shuffling":0,"consensus-state-transition":0.46,"constants":0.8596491228070176,"cross-fork":0.5,"eip-interactions":1,"evm-execution":0.5333333333333333,"trick":0.5}},{"model_id":"grok-4.20-beta-high","display_name":"grok-4.20-beta","lab":"x-ai","prompt":"bare","release_date":"2026-03-12","precision":"unknown","base_model":"grok-4.20-beta","reasoning_effort":"high","provider":"xai","notes":null,"pass_rate":0.5632911392405063,"ci_lower":0.5081691907696114,"ci_upper":0.616892714873682,"total_questions":316,"total_runs":316,"cost_avg":0.016590727848101276,"cost_total":5.242670000000003,"latency_p50":25072.5,"category_pass_rates":{"calculations":1,"conceptual":0.8333333333333334,"consensus-epoch-processing":0.36538461538461536,"consensus-fork-choice":0.7333333333333333,"consensus-shuffling":0,"consensus-state-transition":0.56,"constants":0.8771929824561403,"cross-fork":0.75,"eip-interactions":0,"evm-execution":0.475,"trick":0.5}},{"model_id":"glm-5-high","display_name":"glm-5","lab":"z-ai","prompt":"bare","release_date":"2026-02-12","precision":"unknown","base_model":"glm-5","reasoning_effort":"high","provider":"z-ai","notes":null,"pass_rate":0.560126582278481,"ci_lower":0.5050000317023345,"ci_upper":0.6138087786587942,"total_questions":316,"total_runs":316,"cost_avg":0.027856576582278476,"cost_total":8.802678199999999,"latency_p50":200202,"category_pass_rates":{"calculations":0.5,"conceptual":0.8333333333333334,"consensus-epoch-processing":0.3076923076923077,"consensus-fork-choice":0.8666666666666667,"consensus-shuffling":0,"consensus-state-transition":0.46,"constants":0.8070175438596491,"cross-fork":0.75,"eip-interactions":0,"evm-execution":0.55,"trick":0.6666666666666666}},{"model_id":"mistral-large-4-0-high","display_name":"mistral-large-4-0","lab":"mistral","prompt":"bare","release_date":"2026-10-06","precision":"unknown","base_model":"mistral-large-4-0","reasoning_effort":"high","provider":"mistral","notes":null,"pass_rate":0.560126582278481,"ci_lower":0.5050000317023345,"ci_upper":0.6138087786587942,"total_questions":316,"total_runs":316,"cost_avg":0.022758730854430375,"cost_total":7.191758949999999,"latency_p50":102837,"category_pass_rates":{"calculations":1,"conceptual":0.6666666666666666,"consensus-epoch-processing":0.40384615384615385,"consensus-fork-choice":0.6,"consensus-shuffling":0,"consensus-state-transition":0.52,"constants":0.8421052631578947,"cross-fork":0.75,"eip-interactions":0,"evm-execution":0.5083333333333333,"trick":0.5}},{"model_id":"glm-5.3-high","display_name":"glm-5.3","lab":"z-ai","prompt":"bare","release_date":"2026-08-14","precision":"fp8","base_model":"glm-5.3","reasoning_effort":"high","provider":"z-ai","notes":null,"pass_rate":0.5569620253164557,"ci_lower":0.5018330886648706,"ci_upper":0.6107226264140934,"total_questions":316,"total_runs":316,"cost_avg":0.036696255063291154,"cost_total":11.596016600000006,"latency_p50":23254,"category_pass_rates":{"calculations":1,"conceptual":0.8333333333333334,"consensus-epoch-processing":0.34615384615384615,"consensus-fork-choice":0.7333333333333333,"consensus-shuffling":0,"consensus-state-transition":0.5,"constants":0.7543859649122807,"cross-fork":0.75,"eip-interactions":1,"evm-execution":0.5333333333333333,"trick":0.6666666666666666}},{"model_id":"mimo-v2.5-pro-high","display_name":"mimo-v2.5-pro","lab":"xiaomi","prompt":"bare","release_date":"2026-04-22","precision":"fp8","base_model":"mimo-v2.5-pro","reasoning_effort":"high","provider":"xiaomi","notes":null,"pass_rate":0.5506329113924051,"ci_lower":0.4955058312461932,"ci_upper":0.6045436932684417,"total_questions":316,"total_runs":316,"cost_avg":0.0065640508860759506,"cost_total":2.0742400800000005,"latency_p50":70366.5,"category_pass_rates":{"calculations":1,"conceptual":0.3333333333333333,"consensus-epoch-processing":0.40384615384615385,"consensus-fork-choice":0.7333333333333333,"consensus-shuffling":0,"consensus-state-transition":0.52,"constants":0.7192982456140351,"cross-fork":0.75,"eip-interactions":0,"evm-execution":0.5583333333333333,"trick":0.16666666666666666}},{"model_id":"deepseek-v4-flash","display_name":"deepseek-v4-flash","lab":"deepseek","prompt":"bare","release_date":null,"precision":"unknown","base_model":"deepseek-v4-flash","reasoning_effort":"max","provider":"starflinger","notes":null,"pass_rate":0.5474683544303798,"ci_lower":0.49234550785359626,"ci_upper":0.6014509213788739,"total_questions":316,"total_runs":316,"cost_avg":0,"cost_total":0,"latency_p50":116297,"category_pass_rates":{"calculations":1,"conceptual":0.3333333333333333,"consensus-epoch-processing":0.36538461538461536,"consensus-fork-choice":0.8,"consensus-shuffling":0,"consensus-state-transition":0.42,"constants":0.6842105263157895,"cross-fork":0.75,"eip-interactions":1,"evm-execution":0.5916666666666667,"trick":0.5}},{"model_id":"minimax-m3-high","display_name":"minimax-m3","lab":"minimax","prompt":"bare","release_date":"2026-06-01","precision":"fp8","base_model":"minimax-m3","reasoning_effort":"high","provider":"minimax","notes":null,"pass_rate":0.5443037974683544,"ci_lower":0.4891873824624862,"ci_upper":0.5983559514878193,"total_questions":316,"total_runs":316,"cost_avg":0.016495246518987334,"cost_total":5.212497899999997,"latency_p50":144474.5,"category_pass_rates":{"calculations":1,"conceptual":0.8333333333333334,"consensus-epoch-processing":0.40384615384615385,"consensus-fork-choice":0.6666666666666666,"consensus-shuffling":0,"consensus-state-transition":0.5,"constants":0.7894736842105263,"cross-fork":0.75,"eip-interactions":0,"evm-execution":0.475,"trick":0.6666666666666666}},{"model_id":"glm-5.3-flash-nvfp4-sglang-h-high-65k","display_name":"glm-5.3-flash","lab":"z-ai","prompt":"bare","release_date":"2026-09-07","precision":"nvfp4-w4a16-k32","base_model":"glm-5.3-flash","reasoning_effort":"high","provider":"starflinger","notes":"Starflinger patched SGLang v0.6.0 h; W4A16 NVFP4 K32. High reasoning, 65536 output tokens per attempt, concurrency 2. Historical baseline; separate from the deferred 131072-token rerun.","pass_rate":0.5443037974683544,"ci_lower":0.4891873824624862,"ci_upper":0.5983559514878193,"total_questions":316,"total_runs":316,"cost_avg":0,"cost_total":0,"latency_p50":11558.5,"category_pass_rates":{"calculations":1,"conceptual":0.5,"consensus-epoch-processing":0.3076923076923077,"consensus-fork-choice":0.5333333333333333,"consensus-shuffling":0,"consensus-state-transition":0.6,"constants":0.8245614035087719,"cross-fork":1,"eip-interactions":1,"evm-execution":0.4583333333333333,"trick":1}},{"model_id":"claude-3.7-sonnet-thinking","display_name":"claude-3.7-sonnet","lab":"anthropic","prompt":"bare","release_date":"2025-02-24","precision":"unknown","base_model":"claude-3.7-sonnet","reasoning_effort":"high","provider":"google-vertex","notes":null,"pass_rate":0.5411392405063291,"ci_lower":0.48603145126019415,"ci_upper":0.5952587874079466,"total_questions":316,"total_runs":316,"cost_avg":0.14070522151898734,"cost_total":44.46285,"latency_p50":106627,"category_pass_rates":{"calculations":1,"conceptual":0.6666666666666666,"consensus-epoch-processing":0.4423076923076923,"consensus-fork-choice":0.6666666666666666,"consensus-shuffling":0,"consensus-state-transition":0.6,"constants":0.6140350877192983,"cross-fork":0.5,"eip-interactions":0,"evm-execution":0.525,"trick":0.3333333333333333}},{"model_id":"deepseek-v4-flash-0731-low","display_name":"deepseek-v4-flash-0731","lab":"deepseek","prompt":"bare","release_date":"2026-07-31","precision":"fp8-fp4experts","base_model":"deepseek-v4-flash-0731","reasoning_effort":"low","provider":"starflinger","notes":null,"pass_rate":0.5348101265822784,"ci_lower":0.4797261575334397,"ci_upper":0.5890578905703715,"total_questions":316,"total_runs":316,"cost_avg":0,"cost_total":0,"latency_p50":221774.5,"category_pass_rates":{"calculations":1,"conceptual":0.6666666666666666,"consensus-epoch-processing":0.25,"consensus-fork-choice":0.7333333333333333,"consensus-shuffling":0,"consensus-state-transition":0.5,"constants":0.7368421052631579,"cross-fork":0.75,"eip-interactions":1,"evm-execution":0.5416666666666666,"trick":0.5}},{"model_id":"qwen3.5-397b-a17b","display_name":"qwen3.5-397b-a17b","lab":"qwen","prompt":"bare","release_date":"2026-02-15","precision":"unknown","base_model":"qwen3.5-397b-a17b","reasoning_effort":null,"provider":"alibaba","notes":null,"pass_rate":0.5284810126582279,"ci_lower":0.47342960159124187,"ci_upper":0.5828482559482402,"total_questions":316,"total_runs":316,"cost_avg":0.016885327689873407,"cost_total":5.335763549999997,"latency_p50":84260.5,"category_pass_rates":{"calculations":1,"conceptual":0.6666666666666666,"consensus-epoch-processing":0.25,"consensus-fork-choice":0.8,"consensus-shuffling":0,"consensus-state-transition":0.46,"constants":0.8596491228070176,"cross-fork":0.5,"eip-interactions":1,"evm-execution":0.4666666666666667,"trick":0.8333333333333334}},{"model_id":"glm-5.3-flash-nvfp4-qad-kraken-high-131k","display_name":"glm-5.3-flash","lab":"z-ai","prompt":"bare","release_date":"2026-09-07","precision":"nvfp4-csf-mxfp8-qad","base_model":"glm-5.3-flash","reasoning_effort":"high","provider":"starflinger","notes":"Starflinger; Local Inference Lab karmic-kraken-beta; GLM-5.3-Flash NVFP4/MXFP8 CSF QAD, vLLM, TP2. High reasoning, 131072 output tokens per attempt, concurrency 2. Checkpoint revision dec48abd33efa73c3bb7c95b74eee10cad34f9be. Harness 40b7728a28330b0b908809d012dfa300fa185d93. Recorded usage excludes discarded retry attempts.","pass_rate":0.5221518987341772,"ci_lower":0.4671417626231299,"ci_upper":0.5766299043520229,"total_questions":316,"total_runs":316,"cost_avg":0,"cost_total":0,"latency_p50":9809,"category_pass_rates":{"calculations":1,"conceptual":0.6666666666666666,"consensus-epoch-processing":0.25,"consensus-fork-choice":0.6,"consensus-shuffling":0,"consensus-state-transition":0.58,"constants":0.8421052631578947,"cross-fork":1,"eip-interactions":0,"evm-execution":0.425,"trick":0.8333333333333334}},{"model_id":"gemini-3-flash","display_name":"gemini-3-flash","lab":"google","prompt":"bare","release_date":"2025-12-17","precision":"bf16","base_model":"gemini-3-flash","reasoning_effort":null,"provider":"google-vertex","notes":null,"pass_rate":0.5158227848101266,"ci_lower":0.4608626240401377,"ci_upper":0.5704028523706857,"total_questions":316,"total_runs":316,"cost_avg":0.00153563924050633,"cost_total":0.4852620000000003,"latency_p50":4596,"category_pass_rates":{"calculations":1,"conceptual":0.5,"consensus-epoch-processing":0.36538461538461536,"consensus-fork-choice":0.6666666666666666,"consensus-shuffling":0,"consensus-state-transition":0.62,"constants":0.8421052631578947,"cross-fork":0.75,"eip-interactions":1,"evm-execution":0.36666666666666664,"trick":0.3333333333333333}},{"model_id":"deepseek-v4-flash-high","display_name":"deepseek-v4-flash","lab":"deepseek","prompt":"bare","release_date":"2026-04-24","precision":"unknown","base_model":"deepseek-v4-flash","reasoning_effort":"high","provider":"deepseek","notes":null,"pass_rate":0.5158227848101266,"ci_lower":0.4608626240401377,"ci_upper":0.5704028523706857,"total_questions":316,"total_runs":316,"cost_avg":0.0014183820886075962,"cost_total":0.4482087400000004,"latency_p50":42112,"category_pass_rates":{"calculations":1,"conceptual":0.6666666666666666,"consensus-epoch-processing":0.28846153846153844,"consensus-fork-choice":0.8666666666666667,"consensus-shuffling":0,"consensus-state-transition":0.36,"constants":0.7368421052631579,"cross-fork":0.25,"eip-interactions":0,"evm-execution":0.5416666666666666,"trick":0.5}},{"model_id":"gpt-5.4-mini-low","display_name":"gpt-5.4-mini","lab":"openai","prompt":"bare","release_date":"2026-03-17","precision":"unknown","base_model":"gpt-5.4-mini","reasoning_effort":"low","provider":"openai","notes":null,"pass_rate":0.5126582278481012,"ci_lower":0.45772631335333386,"ci_upper":0.5672860677753249,"total_questions":316,"total_runs":316,"cost_avg":0.004448143196202533,"cost_total":1.4056132500000005,"latency_p50":5367.5,"category_pass_rates":{"calculations":1,"conceptual":0.6666666666666666,"consensus-epoch-processing":0.3076923076923077,"consensus-fork-choice":0.7333333333333333,"consensus-shuffling":0,"consensus-state-transition":0.46,"constants":0.8245614035087719,"cross-fork":0.5,"eip-interactions":0,"evm-execution":0.45,"trick":0.5}},{"model_id":"deepseek-r1","display_name":"deepseek-r1","lab":"deepseek","prompt":"bare","release_date":"2025-01-20","precision":"fp8","base_model":"deepseek-r1","reasoning_effort":"high","provider":"novita","notes":null,"pass_rate":0.5063291139240507,"ci_lower":0.45146020338289966,"ci_upper":0.5610459871814297,"total_questions":316,"total_runs":316,"cost_avg":0.013855443354430388,"cost_total":4.3783201000000025,"latency_p50":112822,"category_pass_rates":{"calculations":1,"conceptual":0.6666666666666666,"consensus-epoch-processing":0.25,"consensus-fork-choice":0.6666666666666666,"consensus-shuffling":0,"consensus-state-transition":0.52,"constants":0.631578947368421,"cross-fork":0.25,"eip-interactions":1,"evm-execution":0.5333333333333333,"trick":0.5}},{"model_id":"inkling-high","display_name":"inkling","lab":"thinkingmachines","prompt":"bare","release_date":"2026-07-15","precision":"unknown","base_model":"inkling","reasoning_effort":"high","provider":"together","notes":null,"pass_rate":0.5,"ci_lower":0.4452027705582164,"ci_upper":0.5547972294417836,"total_questions":316,"total_runs":316,"cost_avg":0.05778045348101269,"cost_total":18.25862330000001,"latency_p50":371855.5,"category_pass_rates":{"calculations":1,"conceptual":0.6666666666666666,"consensus-epoch-processing":0.21153846153846154,"consensus-fork-choice":0.6,"consensus-shuffling":0,"consensus-state-transition":0.34,"constants":0.8245614035087719,"cross-fork":0.75,"eip-interactions":1,"evm-execution":0.5,"trick":0.6666666666666666}},{"model_id":"gemini-3.1-flash-lite-preview-high","display_name":"gemini-3.1-flash-lite-preview","lab":"google","prompt":"bare","release_date":"2026-03-03","precision":"unknown","base_model":"gemini-3.1-flash-lite-preview","reasoning_effort":"high","provider":"google-vertex","notes":null,"pass_rate":0.4936708860759494,"ci_lower":0.4389540128185704,"ci_upper":0.5485397966171005,"total_questions":316,"total_runs":316,"cost_avg":0.009212235759493669,"cost_total":2.911066499999999,"latency_p50":13888,"category_pass_rates":{"calculations":1,"conceptual":0.5,"consensus-epoch-processing":0.36538461538461536,"consensus-fork-choice":0.7333333333333333,"consensus-shuffling":0,"consensus-state-transition":0.26,"constants":0.7192982456140351,"cross-fork":0.5,"eip-interactions":1,"evm-execution":0.5,"trick":0.6666666666666666}},{"model_id":"qwen3-235b-a22b-thinking-2507","display_name":"qwen3-235b-a22b-thinking-2507","lab":"qwen","prompt":"bare","release_date":"2025-07-25","precision":"fp8","base_model":"qwen3-235b-a22b-thinking-2507","reasoning_effort":"high","provider":"deepinfra","notes":null,"pass_rate":0.49050632911392406,"ci_lower":0.4358328875885491,"ci_upper":0.545407826564957,"total_questions":316,"total_runs":316,"cost_avg":0.021272523860759507,"cost_total":6.722117540000004,"latency_p50":378629,"category_pass_rates":{"calculations":1,"conceptual":0.5,"consensus-epoch-processing":0.25,"consensus-fork-choice":0.6,"consensus-shuffling":0,"consensus-state-transition":0.48,"constants":0.7368421052631579,"cross-fork":0.75,"eip-interactions":0,"evm-execution":0.4666666666666667,"trick":0.5}},{"model_id":"gpt-5.3-codex-spark-xhigh","display_name":"gpt-5.3-codex-spark","lab":"openai","prompt":"bare","release_date":"2026-02-05","precision":"unknown","base_model":"gpt-5.3-codex-spark","reasoning_effort":"xhigh","provider":"openai","notes":null,"pass_rate":0.4873417721518987,"ci_lower":0.43271393222467525,"ci_upper":0.5422736866466662,"total_questions":316,"total_runs":316,"cost_avg":0.2952708433544305,"cost_total":93.30558650000003,"latency_p50":17917,"category_pass_rates":{"calculations":1,"conceptual":0.6666666666666666,"consensus-epoch-processing":0.2692307692307692,"consensus-fork-choice":0.6666666666666666,"consensus-shuffling":0,"consensus-state-transition":0.38,"constants":0.7543859649122807,"cross-fork":0.25,"eip-interactions":0,"evm-execution":0.48333333333333334,"trick":0.5}},{"model_id":"glm-4.7-high","display_name":"glm-4.7","lab":"z-ai","prompt":"bare","release_date":"2026-01-19","precision":"unknown","base_model":"glm-4.7","reasoning_effort":"high","provider":"z-ai","notes":null,"pass_rate":0.46835443037974683,"ci_lower":0.4140458359177566,"ci_upper":0.5234232112605968,"total_questions":316,"total_runs":316,"cost_avg":0.018582745569620245,"cost_total":5.872147599999997,"latency_p50":216454,"category_pass_rates":{"calculations":1,"conceptual":0.5,"consensus-epoch-processing":0.38461538461538464,"consensus-fork-choice":0.7333333333333333,"consensus-shuffling":0,"consensus-state-transition":0.28,"constants":0.6666666666666666,"cross-fork":0.25,"eip-interactions":1,"evm-execution":0.4583333333333333,"trick":0.5}},{"model_id":"deepseek-r1-0528","display_name":"deepseek-r1-0528","lab":"deepseek","prompt":"bare","release_date":"2025-05-28","precision":"fp8","base_model":"deepseek-r1-0528","reasoning_effort":"high","provider":"siliconflow","notes":null,"pass_rate":0.4620253164556962,"ci_lower":0.40784056732091606,"ci_upper":0.517122289293108,"total_questions":316,"total_runs":316,"cost_avg":0.01070477506329114,"cost_total":3.3827089200000002,"latency_p50":237751.5,"category_pass_rates":{"calculations":1,"conceptual":0.6666666666666666,"consensus-epoch-processing":0.28846153846153844,"consensus-fork-choice":0.8,"consensus-shuffling":0,"consensus-state-transition":0.38,"constants":0.6491228070175439,"cross-fork":0.75,"eip-interactions":1,"evm-execution":0.425,"trick":0.3333333333333333}},{"model_id":"minimax-m3.1-flash-xhigh","display_name":"minimax-m3.1-flash","lab":"minimax","prompt":"bare","release_date":"2026-10-19","precision":"unknown","base_model":"minimax-m3.1-flash","reasoning_effort":"xhigh","provider":"minimax","notes":null,"pass_rate":0.4620253164556962,"ci_lower":0.40784056732091606,"ci_upper":0.517122289293108,"total_questions":316,"total_runs":316,"cost_avg":0,"cost_total":0,"latency_p50":49595,"category_pass_rates":{"calculations":0.5,"conceptual":0.6666666666666666,"consensus-epoch-processing":0.19230769230769232,"consensus-fork-choice":0.6,"consensus-shuffling":0,"consensus-state-transition":0.36,"constants":0.7368421052631579,"cross-fork":1,"eip-interactions":1,"evm-execution":0.475,"trick":0}},{"model_id":"qwen3-max-thinking","display_name":"qwen3-max-thinking","lab":"qwen","prompt":"bare","release_date":"2026-02-09","precision":"unknown","base_model":"qwen3-max-thinking","reasoning_effort":"high","provider":"alibaba","notes":null,"pass_rate":0.4588607594936709,"ci_lower":0.4047412125920535,"ci_upper":0.5139685487398059,"total_questions":316,"total_runs":316,"cost_avg":0.03662561335443038,"cost_total":11.57369382,"latency_p50":192630.5,"category_pass_rates":{"calculations":1,"conceptual":0.8333333333333334,"consensus-epoch-processing":0.25,"consensus-fork-choice":0.6,"consensus-shuffling":0,"consensus-state-transition":0.32,"constants":0.7192982456140351,"cross-fork":0.25,"eip-interactions":1,"evm-execution":0.45,"trick":0.5}},{"model_id":"healer-alpha-high","display_name":"healer-alpha","lab":"openrouter","prompt":"bare","release_date":"2026-03-11","precision":"unknown","base_model":"healer-alpha","reasoning_effort":"high","provider":"stealth","notes":null,"pass_rate":0.45569620253164556,"ci_lower":0.40164404851218083,"ci_upper":0.5108126175375138,"total_questions":316,"total_runs":316,"cost_avg":0,"cost_total":0,"latency_p50":49565.5,"category_pass_rates":{"calculations":1,"conceptual":0.5,"consensus-epoch-processing":0.28846153846153844,"consensus-fork-choice":0.6,"consensus-shuffling":0,"consensus-state-transition":0.3,"constants":0.7192982456140351,"cross-fork":0.75,"eip-interactions":0,"evm-execution":0.44166666666666665,"trick":0.5}},{"model_id":"o4-mini-high","display_name":"o4-mini","lab":"openai","prompt":"bare","release_date":"2025-04-16","precision":"unknown","base_model":"o4-mini","reasoning_effort":"high","provider":"openai","notes":null,"pass_rate":0.4462025316455696,"ci_lower":0.3923657369313149,"ci_upper":0.5013316432718857,"total_questions":316,"total_runs":316,"cost_avg":0.04990317436708865,"cost_total":15.769403100000014,"latency_p50":76725,"category_pass_rates":{"calculations":1,"conceptual":0.5,"consensus-epoch-processing":0.2692307692307692,"consensus-fork-choice":0.6,"consensus-shuffling":0,"consensus-state-transition":0.34,"constants":0.543859649122807,"cross-fork":0.5,"eip-interactions":0,"evm-execution":0.5083333333333333,"trick":0.3333333333333333}},{"model_id":"grok-code-fast-1-high","display_name":"grok-code-fast-1","lab":"x-ai","prompt":"bare","release_date":"2025-08-26","precision":"unknown","base_model":"grok-code-fast-1","reasoning_effort":"high","provider":"xai","notes":null,"pass_rate":0.439873417721519,"ci_lower":0.3861912213412059,"ci_upper":0.49499996829766546,"total_questions":316,"total_runs":316,"cost_avg":0.005564460126582275,"cost_total":1.758369399999999,"latency_p50":27766.5,"category_pass_rates":{"calculations":1,"conceptual":0.5,"consensus-epoch-processing":0.19230769230769232,"consensus-fork-choice":0.4,"consensus-shuffling":0,"consensus-state-transition":0.38,"constants":0.7543859649122807,"cross-fork":0.25,"eip-interactions":0,"evm-execution":0.44166666666666665,"trick":0.3333333333333333}},{"model_id":"qwen3.6-27b-high","display_name":"qwen3.6-27b","lab":"qwen","prompt":"bare","release_date":"2026-04-22","precision":"unknown","base_model":"qwen3.6-27b","reasoning_effort":"high","provider":"alibaba","notes":null,"pass_rate":0.43670886075949367,"ci_lower":0.38310728512631803,"ci_upper":0.49183080923038863,"total_questions":316,"total_runs":316,"cost_avg":0.020546725158227846,"cost_total":6.492765149999999,"latency_p50":1088864.5,"category_pass_rates":{"calculations":1,"conceptual":0.5,"consensus-epoch-processing":0.17307692307692307,"consensus-fork-choice":0.7333333333333333,"consensus-shuffling":0,"consensus-state-transition":0.24,"constants":0.6140350877192983,"cross-fork":0.5,"eip-interactions":1,"evm-execution":0.49166666666666664,"trick":0.6666666666666666}},{"model_id":"claude-haiku-4.5-high","display_name":"claude-haiku-4.5","lab":"anthropic","prompt":"bare","release_date":"2025-10-15","precision":"unknown","base_model":"claude-haiku-4.5","reasoning_effort":"high","provider":"anthropic","notes":null,"pass_rate":0.43670886075949367,"ci_lower":0.38310728512631803,"ci_upper":0.49183080923038863,"total_questions":316,"total_runs":316,"cost_avg":0.06014621835443036,"cost_total":19.006204999999994,"latency_p50":69200.5,"category_pass_rates":{"calculations":1,"conceptual":0.5,"consensus-epoch-processing":0.19230769230769232,"consensus-fork-choice":0.6666666666666666,"consensus-shuffling":0,"consensus-state-transition":0.38,"constants":0.543859649122807,"cross-fork":0.5,"eip-interactions":0,"evm-execution":0.475,"trick":0.6666666666666666}},{"model_id":"qwen3.8-27b","display_name":"qwen3.8-27b","lab":"qwen","prompt":"bare","release_date":"2026-08-14","precision":"bf16","base_model":"qwen3.8-27b","reasoning_effort":"xhigh","provider":"starflinger","notes":null,"pass_rate":0.43354430379746833,"ci_lower":0.3800255701566438,"ci_upper":0.48865942891789815,"total_questions":316,"total_runs":316,"cost_avg":0,"cost_total":0,"latency_p50":1056437,"category_pass_rates":{"calculations":1,"conceptual":0.5,"consensus-epoch-processing":0.21153846153846154,"consensus-fork-choice":0.7333333333333333,"consensus-shuffling":0,"consensus-state-transition":0.24,"constants":0.47368421052631576,"cross-fork":0.25,"eip-interactions":0,"evm-execution":0.5666666666666667,"trick":0.3333333333333333}},{"model_id":"minimax-m2.7-high","display_name":"minimax-m2.7","lab":"minimax","prompt":"bare","release_date":"2026-03-18","precision":"fp8","base_model":"minimax-m2.7","reasoning_effort":"high","provider":"minimax","notes":null,"pass_rate":0.43037974683544306,"ci_lower":0.37694608193713586,"ci_upper":0.4854858218552415,"total_questions":316,"total_runs":316,"cost_avg":0.015772394620253156,"cost_total":4.9840766999999975,"latency_p50":160848,"category_pass_rates":{"calculations":1,"conceptual":0.6666666666666666,"consensus-epoch-processing":0.25,"consensus-fork-choice":0.8,"consensus-shuffling":0,"consensus-state-transition":0.38,"constants":0.5263157894736842,"cross-fork":0.5,"eip-interactions":1,"evm-execution":0.425,"trick":0.3333333333333333}},{"model_id":"qwen3.5-122b-a10b-sf","display_name":"qwen3.5-122b-a10b","lab":"qwen","prompt":"bare","release_date":"2026-02-25","precision":"nvfp4","base_model":"qwen3.5-122b-a10b","reasoning_effort":null,"provider":"starflinger","notes":null,"pass_rate":0.4272151898734177,"ci_lower":0.37386882626575346,"ci_upper":0.4823099822444592,"total_questions":316,"total_runs":316,"cost_avg":0,"cost_total":0,"latency_p50":104286,"category_pass_rates":{"calculations":1,"conceptual":0.5,"consensus-epoch-processing":0.21153846153846154,"consensus-fork-choice":0.6666666666666666,"consensus-shuffling":0,"consensus-state-transition":0.26,"constants":0.6666666666666666,"cross-fork":0.5,"eip-interactions":1,"evm-execution":0.425,"trick":0.6666666666666666}},{"model_id":"qwen3.8-27b-nvfp4-xhigh","display_name":"qwen3.8-27b","lab":"qwen","prompt":"bare","release_date":"2026-08-14","precision":"nvfp4","base_model":"qwen3.8-27b","reasoning_effort":"xhigh","provider":"starflinger","notes":null,"pass_rate":0.4272151898734177,"ci_lower":0.37386882626575346,"ci_upper":0.4823099822444592,"total_questions":316,"total_runs":316,"cost_avg":0,"cost_total":0,"latency_p50":414204,"category_pass_rates":{"calculations":0.5,"conceptual":0.6666666666666666,"consensus-epoch-processing":0.23076923076923078,"consensus-fork-choice":0.7333333333333333,"consensus-shuffling":0,"consensus-state-transition":0.32,"constants":0.5087719298245614,"cross-fork":0.25,"eip-interactions":1,"evm-execution":0.49166666666666664,"trick":0.16666666666666666}},{"model_id":"qwen3-max","display_name":"qwen3-max","lab":"qwen","prompt":"bare","release_date":"2025-09-23","precision":"unknown","base_model":"qwen3-max","reasoning_effort":null,"provider":"alibaba","notes":null,"pass_rate":0.4272151898734177,"ci_lower":0.37386882626575346,"ci_upper":0.4823099822444592,"total_questions":316,"total_runs":316,"cost_avg":0.0027865055696202523,"cost_total":0.8805357599999998,"latency_p50":119030.5,"category_pass_rates":{"calculations":0.5,"conceptual":0.6666666666666666,"consensus-epoch-processing":0.25,"consensus-fork-choice":0.5333333333333333,"consensus-shuffling":0,"consensus-state-transition":0.34,"constants":0.7543859649122807,"cross-fork":0.25,"eip-interactions":1,"evm-execution":0.35833333333333334,"trick":0.6666666666666666}},{"model_id":"claude-haiku-4.5","display_name":"claude-haiku-4.5","lab":"anthropic","prompt":"bare","release_date":"2025-10-15","precision":"unknown","base_model":"claude-haiku-4.5","reasoning_effort":null,"provider":"anthropic","notes":null,"pass_rate":0.4208860759493671,"ci_lower":0.3677210372464032,"ci_upper":0.4759515806994801,"total_questions":316,"total_runs":316,"cost_avg":0.003055553797468357,"cost_total":0.9655550000000008,"latency_p50":4788,"category_pass_rates":{"calculations":0.5,"conceptual":0.3333333333333333,"consensus-epoch-processing":0.2692307692307692,"consensus-fork-choice":0.5333333333333333,"consensus-shuffling":0,"consensus-state-transition":0.34,"constants":0.5087719298245614,"cross-fork":0.5,"eip-interactions":1,"evm-execution":0.45,"trick":0.8333333333333334}},{"model_id":"minimax-m2.5","display_name":"minimax-m2.5","lab":"minimax","prompt":"bare","release_date":"2026-02-12","precision":"fp8","base_model":"minimax-m2.5","reasoning_effort":null,"provider":"minimax","notes":null,"pass_rate":0.40625,"ci_lower":0.35622199632092405,"ci_upper":0.4583022189478922,"total_questions":316,"total_runs":352,"cost_avg":0.010344905113636367,"cost_total":3.641406600000001,"latency_p50":140121.5,"category_pass_rates":{"calculations":1,"conceptual":0.6666666666666666,"consensus-epoch-processing":0.19642857142857142,"consensus-fork-choice":0.6470588235294118,"consensus-shuffling":0,"consensus-state-transition":0.3728813559322034,"constants":0.589041095890411,"cross-fork":0.25,"eip-interactions":0,"evm-execution":0.392,"trick":0}},{"model_id":"gpt-oss-120b-low-ef-ecodev","display_name":"gpt-oss-120b","lab":"openai","prompt":"bare","release_date":"2025-04-16","precision":"unknown","base_model":"gpt-oss-120b","reasoning_effort":"low","provider":"ef-inference","notes":null,"pass_rate":0.4050632911392405,"ci_lower":0.352391098263143,"ci_upper":0.4600160432719169,"total_questions":316,"total_runs":316,"cost_avg":0,"cost_total":0,"latency_p50":22865.5,"category_pass_rates":{"calculations":1,"conceptual":0.5,"consensus-epoch-processing":0.25,"consensus-fork-choice":0.7333333333333333,"consensus-shuffling":0,"consensus-state-transition":0.32,"constants":0.5964912280701754,"cross-fork":0.25,"eip-interactions":0,"evm-execution":0.375,"trick":0.5}},{"model_id":"deepseek-v3.1","display_name":"deepseek-v3.1","lab":"deepseek","prompt":"bare","release_date":"2025-08-21","precision":"fp8","base_model":"deepseek-v3.1","reasoning_effort":null,"provider":"novita","notes":null,"pass_rate":0.39556962025316456,"ci_lower":0.3432205527269568,"ci_upper":0.45042730296160916,"total_questions":316,"total_runs":316,"cost_avg":0.0009654027531645566,"cost_total":0.3050672699999999,"latency_p50":20423,"category_pass_rates":{"calculations":0.5,"conceptual":0.3333333333333333,"consensus-epoch-processing":0.28846153846153844,"consensus-fork-choice":0.5333333333333333,"consensus-shuffling":0,"consensus-state-transition":0.28,"constants":0.6666666666666666,"cross-fork":0.5,"eip-interactions":0,"evm-execution":0.35,"trick":0.5}},{"model_id":"qwen3.5-27b","display_name":"qwen3.5-27b","lab":"qwen","prompt":"bare","release_date":"2026-02-23","precision":"unknown","base_model":"qwen3.5-27b","reasoning_effort":null,"provider":"alibaba","notes":null,"pass_rate":0.39556962025316456,"ci_lower":0.3432205527269568,"ci_upper":0.45042730296160916,"total_questions":316,"total_runs":316,"cost_avg":0.014739138560126586,"cost_total":4.657567785000001,"latency_p50":99722.5,"category_pass_rates":{"calculations":1,"conceptual":0.5,"consensus-epoch-processing":0.21153846153846154,"consensus-fork-choice":0.4666666666666667,"consensus-shuffling":0,"consensus-state-transition":0.22,"constants":0.5614035087719298,"cross-fork":0.5,"eip-interactions":1,"evm-execution":0.43333333333333335,"trick":0.6666666666666666}},{"model_id":"gpt-oss-120b-medium-ef-ecodev","display_name":"gpt-oss-120b","lab":"openai","prompt":"bare","release_date":"2025-04-16","precision":"unknown","base_model":"gpt-oss-120b","reasoning_effort":"medium","provider":"ef-inference","notes":null,"pass_rate":0.39556962025316456,"ci_lower":0.3432205527269568,"ci_upper":0.45042730296160916,"total_questions":316,"total_runs":316,"cost_avg":0,"cost_total":0,"latency_p50":17588,"category_pass_rates":{"calculations":1,"conceptual":0.5,"consensus-epoch-processing":0.21153846153846154,"consensus-fork-choice":0.6666666666666666,"consensus-shuffling":0,"consensus-state-transition":0.34,"constants":0.5964912280701754,"cross-fork":0.5,"eip-interactions":1,"evm-execution":0.35833333333333334,"trick":0.3333333333333333}},{"model_id":"step-3.7-flash-high","display_name":"step-3.7-flash","lab":"stepfun","prompt":"bare","release_date":"2026-05-29","precision":"fp8","base_model":"step-3.7-flash","reasoning_effort":"high","provider":"stepfun","notes":null,"pass_rate":0.38924050632911394,"ci_lower":0.3371184251102828,"ci_upper":0.4440232400139538,"total_questions":316,"total_runs":316,"cost_avg":0.03404386249999997,"cost_total":10.757860549999991,"latency_p50":167546,"category_pass_rates":{"calculations":0.5,"conceptual":0.5,"consensus-epoch-processing":0.25,"consensus-fork-choice":0.6,"consensus-shuffling":0,"consensus-state-transition":0.26,"constants":0.5263157894736842,"cross-fork":0.25,"eip-interactions":1,"evm-execution":0.4166666666666667,"trick":0.3333333333333333}},{"model_id":"gpt-oss-120b-medium","display_name":"gpt-oss-120b","lab":"openai","prompt":"bare","release_date":"2025-08-05","precision":"bf16","base_model":"gpt-oss-120b","reasoning_effort":"medium","provider":"deepinfra","notes":null,"pass_rate":0.3860759493670886,"ci_lower":0.33407086112460926,"ci_upper":0.4408177087174627,"total_questions":316,"total_runs":316,"cost_avg":0.000371764079113924,"cost_total":0.11747744899999997,"latency_p50":7670.5,"category_pass_rates":{"calculations":1,"conceptual":0.5,"consensus-epoch-processing":0.25,"consensus-fork-choice":0.5333333333333333,"consensus-shuffling":0,"consensus-state-transition":0.32,"constants":0.543859649122807,"cross-fork":0.25,"eip-interactions":1,"evm-execution":0.36666666666666664,"trick":0.5}},{"model_id":"deepseek-v3.2","display_name":"deepseek-v3.2","lab":"deepseek","prompt":"bare","release_date":"2025-12-01","precision":"fp8","base_model":"deepseek-v3.2","reasoning_effort":null,"provider":"atlas-cloud","notes":null,"pass_rate":0.3829113924050633,"ci_lower":0.33102564408402607,"ci_upper":0.43760983047588115,"total_questions":316,"total_runs":316,"cost_avg":0.0005485099367088616,"cost_total":0.17332914000000027,"latency_p50":19005.5,"category_pass_rates":{"calculations":1,"conceptual":0.6666666666666666,"consensus-epoch-processing":0.34615384615384615,"consensus-fork-choice":0.5333333333333333,"consensus-shuffling":0,"consensus-state-transition":0.3,"constants":0.543859649122807,"cross-fork":0.25,"eip-interactions":1,"evm-execution":0.325,"trick":0.3333333333333333}},{"model_id":"llama-4-maverick","display_name":"llama-4-maverick","lab":"meta","prompt":"bare","release_date":"2025-04-05","precision":"fp8","base_model":"llama-4-maverick","reasoning_effort":null,"provider":"deepinfra","notes":null,"pass_rate":0.3829113924050633,"ci_lower":0.33102564408402607,"ci_upper":0.43760983047588115,"total_questions":316,"total_runs":316,"cost_avg":0.0004737507911392407,"cost_total":0.14970525000000007,"latency_p50":26417,"category_pass_rates":{"calculations":0.5,"conceptual":0,"consensus-epoch-processing":0.17307692307692307,"consensus-fork-choice":0.6666666666666666,"consensus-shuffling":0,"consensus-state-transition":0.36,"constants":0.6842105263157895,"cross-fork":0.75,"eip-interactions":0,"evm-execution":0.31666666666666665,"trick":0.5}},{"model_id":"muse-glimmer-30b-high","display_name":"muse-glimmer-30b","lab":"meta","prompt":"bare","release_date":"2026-08-10","precision":"unknown","base_model":"muse-glimmer-30b","reasoning_effort":"high","provider":"together","notes":null,"pass_rate":0.37658227848101267,"ci_lower":0.3249422942674039,"ci_upper":0.43118698972817404,"total_questions":316,"total_runs":316,"cost_avg":0.014457886867088626,"cost_total":4.568692250000006,"latency_p50":111556.5,"category_pass_rates":{"calculations":1,"conceptual":0.6666666666666666,"consensus-epoch-processing":0.17307692307692307,"consensus-fork-choice":0.7333333333333333,"consensus-shuffling":0,"consensus-state-transition":0.08,"constants":0.631578947368421,"cross-fork":0.75,"eip-interactions":1,"evm-execution":0.375,"trick":0.6666666666666666}},{"model_id":"qwen3.8-27b-nvfp4-medium","display_name":"qwen3.8-27b","lab":"qwen","prompt":"bare","release_date":"2026-08-14","precision":"nvfp4","base_model":"qwen3.8-27b","reasoning_effort":"medium","provider":"starflinger","notes":null,"pass_rate":0.37658227848101267,"ci_lower":0.3249422942674039,"ci_upper":0.43118698972817404,"total_questions":316,"total_runs":316,"cost_avg":0,"cost_total":0,"latency_p50":28836.5,"category_pass_rates":{"calculations":1,"conceptual":0.6666666666666666,"consensus-epoch-processing":0.23076923076923078,"consensus-fork-choice":0.6666666666666666,"consensus-shuffling":0,"consensus-state-transition":0.24,"constants":0.43859649122807015,"cross-fork":0,"eip-interactions":1,"evm-execution":0.425,"trick":0.3333333333333333}},{"model_id":"deepseek-v3","display_name":"deepseek-v3","lab":"deepseek","prompt":"bare","release_date":"2024-12-26","precision":"fp8","base_model":"deepseek-v3","reasoning_effort":null,"provider":"novita","notes":null,"pass_rate":0.37658227848101267,"ci_lower":0.3249422942674039,"ci_upper":0.43118698972817404,"total_questions":316,"total_runs":316,"cost_avg":0.000548035759493671,"cost_total":0.17317930000000004,"latency_p50":9041,"category_pass_rates":{"calculations":0.5,"conceptual":0.3333333333333333,"consensus-epoch-processing":0.21153846153846154,"consensus-fork-choice":0.6,"consensus-shuffling":0,"consensus-state-transition":0.32,"constants":0.6140350877192983,"cross-fork":0.25,"eip-interactions":1,"evm-execution":0.3416666666666667,"trick":0.3333333333333333}},{"model_id":"gemini-2.5-flash-high","display_name":"gemini-2.5-flash","lab":"google","prompt":"bare","release_date":"2025-06-17","precision":"unknown","base_model":"gemini-2.5-flash","reasoning_effort":"high","provider":"google-vertex","notes":null,"pass_rate":0.37341772151898733,"ci_lower":0.32190418416594274,"ci_upper":0.4279720045474704,"total_questions":316,"total_runs":316,"cost_avg":0.014642784810126585,"cost_total":4.627120000000001,"latency_p50":43965,"category_pass_rates":{"calculations":1,"conceptual":0.5,"consensus-epoch-processing":0.21153846153846154,"consensus-fork-choice":0.6666666666666666,"consensus-shuffling":0,"consensus-state-transition":0.28,"constants":0.5087719298245614,"cross-fork":0.25,"eip-interactions":0,"evm-execution":0.38333333333333336,"trick":0.3333333333333333}},{"model_id":"qwen3.6-35b-a3b-high","display_name":"qwen3.6-35b-a3b","lab":"qwen","prompt":"bare","release_date":"2026-04-17","precision":"fp8","base_model":"qwen3.6-35b-a3b","reasoning_effort":"high","provider":"atlas-cloud","notes":null,"pass_rate":0.37341772151898733,"ci_lower":0.32190418416594274,"ci_upper":0.4279720045474704,"total_questions":316,"total_runs":316,"cost_avg":0.008082024525316455,"cost_total":2.55391975,"latency_p50":42081,"category_pass_rates":{"calculations":1,"conceptual":0.3333333333333333,"consensus-epoch-processing":0.19230769230769232,"consensus-fork-choice":0.6666666666666666,"consensus-shuffling":0,"consensus-state-transition":0.28,"constants":0.543859649122807,"cross-fork":0.75,"eip-interactions":1,"evm-execution":0.35,"trick":0.5}},{"model_id":"gpt-oss-120b-high-ef-ecodev","display_name":"gpt-oss-120b","lab":"openai","prompt":"bare","release_date":"2025-04-16","precision":"unknown","base_model":"gpt-oss-120b","reasoning_effort":"high","provider":"ef-inference","notes":null,"pass_rate":0.37341772151898733,"ci_lower":0.32190418416594274,"ci_upper":0.4279720045474704,"total_questions":316,"total_runs":316,"cost_avg":0,"cost_total":0,"latency_p50":16271,"category_pass_rates":{"calculations":1,"conceptual":0.5,"consensus-epoch-processing":0.23076923076923078,"consensus-fork-choice":0.7333333333333333,"consensus-shuffling":0,"consensus-state-transition":0.3,"constants":0.5614035087719298,"cross-fork":0.25,"eip-interactions":1,"evm-execution":0.31666666666666665,"trick":0.5}},{"model_id":"grok-4.1-fast-high","display_name":"grok-4.1-fast","lab":"x-ai","prompt":"bare","release_date":"2025-11-19","precision":"unknown","base_model":"grok-4.1-fast","reasoning_effort":"high","provider":"xai","notes":null,"pass_rate":0.3670886075949367,"ci_lower":0.31583515322922534,"ci_upper":0.42153484491985854,"total_questions":316,"total_runs":316,"cost_avg":0.004251693987341773,"cost_total":1.3435353000000003,"latency_p50":63215.5,"category_pass_rates":{"calculations":1,"conceptual":0.8333333333333334,"consensus-epoch-processing":0.19230769230769232,"consensus-fork-choice":0.6,"consensus-shuffling":0,"consensus-state-transition":0.2,"constants":0.5263157894736842,"cross-fork":0.5,"eip-interactions":1,"evm-execution":0.35833333333333334,"trick":0.6666666666666666}},{"model_id":"owl-alpha","display_name":"owl-alpha","lab":"openrouter","prompt":"bare","release_date":"2026-04-28","precision":"int8","base_model":"owl-alpha","reasoning_effort":null,"provider":"stealth","notes":null,"pass_rate":0.3639240506329114,"ci_lower":0.31280425747798923,"ci_upper":0.41831264538893,"total_questions":316,"total_runs":316,"cost_avg":0,"cost_total":0,"latency_p50":30458,"category_pass_rates":{"calculations":1,"conceptual":0.16666666666666666,"consensus-epoch-processing":0.1346153846153846,"consensus-fork-choice":0.5333333333333333,"consensus-shuffling":0,"consensus-state-transition":0.22,"constants":0.7368421052631579,"cross-fork":0.5,"eip-interactions":1,"evm-execution":0.31666666666666665,"trick":0.5}},{"model_id":"grok-3","display_name":"grok-3","lab":"x-ai","prompt":"bare","release_date":"2025-06-10","precision":"unknown","base_model":"grok-3","reasoning_effort":null,"provider":"xai","notes":null,"pass_rate":0.3639240506329114,"ci_lower":0.31280425747798923,"ci_upper":0.41831264538893,"total_questions":316,"total_runs":316,"cost_avg":0.005072829113924051,"cost_total":1.603014,"latency_p50":1360,"category_pass_rates":{"calculations":0,"conceptual":0.3333333333333333,"consensus-epoch-processing":0.17307692307692307,"consensus-fork-choice":0.4666666666666667,"consensus-shuffling":0,"consensus-state-transition":0.44,"constants":0.6140350877192983,"cross-fork":0.5,"eip-interactions":1,"evm-execution":0.2833333333333333,"trick":0.5}},{"model_id":"mistral-small-2603-high","display_name":"mistral-small-2603","lab":"mistral","prompt":"bare","release_date":"2026-03-16","precision":"unknown","base_model":"mistral-small-2603","reasoning_effort":"high","provider":"mistral","notes":null,"pass_rate":0.36075949367088606,"ci_lower":0.30977579229519603,"ci_upper":0.41508801528955847,"total_questions":316,"total_runs":316,"cost_avg":0.0027705783227848103,"cost_total":0.8755027500000001,"latency_p50":15144,"category_pass_rates":{"calculations":1,"conceptual":0.5,"consensus-epoch-processing":0.15384615384615385,"consensus-fork-choice":0.6,"consensus-shuffling":0,"consensus-state-transition":0.34,"constants":0.5263157894736842,"cross-fork":0.5,"eip-interactions":1,"evm-execution":0.3416666666666667,"trick":0.16666666666666666}},{"model_id":"claude-3.5-haiku","display_name":"claude-3.5-haiku","lab":"anthropic","prompt":"bare","release_date":"2024-11-04","precision":"unknown","base_model":"claude-3.5-haiku","reasoning_effort":null,"provider":"amazon-bedrock","notes":null,"pass_rate":0.3575949367088608,"ci_lower":0.3067497712890721,"ci_upper":0.4118609410135178,"total_questions":316,"total_runs":316,"cost_avg":0.0011824683544303782,"cost_total":0.3736599999999995,"latency_p50":4194.5,"category_pass_rates":{"calculations":0.5,"conceptual":0.5,"consensus-epoch-processing":0.23076923076923078,"consensus-fork-choice":0.6,"consensus-shuffling":0,"consensus-state-transition":0.28,"constants":0.5614035087719298,"cross-fork":0.25,"eip-interactions":0,"evm-execution":0.325,"trick":0.3333333333333333}},{"model_id":"qwen3.5-35b-a3b","display_name":"qwen3.5-35b-a3b","lab":"qwen","prompt":"bare","release_date":"2026-02-23","precision":"unknown","base_model":"qwen3.5-35b-a3b","reasoning_effort":null,"provider":"alibaba","notes":null,"pass_rate":0.3575949367088608,"ci_lower":0.3067497712890721,"ci_upper":0.4118609410135178,"total_questions":316,"total_runs":316,"cost_avg":0.0139277284414557,"cost_total":4.4011621875000015,"latency_p50":79751,"category_pass_rates":{"calculations":1,"conceptual":0.3333333333333333,"consensus-epoch-processing":0.2692307692307692,"consensus-fork-choice":0.5333333333333333,"consensus-shuffling":0,"consensus-state-transition":0.26,"constants":0.45614035087719296,"cross-fork":0,"eip-interactions":0,"evm-execution":0.36666666666666664,"trick":0.6666666666666666}},{"model_id":"grok-3-mini-high","display_name":"grok-3-mini","lab":"x-ai","prompt":"bare","release_date":"2025-06-10","precision":"unknown","base_model":"grok-3-mini","reasoning_effort":"high","provider":"xai","notes":null,"pass_rate":0.3512658227848101,"ci_lower":0.300705118474274,"ci_upper":0.4053994032639865,"total_questions":316,"total_runs":316,"cost_avg":0.004092142088607594,"cost_total":1.2931168999999998,"latency_p50":56696.5,"category_pass_rates":{"calculations":1,"conceptual":0.5,"consensus-epoch-processing":0.17307692307692307,"consensus-fork-choice":0.7333333333333333,"consensus-shuffling":0,"consensus-state-transition":0.24,"constants":0.543859649122807,"cross-fork":0.5,"eip-interactions":0,"evm-execution":0.31666666666666665,"trick":0.5}},{"model_id":"minimax-m2.5-high","display_name":"minimax-m2.5","lab":"minimax","prompt":"bare","release_date":"2026-02-12","precision":"fp8","base_model":"minimax-m2.5","reasoning_effort":"high","provider":"minimax","notes":null,"pass_rate":0.3449367088607595,"ci_lower":0.29467041699290725,"ci_upper":0.3989279141810239,"total_questions":316,"total_runs":316,"cost_avg":0.011085236392405062,"cost_total":3.5029346999999995,"latency_p50":118952.5,"category_pass_rates":{"calculations":1,"conceptual":0.6666666666666666,"consensus-epoch-processing":0.17307692307692307,"consensus-fork-choice":0.4666666666666667,"consensus-shuffling":0,"consensus-state-transition":0.28,"constants":0.49122807017543857,"cross-fork":0.75,"eip-interactions":0,"evm-execution":0.325,"trick":0.5}},{"model_id":"mistral-large-2512","display_name":"mistral-large-2512","lab":"mistral","prompt":"bare","release_date":"2025-12-02","precision":"unknown","base_model":"mistral-large-2512","reasoning_effort":null,"provider":"mistral","notes":null,"pass_rate":0.34177215189873417,"ci_lower":0.291656836949517,"ci_upper":0.3956883989422495,"total_questions":316,"total_runs":316,"cost_avg":0.0008088338607594938,"cost_total":0.2555915,"latency_p50":5388,"category_pass_rates":{"calculations":0.5,"conceptual":0.6666666666666666,"consensus-epoch-processing":0.1346153846153846,"consensus-fork-choice":0.6,"consensus-shuffling":0,"consensus-state-transition":0.32,"constants":0.6140350877192983,"cross-fork":0.5,"eip-interactions":1,"evm-execution":0.25833333333333336,"trick":0.3333333333333333}},{"model_id":"qwen3.5-27b-uncensored","display_name":"qwen3.5-27b-uncensored-hauhaucs-aggressive","lab":"qwen","prompt":"bare","release_date":"2026-02-24","precision":"q4_k_m","base_model":"qwen3.5-27b-uncensored-hauhaucs-aggressive","reasoning_effort":null,"provider":"selfhosted","notes":null,"pass_rate":0.33860759493670883,"ci_lower":0.2886457924689756,"ci_upper":0.3924463481406262,"total_questions":316,"total_runs":316,"cost_avg":0,"cost_total":0,"latency_p50":267908,"category_pass_rates":{"calculations":1,"conceptual":0.5,"consensus-epoch-processing":0.25,"consensus-fork-choice":0.6666666666666666,"consensus-shuffling":0,"consensus-state-transition":0.28,"constants":0.40350877192982454,"cross-fork":0.25,"eip-interactions":1,"evm-execution":0.31666666666666665,"trick":0.3333333333333333}},{"model_id":"qwen3.6-35b-a3b-q8","display_name":"qwen3.6-35b-a3b","lab":"qwen","prompt":"bare","release_date":"2026-04-16","precision":"q8_0","base_model":"qwen3.6-35b-a3b","reasoning_effort":null,"provider":"selfhosted","notes":null,"pass_rate":0.33860759493670883,"ci_lower":0.2886457924689756,"ci_upper":0.3924463481406262,"total_questions":316,"total_runs":316,"cost_avg":0,"cost_total":0,"latency_p50":293617.5,"category_pass_rates":{"calculations":0.5,"conceptual":0.3333333333333333,"consensus-epoch-processing":0.15384615384615385,"consensus-fork-choice":0.5333333333333333,"consensus-shuffling":0,"consensus-state-transition":0.3,"constants":0.5263157894736842,"cross-fork":0.75,"eip-interactions":0,"evm-execution":0.30833333333333335,"trick":0.5}},{"model_id":"mistral-medium-3.1","display_name":"mistral-medium-3.1","lab":"mistral","prompt":"bare","release_date":"2025-08-13","precision":"unknown","base_model":"mistral-medium-3.1","reasoning_effort":null,"provider":"mistral","notes":null,"pass_rate":0.3322784810126582,"ci_lower":0.2826313785967871,"ci_upper":0.38595457144848533,"total_questions":316,"total_runs":316,"cost_avg":0.0006718582278481005,"cost_total":0.21230719999999978,"latency_p50":2839,"category_pass_rates":{"calculations":0.5,"conceptual":0.3333333333333333,"consensus-epoch-processing":0.1346153846153846,"consensus-fork-choice":0.5333333333333333,"consensus-shuffling":0,"consensus-state-transition":0.3,"constants":0.5789473684210527,"cross-fork":0.5,"eip-interactions":1,"evm-execution":0.2833333333333333,"trick":0.3333333333333333}},{"model_id":"grok-4-fast-high","display_name":"grok-4-fast","lab":"x-ai","prompt":"bare","release_date":"2025-09-19","precision":"unknown","base_model":"grok-4-fast","reasoning_effort":"high","provider":"xai","notes":null,"pass_rate":0.3291139240506329,"ci_lower":0.2796280447495307,"ci_upper":0.3827048100135771,"total_questions":316,"total_runs":316,"cost_avg":0.0035565468354430353,"cost_total":1.1238687999999992,"latency_p50":32218,"category_pass_rates":{"calculations":1,"conceptual":0.5,"consensus-epoch-processing":0.19230769230769232,"consensus-fork-choice":0.6666666666666666,"consensus-shuffling":0,"consensus-state-transition":0.26,"constants":0.42105263157894735,"cross-fork":0.5,"eip-interactions":0,"evm-execution":0.30833333333333335,"trick":0.5}},{"model_id":"laguna-s-2.1-fp8","display_name":"laguna-s-2.1","lab":"starflinger","prompt":"bare","release_date":"2026-07-21","precision":"fp8","base_model":"laguna-s-2.1","reasoning_effort":null,"provider":"starflinger","notes":null,"pass_rate":0.3259493670886076,"ci_lower":0.27662731758415726,"ci_upper":0.37945244189678595,"total_questions":316,"total_runs":316,"cost_avg":0,"cost_total":0,"latency_p50":2938,"category_pass_rates":{"calculations":0,"conceptual":0.3333333333333333,"consensus-epoch-processing":0.19230769230769232,"consensus-fork-choice":0.6,"consensus-shuffling":0,"consensus-state-transition":0.3,"constants":0.42105263157894735,"cross-fork":0.75,"eip-interactions":0,"evm-execution":0.31666666666666665,"trick":0.3333333333333333}},{"model_id":"gpt-oss-120b-low","display_name":"gpt-oss-120b","lab":"openai","prompt":"bare","release_date":"2025-08-05","precision":"bf16","base_model":"gpt-oss-120b","reasoning_effort":"low","provider":"deepinfra","notes":null,"pass_rate":0.3227848101265823,"ci_lower":0.273629216286367,"ci_upper":0.3761974479124115,"total_questions":316,"total_runs":316,"cost_avg":0.00007930436392405053,"cost_total":0.02506017899999997,"latency_p50":1657,"category_pass_rates":{"calculations":1,"conceptual":0.6666666666666666,"consensus-epoch-processing":0.19230769230769232,"consensus-fork-choice":0.4,"consensus-shuffling":0,"consensus-state-transition":0.3,"constants":0.45614035087719296,"cross-fork":0.25,"eip-interactions":1,"evm-execution":0.3,"trick":0.16666666666666666}},{"model_id":"gpt-4o-2024-05-13","display_name":"gpt-4o-2024-05-13","lab":"openai","prompt":"bare","release_date":"2024-05-13","precision":"unknown","base_model":"gpt-4o-2024-05-13","reasoning_effort":null,"provider":"openai","notes":null,"pass_rate":0.31962025316455694,"ci_lower":0.2706337606369862,"ci_upper":0.3729398082796276,"total_questions":316,"total_runs":316,"cost_avg":0.006075348101265821,"cost_total":1.9198099999999994,"latency_p50":1296.5,"category_pass_rates":{"calculations":1,"conceptual":0.16666666666666666,"consensus-epoch-processing":0.11538461538461539,"consensus-fork-choice":0.4666666666666667,"consensus-shuffling":0,"consensus-state-transition":0.3,"constants":0.543859649122807,"cross-fork":0.25,"eip-interactions":0,"evm-execution":0.2833333333333333,"trick":0.6666666666666666}},{"model_id":"gpt-5-nano-high","display_name":"gpt-5-nano","lab":"openai","prompt":"bare","release_date":"2025-08-07","precision":"unknown","base_model":"gpt-5-nano","reasoning_effort":"high","provider":"openai","notes":null,"pass_rate":0.31962025316455694,"ci_lower":0.2706337606369862,"ci_upper":0.3729398082796276,"total_questions":316,"total_runs":316,"cost_avg":0.004733908702531645,"cost_total":1.4959151499999999,"latency_p50":99039.5,"category_pass_rates":{"calculations":1,"conceptual":0.3333333333333333,"consensus-epoch-processing":0.25,"consensus-fork-choice":0.5333333333333333,"consensus-shuffling":0,"consensus-state-transition":0.3,"constants":0.3333333333333333,"cross-fork":0.25,"eip-interactions":0,"evm-execution":0.325,"trick":0.3333333333333333}},{"model_id":"gpt-4o","display_name":"gpt-4o","lab":"openai","prompt":"bare","release_date":"2024-05-13","precision":"unknown","base_model":"gpt-4o","reasoning_effort":null,"provider":"openai","notes":null,"pass_rate":0.31645569620253167,"ci_lower":0.26764097102967865,"ci_upper":0.3696795026047705,"total_questions":316,"total_runs":316,"cost_avg":0.0033973655063291182,"cost_total":1.0735675000000013,"latency_p50":1567.5,"category_pass_rates":{"calculations":1,"conceptual":0.16666666666666666,"consensus-epoch-processing":0.23076923076923078,"consensus-fork-choice":0.13333333333333333,"consensus-shuffling":0,"consensus-state-transition":0.28,"constants":0.5964912280701754,"cross-fork":0.5,"eip-interactions":0,"evm-execution":0.24166666666666667,"trick":0.6666666666666666}},{"model_id":"flint-v1.0","display_name":"flint-v1.0","lab":"recursive","prompt":"bare","release_date":"2026-10-21","precision":"bf16","base_model":"flint-v1.0","reasoning_effort":null,"provider":"recursive","notes":null,"pass_rate":0.31329113924050633,"ci_lower":0.26465086848948566,"ci_upper":0.36641650986279883,"total_questions":316,"total_runs":316,"cost_avg":0,"cost_total":0,"latency_p50":125251,"category_pass_rates":{"calculations":1,"conceptual":0.3333333333333333,"consensus-epoch-processing":0.2692307692307692,"consensus-fork-choice":0.5333333333333333,"consensus-shuffling":0,"consensus-state-transition":0.16,"constants":0.19298245614035087,"cross-fork":0.25,"eip-interactions":0,"evm-execution":0.425,"trick":0.3333333333333333}},{"model_id":"codestral-2508","display_name":"codestral-2508","lab":"mistral","prompt":"bare","release_date":"2025-07-30","precision":"unknown","base_model":"codestral-2508","reasoning_effort":null,"provider":"mistral","notes":null,"pass_rate":0.310126582278481,"ci_lower":0.2616634746922393,"ci_upper":0.3631508083778805,"total_questions":316,"total_runs":316,"cost_avg":0.00039757879746835485,"cost_total":0.12563490000000013,"latency_p50":1000.5,"category_pass_rates":{"calculations":0,"conceptual":0,"consensus-epoch-processing":0.11538461538461539,"consensus-fork-choice":0.5333333333333333,"consensus-shuffling":0,"consensus-state-transition":0.3,"constants":0.5087719298245614,"cross-fork":0.25,"eip-interactions":0,"evm-execution":0.30833333333333335,"trick":0.3333333333333333}},{"model_id":"gpt-4.1-mini","display_name":"gpt-4.1-mini","lab":"openai","prompt":"bare","release_date":"2025-04-14","precision":"unknown","base_model":"gpt-4.1-mini","reasoning_effort":null,"provider":"openai","notes":null,"pass_rate":0.310126582278481,"ci_lower":0.2616634746922393,"ci_upper":0.3631508083778805,"total_questions":316,"total_runs":316,"cost_avg":0.0016058037974683525,"cost_total":0.5074339999999994,"latency_p50":9524.5,"category_pass_rates":{"calculations":0.5,"conceptual":0.3333333333333333,"consensus-epoch-processing":0.21153846153846154,"consensus-fork-choice":0.6666666666666666,"consensus-shuffling":0,"consensus-state-transition":0.2,"constants":0.43859649122807015,"cross-fork":0.5,"eip-interactions":0,"evm-execution":0.2916666666666667,"trick":0.3333333333333333}},{"model_id":"laguna-s-2.1-nvfp4-thinking","display_name":"laguna-s-2.1","lab":"starflinger","prompt":"bare","release_date":"2026-07-21","precision":"nvfp4","base_model":"laguna-s-2.1","reasoning_effort":"high","provider":"starflinger","notes":null,"pass_rate":0.3069620253164557,"ci_lower":0.2586788119848943,"ci_upper":0.35988237580306087,"total_questions":316,"total_runs":316,"cost_avg":0,"cost_total":0,"latency_p50":2195,"category_pass_rates":{"calculations":0,"conceptual":0.5,"consensus-epoch-processing":0.17307692307692307,"consensus-fork-choice":0.5333333333333333,"consensus-shuffling":0,"consensus-state-transition":0.3,"constants":0.40350877192982454,"cross-fork":0.5,"eip-interactions":0,"evm-execution":0.3,"trick":0.16666666666666666}},{"model_id":"mixtral-8x22b-instruct","display_name":"mixtral-8x22b-instruct","lab":"mistral","prompt":"bare","release_date":"2024-04-17","precision":"unknown","base_model":"mixtral-8x22b-instruct","reasoning_effort":null,"provider":"mistral","notes":null,"pass_rate":0.3069620253164557,"ci_lower":0.2586788119848943,"ci_upper":0.35988237580306087,"total_questions":316,"total_runs":316,"cost_avg":0.0018985506329113927,"cost_total":0.5999420000000001,"latency_p50":1077,"category_pass_rates":{"calculations":0,"conceptual":0.16666666666666666,"consensus-epoch-processing":0.11538461538461539,"consensus-fork-choice":0.6666666666666666,"consensus-shuffling":0,"consensus-state-transition":0.36,"constants":0.5087719298245614,"cross-fork":0.5,"eip-interactions":1,"evm-execution":0.25,"trick":0}},{"model_id":"laguna-s-2.1-fp8-thinking","display_name":"laguna-s-2.1","lab":"starflinger","prompt":"bare","release_date":"2026-07-21","precision":"fp8","base_model":"laguna-s-2.1","reasoning_effort":"high","provider":"starflinger","notes":null,"pass_rate":0.29430379746835444,"ci_lower":0.24676794370494787,"ci_upper":0.34678086295434857,"total_questions":316,"total_runs":316,"cost_avg":0,"cost_total":0,"latency_p50":1241,"category_pass_rates":{"calculations":0.5,"conceptual":0.5,"consensus-epoch-processing":0.11538461538461539,"consensus-fork-choice":0.5333333333333333,"consensus-shuffling":0,"consensus-state-transition":0.34,"constants":0.42105263157894735,"cross-fork":0,"eip-interactions":0,"evm-execution":0.275,"trick":0.16666666666666666}},{"model_id":"gemma-4-31b-it-high","display_name":"gemma-4-31b-it","lab":"google","prompt":"bare","release_date":"2026-04-02","precision":"bf16","base_model":"gemma-4-31b-it","reasoning_effort":"high","provider":"parasail","notes":null,"pass_rate":0.2911392405063291,"ci_lower":0.24379729669081032,"ci_upper":0.3434984146863215,"total_questions":316,"total_runs":316,"cost_avg":0.0018820065822784822,"cost_total":0.5947140800000004,"latency_p50":92587.5,"category_pass_rates":{"calculations":0.5,"conceptual":0.3333333333333333,"consensus-epoch-processing":0.17307692307692307,"consensus-fork-choice":0.6,"consensus-shuffling":0,"consensus-state-transition":0.12,"constants":0.24561403508771928,"cross-fork":0.75,"eip-interactions":0,"evm-execution":0.38333333333333336,"trick":0.3333333333333333}},{"model_id":"gpt-oss-20b-high","display_name":"gpt-oss-20b","lab":"openai","prompt":"bare","release_date":"2025-08-05","precision":"bf16","base_model":"gpt-oss-20b","reasoning_effort":"high","provider":"deepinfra","notes":null,"pass_rate":0.2879746835443038,"ci_lower":0.24082953020974499,"ci_upper":0.34021308588522203,"total_questions":316,"total_runs":316,"cost_avg":0.0028686059177215214,"cost_total":0.9064794700000007,"latency_p50":161619.5,"category_pass_rates":{"calculations":1,"conceptual":0.6666666666666666,"consensus-epoch-processing":0.17307692307692307,"consensus-fork-choice":0.5333333333333333,"consensus-shuffling":0,"consensus-state-transition":0.24,"constants":0.2982456140350877,"cross-fork":0,"eip-interactions":0,"evm-execution":0.30833333333333335,"trick":0.3333333333333333}},{"model_id":"devstral-medium","display_name":"devstral-medium","lab":"mistral","prompt":"bare","release_date":"2025-07-10","precision":"unknown","base_model":"devstral-medium","reasoning_effort":null,"provider":"mistral","notes":null,"pass_rate":0.2879746835443038,"ci_lower":0.24082953020974499,"ci_upper":0.34021308588522203,"total_questions":316,"total_runs":316,"cost_avg":0.000981813924050633,"cost_total":0.31025320000000006,"latency_p50":1632.5,"category_pass_rates":{"calculations":0.5,"conceptual":0.16666666666666666,"consensus-epoch-processing":0.1346153846153846,"consensus-fork-choice":0.13333333333333333,"consensus-shuffling":0,"consensus-state-transition":0.24,"constants":0.6491228070175439,"cross-fork":0.25,"eip-interactions":0,"evm-execution":0.25,"trick":0}},{"model_id":"gpt-oss-20b-medium","display_name":"gpt-oss-20b","lab":"openai","prompt":"bare","release_date":"2025-08-05","precision":"bf16","base_model":"gpt-oss-20b","reasoning_effort":"medium","provider":"deepinfra","notes":null,"pass_rate":0.2848101265822785,"ci_lower":0.23786467196456465,"ci_upper":0.33692484884823776,"total_questions":316,"total_runs":316,"cost_avg":0.0005621599683544301,"cost_total":0.1776425499999999,"latency_p50":36608.5,"category_pass_rates":{"calculations":0.5,"conceptual":0.6666666666666666,"consensus-epoch-processing":0.19230769230769232,"consensus-fork-choice":0.4666666666666667,"consensus-shuffling":0,"consensus-state-transition":0.26,"constants":0.2807017543859649,"cross-fork":0,"eip-interactions":0,"evm-execution":0.30833333333333335,"trick":0.3333333333333333}},{"model_id":"laguna-s-2.1-nvfp4","display_name":"laguna-s-2.1","lab":"starflinger","prompt":"bare","release_date":"2026-07-21","precision":"nvfp4","base_model":"laguna-s-2.1","reasoning_effort":null,"provider":"starflinger","notes":null,"pass_rate":0.28164556962025317,"ci_lower":0.23490275053177267,"ci_upper":0.33363367499886504,"total_questions":316,"total_runs":316,"cost_avg":0,"cost_total":0,"latency_p50":1986.5,"category_pass_rates":{"calculations":0,"conceptual":0.6666666666666666,"consensus-epoch-processing":0.15384615384615385,"consensus-fork-choice":0.4666666666666667,"consensus-shuffling":0,"consensus-state-transition":0.26,"constants":0.43859649122807015,"cross-fork":0.25,"eip-interactions":0,"evm-execution":0.24166666666666667,"trick":0.3333333333333333}},{"model_id":"qwq-32b","display_name":"qwq-32b","lab":"qwen","prompt":"bare","release_date":"2025-03-05","precision":"fp8","base_model":"qwq-32b","reasoning_effort":"high","provider":"siliconflow","notes":null,"pass_rate":0.27531645569620256,"ci_lower":0.22898783696744218,"ci_upper":0.32704239799886625,"total_questions":316,"total_runs":316,"cost_avg":0.004423888987341768,"cost_total":1.3979489199999988,"latency_p50":131239.5,"category_pass_rates":{"calculations":1,"conceptual":0.3333333333333333,"consensus-epoch-processing":0.09615384615384616,"consensus-fork-choice":0.6,"consensus-shuffling":0,"consensus-state-transition":0.22,"constants":0.38596491228070173,"cross-fork":0.25,"eip-interactions":0,"evm-execution":0.2916666666666667,"trick":0}},{"model_id":"mistral-large-2411","display_name":"mistral-large-2411","lab":"mistral","prompt":"bare","release_date":"2024-11-19","precision":"unknown","base_model":"mistral-large-2411","reasoning_effort":null,"provider":"mistral","notes":null,"pass_rate":0.2721518987341772,"ci_lower":0.22603490664753303,"ci_upper":0.3237422330366107,"total_questions":316,"total_runs":316,"cost_avg":0.004633436708860762,"cost_total":1.4641660000000007,"latency_p50":8972,"category_pass_rates":{"calculations":0.5,"conceptual":0.16666666666666666,"consensus-epoch-processing":0.11538461538461539,"consensus-fork-choice":0.5333333333333333,"consensus-shuffling":0,"consensus-state-transition":0.28,"constants":0.47368421052631576,"cross-fork":0.25,"eip-interactions":0,"evm-execution":0.23333333333333334,"trick":0}},{"model_id":"gemini-2.0-flash","display_name":"gemini-2.0-flash","lab":"google","prompt":"bare","release_date":"2025-02-05","precision":"unknown","base_model":"gemini-2.0-flash","reasoning_effort":null,"provider":"google-vertex","notes":null,"pass_rate":0.2689873417721519,"ci_lower":0.22308503683440079,"ci_upper":0.3204390075675782,"total_questions":316,"total_runs":316,"cost_avg":0.00024798101265822807,"cost_total":0.07836200000000007,"latency_p50":2500.5,"category_pass_rates":{"calculations":0,"conceptual":0.16666666666666666,"consensus-epoch-processing":0.19230769230769232,"consensus-fork-choice":0.6,"consensus-shuffling":0,"consensus-state-transition":0.16,"constants":0.42105263157894735,"cross-fork":0.5,"eip-interactions":0,"evm-execution":0.24166666666666667,"trick":0.3333333333333333}},{"model_id":"qwen-2.5-72b-instruct","display_name":"qwen-2.5-72b-instruct","lab":"qwen","prompt":"bare","release_date":"2024-09-19","precision":"fp8","base_model":"qwen-2.5-72b-instruct","reasoning_effort":null,"provider":"deepinfra","notes":null,"pass_rate":0.2626582278481013,"ci_lower":0.2171946136113794,"ci_upper":0.31382324022627034,"total_questions":316,"total_runs":316,"cost_avg":0.0003804845569620254,"cost_total":0.12023312000000003,"latency_p50":2286.5,"category_pass_rates":{"calculations":0,"conceptual":0,"consensus-epoch-processing":0.19230769230769232,"consensus-fork-choice":0.5333333333333333,"consensus-shuffling":0,"consensus-state-transition":0.2,"constants":0.49122807017543857,"cross-fork":0,"eip-interactions":0,"evm-execution":0.21666666666666667,"trick":0.16666666666666666}},{"model_id":"gpt-4-turbo","display_name":"gpt-4-turbo","lab":"openai","prompt":"bare","release_date":"2024-04-09","precision":"unknown","base_model":"gpt-4-turbo","reasoning_effort":null,"provider":"openai","notes":null,"pass_rate":0.2626582278481013,"ci_lower":0.2171946136113794,"ci_upper":0.31382324022627034,"total_questions":316,"total_runs":316,"cost_avg":0.015005949367088607,"cost_total":4.74188,"latency_p50":17998.5,"category_pass_rates":{"calculations":0.5,"conceptual":0.3333333333333333,"consensus-epoch-processing":0.15384615384615385,"consensus-fork-choice":0.3333333333333333,"consensus-shuffling":0,"consensus-state-transition":0.12,"constants":0.43859649122807015,"cross-fork":0.25,"eip-interactions":0,"evm-execution":0.275,"trick":0.3333333333333333}},{"model_id":"llama-4-scout","display_name":"llama-4-scout","lab":"meta","prompt":"bare","release_date":"2025-04-05","precision":"fp8","base_model":"llama-4-scout","reasoning_effort":null,"provider":"deepinfra","notes":null,"pass_rate":0.25316455696202533,"ci_lower":0.20838280506712323,"ci_upper":0.3038757629240325,"total_questions":316,"total_runs":316,"cost_avg":0.00021680537974683536,"cost_total":0.06851049999999997,"latency_p50":10931,"category_pass_rates":{"calculations":0,"conceptual":0.16666666666666666,"consensus-epoch-processing":0.25,"consensus-fork-choice":0.5333333333333333,"consensus-shuffling":0,"consensus-state-transition":0.28,"constants":0.2631578947368421,"cross-fork":0.25,"eip-interactions":0,"evm-execution":0.23333333333333334,"trick":0}},{"model_id":"gemma-4-26b-a4b-it-high","display_name":"gemma-4-26b-a4b-it","lab":"google","prompt":"bare","release_date":"2026-04-02","precision":"bf16","base_model":"gemma-4-26b-a4b-it","reasoning_effort":"high","provider":"parasail","notes":null,"pass_rate":0.24367088607594936,"ci_lower":0.199600510151268,"ci_upper":0.29389877199339365,"total_questions":316,"total_runs":316,"cost_avg":0.0036450802531645555,"cost_total":1.1518453599999996,"latency_p50":217952.5,"category_pass_rates":{"calculations":0.5,"conceptual":0.3333333333333333,"consensus-epoch-processing":0.1346153846153846,"consensus-fork-choice":0.6,"consensus-shuffling":0,"consensus-state-transition":0.28,"constants":0.17543859649122806,"cross-fork":0,"eip-interactions":0,"evm-execution":0.25833333333333336,"trick":0.5}},{"model_id":"hermes-4-405b","display_name":"hermes-4-405b","lab":"nousresearch","prompt":"bare","release_date":"2025-08-26","precision":"fp8","base_model":"hermes-4-405b","reasoning_effort":null,"provider":"nebius","notes":null,"pass_rate":0.24367088607594936,"ci_lower":0.199600510151268,"ci_upper":0.29389877199339365,"total_questions":316,"total_runs":316,"cost_avg":0.000858835443037975,"cost_total":0.2713920000000001,"latency_p50":2599,"category_pass_rates":{"calculations":0.5,"conceptual":0.5,"consensus-epoch-processing":0.15384615384615385,"consensus-fork-choice":0.5333333333333333,"consensus-shuffling":0,"consensus-state-transition":0.24,"constants":0.3684210526315789,"cross-fork":0,"eip-interactions":0,"evm-execution":0.19166666666666668,"trick":0.16666666666666666}},{"model_id":"command-a","display_name":"command-a","lab":"cohere","prompt":"bare","release_date":"2025-03-13","precision":"unknown","base_model":"command-a","reasoning_effort":null,"provider":"cohere","notes":null,"pass_rate":0.23734177215189872,"ci_lower":0.19376259990374478,"ci_upper":0.28723049167658754,"total_questions":316,"total_runs":316,"cost_avg":0.004271368670886079,"cost_total":1.3497525000000008,"latency_p50":5349.5,"category_pass_rates":{"calculations":0,"conceptual":0.5,"consensus-epoch-processing":0.1346153846153846,"consensus-fork-choice":0.6666666666666666,"consensus-shuffling":0,"consensus-state-transition":0.16,"constants":0.43859649122807015,"cross-fork":0.5,"eip-interactions":0,"evm-execution":0.15,"trick":0.3333333333333333}},{"model_id":"glm-4.7-flash-high","display_name":"glm-4.7-flash","lab":"z-ai","prompt":"bare","release_date":"2026-01-19","precision":"unknown","base_model":"glm-4.7-flash","reasoning_effort":"high","provider":"z-ai","notes":null,"pass_rate":0.23734177215189872,"ci_lower":0.19376259990374478,"ci_upper":0.28723049167658754,"total_questions":316,"total_runs":316,"cost_avg":0.003164062215189875,"cost_total":0.9998436600000006,"latency_p50":130177,"category_pass_rates":{"calculations":0.5,"conceptual":0,"consensus-epoch-processing":0.25,"consensus-fork-choice":0.6,"consensus-shuffling":0,"consensus-state-transition":0.2,"constants":0.2807017543859649,"cross-fork":0.25,"eip-interactions":1,"evm-execution":0.2,"trick":0}},{"model_id":"claude-3-haiku","display_name":"claude-3-haiku","lab":"anthropic","prompt":"bare","release_date":"2024-03-07","precision":"unknown","base_model":"claude-3-haiku","reasoning_effort":null,"provider":"amazon-bedrock","notes":null,"pass_rate":0.23734177215189872,"ci_lower":0.20582067540011031,"ci_upper":0.2720367029154953,"total_questions":316,"total_runs":632,"cost_avg":0.0003655181962025321,"cost_total":0.23100750000000028,"latency_p50":1985.5,"category_pass_rates":{"calculations":0,"conceptual":0.08333333333333333,"consensus-epoch-processing":0.125,"consensus-fork-choice":0.6666666666666666,"consensus-shuffling":0,"consensus-state-transition":0.28,"constants":0.38596491228070173,"cross-fork":0.25,"eip-interactions":0,"evm-execution":0.16666666666666666,"trick":0.16666666666666666}},{"model_id":"llama-3.3-70b-instruct","display_name":"llama-3.3-70b-instruct","lab":"meta","prompt":"bare","release_date":"2024-12-06","precision":"bf16","base_model":"llama-3.3-70b-instruct","reasoning_effort":null,"provider":"novita","notes":null,"pass_rate":0.2310126582278481,"ci_lower":0.18793867595940303,"ci_upper":0.2805482250566,"total_questions":316,"total_runs":316,"cost_avg":0.0002519334651898734,"cost_total":0.079610975,"latency_p50":10187,"category_pass_rates":{"calculations":0,"conceptual":0,"consensus-epoch-processing":0.11538461538461539,"consensus-fork-choice":0.6,"consensus-shuffling":0,"consensus-state-transition":0.2,"constants":0.3508771929824561,"cross-fork":0,"eip-interactions":0,"evm-execution":0.225,"trick":0.16666666666666666}},{"model_id":"gemini-2.5-flash-lite","display_name":"gemini-2.5-flash-lite","lab":"google","prompt":"bare","release_date":"2025-06-17","precision":"unknown","base_model":"gemini-2.5-flash-lite","reasoning_effort":null,"provider":"google-vertex","notes":null,"pass_rate":0.2120253164556962,"ci_lower":0.1705548950490372,"ci_upper":0.26041343427397773,"total_questions":316,"total_runs":316,"cost_avg":0.00044735569620253147,"cost_total":0.14136439999999995,"latency_p50":1801,"category_pass_rates":{"calculations":0,"conceptual":0.3333333333333333,"consensus-epoch-processing":0.057692307692307696,"consensus-fork-choice":0.6,"consensus-shuffling":0,"consensus-state-transition":0.22,"constants":0.2982456140350877,"cross-fork":0.25,"eip-interactions":0,"evm-execution":0.18333333333333332,"trick":0.3333333333333333}},{"model_id":"gpt-4.1-nano","display_name":"gpt-4.1-nano","lab":"openai","prompt":"bare","release_date":"2025-04-14","precision":"unknown","base_model":"gpt-4.1-nano","reasoning_effort":null,"provider":"openai","notes":null,"pass_rate":0.20253164556962025,"ci_lower":0.16191533523134458,"ci_upper":0.2502937082451763,"total_questions":316,"total_runs":316,"cost_avg":0.00017584651898734185,"cost_total":0.05556750000000003,"latency_p50":2321.5,"category_pass_rates":{"calculations":0,"conceptual":0,"consensus-epoch-processing":0.19230769230769232,"consensus-fork-choice":0.3333333333333333,"consensus-shuffling":0,"consensus-state-transition":0.18,"constants":0.3684210526315789,"cross-fork":0.25,"eip-interactions":0,"evm-execution":0.15,"trick":0}},{"model_id":"llama-3.1-instruct-70b","display_name":"llama-3.1-instruct-70b","lab":"meta","prompt":"bare","release_date":"2024-07-23","precision":"bf16","base_model":"llama-3.1-instruct-70b","reasoning_effort":null,"provider":"deepinfra","notes":null,"pass_rate":0.19936708860759494,"ci_lower":0.1590436614124851,"ci_upper":0.24691228678187116,"total_questions":316,"total_runs":316,"cost_avg":0.0003199645569620255,"cost_total":0.10110880000000005,"latency_p50":15445.5,"category_pass_rates":{"calculations":0,"conceptual":0.16666666666666666,"consensus-epoch-processing":0.09615384615384616,"consensus-fork-choice":0.5333333333333333,"consensus-shuffling":0,"consensus-state-transition":0.2,"constants":0.24561403508771928,"cross-fork":0.25,"eip-interactions":0,"evm-execution":0.2,"trick":0}},{"model_id":"gpt-4o-mini","display_name":"gpt-4o-mini","lab":"openai","prompt":"bare","release_date":"2024-07-18","precision":"unknown","base_model":"gpt-4o-mini","reasoning_effort":null,"provider":"openai","notes":null,"pass_rate":0.18354430379746836,"ci_lower":0.14474996823019048,"ci_upper":0.22994050355334242,"total_questions":316,"total_runs":316,"cost_avg":0.0001934235759493671,"cost_total":0.061121850000000005,"latency_p50":1526.5,"category_pass_rates":{"calculations":0,"conceptual":0,"consensus-epoch-processing":0.21153846153846154,"consensus-fork-choice":0.4666666666666667,"consensus-shuffling":0,"consensus-state-transition":0.24,"constants":0.22807017543859648,"cross-fork":0.25,"eip-interactions":0,"evm-execution":0.10833333333333334,"trick":0.16666666666666666}},{"model_id":"command-r-plus","display_name":"command-r-plus","lab":"cohere","prompt":"bare","release_date":"2024-08-30","precision":"unknown","base_model":"command-r-plus","reasoning_effort":null,"provider":"cohere","notes":null,"pass_rate":0.18037974683544303,"ci_lower":0.14190473784822366,"ci_upper":0.22653263865314452,"total_questions":316,"total_runs":316,"cost_avg":0.002114691455696203,"cost_total":0.6682425000000002,"latency_p50":1851.5,"category_pass_rates":{"calculations":0,"conceptual":0.16666666666666666,"consensus-epoch-processing":0.1346153846153846,"consensus-fork-choice":0.4666666666666667,"consensus-shuffling":0,"consensus-state-transition":0.2,"constants":0.24561403508771928,"cross-fork":0.25,"eip-interactions":0,"evm-execution":0.13333333333333333,"trick":0.16666666666666666}},{"model_id":"gpt-3.5-turbo","display_name":"gpt-3.5-turbo","lab":"openai","prompt":"bare","release_date":"2023-03-01","precision":"unknown","base_model":"gpt-3.5-turbo","reasoning_effort":null,"provider":"openai","notes":null,"pass_rate":0.17405063291139242,"ci_lower":0.13622850497778094,"ci_upper":0.21970268095925796,"total_questions":316,"total_runs":316,"cost_avg":0.0002940411392405057,"cost_total":0.0929169999999998,"latency_p50":887,"category_pass_rates":{"calculations":0,"conceptual":0.16666666666666666,"consensus-epoch-processing":0.057692307692307696,"consensus-fork-choice":0.6666666666666666,"consensus-shuffling":0,"consensus-state-transition":0.26,"constants":0.2631578947368421,"cross-fork":0,"eip-interactions":0,"evm-execution":0.1,"trick":0.16666666666666666}},{"model_id":"command-r","display_name":"command-r","lab":"cohere","prompt":"bare","release_date":"2024-08-30","precision":"unknown","base_model":"command-r","reasoning_effort":null,"provider":"cohere","notes":null,"pass_rate":0.17088607594936708,"ci_lower":0.13339770054072886,"ci_upper":0.21628039011414535,"total_questions":316,"total_runs":316,"cost_avg":0.00012854810126582282,"cost_total":0.04062120000000001,"latency_p50":2468,"category_pass_rates":{"calculations":0,"conceptual":0,"consensus-epoch-processing":0.057692307692307696,"consensus-fork-choice":0.5333333333333333,"consensus-shuffling":0,"consensus-state-transition":0.22,"constants":0.2807017543859649,"cross-fork":0.25,"eip-interactions":0,"evm-execution":0.11666666666666667,"trick":0.16666666666666666}},{"model_id":"hermes-4-70b","display_name":"hermes-4-70b","lab":"nousresearch","prompt":"bare","release_date":"2025-08-26","precision":"fp8","base_model":"hermes-4-70b","reasoning_effort":null,"provider":"nebius","notes":null,"pass_rate":0.16455696202531644,"ci_lower":0.12775124522715484,"ci_upper":0.20942065486338998,"total_questions":316,"total_runs":316,"cost_avg":0.00008327015822784815,"cost_total":0.026313370000000016,"latency_p50":2497.5,"category_pass_rates":{"calculations":0,"conceptual":0.16666666666666666,"consensus-epoch-processing":0.07692307692307693,"consensus-fork-choice":0.6666666666666666,"consensus-shuffling":0,"consensus-state-transition":0.2,"constants":0.21052631578947367,"cross-fork":0,"eip-interactions":0,"evm-execution":0.11666666666666667,"trick":0.16666666666666666}},{"model_id":"phi-4","display_name":"phi-4","lab":"microsoft","prompt":"bare","release_date":"2025-01-10","precision":"bf16","base_model":"phi-4","reasoning_effort":null,"provider":"deepinfra","notes":null,"pass_rate":0.1550632911392405,"ci_lower":0.11932118911957067,"ci_upper":0.19909142512448016,"total_questions":316,"total_runs":316,"cost_avg":0.00010118743670886082,"cost_total":0.03197523000000002,"latency_p50":7646.5,"category_pass_rates":{"calculations":0,"conceptual":0.16666666666666666,"consensus-epoch-processing":0.15384615384615385,"consensus-fork-choice":0.26666666666666666,"consensus-shuffling":0,"consensus-state-transition":0.16,"constants":0.12280701754385964,"cross-fork":0,"eip-interactions":0,"evm-execution":0.175,"trick":0}},{"model_id":"llama-3-instruct-8b","display_name":"llama-3-instruct-8b","lab":"meta","prompt":"bare","release_date":"2024-04-18","precision":"bf16","base_model":"llama-3-instruct-8b","reasoning_effort":null,"provider":"deepinfra","notes":null,"pass_rate":0.10759493670886076,"ci_lower":0.0780227604632617,"ci_upper":0.14659342454831908,"total_questions":316,"total_runs":316,"cost_avg":0.00001968933544303795,"cost_total":0.0062218299999999924,"latency_p50":1438.5,"category_pass_rates":{"calculations":0,"conceptual":0,"consensus-epoch-processing":0.057692307692307696,"consensus-fork-choice":0.4,"consensus-shuffling":0,"consensus-state-transition":0.18,"constants":0.05263157894736842,"cross-fork":0,"eip-interactions":0,"evm-execution":0.1,"trick":0.16666666666666666}},{"model_id":"llama-3.1-instruct-8b","display_name":"llama-3.1-instruct-8b","lab":"meta","prompt":"bare","release_date":"2024-07-23","precision":"bf16","base_model":"llama-3.1-instruct-8b","reasoning_effort":null,"provider":"deepinfra","notes":null,"pass_rate":0.0949367088607595,"ci_lower":0.06731043634030419,"ci_upper":0.1322933675426179,"total_questions":316,"total_runs":316,"cost_avg":0.000020104367088607574,"cost_total":0.006352979999999993,"latency_p50":1468.5,"category_pass_rates":{"calculations":0,"conceptual":0,"consensus-epoch-processing":0.057692307692307696,"consensus-fork-choice":0.3333333333333333,"consensus-shuffling":0,"consensus-state-transition":0.14,"constants":0.08771929824561403,"cross-fork":0,"eip-interactions":0,"evm-execution":0.08333333333333333,"trick":0}},{"model_id":"llama-3.2-instruct-3b","display_name":"llama-3.2-instruct-3b","lab":"meta","prompt":"bare","release_date":"2024-09-25","precision":"unknown","base_model":"llama-3.2-instruct-3b","reasoning_effort":null,"provider":"cloudflare","notes":null,"pass_rate":0.06962025316455696,"ci_lower":0.04642211732326561,"ci_upper":0.10315692430233908,"total_questions":316,"total_runs":316,"cost_avg":0.00013941542721518976,"cost_total":0.04405527499999996,"latency_p50":1254,"category_pass_rates":{"calculations":0,"conceptual":0,"consensus-epoch-processing":0.07692307692307693,"consensus-fork-choice":0.13333333333333333,"consensus-shuffling":0,"consensus-state-transition":0.18,"constants":0.017543859649122806,"cross-fork":0,"eip-interactions":0,"evm-execution":0.05,"trick":0}},{"model_id":"mixtral-8x7b-instruct","display_name":"mixtral-8x7b-instruct","lab":"mistral","prompt":"bare","release_date":"2023-12-10","precision":"fp8","base_model":"mixtral-8x7b-instruct","reasoning_effort":null,"provider":"deepinfra","notes":null,"pass_rate":0.060126582278481014,"ci_lower":0.03882664502120181,"ci_upper":0.09199311075790886,"total_questions":316,"total_runs":316,"cost_avg":0.0005237931645569622,"cost_total":0.16551864000000005,"latency_p50":3147,"category_pass_rates":{"calculations":0,"conceptual":0,"consensus-epoch-processing":0.057692307692307696,"consensus-fork-choice":0,"consensus-shuffling":0,"consensus-state-transition":0.06,"constants":0.12280701754385964,"cross-fork":0,"eip-interactions":0,"evm-execution":0.041666666666666664,"trick":0.16666666666666666}},{"model_id":"llama-3.2-instruct-1b","display_name":"llama-3.2-instruct-1b","lab":"meta","prompt":"bare","release_date":"2024-09-25","precision":"unknown","base_model":"llama-3.2-instruct-1b","reasoning_effort":null,"provider":"cloudflare","notes":null,"pass_rate":0.006329113924050633,"ci_lower":0.0017373543361146123,"ci_upper":0.022779781646196633,"total_questions":316,"total_runs":316,"cost_avg":0.0002661527563291137,"cost_total":0.08410427099999994,"latency_p50":2083,"category_pass_rates":{"calculations":0,"conceptual":0,"consensus-epoch-processing":0,"consensus-fork-choice":0.13333333333333333,"consensus-shuffling":0,"consensus-state-transition":0,"constants":0,"cross-fork":0,"eip-interactions":0,"evm-execution":0,"trick":0}}],"categories":[{"name":"calculations","pass_rate":0.8227,"ci_lower":0.7842946693882629,"ci_upper":0.8555732474541422,"total":440,"passed":362},{"name":"constants","pass_rate":0.7384,"ci_lower":0.7306056955072958,"ci_upper":0.7459733745845586,"total":12567,"passed":9279},{"name":"consensus-fork-choice","pass_rate":0.7069,"ci_lower":0.6911768984999016,"ci_upper":0.722208490144054,"total":3303,"passed":2335},{"name":"cross-fork","pass_rate":0.63,"ci_lower":0.597583830047457,"ci_upper":0.661219553518596,"total":881,"passed":555},{"name":"conceptual","pass_rate":0.6117,"ci_lower":0.585087728352306,"ci_upper":0.6375804004262589,"total":1321,"passed":808},{"name":"consensus-state-transition","pass_rate":0.6014,"ci_lower":0.5921857521822793,"ci_upper":0.6104659259768118,"total":11020,"passed":6627},{"name":"trick","pass_rate":0.5548,"ci_lower":0.5278986591793131,"ci_upper":0.5813834133755421,"total":1323,"passed":734},{"name":"evm-execution","pass_rate":0.5191,"ci_lower":0.5130639812057771,"ci_upper":0.5251111179934856,"total":26427,"passed":13718},{"name":"eip-interactions","pass_rate":0.5182,"ci_lower":0.4524101890969525,"ci_upper":0.5833293695909767,"total":220,"passed":114},{"name":"consensus-epoch-processing","pass_rate":0.4268,"ci_lower":0.4177319758370983,"ci_upper":0.43584366171796185,"total":11456,"passed":4889},{"name":"consensus-shuffling","pass_rate":0,"ci_lower":0,"ci_upper":0.005786922663478757,"total":660,"passed":0}],"total_available_questions":316}