Gahow Wang
9b6091fe6e
Extends exp(c) (dispatch ablation, 1 round-robin policy) to the full 5-policy routing comparison, both modes on the SAME ttp trace (807 reqs, fresh vLLM/arm, dash0 8xH20). Confirms exp(c)'s prediction and finds something stronger: the dispatch mode FLIPS which policy wins. - thinktime helps every policy but helps LPWL most (TTFT p90 -40%, E2E mean -31% vs -3..-16% for the rest): tracets bursts punish prefill-spreading. - Ranking flip: tracets -> LPWL only ties unified_ab on TTFT p90 and is 3rd on E2E mean; thinktime -> LPWL is 1st on both (TTFT p90 -31%, best TPOT/balance, zero knobs) vs the tuned unified+A+B. - => benchmark agentic routing with thinktime; tracets' burst artifact erases LPWL's advantage. Caveat n=1: tracets ranking is run-sensitive (does not reproduce dash1 lpwl_5policy_600s.md), the thinktime advantage is the robust signal (appears in both environments). README + grouped-bar fig (figs/exp_d_policy_dispatch.png) + bench_report summaries in results/. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
803 lines
21 KiB
JSON
803 lines
21 KiB
JSON
{
|
|
"leastwork": {
|
|
"n_total": 807,
|
|
"n_ok": 807,
|
|
"window_s": 1045.237051486969,
|
|
"ttft_ms": {
|
|
"n": 807,
|
|
"mean": 4029.99802739754,
|
|
"p50": 856.9921070011333,
|
|
"p90": 11099.306205986068,
|
|
"p99": 43400.520397000946
|
|
},
|
|
"tpot_ms": {
|
|
"n": 806,
|
|
"mean": 21.48754069144944,
|
|
"p50": 8.545071840088175,
|
|
"p90": 33.14273249998223,
|
|
"p99": 221.47291811146448
|
|
},
|
|
"e2e_ms": {
|
|
"n": 807,
|
|
"mean": 9827.03743343642,
|
|
"p50": 2474.214674992254,
|
|
"p90": 25365.627319028135,
|
|
"p99": 93929.44298699149
|
|
},
|
|
"throughput": {
|
|
"decode_tps": 220.78532297692573,
|
|
"prefill_tps": 8826.999566150593,
|
|
"total_tps": 9047.784889127519,
|
|
"total_output_tokens": 230773,
|
|
"total_new_prefill_tokens": 9226307
|
|
},
|
|
"apc": 0.6495987515101673,
|
|
"per_worker": {
|
|
"0": {
|
|
"n": 95,
|
|
"decode_tps": 26.681028920976477,
|
|
"prefill_tps": 996.0917463831219,
|
|
"ttft_p90_ms": 5435.623251018114,
|
|
"gpu_util_mean": 40.30392156862745,
|
|
"gpu_util_max": 100.0,
|
|
"gpu_mem_max_mb": 89575.0
|
|
},
|
|
"1": {
|
|
"n": 94,
|
|
"decode_tps": 53.13244485640239,
|
|
"prefill_tps": 951.7659162433571,
|
|
"ttft_p90_ms": 10786.369787005242,
|
|
"gpu_util_mean": 61.27450980392157,
|
|
"gpu_util_max": 100.0,
|
|
"gpu_mem_max_mb": 89575.0
|
|
},
|
|
"2": {
|
|
"n": 89,
|
|
"decode_tps": 19.963892372849116,
|
|
"prefill_tps": 1100.5666115294052,
|
|
"ttft_p90_ms": 4944.386984978337,
|
|
"gpu_util_mean": 36.745098039215684,
|
|
"gpu_util_max": 100.0,
|
|
"gpu_mem_max_mb": 89575.0
|
|
},
|
|
"3": {
|
|
"n": 117,
|
|
"decode_tps": 23.37842881214075,
|
|
"prefill_tps": 1155.5378737117585,
|
|
"ttft_p90_ms": 6188.670104980702,
|
|
"gpu_util_mean": 40.76470588235294,
|
|
"gpu_util_max": 100.0,
|
|
"gpu_mem_max_mb": 89575.0
|
|
},
|
|
"4": {
|
|
"n": 108,
|
|
"decode_tps": 34.993018997907186,
|
|
"prefill_tps": 947.086594941991,
|
|
"ttft_p90_ms": 4642.632269999012,
|
|
"gpu_util_mean": 40.19607843137255,
|
|
"gpu_util_max": 100.0,
|
|
"gpu_mem_max_mb": 89575.0
|
|
},
|
|
"5": {
|
|
"n": 87,
|
|
"decode_tps": 10.190989675352892,
|
|
"prefill_tps": 1130.268964651918,
|
|
"ttft_p90_ms": 9265.34449501196,
|
|
"gpu_util_mean": 29.41176470588235,
|
|
"gpu_util_max": 100.0,
|
|
"gpu_mem_max_mb": 89575.0
|
|
},
|
|
"6": {
|
|
"n": 130,
|
|
"decode_tps": 20.925396749842136,
|
|
"prefill_tps": 1646.6073390256745,
|
|
"ttft_p90_ms": 38816.55501498608,
|
|
"gpu_util_mean": 81.27450980392157,
|
|
"gpu_util_max": 100.0,
|
|
"gpu_mem_max_mb": 89575.0
|
|
},
|
|
"7": {
|
|
"n": 87,
|
|
"decode_tps": 31.520122591454786,
|
|
"prefill_tps": 899.0745196633663,
|
|
"ttft_p90_ms": 7304.075189982541,
|
|
"gpu_util_mean": 42.26960784313726,
|
|
"gpu_util_max": 100.0,
|
|
"gpu_mem_max_mb": 89575.0
|
|
}
|
|
},
|
|
"decisions": {},
|
|
"gpu_captured": true,
|
|
"spread": {
|
|
"n_ratio": 1.4942528735632183,
|
|
"ttft_p90_ratio": 8.3608937252733,
|
|
"gpu_util_ratio": 2.7633333333333336,
|
|
"gpu_util_min": 29.41176470588235,
|
|
"gpu_util_max": 81.27450980392157
|
|
},
|
|
"per_class": {
|
|
"WARM<5k": {
|
|
"n": 92,
|
|
"ttft_ms": {
|
|
"n": 92,
|
|
"mean": 2123.6411421857115,
|
|
"p50": 202.10654201218858,
|
|
"p90": 478.5369329911191,
|
|
"p99": 39781.0776779952
|
|
}
|
|
},
|
|
"MED5-20k": {
|
|
"n": 278,
|
|
"ttft_ms": {
|
|
"n": 278,
|
|
"mean": 1228.5320352953054,
|
|
"p50": 757.9952410014812,
|
|
"p90": 1679.417210019892,
|
|
"p99": 15248.28791298205
|
|
}
|
|
},
|
|
"HEAVY20-50k": {
|
|
"n": 248,
|
|
"ttft_ms": {
|
|
"n": 248,
|
|
"mean": 2886.291041083525,
|
|
"p50": 1301.9832599966321,
|
|
"p90": 5246.098520990927,
|
|
"p99": 39812.045788014075
|
|
}
|
|
},
|
|
"HEAVY+>50k": {
|
|
"n": 189,
|
|
"ttft_ms": {
|
|
"n": 189,
|
|
"mean": 10579.37216416889,
|
|
"p50": 2986.336703004781,
|
|
"p90": 34044.874527986394,
|
|
"p99": 51031.28803099389
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"unified_ab": {
|
|
"n_total": 807,
|
|
"n_ok": 807,
|
|
"window_s": 1081.3928244113922,
|
|
"ttft_ms": {
|
|
"n": 807,
|
|
"mean": 4003.8645795703064,
|
|
"p50": 745.6592369999271,
|
|
"p90": 10783.10890001012,
|
|
"p99": 46727.02033401583
|
|
},
|
|
"tpot_ms": {
|
|
"n": 806,
|
|
"mean": 18.129335403553664,
|
|
"p50": 8.004697278213117,
|
|
"p90": 20.508462421730655,
|
|
"p99": 188.8185092436804
|
|
},
|
|
"e2e_ms": {
|
|
"n": 807,
|
|
"mean": 8531.231209817679,
|
|
"p50": 2227.301309001632,
|
|
"p90": 22062.78157699853,
|
|
"p99": 75419.32771002757
|
|
},
|
|
"throughput": {
|
|
"decode_tps": 213.4034874196719,
|
|
"prefill_tps": 8110.4912128244405,
|
|
"total_tps": 8323.894700244113,
|
|
"total_output_tokens": 230773,
|
|
"total_new_prefill_tokens": 8770627
|
|
},
|
|
"apc": 0.66690479182639,
|
|
"per_worker": {
|
|
"0": {
|
|
"n": 119,
|
|
"decode_tps": 36.19868665330458,
|
|
"prefill_tps": 1627.2449384503814,
|
|
"ttft_p90_ms": 28329.616097005783,
|
|
"gpu_util_mean": 85.30805687203791,
|
|
"gpu_util_max": 100.0,
|
|
"gpu_mem_max_mb": 89575.0
|
|
},
|
|
"1": {
|
|
"n": 94,
|
|
"decode_tps": 30.323855734708484,
|
|
"prefill_tps": 977.303507238684,
|
|
"ttft_p90_ms": 5139.202910999302,
|
|
"gpu_util_mean": 41.165876777251185,
|
|
"gpu_util_max": 100.0,
|
|
"gpu_mem_max_mb": 89575.0
|
|
},
|
|
"2": {
|
|
"n": 79,
|
|
"decode_tps": 13.535321919642842,
|
|
"prefill_tps": 1030.6661694437062,
|
|
"ttft_p90_ms": 16363.982771988958,
|
|
"gpu_util_mean": 35.014218009478675,
|
|
"gpu_util_max": 100.0,
|
|
"gpu_mem_max_mb": 89575.0
|
|
},
|
|
"3": {
|
|
"n": 122,
|
|
"decode_tps": 25.46530675842895,
|
|
"prefill_tps": 900.9917376974744,
|
|
"ttft_p90_ms": 5133.929038012866,
|
|
"gpu_util_mean": 35.85781990521327,
|
|
"gpu_util_max": 100.0,
|
|
"gpu_mem_max_mb": 89575.0
|
|
},
|
|
"4": {
|
|
"n": 101,
|
|
"decode_tps": 39.54437188287811,
|
|
"prefill_tps": 731.5889121334044,
|
|
"ttft_p90_ms": 8783.158237987664,
|
|
"gpu_util_mean": 45.843601895734594,
|
|
"gpu_util_max": 100.0,
|
|
"gpu_mem_max_mb": 89575.0
|
|
},
|
|
"5": {
|
|
"n": 109,
|
|
"decode_tps": 32.10951581715914,
|
|
"prefill_tps": 842.2332564447569,
|
|
"ttft_p90_ms": 4199.806818010984,
|
|
"gpu_util_mean": 37.32701421800948,
|
|
"gpu_util_max": 100.0,
|
|
"gpu_mem_max_mb": 89575.0
|
|
},
|
|
"6": {
|
|
"n": 89,
|
|
"decode_tps": 8.69804181021798,
|
|
"prefill_tps": 1146.4917946675243,
|
|
"ttft_p90_ms": 11112.522551004076,
|
|
"gpu_util_mean": 30.95734597156398,
|
|
"gpu_util_max": 100.0,
|
|
"gpu_mem_max_mb": 89575.0
|
|
},
|
|
"7": {
|
|
"n": 94,
|
|
"decode_tps": 27.528386843331813,
|
|
"prefill_tps": 853.9708967485094,
|
|
"ttft_p90_ms": 10584.918729990022,
|
|
"gpu_util_mean": 47.426540284360186,
|
|
"gpu_util_max": 100.0,
|
|
"gpu_mem_max_mb": 89575.0
|
|
}
|
|
},
|
|
"decisions": {
|
|
"lmetric_fallback": 394,
|
|
"affinity": 413
|
|
},
|
|
"gpu_captured": true,
|
|
"spread": {
|
|
"n_ratio": 1.5443037974683544,
|
|
"ttft_p90_ratio": 6.745456951856325,
|
|
"gpu_util_ratio": 2.7556644213104713,
|
|
"gpu_util_min": 30.95734597156398,
|
|
"gpu_util_max": 85.30805687203791
|
|
},
|
|
"per_class": {
|
|
"WARM<5k": {
|
|
"n": 92,
|
|
"ttft_ms": {
|
|
"n": 92,
|
|
"mean": 473.4928183806124,
|
|
"p50": 193.41183800133877,
|
|
"p90": 326.81434298865497,
|
|
"p99": 5278.063865000149
|
|
}
|
|
},
|
|
"MED5-20k": {
|
|
"n": 278,
|
|
"ttft_ms": {
|
|
"n": 278,
|
|
"mean": 1676.7254021373014,
|
|
"p50": 733.51754702162,
|
|
"p90": 1942.2162729897536,
|
|
"p99": 28329.616097005783
|
|
}
|
|
},
|
|
"HEAVY20-50k": {
|
|
"n": 248,
|
|
"ttft_ms": {
|
|
"n": 248,
|
|
"mean": 2520.2562936371373,
|
|
"p50": 1149.4361000077333,
|
|
"p90": 5139.202910999302,
|
|
"p99": 26739.575799991144
|
|
}
|
|
},
|
|
"HEAVY+>50k": {
|
|
"n": 189,
|
|
"ttft_ms": {
|
|
"n": 189,
|
|
"mean": 11092.085469873233,
|
|
"p50": 2945.923403982306,
|
|
"p90": 38718.26310700271,
|
|
"p99": 51830.85186799872
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"unified_def": {
|
|
"n_total": 807,
|
|
"n_ok": 807,
|
|
"window_s": 912.5732414722443,
|
|
"ttft_ms": {
|
|
"n": 807,
|
|
"mean": 4275.594404255811,
|
|
"p50": 757.1689730102662,
|
|
"p90": 12997.265826008515,
|
|
"p99": 47988.61391301034
|
|
},
|
|
"tpot_ms": {
|
|
"n": 806,
|
|
"mean": 18.24678916181055,
|
|
"p50": 8.282395397119393,
|
|
"p90": 19.536432251223843,
|
|
"p99": 127.01842809143604
|
|
},
|
|
"e2e_ms": {
|
|
"n": 807,
|
|
"mean": 8365.99611452138,
|
|
"p50": 2119.7480160044506,
|
|
"p90": 22818.839199026115,
|
|
"p99": 82257.18197401147
|
|
},
|
|
"throughput": {
|
|
"decode_tps": 252.8816203592563,
|
|
"prefill_tps": 8854.578057717206,
|
|
"total_tps": 9107.459678076462,
|
|
"total_output_tokens": 230773,
|
|
"total_new_prefill_tokens": 8080451
|
|
},
|
|
"apc": 0.6931166371592755,
|
|
"per_worker": {
|
|
"0": {
|
|
"n": 86,
|
|
"decode_tps": 48.0925782233062,
|
|
"prefill_tps": 878.7174152798008,
|
|
"ttft_p90_ms": 9217.135582002811,
|
|
"gpu_util_mean": 49.02808988764045,
|
|
"gpu_util_max": 100.0,
|
|
"gpu_mem_max_mb": 89575.0
|
|
},
|
|
"1": {
|
|
"n": 114,
|
|
"decode_tps": 36.67650822853761,
|
|
"prefill_tps": 962.1616765613936,
|
|
"ttft_p90_ms": 5333.489734999603,
|
|
"gpu_util_mean": 43.674157303370784,
|
|
"gpu_util_max": 100.0,
|
|
"gpu_mem_max_mb": 89575.0
|
|
},
|
|
"2": {
|
|
"n": 75,
|
|
"decode_tps": 21.302399759867672,
|
|
"prefill_tps": 1132.1228292133994,
|
|
"ttft_p90_ms": 16419.932407996384,
|
|
"gpu_util_mean": 44.235955056179776,
|
|
"gpu_util_max": 100.0,
|
|
"gpu_mem_max_mb": 89575.0
|
|
},
|
|
"3": {
|
|
"n": 94,
|
|
"decode_tps": 39.351362025545676,
|
|
"prefill_tps": 918.7383126064411,
|
|
"ttft_p90_ms": 14089.503638009774,
|
|
"gpu_util_mean": 57.30898876404494,
|
|
"gpu_util_max": 100.0,
|
|
"gpu_mem_max_mb": 89575.0
|
|
},
|
|
"4": {
|
|
"n": 103,
|
|
"decode_tps": 23.649608622297535,
|
|
"prefill_tps": 1200.1765449894706,
|
|
"ttft_p90_ms": 12823.167912021745,
|
|
"gpu_util_mean": 43.82022471910113,
|
|
"gpu_util_max": 100.0,
|
|
"gpu_mem_max_mb": 89575.0
|
|
},
|
|
"5": {
|
|
"n": 104,
|
|
"decode_tps": 25.253863419028313,
|
|
"prefill_tps": 1067.8562067279715,
|
|
"ttft_p90_ms": 18659.113589994377,
|
|
"gpu_util_mean": 52.449438202247194,
|
|
"gpu_util_max": 100.0,
|
|
"gpu_mem_max_mb": 89575.0
|
|
},
|
|
"6": {
|
|
"n": 114,
|
|
"decode_tps": 27.64600018218629,
|
|
"prefill_tps": 1278.7061322523903,
|
|
"ttft_p90_ms": 7502.1790039900225,
|
|
"gpu_util_mean": 42.62359550561798,
|
|
"gpu_util_max": 100.0,
|
|
"gpu_mem_max_mb": 89575.0
|
|
},
|
|
"7": {
|
|
"n": 117,
|
|
"decode_tps": 30.90929989848701,
|
|
"prefill_tps": 1416.0989400863393,
|
|
"ttft_p90_ms": 15775.390956026968,
|
|
"gpu_util_mean": 79.19101123595506,
|
|
"gpu_util_max": 100.0,
|
|
"gpu_mem_max_mb": 89575.0
|
|
}
|
|
},
|
|
"decisions": {
|
|
"lmetric_fallback": 353,
|
|
"affinity": 454
|
|
},
|
|
"gpu_captured": true,
|
|
"spread": {
|
|
"n_ratio": 1.56,
|
|
"ttft_p90_ratio": 3.4984812040696216,
|
|
"gpu_util_ratio": 1.8579148543561357,
|
|
"gpu_util_min": 42.62359550561798,
|
|
"gpu_util_max": 79.19101123595506
|
|
},
|
|
"per_class": {
|
|
"WARM<5k": {
|
|
"n": 92,
|
|
"ttft_ms": {
|
|
"n": 92,
|
|
"mean": 187.7879224041902,
|
|
"p50": 171.855135995429,
|
|
"p90": 318.49737899028696,
|
|
"p99": 406.7676870035939
|
|
}
|
|
},
|
|
"MED5-20k": {
|
|
"n": 278,
|
|
"ttft_ms": {
|
|
"n": 278,
|
|
"mean": 2014.1915133893583,
|
|
"p50": 737.6636109838728,
|
|
"p90": 2057.0135610178113,
|
|
"p99": 29051.10613000579
|
|
}
|
|
},
|
|
"HEAVY20-50k": {
|
|
"n": 248,
|
|
"ttft_ms": {
|
|
"n": 248,
|
|
"mean": 3298.1720407103326,
|
|
"p50": 1321.936836000532,
|
|
"p90": 7205.449934001081,
|
|
"p99": 36468.73455500463
|
|
}
|
|
},
|
|
"HEAVY+>50k": {
|
|
"n": 189,
|
|
"ttft_ms": {
|
|
"n": 189,
|
|
"mean": 10874.266077009788,
|
|
"p50": 2567.718550999416,
|
|
"p90": 35562.78318798286,
|
|
"p99": 61205.50292698317
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"lmetric": {
|
|
"n_total": 807,
|
|
"n_ok": 807,
|
|
"window_s": 1077.4112372398376,
|
|
"ttft_ms": {
|
|
"n": 807,
|
|
"mean": 5152.329462028203,
|
|
"p50": 1274.1917690145783,
|
|
"p90": 16492.156354011968,
|
|
"p99": 46248.138958995696
|
|
},
|
|
"tpot_ms": {
|
|
"n": 806,
|
|
"mean": 24.0370380963572,
|
|
"p50": 10.534365684851883,
|
|
"p90": 38.95731354601127,
|
|
"p99": 231.34709527787183
|
|
},
|
|
"e2e_ms": {
|
|
"n": 807,
|
|
"mean": 10774.921962922257,
|
|
"p50": 3460.944951977581,
|
|
"p90": 27791.26176200225,
|
|
"p99": 99231.47636200883
|
|
},
|
|
"throughput": {
|
|
"decode_tps": 214.19212276939402,
|
|
"prefill_tps": 12337.071064946676,
|
|
"total_tps": 12551.263187716071,
|
|
"total_output_tokens": 230773,
|
|
"total_new_prefill_tokens": 13292099
|
|
},
|
|
"apc": 0.49518609291339905,
|
|
"per_worker": {
|
|
"0": {
|
|
"n": 140,
|
|
"decode_tps": 40.62422823074453,
|
|
"prefill_tps": 2047.015033600442,
|
|
"ttft_p90_ms": 13910.764692991506,
|
|
"gpu_util_mean": 92.98571428571428,
|
|
"gpu_util_max": 100.0,
|
|
"gpu_mem_max_mb": 89575.0
|
|
},
|
|
"1": {
|
|
"n": 89,
|
|
"decode_tps": 23.129515582036454,
|
|
"prefill_tps": 1463.9739641483652,
|
|
"ttft_p90_ms": 17954.478895000648,
|
|
"gpu_util_mean": 60.319047619047616,
|
|
"gpu_util_max": 100.0,
|
|
"gpu_mem_max_mb": 89575.0
|
|
},
|
|
"2": {
|
|
"n": 92,
|
|
"decode_tps": 18.14256184117442,
|
|
"prefill_tps": 1663.4864553589516,
|
|
"ttft_p90_ms": 15653.674011002295,
|
|
"gpu_util_mean": 55.94761904761905,
|
|
"gpu_util_max": 100.0,
|
|
"gpu_mem_max_mb": 89575.0
|
|
},
|
|
"3": {
|
|
"n": 92,
|
|
"decode_tps": 43.91637878329201,
|
|
"prefill_tps": 1313.127198881315,
|
|
"ttft_p90_ms": 13397.495551005704,
|
|
"gpu_util_mean": 54.385714285714286,
|
|
"gpu_util_max": 100.0,
|
|
"gpu_mem_max_mb": 89575.0
|
|
},
|
|
"4": {
|
|
"n": 134,
|
|
"decode_tps": 21.903428500019192,
|
|
"prefill_tps": 1412.6889969137978,
|
|
"ttft_p90_ms": 21995.840935007436,
|
|
"gpu_util_mean": 51.51428571428571,
|
|
"gpu_util_max": 100.0,
|
|
"gpu_mem_max_mb": 89575.0
|
|
},
|
|
"5": {
|
|
"n": 88,
|
|
"decode_tps": 16.934109622562403,
|
|
"prefill_tps": 1474.5493132872787,
|
|
"ttft_p90_ms": 11013.645827013534,
|
|
"gpu_util_mean": 42.55238095238095,
|
|
"gpu_util_max": 100.0,
|
|
"gpu_mem_max_mb": 89575.0
|
|
},
|
|
"6": {
|
|
"n": 108,
|
|
"decode_tps": 30.606697665860118,
|
|
"prefill_tps": 1651.413070981298,
|
|
"ttft_p90_ms": 18613.971301994752,
|
|
"gpu_util_mean": 52.67619047619048,
|
|
"gpu_util_max": 100.0,
|
|
"gpu_mem_max_mb": 89575.0
|
|
},
|
|
"7": {
|
|
"n": 64,
|
|
"decode_tps": 18.935202543704882,
|
|
"prefill_tps": 1310.817031775228,
|
|
"ttft_p90_ms": 15090.364522009622,
|
|
"gpu_util_mean": 41.904761904761905,
|
|
"gpu_util_max": 100.0,
|
|
"gpu_mem_max_mb": 89575.0
|
|
}
|
|
},
|
|
"decisions": {},
|
|
"gpu_captured": true,
|
|
"spread": {
|
|
"n_ratio": 2.1875,
|
|
"ttft_p90_ratio": 1.997144386199301,
|
|
"gpu_util_ratio": 2.2189772727272725,
|
|
"gpu_util_min": 41.904761904761905,
|
|
"gpu_util_max": 92.98571428571428
|
|
},
|
|
"per_class": {
|
|
"WARM<5k": {
|
|
"n": 92,
|
|
"ttft_ms": {
|
|
"n": 92,
|
|
"mean": 1789.4509492392028,
|
|
"p50": 262.9711049958132,
|
|
"p90": 1572.630943992408,
|
|
"p99": 24139.729285001522
|
|
}
|
|
},
|
|
"MED5-20k": {
|
|
"n": 278,
|
|
"ttft_ms": {
|
|
"n": 278,
|
|
"mean": 1697.68317032391,
|
|
"p50": 920.4712719947565,
|
|
"p90": 2029.556789988419,
|
|
"p99": 22497.115491016302
|
|
}
|
|
},
|
|
"HEAVY20-50k": {
|
|
"n": 248,
|
|
"ttft_ms": {
|
|
"n": 248,
|
|
"mean": 3424.9239484553045,
|
|
"p50": 2699.253859987948,
|
|
"p90": 6799.459913017927,
|
|
"p99": 24401.87052200781
|
|
}
|
|
},
|
|
"HEAVY+>50k": {
|
|
"n": 189,
|
|
"ttft_ms": {
|
|
"n": 189,
|
|
"mean": 14137.37210560736,
|
|
"p50": 11013.645827013534,
|
|
"p90": 35319.35577199329,
|
|
"p99": 51099.66781400726
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"sticky": {
|
|
"n_total": 807,
|
|
"n_ok": 807,
|
|
"window_s": 925.9680500030518,
|
|
"ttft_ms": {
|
|
"n": 807,
|
|
"mean": 4914.133386887214,
|
|
"p50": 906.6202020039782,
|
|
"p90": 15236.451414995827,
|
|
"p99": 46771.370519010816
|
|
},
|
|
"tpot_ms": {
|
|
"n": 806,
|
|
"mean": 22.62391467634138,
|
|
"p50": 9.728358783056292,
|
|
"p90": 30.957536839455965,
|
|
"p99": 231.30005976865783
|
|
},
|
|
"e2e_ms": {
|
|
"n": 807,
|
|
"mean": 10139.474540275223,
|
|
"p50": 2597.957960999338,
|
|
"p90": 27973.595037998166,
|
|
"p99": 82362.0547579776
|
|
},
|
|
"throughput": {
|
|
"decode_tps": 249.22350182518656,
|
|
"prefill_tps": 8743.631057219865,
|
|
"total_tps": 8992.854559045052,
|
|
"total_output_tokens": 230773,
|
|
"total_new_prefill_tokens": 8096323
|
|
},
|
|
"apc": 0.6925138424965755,
|
|
"per_worker": {
|
|
"0": {
|
|
"n": 136,
|
|
"decode_tps": 55.41875877880546,
|
|
"prefill_tps": 1434.2957081463262,
|
|
"ttft_p90_ms": 11807.82909199479,
|
|
"gpu_util_mean": 65.71270718232044,
|
|
"gpu_util_max": 100.0,
|
|
"gpu_mem_max_mb": 89575.0
|
|
},
|
|
"1": {
|
|
"n": 129,
|
|
"decode_tps": 38.47432964872015,
|
|
"prefill_tps": 1709.658340797809,
|
|
"ttft_p90_ms": 23309.77585897199,
|
|
"gpu_util_mean": 86.77900552486187,
|
|
"gpu_util_max": 100.0,
|
|
"gpu_mem_max_mb": 89575.0
|
|
},
|
|
"2": {
|
|
"n": 108,
|
|
"decode_tps": 47.74786775834683,
|
|
"prefill_tps": 1144.2727424520835,
|
|
"ttft_p90_ms": 30574.395572999492,
|
|
"gpu_util_mean": 63.50828729281768,
|
|
"gpu_util_max": 100.0,
|
|
"gpu_mem_max_mb": 89575.0
|
|
},
|
|
"3": {
|
|
"n": 78,
|
|
"decode_tps": 25.291369394357414,
|
|
"prefill_tps": 1245.5040970325047,
|
|
"ttft_p90_ms": 26070.17321899184,
|
|
"gpu_util_mean": 51.87845303867403,
|
|
"gpu_util_max": 100.0,
|
|
"gpu_mem_max_mb": 89575.0
|
|
},
|
|
"4": {
|
|
"n": 132,
|
|
"decode_tps": 40.39124244068328,
|
|
"prefill_tps": 941.3586138281199,
|
|
"ttft_p90_ms": 5989.128820016049,
|
|
"gpu_util_mean": 43.0828729281768,
|
|
"gpu_util_max": 100.0,
|
|
"gpu_mem_max_mb": 89575.0
|
|
},
|
|
"5": {
|
|
"n": 84,
|
|
"decode_tps": 22.365782491017637,
|
|
"prefill_tps": 1030.0301398054248,
|
|
"ttft_p90_ms": 14970.723142003408,
|
|
"gpu_util_mean": 41.43646408839779,
|
|
"gpu_util_max": 100.0,
|
|
"gpu_mem_max_mb": 89575.0
|
|
},
|
|
"6": {
|
|
"n": 66,
|
|
"decode_tps": 9.593203566765315,
|
|
"prefill_tps": 669.7250515262986,
|
|
"ttft_p90_ms": 13777.997018012684,
|
|
"gpu_util_mean": 20.994475138121548,
|
|
"gpu_util_max": 100.0,
|
|
"gpu_mem_max_mb": 89575.0
|
|
},
|
|
"7": {
|
|
"n": 74,
|
|
"decode_tps": 9.940947746490457,
|
|
"prefill_tps": 568.7863636312983,
|
|
"ttft_p90_ms": 7027.451686997665,
|
|
"gpu_util_mean": 18.50828729281768,
|
|
"gpu_util_max": 100.0,
|
|
"gpu_mem_max_mb": 89575.0
|
|
}
|
|
},
|
|
"decisions": {},
|
|
"gpu_captured": true,
|
|
"spread": {
|
|
"n_ratio": 2.0606060606060606,
|
|
"ttft_p90_ratio": 5.104982125416625,
|
|
"gpu_util_ratio": 4.68865671641791,
|
|
"gpu_util_min": 18.50828729281768,
|
|
"gpu_util_max": 86.77900552486187
|
|
},
|
|
"per_class": {
|
|
"WARM<5k": {
|
|
"n": 92,
|
|
"ttft_ms": {
|
|
"n": 92,
|
|
"mean": 892.6154872479738,
|
|
"p50": 207.1402580186259,
|
|
"p90": 375.00955499126576,
|
|
"p99": 11232.500832004007
|
|
}
|
|
},
|
|
"MED5-20k": {
|
|
"n": 278,
|
|
"ttft_ms": {
|
|
"n": 278,
|
|
"mean": 2908.9762750470118,
|
|
"p50": 770.736623002449,
|
|
"p90": 4912.921145994915,
|
|
"p99": 42022.69450199674
|
|
}
|
|
},
|
|
"HEAVY20-50k": {
|
|
"n": 248,
|
|
"ttft_ms": {
|
|
"n": 248,
|
|
"mean": 4250.573046338691,
|
|
"p50": 1623.4680919733364,
|
|
"p90": 11137.098645005608,
|
|
"p99": 39817.45037299697
|
|
}
|
|
},
|
|
"HEAVY+>50k": {
|
|
"n": 189,
|
|
"ttft_ms": {
|
|
"n": 189,
|
|
"mean": 10691.78570601113,
|
|
"p50": 2671.919913002057,
|
|
"p90": 36922.92091701529,
|
|
"p99": 53025.03776800586
|
|
}
|
|
}
|
|
}
|
|
}
|
|
} |