{ "leastwork": { "n_total": 807, "n_ok": 807, "window_s": 1045.237051486969, "ttft_ms": { "n": 807, "mean": 4029.99802739754, "p50": 856.9921070011333, "p90": 11099.306205986068, "p99": 43400.520397000946 }, "tpot_ms": { "n": 806, "mean": 21.48754069144944, "p50": 8.545071840088175, "p90": 33.14273249998223, "p99": 221.47291811146448 }, "e2e_ms": { "n": 807, "mean": 9827.03743343642, "p50": 2474.214674992254, "p90": 25365.627319028135, "p99": 93929.44298699149 }, "throughput": { "decode_tps": 220.78532297692573, "prefill_tps": 8826.999566150593, "total_tps": 9047.784889127519, "total_output_tokens": 230773, "total_new_prefill_tokens": 9226307 }, "apc": 0.6495987515101673, "per_worker": { "0": { "n": 95, "decode_tps": 26.681028920976477, "prefill_tps": 996.0917463831219, "ttft_p90_ms": 5435.623251018114, "gpu_util_mean": 40.30392156862745, "gpu_util_max": 100.0, "gpu_mem_max_mb": 89575.0 }, "1": { "n": 94, "decode_tps": 53.13244485640239, "prefill_tps": 951.7659162433571, "ttft_p90_ms": 10786.369787005242, "gpu_util_mean": 61.27450980392157, "gpu_util_max": 100.0, "gpu_mem_max_mb": 89575.0 }, "2": { "n": 89, "decode_tps": 19.963892372849116, "prefill_tps": 1100.5666115294052, "ttft_p90_ms": 4944.386984978337, "gpu_util_mean": 36.745098039215684, "gpu_util_max": 100.0, "gpu_mem_max_mb": 89575.0 }, "3": { "n": 117, "decode_tps": 23.37842881214075, "prefill_tps": 1155.5378737117585, "ttft_p90_ms": 6188.670104980702, "gpu_util_mean": 40.76470588235294, "gpu_util_max": 100.0, "gpu_mem_max_mb": 89575.0 }, "4": { "n": 108, "decode_tps": 34.993018997907186, "prefill_tps": 947.086594941991, "ttft_p90_ms": 4642.632269999012, "gpu_util_mean": 40.19607843137255, "gpu_util_max": 100.0, "gpu_mem_max_mb": 89575.0 }, "5": { "n": 87, "decode_tps": 10.190989675352892, "prefill_tps": 1130.268964651918, "ttft_p90_ms": 9265.34449501196, "gpu_util_mean": 29.41176470588235, "gpu_util_max": 100.0, "gpu_mem_max_mb": 89575.0 }, "6": { "n": 130, "decode_tps": 20.925396749842136, "prefill_tps": 1646.6073390256745, "ttft_p90_ms": 38816.55501498608, "gpu_util_mean": 81.27450980392157, "gpu_util_max": 100.0, "gpu_mem_max_mb": 89575.0 }, "7": { "n": 87, "decode_tps": 31.520122591454786, "prefill_tps": 899.0745196633663, "ttft_p90_ms": 7304.075189982541, "gpu_util_mean": 42.26960784313726, "gpu_util_max": 100.0, "gpu_mem_max_mb": 89575.0 } }, "decisions": {}, "gpu_captured": true, "spread": { "n_ratio": 1.4942528735632183, "ttft_p90_ratio": 8.3608937252733, "gpu_util_ratio": 2.7633333333333336, "gpu_util_min": 29.41176470588235, "gpu_util_max": 81.27450980392157 }, "per_class": { "WARM<5k": { "n": 92, "ttft_ms": { "n": 92, "mean": 2123.6411421857115, "p50": 202.10654201218858, "p90": 478.5369329911191, "p99": 39781.0776779952 } }, "MED5-20k": { "n": 278, "ttft_ms": { "n": 278, "mean": 1228.5320352953054, "p50": 757.9952410014812, "p90": 1679.417210019892, "p99": 15248.28791298205 } }, "HEAVY20-50k": { "n": 248, "ttft_ms": { "n": 248, "mean": 2886.291041083525, "p50": 1301.9832599966321, "p90": 5246.098520990927, "p99": 39812.045788014075 } }, "HEAVY+>50k": { "n": 189, "ttft_ms": { "n": 189, "mean": 10579.37216416889, "p50": 2986.336703004781, "p90": 34044.874527986394, "p99": 51031.28803099389 } } } }, "unified_ab": { "n_total": 807, "n_ok": 807, "window_s": 1081.3928244113922, "ttft_ms": { "n": 807, "mean": 4003.8645795703064, "p50": 745.6592369999271, "p90": 10783.10890001012, "p99": 46727.02033401583 }, "tpot_ms": { "n": 806, "mean": 18.129335403553664, "p50": 8.004697278213117, "p90": 20.508462421730655, "p99": 188.8185092436804 }, "e2e_ms": { "n": 807, "mean": 8531.231209817679, "p50": 2227.301309001632, "p90": 22062.78157699853, "p99": 75419.32771002757 }, "throughput": { "decode_tps": 213.4034874196719, "prefill_tps": 8110.4912128244405, "total_tps": 8323.894700244113, "total_output_tokens": 230773, "total_new_prefill_tokens": 8770627 }, "apc": 0.66690479182639, "per_worker": { "0": { "n": 119, "decode_tps": 36.19868665330458, "prefill_tps": 1627.2449384503814, "ttft_p90_ms": 28329.616097005783, "gpu_util_mean": 85.30805687203791, "gpu_util_max": 100.0, "gpu_mem_max_mb": 89575.0 }, "1": { "n": 94, "decode_tps": 30.323855734708484, "prefill_tps": 977.303507238684, "ttft_p90_ms": 5139.202910999302, "gpu_util_mean": 41.165876777251185, "gpu_util_max": 100.0, "gpu_mem_max_mb": 89575.0 }, "2": { "n": 79, "decode_tps": 13.535321919642842, "prefill_tps": 1030.6661694437062, "ttft_p90_ms": 16363.982771988958, "gpu_util_mean": 35.014218009478675, "gpu_util_max": 100.0, "gpu_mem_max_mb": 89575.0 }, "3": { "n": 122, "decode_tps": 25.46530675842895, "prefill_tps": 900.9917376974744, "ttft_p90_ms": 5133.929038012866, "gpu_util_mean": 35.85781990521327, "gpu_util_max": 100.0, "gpu_mem_max_mb": 89575.0 }, "4": { "n": 101, "decode_tps": 39.54437188287811, "prefill_tps": 731.5889121334044, "ttft_p90_ms": 8783.158237987664, "gpu_util_mean": 45.843601895734594, "gpu_util_max": 100.0, "gpu_mem_max_mb": 89575.0 }, "5": { "n": 109, "decode_tps": 32.10951581715914, "prefill_tps": 842.2332564447569, "ttft_p90_ms": 4199.806818010984, "gpu_util_mean": 37.32701421800948, "gpu_util_max": 100.0, "gpu_mem_max_mb": 89575.0 }, "6": { "n": 89, "decode_tps": 8.69804181021798, "prefill_tps": 1146.4917946675243, "ttft_p90_ms": 11112.522551004076, "gpu_util_mean": 30.95734597156398, "gpu_util_max": 100.0, "gpu_mem_max_mb": 89575.0 }, "7": { "n": 94, "decode_tps": 27.528386843331813, "prefill_tps": 853.9708967485094, "ttft_p90_ms": 10584.918729990022, "gpu_util_mean": 47.426540284360186, "gpu_util_max": 100.0, "gpu_mem_max_mb": 89575.0 } }, "decisions": { "lmetric_fallback": 394, "affinity": 413 }, "gpu_captured": true, "spread": { "n_ratio": 1.5443037974683544, "ttft_p90_ratio": 6.745456951856325, "gpu_util_ratio": 2.7556644213104713, "gpu_util_min": 30.95734597156398, "gpu_util_max": 85.30805687203791 }, "per_class": { "WARM<5k": { "n": 92, "ttft_ms": { "n": 92, "mean": 473.4928183806124, "p50": 193.41183800133877, "p90": 326.81434298865497, "p99": 5278.063865000149 } }, "MED5-20k": { "n": 278, "ttft_ms": { "n": 278, "mean": 1676.7254021373014, "p50": 733.51754702162, "p90": 1942.2162729897536, "p99": 28329.616097005783 } }, "HEAVY20-50k": { "n": 248, "ttft_ms": { "n": 248, "mean": 2520.2562936371373, "p50": 1149.4361000077333, "p90": 5139.202910999302, "p99": 26739.575799991144 } }, "HEAVY+>50k": { "n": 189, "ttft_ms": { "n": 189, "mean": 11092.085469873233, "p50": 2945.923403982306, "p90": 38718.26310700271, "p99": 51830.85186799872 } } } }, "unified_def": { "n_total": 807, "n_ok": 807, "window_s": 912.5732414722443, "ttft_ms": { "n": 807, "mean": 4275.594404255811, "p50": 757.1689730102662, "p90": 12997.265826008515, "p99": 47988.61391301034 }, "tpot_ms": { "n": 806, "mean": 18.24678916181055, "p50": 8.282395397119393, "p90": 19.536432251223843, "p99": 127.01842809143604 }, "e2e_ms": { "n": 807, "mean": 8365.99611452138, "p50": 2119.7480160044506, "p90": 22818.839199026115, "p99": 82257.18197401147 }, "throughput": { "decode_tps": 252.8816203592563, "prefill_tps": 8854.578057717206, "total_tps": 9107.459678076462, "total_output_tokens": 230773, "total_new_prefill_tokens": 8080451 }, "apc": 0.6931166371592755, "per_worker": { "0": { "n": 86, "decode_tps": 48.0925782233062, "prefill_tps": 878.7174152798008, "ttft_p90_ms": 9217.135582002811, "gpu_util_mean": 49.02808988764045, "gpu_util_max": 100.0, "gpu_mem_max_mb": 89575.0 }, "1": { "n": 114, "decode_tps": 36.67650822853761, "prefill_tps": 962.1616765613936, "ttft_p90_ms": 5333.489734999603, "gpu_util_mean": 43.674157303370784, "gpu_util_max": 100.0, "gpu_mem_max_mb": 89575.0 }, "2": { "n": 75, "decode_tps": 21.302399759867672, "prefill_tps": 1132.1228292133994, "ttft_p90_ms": 16419.932407996384, "gpu_util_mean": 44.235955056179776, "gpu_util_max": 100.0, "gpu_mem_max_mb": 89575.0 }, "3": { "n": 94, "decode_tps": 39.351362025545676, "prefill_tps": 918.7383126064411, "ttft_p90_ms": 14089.503638009774, "gpu_util_mean": 57.30898876404494, "gpu_util_max": 100.0, "gpu_mem_max_mb": 89575.0 }, "4": { "n": 103, "decode_tps": 23.649608622297535, "prefill_tps": 1200.1765449894706, "ttft_p90_ms": 12823.167912021745, "gpu_util_mean": 43.82022471910113, "gpu_util_max": 100.0, "gpu_mem_max_mb": 89575.0 }, "5": { "n": 104, "decode_tps": 25.253863419028313, "prefill_tps": 1067.8562067279715, "ttft_p90_ms": 18659.113589994377, "gpu_util_mean": 52.449438202247194, "gpu_util_max": 100.0, "gpu_mem_max_mb": 89575.0 }, "6": { "n": 114, "decode_tps": 27.64600018218629, "prefill_tps": 1278.7061322523903, "ttft_p90_ms": 7502.1790039900225, "gpu_util_mean": 42.62359550561798, "gpu_util_max": 100.0, "gpu_mem_max_mb": 89575.0 }, "7": { "n": 117, "decode_tps": 30.90929989848701, "prefill_tps": 1416.0989400863393, "ttft_p90_ms": 15775.390956026968, "gpu_util_mean": 79.19101123595506, "gpu_util_max": 100.0, "gpu_mem_max_mb": 89575.0 } }, "decisions": { "lmetric_fallback": 353, "affinity": 454 }, "gpu_captured": true, "spread": { "n_ratio": 1.56, "ttft_p90_ratio": 3.4984812040696216, "gpu_util_ratio": 1.8579148543561357, "gpu_util_min": 42.62359550561798, "gpu_util_max": 79.19101123595506 }, "per_class": { "WARM<5k": { "n": 92, "ttft_ms": { "n": 92, "mean": 187.7879224041902, "p50": 171.855135995429, "p90": 318.49737899028696, "p99": 406.7676870035939 } }, "MED5-20k": { "n": 278, "ttft_ms": { "n": 278, "mean": 2014.1915133893583, "p50": 737.6636109838728, "p90": 2057.0135610178113, "p99": 29051.10613000579 } }, "HEAVY20-50k": { "n": 248, "ttft_ms": { "n": 248, "mean": 3298.1720407103326, "p50": 1321.936836000532, "p90": 7205.449934001081, "p99": 36468.73455500463 } }, "HEAVY+>50k": { "n": 189, "ttft_ms": { "n": 189, "mean": 10874.266077009788, "p50": 2567.718550999416, "p90": 35562.78318798286, "p99": 61205.50292698317 } } } }, "lmetric": { "n_total": 807, "n_ok": 807, "window_s": 1077.4112372398376, "ttft_ms": { "n": 807, "mean": 5152.329462028203, "p50": 1274.1917690145783, "p90": 16492.156354011968, "p99": 46248.138958995696 }, "tpot_ms": { "n": 806, "mean": 24.0370380963572, "p50": 10.534365684851883, "p90": 38.95731354601127, "p99": 231.34709527787183 }, "e2e_ms": { "n": 807, "mean": 10774.921962922257, "p50": 3460.944951977581, "p90": 27791.26176200225, "p99": 99231.47636200883 }, "throughput": { "decode_tps": 214.19212276939402, "prefill_tps": 12337.071064946676, "total_tps": 12551.263187716071, "total_output_tokens": 230773, "total_new_prefill_tokens": 13292099 }, "apc": 0.49518609291339905, "per_worker": { "0": { "n": 140, "decode_tps": 40.62422823074453, "prefill_tps": 2047.015033600442, "ttft_p90_ms": 13910.764692991506, "gpu_util_mean": 92.98571428571428, "gpu_util_max": 100.0, "gpu_mem_max_mb": 89575.0 }, "1": { "n": 89, "decode_tps": 23.129515582036454, "prefill_tps": 1463.9739641483652, "ttft_p90_ms": 17954.478895000648, "gpu_util_mean": 60.319047619047616, "gpu_util_max": 100.0, "gpu_mem_max_mb": 89575.0 }, "2": { "n": 92, "decode_tps": 18.14256184117442, "prefill_tps": 1663.4864553589516, "ttft_p90_ms": 15653.674011002295, "gpu_util_mean": 55.94761904761905, "gpu_util_max": 100.0, "gpu_mem_max_mb": 89575.0 }, "3": { "n": 92, "decode_tps": 43.91637878329201, "prefill_tps": 1313.127198881315, "ttft_p90_ms": 13397.495551005704, "gpu_util_mean": 54.385714285714286, "gpu_util_max": 100.0, "gpu_mem_max_mb": 89575.0 }, "4": { "n": 134, "decode_tps": 21.903428500019192, "prefill_tps": 1412.6889969137978, "ttft_p90_ms": 21995.840935007436, "gpu_util_mean": 51.51428571428571, "gpu_util_max": 100.0, "gpu_mem_max_mb": 89575.0 }, "5": { "n": 88, "decode_tps": 16.934109622562403, "prefill_tps": 1474.5493132872787, "ttft_p90_ms": 11013.645827013534, "gpu_util_mean": 42.55238095238095, "gpu_util_max": 100.0, "gpu_mem_max_mb": 89575.0 }, "6": { "n": 108, "decode_tps": 30.606697665860118, "prefill_tps": 1651.413070981298, "ttft_p90_ms": 18613.971301994752, "gpu_util_mean": 52.67619047619048, "gpu_util_max": 100.0, "gpu_mem_max_mb": 89575.0 }, "7": { "n": 64, "decode_tps": 18.935202543704882, "prefill_tps": 1310.817031775228, "ttft_p90_ms": 15090.364522009622, "gpu_util_mean": 41.904761904761905, "gpu_util_max": 100.0, "gpu_mem_max_mb": 89575.0 } }, "decisions": {}, "gpu_captured": true, "spread": { "n_ratio": 2.1875, "ttft_p90_ratio": 1.997144386199301, "gpu_util_ratio": 2.2189772727272725, "gpu_util_min": 41.904761904761905, "gpu_util_max": 92.98571428571428 }, "per_class": { "WARM<5k": { "n": 92, "ttft_ms": { "n": 92, "mean": 1789.4509492392028, "p50": 262.9711049958132, "p90": 1572.630943992408, "p99": 24139.729285001522 } }, "MED5-20k": { "n": 278, "ttft_ms": { "n": 278, "mean": 1697.68317032391, "p50": 920.4712719947565, "p90": 2029.556789988419, "p99": 22497.115491016302 } }, "HEAVY20-50k": { "n": 248, "ttft_ms": { "n": 248, "mean": 3424.9239484553045, "p50": 2699.253859987948, "p90": 6799.459913017927, "p99": 24401.87052200781 } }, "HEAVY+>50k": { "n": 189, "ttft_ms": { "n": 189, "mean": 14137.37210560736, "p50": 11013.645827013534, "p90": 35319.35577199329, "p99": 51099.66781400726 } } } }, "sticky": { "n_total": 807, "n_ok": 807, "window_s": 925.9680500030518, "ttft_ms": { "n": 807, "mean": 4914.133386887214, "p50": 906.6202020039782, "p90": 15236.451414995827, "p99": 46771.370519010816 }, "tpot_ms": { "n": 806, "mean": 22.62391467634138, "p50": 9.728358783056292, "p90": 30.957536839455965, "p99": 231.30005976865783 }, "e2e_ms": { "n": 807, "mean": 10139.474540275223, "p50": 2597.957960999338, "p90": 27973.595037998166, "p99": 82362.0547579776 }, "throughput": { "decode_tps": 249.22350182518656, "prefill_tps": 8743.631057219865, "total_tps": 8992.854559045052, "total_output_tokens": 230773, "total_new_prefill_tokens": 8096323 }, "apc": 0.6925138424965755, "per_worker": { "0": { "n": 136, "decode_tps": 55.41875877880546, "prefill_tps": 1434.2957081463262, "ttft_p90_ms": 11807.82909199479, "gpu_util_mean": 65.71270718232044, "gpu_util_max": 100.0, "gpu_mem_max_mb": 89575.0 }, "1": { "n": 129, "decode_tps": 38.47432964872015, "prefill_tps": 1709.658340797809, "ttft_p90_ms": 23309.77585897199, "gpu_util_mean": 86.77900552486187, "gpu_util_max": 100.0, "gpu_mem_max_mb": 89575.0 }, "2": { "n": 108, "decode_tps": 47.74786775834683, "prefill_tps": 1144.2727424520835, "ttft_p90_ms": 30574.395572999492, "gpu_util_mean": 63.50828729281768, "gpu_util_max": 100.0, "gpu_mem_max_mb": 89575.0 }, "3": { "n": 78, "decode_tps": 25.291369394357414, "prefill_tps": 1245.5040970325047, "ttft_p90_ms": 26070.17321899184, "gpu_util_mean": 51.87845303867403, "gpu_util_max": 100.0, "gpu_mem_max_mb": 89575.0 }, "4": { "n": 132, "decode_tps": 40.39124244068328, "prefill_tps": 941.3586138281199, "ttft_p90_ms": 5989.128820016049, "gpu_util_mean": 43.0828729281768, "gpu_util_max": 100.0, "gpu_mem_max_mb": 89575.0 }, "5": { "n": 84, "decode_tps": 22.365782491017637, "prefill_tps": 1030.0301398054248, "ttft_p90_ms": 14970.723142003408, "gpu_util_mean": 41.43646408839779, "gpu_util_max": 100.0, "gpu_mem_max_mb": 89575.0 }, "6": { "n": 66, "decode_tps": 9.593203566765315, "prefill_tps": 669.7250515262986, "ttft_p90_ms": 13777.997018012684, "gpu_util_mean": 20.994475138121548, "gpu_util_max": 100.0, "gpu_mem_max_mb": 89575.0 }, "7": { "n": 74, "decode_tps": 9.940947746490457, "prefill_tps": 568.7863636312983, "ttft_p90_ms": 7027.451686997665, "gpu_util_mean": 18.50828729281768, "gpu_util_max": 100.0, "gpu_mem_max_mb": 89575.0 } }, "decisions": {}, "gpu_captured": true, "spread": { "n_ratio": 2.0606060606060606, "ttft_p90_ratio": 5.104982125416625, "gpu_util_ratio": 4.68865671641791, "gpu_util_min": 18.50828729281768, "gpu_util_max": 86.77900552486187 }, "per_class": { "WARM<5k": { "n": 92, "ttft_ms": { "n": 92, "mean": 892.6154872479738, "p50": 207.1402580186259, "p90": 375.00955499126576, "p99": 11232.500832004007 } }, "MED5-20k": { "n": 278, "ttft_ms": { "n": 278, "mean": 2908.9762750470118, "p50": 770.736623002449, "p90": 4912.921145994915, "p99": 42022.69450199674 } }, "HEAVY20-50k": { "n": 248, "ttft_ms": { "n": 248, "mean": 4250.573046338691, "p50": 1623.4680919733364, "p90": 11137.098645005608, "p99": 39817.45037299697 } }, "HEAVY+>50k": { "n": 189, "ttft_ms": { "n": 189, "mean": 10691.78570601113, "p50": 2671.919913002057, "p90": 36922.92091701529, "p99": 53025.03776800586 } } } } }