2172 lines
55 KiB
JSON
2172 lines
55 KiB
JSON
{
|
|
"cell_root": "/home/admin/cpfs/wjh/fidelity-prefix-pilot-20260714/runs/p1b/cells",
|
|
"cells": [
|
|
{
|
|
"cell": "tp1_mns64",
|
|
"cudagraph_runtime_mode_counts": {
|
|
"FULL": 26305,
|
|
"NONE": 698,
|
|
"PIECEWISE": 141
|
|
},
|
|
"dropped_records_before_max": 0,
|
|
"groups": [
|
|
{
|
|
"cudagraph_runtime_mode": "FULL",
|
|
"decode_tokens": {
|
|
"max": 33,
|
|
"mean": 4.150661496350365,
|
|
"min": 1,
|
|
"p50": 3,
|
|
"p95": 13,
|
|
"p99": 21
|
|
},
|
|
"kv_usage": {
|
|
"max": 0.35797781996203415,
|
|
"mean": 0.03727774678883895,
|
|
"min": 0.00029973024278151517,
|
|
"p50": 0.023728644220201822,
|
|
"p95": 0.13018283544809672,
|
|
"p99": 0.21860325706863826
|
|
},
|
|
"phase": "pure_decode",
|
|
"prefill_tokens": {
|
|
"max": 0,
|
|
"mean": 0.0,
|
|
"min": 0,
|
|
"p50": 0,
|
|
"p95": 0,
|
|
"p99": 0
|
|
},
|
|
"queue_waiting": {
|
|
"max": 0,
|
|
"mean": 0.0,
|
|
"min": 0,
|
|
"p50": 0,
|
|
"p95": 0,
|
|
"p99": 0
|
|
},
|
|
"scheduled_requests": {
|
|
"max": 33,
|
|
"mean": 4.150661496350365,
|
|
"min": 1,
|
|
"p50": 3,
|
|
"p95": 13,
|
|
"p99": 21
|
|
},
|
|
"steps": 26304,
|
|
"submit_to_complete_ms": {
|
|
"max": 771.351779,
|
|
"mean": 20.43450573646594,
|
|
"min": 5.345856,
|
|
"p50": 15.161645,
|
|
"p95": 35.769253,
|
|
"p99": 179.487345
|
|
},
|
|
"total_tokens": {
|
|
"max": 33,
|
|
"mean": 4.150661496350365,
|
|
"min": 1,
|
|
"p50": 3,
|
|
"p95": 13,
|
|
"p99": 21
|
|
}
|
|
},
|
|
{
|
|
"cudagraph_runtime_mode": "FULL",
|
|
"decode_tokens": {
|
|
"max": 0,
|
|
"mean": 0.0,
|
|
"min": 0,
|
|
"p50": 0,
|
|
"p95": 0,
|
|
"p99": 0
|
|
},
|
|
"kv_usage": {
|
|
"max": 0.02532720551503642,
|
|
"mean": 0.02532720551503642,
|
|
"min": 0.02532720551503642,
|
|
"p50": 0.02532720551503642,
|
|
"p95": 0.02532720551503642,
|
|
"p99": 0.02532720551503642
|
|
},
|
|
"phase": "pure_prefill",
|
|
"prefill_tokens": {
|
|
"max": 1,
|
|
"mean": 1.0,
|
|
"min": 1,
|
|
"p50": 1,
|
|
"p95": 1,
|
|
"p99": 1
|
|
},
|
|
"queue_waiting": {
|
|
"max": 0,
|
|
"mean": 0.0,
|
|
"min": 0,
|
|
"p50": 0,
|
|
"p95": 0,
|
|
"p99": 0
|
|
},
|
|
"scheduled_requests": {
|
|
"max": 1,
|
|
"mean": 1.0,
|
|
"min": 1,
|
|
"p50": 1,
|
|
"p95": 1,
|
|
"p99": 1
|
|
},
|
|
"steps": 1,
|
|
"submit_to_complete_ms": {
|
|
"max": 7.226884,
|
|
"mean": 7.226884,
|
|
"min": 7.226884,
|
|
"p50": 7.226884,
|
|
"p95": 7.226884,
|
|
"p99": 7.226884
|
|
},
|
|
"total_tokens": {
|
|
"max": 1,
|
|
"mean": 1.0,
|
|
"min": 1,
|
|
"p50": 1,
|
|
"p95": 1,
|
|
"p99": 1
|
|
}
|
|
},
|
|
{
|
|
"cudagraph_runtime_mode": "NONE",
|
|
"decode_tokens": {
|
|
"max": 0,
|
|
"mean": 0.0,
|
|
"min": 0,
|
|
"p50": 0,
|
|
"p95": 0,
|
|
"p99": 0
|
|
},
|
|
"kv_usage": {
|
|
"max": 0.022329903087221492,
|
|
"mean": 0.00819839067915799,
|
|
"min": 0.000549505445099463,
|
|
"p50": 0.007043660705365218,
|
|
"p95": 0.020731341792386893,
|
|
"p99": 0.022329903087221492
|
|
},
|
|
"phase": "pure_prefill",
|
|
"prefill_tokens": {
|
|
"max": 7089,
|
|
"mean": 2337.153846153846,
|
|
"min": 163,
|
|
"p50": 1828,
|
|
"p95": 6569,
|
|
"p99": 7089
|
|
},
|
|
"queue_waiting": {
|
|
"max": 0,
|
|
"mean": 0.0,
|
|
"min": 0,
|
|
"p50": 0,
|
|
"p95": 0,
|
|
"p99": 0
|
|
},
|
|
"scheduled_requests": {
|
|
"max": 1,
|
|
"mean": 1.0,
|
|
"min": 1,
|
|
"p50": 1,
|
|
"p95": 1,
|
|
"p99": 1
|
|
},
|
|
"steps": 52,
|
|
"submit_to_complete_ms": {
|
|
"max": 462.299574,
|
|
"mean": 154.1463595769231,
|
|
"min": 41.988631,
|
|
"p50": 121.390432,
|
|
"p95": 426.234755,
|
|
"p99": 462.299574
|
|
},
|
|
"total_tokens": {
|
|
"max": 7089,
|
|
"mean": 2337.153846153846,
|
|
"min": 163,
|
|
"p50": 1828,
|
|
"p95": 6569,
|
|
"p99": 7089
|
|
}
|
|
},
|
|
{
|
|
"cudagraph_runtime_mode": "PIECEWISE",
|
|
"decode_tokens": {
|
|
"max": 0,
|
|
"mean": 0.0,
|
|
"min": 0,
|
|
"p50": 0,
|
|
"p95": 0,
|
|
"p99": 0
|
|
},
|
|
"kv_usage": {
|
|
"max": 0.011539614347087612,
|
|
"mean": 0.002140381349093522,
|
|
"min": 0.00029973024278151517,
|
|
"p50": 0.0007992806474172998,
|
|
"p95": 0.011539614347087612,
|
|
"p99": 0.011539614347087612
|
|
},
|
|
"phase": "pure_prefill",
|
|
"prefill_tokens": {
|
|
"max": 118,
|
|
"mean": 43.07692307692308,
|
|
"min": 6,
|
|
"p50": 27,
|
|
"p95": 118,
|
|
"p99": 118
|
|
},
|
|
"queue_waiting": {
|
|
"max": 0,
|
|
"mean": 0.0,
|
|
"min": 0,
|
|
"p50": 0,
|
|
"p95": 0,
|
|
"p99": 0
|
|
},
|
|
"scheduled_requests": {
|
|
"max": 1,
|
|
"mean": 1.0,
|
|
"min": 1,
|
|
"p50": 1,
|
|
"p95": 1,
|
|
"p99": 1
|
|
},
|
|
"steps": 13,
|
|
"submit_to_complete_ms": {
|
|
"max": 20.969793,
|
|
"mean": 15.511612692307692,
|
|
"min": 10.185672,
|
|
"p50": 15.302969,
|
|
"p95": 20.969793,
|
|
"p99": 20.969793
|
|
},
|
|
"total_tokens": {
|
|
"max": 118,
|
|
"mean": 43.07692307692308,
|
|
"min": 6,
|
|
"p50": 27,
|
|
"p95": 118,
|
|
"p99": 118
|
|
}
|
|
},
|
|
{
|
|
"cudagraph_runtime_mode": "NONE",
|
|
"decode_tokens": {
|
|
"max": 32,
|
|
"mean": 7.1671826625387,
|
|
"min": 1,
|
|
"p50": 5,
|
|
"p95": 20,
|
|
"p99": 27
|
|
},
|
|
"kv_usage": {
|
|
"max": 0.35622939354580874,
|
|
"mean": 0.08236302498030411,
|
|
"min": 0.003446897791987258,
|
|
"p50": 0.06049555400139872,
|
|
"p95": 0.2183534818663203,
|
|
"p99": 0.3386452193026276
|
|
},
|
|
"phase": "true_mixed",
|
|
"prefill_tokens": {
|
|
"max": 8189,
|
|
"mean": 3407.3668730650156,
|
|
"min": 119,
|
|
"p50": 2823,
|
|
"p95": 7968,
|
|
"p99": 8188
|
|
},
|
|
"queue_waiting": {
|
|
"max": 1,
|
|
"mean": 0.009287925696594427,
|
|
"min": 0,
|
|
"p50": 0,
|
|
"p95": 0,
|
|
"p99": 0
|
|
},
|
|
"scheduled_requests": {
|
|
"max": 33,
|
|
"mean": 8.292569659442725,
|
|
"min": 2,
|
|
"p50": 6,
|
|
"p95": 21,
|
|
"p99": 28
|
|
},
|
|
"steps": 646,
|
|
"submit_to_complete_ms": {
|
|
"max": 1031.099195,
|
|
"mean": 262.67329438235294,
|
|
"min": 41.593929,
|
|
"p50": 208.770469,
|
|
"p95": 560.110521,
|
|
"p99": 836.430251
|
|
},
|
|
"total_tokens": {
|
|
"max": 8192,
|
|
"mean": 3414.534055727554,
|
|
"min": 131,
|
|
"p50": 2838,
|
|
"p95": 7973,
|
|
"p99": 8192
|
|
}
|
|
},
|
|
{
|
|
"cudagraph_runtime_mode": "PIECEWISE",
|
|
"decode_tokens": {
|
|
"max": 26,
|
|
"mean": 5.8203125,
|
|
"min": 1,
|
|
"p50": 4,
|
|
"p95": 15,
|
|
"p99": 25
|
|
},
|
|
"kv_usage": {
|
|
"max": 0.31431711459686285,
|
|
"mean": 0.05598086721950245,
|
|
"min": 0.0004995504046357846,
|
|
"p50": 0.03536816864821657,
|
|
"p95": 0.13977420321710465,
|
|
"p99": 0.29343590768308525
|
|
},
|
|
"phase": "true_mixed",
|
|
"prefill_tokens": {
|
|
"max": 122,
|
|
"mean": 38.9140625,
|
|
"min": 2,
|
|
"p50": 27,
|
|
"p95": 94,
|
|
"p99": 119
|
|
},
|
|
"queue_waiting": {
|
|
"max": 0,
|
|
"mean": 0.0,
|
|
"min": 0,
|
|
"p50": 0,
|
|
"p95": 0,
|
|
"p99": 0
|
|
},
|
|
"scheduled_requests": {
|
|
"max": 27,
|
|
"mean": 6.8203125,
|
|
"min": 2,
|
|
"p50": 5,
|
|
"p95": 16,
|
|
"p99": 26
|
|
},
|
|
"steps": 128,
|
|
"submit_to_complete_ms": {
|
|
"max": 549.993756,
|
|
"mean": 38.9609057265625,
|
|
"min": 13.155768,
|
|
"p50": 24.716054,
|
|
"p95": 86.104443,
|
|
"p99": 407.701862
|
|
},
|
|
"total_tokens": {
|
|
"max": 123,
|
|
"mean": 44.734375,
|
|
"min": 3,
|
|
"p50": 35,
|
|
"p95": 105,
|
|
"p99": 120
|
|
}
|
|
}
|
|
],
|
|
"max_num_seqs": 64,
|
|
"phase_counts": {
|
|
"pure_decode": 26304,
|
|
"pure_prefill": 66,
|
|
"true_mixed": 774
|
|
},
|
|
"prefix": {
|
|
"hit_rate": 0.1199154023961761,
|
|
"hits": 317232,
|
|
"queries": 2645465
|
|
},
|
|
"records": 27144,
|
|
"stream": "/home/admin/cpfs/wjh/fidelity-prefix-pilot-20260714/runs/p1b/cells/tp1_mns64/opprof/opprof-v1-dp0-pid234511-1784005631788937266.jsonl",
|
|
"tensor_parallel_size": 1
|
|
},
|
|
{
|
|
"cell": "tp1_mns8",
|
|
"cudagraph_runtime_mode_counts": {
|
|
"FULL": 27496,
|
|
"NONE": 800,
|
|
"PIECEWISE": 13
|
|
},
|
|
"dropped_records_before_max": 0,
|
|
"groups": [
|
|
{
|
|
"cudagraph_runtime_mode": "FULL",
|
|
"decode_tokens": {
|
|
"max": 8,
|
|
"mean": 3.827968721585743,
|
|
"min": 1,
|
|
"p50": 3,
|
|
"p95": 8,
|
|
"p99": 8
|
|
},
|
|
"kv_usage": {
|
|
"max": 0.13572814567888947,
|
|
"mean": 0.03436450743578334,
|
|
"min": 0.0002985223145429705,
|
|
"p50": 0.02666799343250903,
|
|
"p95": 0.09512911090103982,
|
|
"p99": 0.11562764316632668
|
|
},
|
|
"phase": "pure_decode",
|
|
"prefill_tokens": {
|
|
"max": 0,
|
|
"mean": 0.0,
|
|
"min": 0,
|
|
"p50": 0,
|
|
"p95": 0,
|
|
"p99": 0
|
|
},
|
|
"queue_waiting": {
|
|
"max": 13,
|
|
"mean": 0.7083106019276232,
|
|
"min": 0,
|
|
"p50": 0,
|
|
"p95": 5,
|
|
"p99": 9
|
|
},
|
|
"scheduled_requests": {
|
|
"max": 8,
|
|
"mean": 3.827968721585743,
|
|
"min": 1,
|
|
"p50": 3,
|
|
"p95": 8,
|
|
"p99": 8
|
|
},
|
|
"steps": 27495,
|
|
"submit_to_complete_ms": {
|
|
"max": 561.758474,
|
|
"mean": 19.407119342862337,
|
|
"min": 5.267886,
|
|
"p50": 15.55276,
|
|
"p95": 24.123839,
|
|
"p99": 151.668642
|
|
},
|
|
"total_tokens": {
|
|
"max": 8,
|
|
"mean": 3.827968721585743,
|
|
"min": 1,
|
|
"p50": 3,
|
|
"p95": 8,
|
|
"p99": 8
|
|
}
|
|
},
|
|
{
|
|
"cudagraph_runtime_mode": "FULL",
|
|
"decode_tokens": {
|
|
"max": 0,
|
|
"mean": 0.0,
|
|
"min": 0,
|
|
"p50": 0,
|
|
"p95": 0,
|
|
"p99": 0
|
|
},
|
|
"kv_usage": {
|
|
"max": 0.025225135578884506,
|
|
"mean": 0.025225135578884506,
|
|
"min": 0.025225135578884506,
|
|
"p50": 0.025225135578884506,
|
|
"p95": 0.025225135578884506,
|
|
"p99": 0.025225135578884506
|
|
},
|
|
"phase": "pure_prefill",
|
|
"prefill_tokens": {
|
|
"max": 1,
|
|
"mean": 1.0,
|
|
"min": 1,
|
|
"p50": 1,
|
|
"p95": 1,
|
|
"p99": 1
|
|
},
|
|
"queue_waiting": {
|
|
"max": 0,
|
|
"mean": 0.0,
|
|
"min": 0,
|
|
"p50": 0,
|
|
"p95": 0,
|
|
"p99": 0
|
|
},
|
|
"scheduled_requests": {
|
|
"max": 1,
|
|
"mean": 1.0,
|
|
"min": 1,
|
|
"p50": 1,
|
|
"p95": 1,
|
|
"p99": 1
|
|
},
|
|
"steps": 1,
|
|
"submit_to_complete_ms": {
|
|
"max": 7.195164,
|
|
"mean": 7.195164,
|
|
"min": 7.195164,
|
|
"p50": 7.195164,
|
|
"p95": 7.195164,
|
|
"p99": 7.195164
|
|
},
|
|
"total_tokens": {
|
|
"max": 1,
|
|
"mean": 1.0,
|
|
"min": 1,
|
|
"p50": 1,
|
|
"p95": 1,
|
|
"p99": 1
|
|
}
|
|
},
|
|
{
|
|
"cudagraph_runtime_mode": "NONE",
|
|
"decode_tokens": {
|
|
"max": 0,
|
|
"mean": 0.0,
|
|
"min": 0,
|
|
"p50": 0,
|
|
"p95": 0,
|
|
"p99": 0
|
|
},
|
|
"kv_usage": {
|
|
"max": 0.02318523309617393,
|
|
"mean": 0.007211759417999546,
|
|
"min": 0.0002985223145429705,
|
|
"p50": 0.005373401661774246,
|
|
"p95": 0.02064779342255829,
|
|
"p99": 0.02318523309617393
|
|
},
|
|
"phase": "pure_prefill",
|
|
"prefill_tokens": {
|
|
"max": 7089,
|
|
"mean": 2017.593220338983,
|
|
"min": 18,
|
|
"p50": 1501,
|
|
"p95": 6569,
|
|
"p99": 7089
|
|
},
|
|
"queue_waiting": {
|
|
"max": 0,
|
|
"mean": 0.0,
|
|
"min": 0,
|
|
"p50": 0,
|
|
"p95": 0,
|
|
"p99": 0
|
|
},
|
|
"scheduled_requests": {
|
|
"max": 1,
|
|
"mean": 1.0,
|
|
"min": 1,
|
|
"p50": 1,
|
|
"p95": 1,
|
|
"p99": 1
|
|
},
|
|
"steps": 59,
|
|
"submit_to_complete_ms": {
|
|
"max": 463.900987,
|
|
"mean": 140.730381559322,
|
|
"min": 41.733004,
|
|
"p50": 101.529539,
|
|
"p95": 425.237592,
|
|
"p99": 463.900987
|
|
},
|
|
"total_tokens": {
|
|
"max": 7089,
|
|
"mean": 2017.593220338983,
|
|
"min": 18,
|
|
"p50": 1501,
|
|
"p95": 6569,
|
|
"p99": 7089
|
|
}
|
|
},
|
|
{
|
|
"cudagraph_runtime_mode": "PIECEWISE",
|
|
"decode_tokens": {
|
|
"max": 0,
|
|
"mean": 0.0,
|
|
"min": 0,
|
|
"p50": 0,
|
|
"p95": 0,
|
|
"p99": 0
|
|
},
|
|
"kv_usage": {
|
|
"max": 0.011493109109905975,
|
|
"mean": 0.004925618189959698,
|
|
"min": 0.0005472909099955015,
|
|
"p50": 0.0027364545499776183,
|
|
"p95": 0.011493109109905975,
|
|
"p99": 0.011493109109905975
|
|
},
|
|
"phase": "pure_prefill",
|
|
"prefill_tokens": {
|
|
"max": 10,
|
|
"mean": 8.0,
|
|
"min": 6,
|
|
"p50": 8,
|
|
"p95": 10,
|
|
"p99": 10
|
|
},
|
|
"queue_waiting": {
|
|
"max": 0,
|
|
"mean": 0.0,
|
|
"min": 0,
|
|
"p50": 0,
|
|
"p95": 0,
|
|
"p99": 0
|
|
},
|
|
"scheduled_requests": {
|
|
"max": 1,
|
|
"mean": 1.0,
|
|
"min": 1,
|
|
"p50": 1,
|
|
"p95": 1,
|
|
"p99": 1
|
|
},
|
|
"steps": 3,
|
|
"submit_to_complete_ms": {
|
|
"max": 11.597111,
|
|
"mean": 10.493325333333333,
|
|
"min": 9.464417,
|
|
"p50": 10.418448,
|
|
"p95": 11.597111,
|
|
"p99": 11.597111
|
|
},
|
|
"total_tokens": {
|
|
"max": 10,
|
|
"mean": 8.0,
|
|
"min": 6,
|
|
"p50": 8,
|
|
"p95": 10,
|
|
"p99": 10
|
|
}
|
|
},
|
|
{
|
|
"cudagraph_runtime_mode": "NONE",
|
|
"decode_tokens": {
|
|
"max": 7,
|
|
"mean": 4.6396761133603235,
|
|
"min": 1,
|
|
"p50": 5,
|
|
"p95": 7,
|
|
"p99": 7
|
|
},
|
|
"kv_usage": {
|
|
"max": 0.13562863824070848,
|
|
"mean": 0.05433522417609084,
|
|
"min": 0.0004477834718145113,
|
|
"p50": 0.05353500174138015,
|
|
"p95": 0.10617443653913128,
|
|
"p99": 0.12702124483805166
|
|
},
|
|
"phase": "true_mixed",
|
|
"prefill_tokens": {
|
|
"max": 8189,
|
|
"mean": 2800.905533063428,
|
|
"min": 13,
|
|
"p50": 1881,
|
|
"p95": 7847,
|
|
"p99": 8186
|
|
},
|
|
"queue_waiting": {
|
|
"max": 12,
|
|
"mean": 1.2199730094466936,
|
|
"min": 0,
|
|
"p50": 0,
|
|
"p95": 7,
|
|
"p99": 10
|
|
},
|
|
"scheduled_requests": {
|
|
"max": 8,
|
|
"mean": 5.711201079622132,
|
|
"min": 2,
|
|
"p50": 6,
|
|
"p95": 8,
|
|
"p99": 8
|
|
},
|
|
"steps": 741,
|
|
"submit_to_complete_ms": {
|
|
"max": 1024.27287,
|
|
"mean": 224.85357696356274,
|
|
"min": 40.32457,
|
|
"p50": 144.763835,
|
|
"p95": 549.585886,
|
|
"p99": 785.810047
|
|
},
|
|
"total_tokens": {
|
|
"max": 8192,
|
|
"mean": 2805.545209176788,
|
|
"min": 17,
|
|
"p50": 1884,
|
|
"p95": 7854,
|
|
"p99": 8192
|
|
}
|
|
},
|
|
{
|
|
"cudagraph_runtime_mode": "PIECEWISE",
|
|
"decode_tokens": {
|
|
"max": 5,
|
|
"mean": 2.4,
|
|
"min": 1,
|
|
"p50": 2,
|
|
"p95": 5,
|
|
"p99": 5
|
|
},
|
|
"kv_usage": {
|
|
"max": 0.06567490919946262,
|
|
"mean": 0.033126026170456235,
|
|
"min": 0.008557639683566376,
|
|
"p50": 0.02617045624160408,
|
|
"p95": 0.06567490919946262,
|
|
"p99": 0.06567490919946262
|
|
},
|
|
"phase": "true_mixed",
|
|
"prefill_tokens": {
|
|
"max": 13,
|
|
"mean": 7.5,
|
|
"min": 2,
|
|
"p50": 7,
|
|
"p95": 13,
|
|
"p99": 13
|
|
},
|
|
"queue_waiting": {
|
|
"max": 0,
|
|
"mean": 0.0,
|
|
"min": 0,
|
|
"p50": 0,
|
|
"p95": 0,
|
|
"p99": 0
|
|
},
|
|
"scheduled_requests": {
|
|
"max": 6,
|
|
"mean": 3.4,
|
|
"min": 2,
|
|
"p50": 3,
|
|
"p95": 6,
|
|
"p99": 6
|
|
},
|
|
"steps": 10,
|
|
"submit_to_complete_ms": {
|
|
"max": 22.147232,
|
|
"mean": 17.1428369,
|
|
"min": 13.210445,
|
|
"p50": 15.229545,
|
|
"p95": 22.147232,
|
|
"p99": 22.147232
|
|
},
|
|
"total_tokens": {
|
|
"max": 15,
|
|
"mean": 9.9,
|
|
"min": 3,
|
|
"p50": 9,
|
|
"p95": 15,
|
|
"p99": 15
|
|
}
|
|
}
|
|
],
|
|
"max_num_seqs": 8,
|
|
"phase_counts": {
|
|
"pure_decode": 27495,
|
|
"pure_prefill": 63,
|
|
"true_mixed": 751
|
|
},
|
|
"prefix": {
|
|
"hit_rate": 0.12292016112369349,
|
|
"hits": 307568,
|
|
"queries": 2502177
|
|
},
|
|
"records": 28309,
|
|
"stream": "/home/admin/cpfs/wjh/fidelity-prefix-pilot-20260714/runs/p1b/cells/tp1_mns8/opprof/opprof-v1-dp0-pid227734-1784005177936445360.jsonl",
|
|
"tensor_parallel_size": 1
|
|
},
|
|
{
|
|
"cell": "tp2_mns64",
|
|
"cudagraph_runtime_mode_counts": {
|
|
"FULL": 30746,
|
|
"NONE": 1555,
|
|
"PIECEWISE": 298
|
|
},
|
|
"dropped_records_before_max": 0,
|
|
"groups": [
|
|
{
|
|
"cudagraph_runtime_mode": "FULL",
|
|
"decode_tokens": {
|
|
"max": 39,
|
|
"mean": 7.531678917582775,
|
|
"min": 1,
|
|
"p50": 6,
|
|
"p95": 21,
|
|
"p99": 30
|
|
},
|
|
"kv_usage": {
|
|
"max": 0.10309634347542496,
|
|
"mean": 0.016984430152386328,
|
|
"min": 7.673713693745476e-05,
|
|
"p50": 0.011983782885060523,
|
|
"p95": 0.05189988361534226,
|
|
"p99": 0.07517681515302665
|
|
},
|
|
"phase": "pure_decode",
|
|
"prefill_tokens": {
|
|
"max": 0,
|
|
"mean": 0.0,
|
|
"min": 0,
|
|
"p50": 0,
|
|
"p95": 0,
|
|
"p99": 0
|
|
},
|
|
"queue_waiting": {
|
|
"max": 0,
|
|
"mean": 0.0,
|
|
"min": 0,
|
|
"p50": 0,
|
|
"p95": 0,
|
|
"p99": 0
|
|
},
|
|
"scheduled_requests": {
|
|
"max": 39,
|
|
"mean": 7.531678917582775,
|
|
"min": 1,
|
|
"p50": 6,
|
|
"p95": 21,
|
|
"p99": 30
|
|
},
|
|
"steps": 30746,
|
|
"submit_to_complete_ms": {
|
|
"max": 327.486803,
|
|
"mean": 19.866698748357507,
|
|
"min": 6.465612,
|
|
"p50": 13.818435,
|
|
"p95": 29.678643,
|
|
"p99": 212.284891
|
|
},
|
|
"total_tokens": {
|
|
"max": 39,
|
|
"mean": 7.531678917582775,
|
|
"min": 1,
|
|
"p50": 6,
|
|
"p95": 21,
|
|
"p99": 30
|
|
}
|
|
},
|
|
{
|
|
"cudagraph_runtime_mode": "NONE",
|
|
"decode_tokens": {
|
|
"max": 0,
|
|
"mean": 0.0,
|
|
"min": 0,
|
|
"p50": 0,
|
|
"p95": 0,
|
|
"p99": 0
|
|
},
|
|
"kv_usage": {
|
|
"max": 0.0061261814321708075,
|
|
"mean": 0.0026679719731172026,
|
|
"min": 0.00014068475105188938,
|
|
"p50": 0.0020719026973103905,
|
|
"p95": 0.005857601452889827,
|
|
"p99": 0.0061261814321708075
|
|
},
|
|
"phase": "pure_prefill",
|
|
"prefill_tokens": {
|
|
"max": 7612,
|
|
"mean": 2967.242424242424,
|
|
"min": 129,
|
|
"p50": 2331,
|
|
"p95": 7273,
|
|
"p99": 7612
|
|
},
|
|
"queue_waiting": {
|
|
"max": 0,
|
|
"mean": 0.0,
|
|
"min": 0,
|
|
"p50": 0,
|
|
"p95": 0,
|
|
"p99": 0
|
|
},
|
|
"scheduled_requests": {
|
|
"max": 1,
|
|
"mean": 1.0,
|
|
"min": 1,
|
|
"p50": 1,
|
|
"p95": 1,
|
|
"p99": 1
|
|
},
|
|
"steps": 33,
|
|
"submit_to_complete_ms": {
|
|
"max": 285.377954,
|
|
"mean": 125.60165333333333,
|
|
"min": 58.085137,
|
|
"p50": 87.591746,
|
|
"p95": 269.63345,
|
|
"p99": 285.377954
|
|
},
|
|
"total_tokens": {
|
|
"max": 7612,
|
|
"mean": 2967.242424242424,
|
|
"min": 129,
|
|
"p50": 2331,
|
|
"p95": 7273,
|
|
"p99": 7612
|
|
}
|
|
},
|
|
{
|
|
"cudagraph_runtime_mode": "PIECEWISE",
|
|
"decode_tokens": {
|
|
"max": 0,
|
|
"mean": 0.0,
|
|
"min": 0,
|
|
"p50": 0,
|
|
"p95": 0,
|
|
"p99": 0
|
|
},
|
|
"kv_usage": {
|
|
"max": 0.0007545818465513054,
|
|
"mean": 0.00022381664940082646,
|
|
"min": 7.673713693745476e-05,
|
|
"p50": 8.952665976036389e-05,
|
|
"p95": 0.0007545818465513054,
|
|
"p99": 0.0007545818465513054
|
|
},
|
|
"phase": "pure_prefill",
|
|
"prefill_tokens": {
|
|
"max": 93,
|
|
"mean": 46.25,
|
|
"min": 10,
|
|
"p50": 32,
|
|
"p95": 93,
|
|
"p99": 93
|
|
},
|
|
"queue_waiting": {
|
|
"max": 0,
|
|
"mean": 0.0,
|
|
"min": 0,
|
|
"p50": 0,
|
|
"p95": 0,
|
|
"p99": 0
|
|
},
|
|
"scheduled_requests": {
|
|
"max": 1,
|
|
"mean": 1.0,
|
|
"min": 1,
|
|
"p50": 1,
|
|
"p95": 1,
|
|
"p99": 1
|
|
},
|
|
"steps": 8,
|
|
"submit_to_complete_ms": {
|
|
"max": 17.211201,
|
|
"mean": 12.989633,
|
|
"min": 10.521338,
|
|
"p50": 13.034104,
|
|
"p95": 17.211201,
|
|
"p99": 17.211201
|
|
},
|
|
"total_tokens": {
|
|
"max": 93,
|
|
"mean": 46.25,
|
|
"min": 10,
|
|
"p50": 32,
|
|
"p95": 93,
|
|
"p99": 93
|
|
}
|
|
},
|
|
{
|
|
"cudagraph_runtime_mode": "NONE",
|
|
"decode_tokens": {
|
|
"max": 37,
|
|
"mean": 11.961892247043364,
|
|
"min": 1,
|
|
"p50": 11,
|
|
"p95": 28,
|
|
"p99": 32
|
|
},
|
|
"kv_usage": {
|
|
"max": 0.10278939492767525,
|
|
"mean": 0.03137185077117546,
|
|
"min": 0.0009336351660719222,
|
|
"p50": 0.02681962935962856,
|
|
"p95": 0.07204338206141525,
|
|
"p99": 0.08896392075611659
|
|
},
|
|
"phase": "true_mixed",
|
|
"prefill_tokens": {
|
|
"max": 8191,
|
|
"mean": 3282.0696452036796,
|
|
"min": 114,
|
|
"p50": 2770,
|
|
"p95": 7855,
|
|
"p99": 8176
|
|
},
|
|
"queue_waiting": {
|
|
"max": 1,
|
|
"mean": 0.0032851511169513796,
|
|
"min": 0,
|
|
"p50": 0,
|
|
"p95": 0,
|
|
"p99": 0
|
|
},
|
|
"scheduled_requests": {
|
|
"max": 39,
|
|
"mean": 13.06898817345598,
|
|
"min": 2,
|
|
"p50": 12,
|
|
"p95": 29,
|
|
"p99": 34
|
|
},
|
|
"steps": 1522,
|
|
"submit_to_complete_ms": {
|
|
"max": 560.812857,
|
|
"mean": 146.33081431603154,
|
|
"min": 56.009573,
|
|
"p50": 112.614117,
|
|
"p95": 307.930472,
|
|
"p99": 403.669994
|
|
},
|
|
"total_tokens": {
|
|
"max": 8192,
|
|
"mean": 3294.0315374507227,
|
|
"min": 130,
|
|
"p50": 2779,
|
|
"p95": 7865,
|
|
"p99": 8192
|
|
}
|
|
},
|
|
{
|
|
"cudagraph_runtime_mode": "PIECEWISE",
|
|
"decode_tokens": {
|
|
"max": 34,
|
|
"mean": 10.627586206896552,
|
|
"min": 1,
|
|
"p50": 9,
|
|
"p95": 25,
|
|
"p99": 33
|
|
},
|
|
"kv_usage": {
|
|
"max": 0.10290450063308143,
|
|
"mean": 0.025070992877117823,
|
|
"min": 0.0008313189834887602,
|
|
"p50": 0.021217818363196916,
|
|
"p95": 0.06711941577459746,
|
|
"p99": 0.08507590581795388
|
|
},
|
|
"phase": "true_mixed",
|
|
"prefill_tokens": {
|
|
"max": 122,
|
|
"mean": 38.53793103448276,
|
|
"min": 2,
|
|
"p50": 27,
|
|
"p95": 96,
|
|
"p99": 119
|
|
},
|
|
"queue_waiting": {
|
|
"max": 0,
|
|
"mean": 0.0,
|
|
"min": 0,
|
|
"p50": 0,
|
|
"p95": 0,
|
|
"p99": 0
|
|
},
|
|
"scheduled_requests": {
|
|
"max": 35,
|
|
"mean": 11.63448275862069,
|
|
"min": 2,
|
|
"p50": 10,
|
|
"p95": 26,
|
|
"p99": 34
|
|
},
|
|
"steps": 290,
|
|
"submit_to_complete_ms": {
|
|
"max": 310.419123,
|
|
"mean": 25.218022886206896,
|
|
"min": 10.997474,
|
|
"p50": 20.06169,
|
|
"p95": 35.895051,
|
|
"p99": 196.114372
|
|
},
|
|
"total_tokens": {
|
|
"max": 127,
|
|
"mean": 49.16551724137931,
|
|
"min": 5,
|
|
"p50": 41,
|
|
"p95": 104,
|
|
"p99": 124
|
|
}
|
|
}
|
|
],
|
|
"max_num_seqs": 64,
|
|
"phase_counts": {
|
|
"pure_decode": 30746,
|
|
"pure_prefill": 41,
|
|
"true_mixed": 1812
|
|
},
|
|
"prefix": {
|
|
"hit_rate": 0.1232028946400688,
|
|
"hits": 717296,
|
|
"queries": 5822071
|
|
},
|
|
"records": 32599,
|
|
"stream": "/home/admin/cpfs/wjh/fidelity-prefix-pilot-20260714/runs/p1b/cells/tp2_mns64/opprof/opprof-v1-dp0-pid247791-1784006495467408663.jsonl",
|
|
"tensor_parallel_size": 2
|
|
},
|
|
{
|
|
"cell": "tp2_mns8",
|
|
"cudagraph_runtime_mode_counts": {
|
|
"FULL": 27697,
|
|
"NONE": 1201,
|
|
"PIECEWISE": 13
|
|
},
|
|
"dropped_records_before_max": 0,
|
|
"groups": [
|
|
{
|
|
"cudagraph_runtime_mode": "FULL",
|
|
"decode_tokens": {
|
|
"max": 8,
|
|
"mean": 5.417373722785861,
|
|
"min": 1,
|
|
"p50": 6,
|
|
"p95": 8,
|
|
"p99": 8
|
|
},
|
|
"kv_usage": {
|
|
"max": 0.03395735232188657,
|
|
"mean": 0.012474261139679475,
|
|
"min": 7.656674706180144e-05,
|
|
"p50": 0.011918890292612616,
|
|
"p95": 0.026530377856896714,
|
|
"p99": 0.03005244822173725
|
|
},
|
|
"phase": "pure_decode",
|
|
"prefill_tokens": {
|
|
"max": 0,
|
|
"mean": 0.0,
|
|
"min": 0,
|
|
"p50": 0,
|
|
"p95": 0,
|
|
"p99": 0
|
|
},
|
|
"queue_waiting": {
|
|
"max": 24,
|
|
"mean": 2.201646387695418,
|
|
"min": 0,
|
|
"p50": 0,
|
|
"p95": 13,
|
|
"p99": 20
|
|
},
|
|
"scheduled_requests": {
|
|
"max": 8,
|
|
"mean": 5.417373722785861,
|
|
"min": 1,
|
|
"p50": 6,
|
|
"p95": 8,
|
|
"p99": 8
|
|
},
|
|
"steps": 27697,
|
|
"submit_to_complete_ms": {
|
|
"max": 314.563834,
|
|
"mean": 16.962280882044986,
|
|
"min": 6.575488,
|
|
"p50": 13.532942,
|
|
"p95": 15.503119,
|
|
"p99": 174.775316
|
|
},
|
|
"total_tokens": {
|
|
"max": 8,
|
|
"mean": 5.417373722785861,
|
|
"min": 1,
|
|
"p50": 6,
|
|
"p95": 8,
|
|
"p99": 8
|
|
}
|
|
},
|
|
{
|
|
"cudagraph_runtime_mode": "NONE",
|
|
"decode_tokens": {
|
|
"max": 0,
|
|
"mean": 0.0,
|
|
"min": 0,
|
|
"p50": 0,
|
|
"p95": 0,
|
|
"p99": 0
|
|
},
|
|
"kv_usage": {
|
|
"max": 0.005844595025713661,
|
|
"mean": 0.0019588326123298008,
|
|
"min": 7.656674706180144e-05,
|
|
"p50": 0.001365440322601219,
|
|
"p95": 0.005704222656100488,
|
|
"p99": 0.005844595025713661
|
|
},
|
|
"phase": "pure_prefill",
|
|
"prefill_tokens": {
|
|
"max": 7273,
|
|
"mean": 2182.5666666666666,
|
|
"min": 24,
|
|
"p50": 1060,
|
|
"p95": 7073,
|
|
"p99": 7273
|
|
},
|
|
"queue_waiting": {
|
|
"max": 0,
|
|
"mean": 0.0,
|
|
"min": 0,
|
|
"p50": 0,
|
|
"p95": 0,
|
|
"p99": 0
|
|
},
|
|
"scheduled_requests": {
|
|
"max": 1,
|
|
"mean": 1.0,
|
|
"min": 1,
|
|
"p50": 1,
|
|
"p95": 1,
|
|
"p99": 1
|
|
},
|
|
"steps": 30,
|
|
"submit_to_complete_ms": {
|
|
"max": 269.442406,
|
|
"mean": 103.43858230000001,
|
|
"min": 55.279405,
|
|
"p50": 66.596689,
|
|
"p95": 260.579,
|
|
"p99": 269.442406
|
|
},
|
|
"total_tokens": {
|
|
"max": 7273,
|
|
"mean": 2182.5666666666666,
|
|
"min": 24,
|
|
"p50": 1060,
|
|
"p95": 7073,
|
|
"p99": 7273
|
|
}
|
|
},
|
|
{
|
|
"cudagraph_runtime_mode": "PIECEWISE",
|
|
"decode_tokens": {
|
|
"max": 0,
|
|
"mean": 0.0,
|
|
"min": 0,
|
|
"p50": 0,
|
|
"p95": 0,
|
|
"p99": 0
|
|
},
|
|
"kv_usage": {
|
|
"max": 0.00014037236961317312,
|
|
"mean": 0.00014037236961317312,
|
|
"min": 0.00014037236961317312,
|
|
"p50": 0.00014037236961317312,
|
|
"p95": 0.00014037236961317312,
|
|
"p99": 0.00014037236961317312
|
|
},
|
|
"phase": "pure_prefill",
|
|
"prefill_tokens": {
|
|
"max": 10,
|
|
"mean": 10.0,
|
|
"min": 10,
|
|
"p50": 10,
|
|
"p95": 10,
|
|
"p99": 10
|
|
},
|
|
"queue_waiting": {
|
|
"max": 0,
|
|
"mean": 0.0,
|
|
"min": 0,
|
|
"p50": 0,
|
|
"p95": 0,
|
|
"p99": 0
|
|
},
|
|
"scheduled_requests": {
|
|
"max": 1,
|
|
"mean": 1.0,
|
|
"min": 1,
|
|
"p50": 1,
|
|
"p95": 1,
|
|
"p99": 1
|
|
},
|
|
"steps": 1,
|
|
"submit_to_complete_ms": {
|
|
"max": 10.336689,
|
|
"mean": 10.336689,
|
|
"min": 10.336689,
|
|
"p50": 10.336689,
|
|
"p95": 10.336689,
|
|
"p99": 10.336689
|
|
},
|
|
"total_tokens": {
|
|
"max": 10,
|
|
"mean": 10.0,
|
|
"min": 10,
|
|
"p50": 10,
|
|
"p95": 10,
|
|
"p99": 10
|
|
}
|
|
},
|
|
{
|
|
"cudagraph_runtime_mode": "NONE",
|
|
"decode_tokens": {
|
|
"max": 7,
|
|
"mean": 5.772843723313407,
|
|
"min": 1,
|
|
"p50": 7,
|
|
"p95": 7,
|
|
"p99": 7
|
|
},
|
|
"kv_usage": {
|
|
"max": 0.03394459119737636,
|
|
"mean": 0.0159228647453862,
|
|
"min": 0.0008167119686586233,
|
|
"p50": 0.016079016882967756,
|
|
"p95": 0.02733432870104513,
|
|
"p99": 0.0316731110345444
|
|
},
|
|
"phase": "true_mixed",
|
|
"prefill_tokens": {
|
|
"max": 8191,
|
|
"mean": 2652.5926558497013,
|
|
"min": 13,
|
|
"p50": 1837,
|
|
"p95": 7634,
|
|
"p99": 8066
|
|
},
|
|
"queue_waiting": {
|
|
"max": 23,
|
|
"mean": 2.9812126387702818,
|
|
"min": 0,
|
|
"p50": 1,
|
|
"p95": 13,
|
|
"p99": 21
|
|
},
|
|
"scheduled_requests": {
|
|
"max": 8,
|
|
"mean": 6.7933390264731,
|
|
"min": 2,
|
|
"p50": 8,
|
|
"p95": 8,
|
|
"p99": 8
|
|
},
|
|
"steps": 1171,
|
|
"submit_to_complete_ms": {
|
|
"max": 506.291757,
|
|
"mean": 121.45648444577284,
|
|
"min": 53.060612,
|
|
"p50": 81.564408,
|
|
"p95": 292.93805,
|
|
"p99": 310.013083
|
|
},
|
|
"total_tokens": {
|
|
"max": 8192,
|
|
"mean": 2658.3654995730144,
|
|
"min": 17,
|
|
"p50": 1840,
|
|
"p95": 7641,
|
|
"p99": 8073
|
|
}
|
|
},
|
|
{
|
|
"cudagraph_runtime_mode": "PIECEWISE",
|
|
"decode_tokens": {
|
|
"max": 7,
|
|
"mean": 3.3333333333333335,
|
|
"min": 1,
|
|
"p50": 3,
|
|
"p95": 7,
|
|
"p99": 7
|
|
},
|
|
"kv_usage": {
|
|
"max": 0.01494327680155172,
|
|
"mean": 0.009233737010238682,
|
|
"min": 0.0034837869913096897,
|
|
"p50": 0.009315620892513032,
|
|
"p95": 0.01494327680155172,
|
|
"p99": 0.01494327680155172
|
|
},
|
|
"phase": "true_mixed",
|
|
"prefill_tokens": {
|
|
"max": 13,
|
|
"mean": 6.916666666666667,
|
|
"min": 2,
|
|
"p50": 6,
|
|
"p95": 13,
|
|
"p99": 13
|
|
},
|
|
"queue_waiting": {
|
|
"max": 11,
|
|
"mean": 0.9166666666666666,
|
|
"min": 0,
|
|
"p50": 0,
|
|
"p95": 11,
|
|
"p99": 11
|
|
},
|
|
"scheduled_requests": {
|
|
"max": 8,
|
|
"mean": 4.333333333333333,
|
|
"min": 2,
|
|
"p50": 4,
|
|
"p95": 8,
|
|
"p99": 8
|
|
},
|
|
"steps": 12,
|
|
"submit_to_complete_ms": {
|
|
"max": 15.924106,
|
|
"mean": 13.055620500000002,
|
|
"min": 10.64629,
|
|
"p50": 12.616157,
|
|
"p95": 15.924106,
|
|
"p99": 15.924106
|
|
},
|
|
"total_tokens": {
|
|
"max": 16,
|
|
"mean": 10.25,
|
|
"min": 5,
|
|
"p50": 9,
|
|
"p95": 16,
|
|
"p99": 16
|
|
}
|
|
}
|
|
],
|
|
"max_num_seqs": 8,
|
|
"phase_counts": {
|
|
"pure_decode": 27697,
|
|
"pure_prefill": 31,
|
|
"true_mixed": 1183
|
|
},
|
|
"prefix": {
|
|
"hit_rate": 0.12439998012356517,
|
|
"hits": 450624,
|
|
"queries": 3622380
|
|
},
|
|
"records": 28911,
|
|
"stream": "/home/admin/cpfs/wjh/fidelity-prefix-pilot-20260714/runs/p1b/cells/tp2_mns8/opprof/opprof-v1-dp0-pid241594-1784006096655191299.jsonl",
|
|
"tensor_parallel_size": 2
|
|
},
|
|
{
|
|
"cell": "tp4_mns16",
|
|
"cudagraph_runtime_mode_counts": {
|
|
"FULL": 17418,
|
|
"NONE": 1752,
|
|
"PIECEWISE": 53
|
|
},
|
|
"dropped_records_before_max": 0,
|
|
"groups": [
|
|
{
|
|
"cudagraph_runtime_mode": "FULL",
|
|
"decode_tokens": {
|
|
"max": 16,
|
|
"mean": 13.292530286501695,
|
|
"min": 1,
|
|
"p50": 16,
|
|
"p95": 16,
|
|
"p99": 16
|
|
},
|
|
"kv_usage": {
|
|
"max": 0.02384243790712981,
|
|
"mean": 0.011910732119220589,
|
|
"min": 0.00021424527002555926,
|
|
"p50": 0.012732290332947316,
|
|
"p95": 0.01879237082795593,
|
|
"p99": 0.02112866448680606
|
|
},
|
|
"phase": "pure_decode",
|
|
"prefill_tokens": {
|
|
"max": 0,
|
|
"mean": 0.0,
|
|
"min": 0,
|
|
"p50": 0,
|
|
"p95": 0,
|
|
"p99": 0
|
|
},
|
|
"queue_waiting": {
|
|
"max": 48,
|
|
"mean": 11.14313601653557,
|
|
"min": 0,
|
|
"p50": 5,
|
|
"p95": 48,
|
|
"p99": 48
|
|
},
|
|
"scheduled_requests": {
|
|
"max": 16,
|
|
"mean": 13.292530286501695,
|
|
"min": 1,
|
|
"p50": 16,
|
|
"p95": 16,
|
|
"p99": 16
|
|
},
|
|
"steps": 17417,
|
|
"submit_to_complete_ms": {
|
|
"max": 185.583096,
|
|
"mean": 18.308918529310446,
|
|
"min": 6.71116,
|
|
"p50": 12.899641,
|
|
"p95": 64.79132,
|
|
"p99": 141.517546
|
|
},
|
|
"total_tokens": {
|
|
"max": 16,
|
|
"mean": 13.292530286501695,
|
|
"min": 1,
|
|
"p50": 16,
|
|
"p95": 16,
|
|
"p99": 16
|
|
}
|
|
},
|
|
{
|
|
"cudagraph_runtime_mode": "NONE",
|
|
"decode_tokens": {
|
|
"max": 0,
|
|
"mean": 0.0,
|
|
"min": 0,
|
|
"p50": 0,
|
|
"p95": 0,
|
|
"p99": 0
|
|
},
|
|
"kv_usage": {
|
|
"max": 0.0014436050337436335,
|
|
"mean": 0.0005439024265827402,
|
|
"min": 0.0003213679050383389,
|
|
"p50": 0.0003570754500425988,
|
|
"p95": 0.0014436050337436335,
|
|
"p99": 0.0014436050337436335
|
|
},
|
|
"phase": "pure_prefill",
|
|
"prefill_tokens": {
|
|
"max": 4459,
|
|
"mean": 1152.125,
|
|
"min": 201,
|
|
"p50": 699,
|
|
"p95": 4459,
|
|
"p99": 4459
|
|
},
|
|
"queue_waiting": {
|
|
"max": 0,
|
|
"mean": 0.0,
|
|
"min": 0,
|
|
"p50": 0,
|
|
"p95": 0,
|
|
"p99": 0
|
|
},
|
|
"scheduled_requests": {
|
|
"max": 1,
|
|
"mean": 1.0,
|
|
"min": 1,
|
|
"p50": 1,
|
|
"p95": 1,
|
|
"p99": 1
|
|
},
|
|
"steps": 8,
|
|
"submit_to_complete_ms": {
|
|
"max": 169.859824,
|
|
"mean": 79.969349875,
|
|
"min": 59.109261,
|
|
"p50": 63.446202,
|
|
"p95": 169.859824,
|
|
"p99": 169.859824
|
|
},
|
|
"total_tokens": {
|
|
"max": 4459,
|
|
"mean": 1152.125,
|
|
"min": 201,
|
|
"p50": 699,
|
|
"p95": 4459,
|
|
"p99": 4459
|
|
}
|
|
},
|
|
{
|
|
"cudagraph_runtime_mode": "PIECEWISE",
|
|
"decode_tokens": {
|
|
"max": 0,
|
|
"mean": 0.0,
|
|
"min": 0,
|
|
"p50": 0,
|
|
"p95": 0,
|
|
"p99": 0
|
|
},
|
|
"kv_usage": {
|
|
"max": 0.0014334028780281782,
|
|
"mean": 0.0006869451515105393,
|
|
"min": 0.0002805592821762959,
|
|
"p50": 0.00034687329432714353,
|
|
"p95": 0.0014334028780281782,
|
|
"p99": 0.0014334028780281782
|
|
},
|
|
"phase": "pure_prefill",
|
|
"prefill_tokens": {
|
|
"max": 23,
|
|
"mean": 13.0,
|
|
"min": 3,
|
|
"p50": 13,
|
|
"p95": 23,
|
|
"p99": 23
|
|
},
|
|
"queue_waiting": {
|
|
"max": 0,
|
|
"mean": 0.0,
|
|
"min": 0,
|
|
"p50": 0,
|
|
"p95": 0,
|
|
"p99": 0
|
|
},
|
|
"scheduled_requests": {
|
|
"max": 1,
|
|
"mean": 1.0,
|
|
"min": 1,
|
|
"p50": 1,
|
|
"p95": 1,
|
|
"p99": 1
|
|
},
|
|
"steps": 3,
|
|
"submit_to_complete_ms": {
|
|
"max": 13.064208,
|
|
"mean": 12.249279000000001,
|
|
"min": 10.993105,
|
|
"p50": 12.690524,
|
|
"p95": 13.064208,
|
|
"p99": 13.064208
|
|
},
|
|
"total_tokens": {
|
|
"max": 23,
|
|
"mean": 13.0,
|
|
"min": 3,
|
|
"p50": 13,
|
|
"p95": 23,
|
|
"p99": 23
|
|
}
|
|
},
|
|
{
|
|
"cudagraph_runtime_mode": "FULL",
|
|
"decode_tokens": {
|
|
"max": 15,
|
|
"mean": 15.0,
|
|
"min": 15,
|
|
"p50": 15,
|
|
"p95": 15,
|
|
"p99": 15
|
|
},
|
|
"kv_usage": {
|
|
"max": 0.014119783510255668,
|
|
"mean": 0.014119783510255668,
|
|
"min": 0.014119783510255668,
|
|
"p50": 0.014119783510255668,
|
|
"p95": 0.014119783510255668,
|
|
"p99": 0.014119783510255668
|
|
},
|
|
"phase": "true_mixed",
|
|
"prefill_tokens": {
|
|
"max": 1,
|
|
"mean": 1.0,
|
|
"min": 1,
|
|
"p50": 1,
|
|
"p95": 1,
|
|
"p99": 1
|
|
},
|
|
"queue_waiting": {
|
|
"max": 3,
|
|
"mean": 3.0,
|
|
"min": 3,
|
|
"p50": 3,
|
|
"p95": 3,
|
|
"p99": 3
|
|
},
|
|
"scheduled_requests": {
|
|
"max": 16,
|
|
"mean": 16.0,
|
|
"min": 16,
|
|
"p50": 16,
|
|
"p95": 16,
|
|
"p99": 16
|
|
},
|
|
"steps": 1,
|
|
"submit_to_complete_ms": {
|
|
"max": 31.93902,
|
|
"mean": 31.93902,
|
|
"min": 31.93902,
|
|
"p50": 31.93902,
|
|
"p95": 31.93902,
|
|
"p99": 31.93902
|
|
},
|
|
"total_tokens": {
|
|
"max": 16,
|
|
"mean": 16.0,
|
|
"min": 16,
|
|
"p50": 16,
|
|
"p95": 16,
|
|
"p99": 16
|
|
}
|
|
},
|
|
{
|
|
"cudagraph_runtime_mode": "NONE",
|
|
"decode_tokens": {
|
|
"max": 15,
|
|
"mean": 13.41112385321101,
|
|
"min": 1,
|
|
"p50": 15,
|
|
"p95": 15,
|
|
"p99": 15
|
|
},
|
|
"kv_usage": {
|
|
"max": 0.02384243790712981,
|
|
"mean": 0.013269559018371043,
|
|
"min": 0.0002397506593143639,
|
|
"p50": 0.013645383269484834,
|
|
"p95": 0.019067829032274553,
|
|
"p99": 0.021705086284732
|
|
},
|
|
"phase": "true_mixed",
|
|
"prefill_tokens": {
|
|
"max": 8191,
|
|
"mean": 2915.523509174312,
|
|
"min": 18,
|
|
"p50": 2078,
|
|
"p95": 7887,
|
|
"p99": 8180
|
|
},
|
|
"queue_waiting": {
|
|
"max": 48,
|
|
"mean": 13.461582568807339,
|
|
"min": 0,
|
|
"p50": 10,
|
|
"p95": 47,
|
|
"p99": 47
|
|
},
|
|
"scheduled_requests": {
|
|
"max": 16,
|
|
"mean": 14.555045871559633,
|
|
"min": 2,
|
|
"p50": 16,
|
|
"p95": 16,
|
|
"p99": 16
|
|
},
|
|
"steps": 1744,
|
|
"submit_to_complete_ms": {
|
|
"max": 379.907708,
|
|
"mean": 108.41418502752293,
|
|
"min": 56.308147,
|
|
"p50": 83.525673,
|
|
"p95": 217.228507,
|
|
"p99": 297.197569
|
|
},
|
|
"total_tokens": {
|
|
"max": 8192,
|
|
"mean": 2928.934633027523,
|
|
"min": 33,
|
|
"p50": 2093,
|
|
"p95": 7900,
|
|
"p99": 8192
|
|
}
|
|
},
|
|
{
|
|
"cudagraph_runtime_mode": "PIECEWISE",
|
|
"decode_tokens": {
|
|
"max": 15,
|
|
"mean": 9.54,
|
|
"min": 1,
|
|
"p50": 9,
|
|
"p95": 15,
|
|
"p99": 15
|
|
},
|
|
"kv_usage": {
|
|
"max": 0.022806919102006273,
|
|
"mean": 0.009802027168340671,
|
|
"min": 0.0015405255130409579,
|
|
"p50": 0.008386171998143177,
|
|
"p95": 0.018516912623637416,
|
|
"p99": 0.022806919102006273
|
|
},
|
|
"phase": "true_mixed",
|
|
"prefill_tokens": {
|
|
"max": 25,
|
|
"mean": 13.94,
|
|
"min": 2,
|
|
"p50": 13,
|
|
"p95": 23,
|
|
"p99": 25
|
|
},
|
|
"queue_waiting": {
|
|
"max": 47,
|
|
"mean": 5.82,
|
|
"min": 0,
|
|
"p50": 0,
|
|
"p95": 47,
|
|
"p99": 47
|
|
},
|
|
"scheduled_requests": {
|
|
"max": 16,
|
|
"mean": 10.54,
|
|
"min": 2,
|
|
"p50": 10,
|
|
"p95": 16,
|
|
"p99": 16
|
|
},
|
|
"steps": 50,
|
|
"submit_to_complete_ms": {
|
|
"max": 91.408766,
|
|
"mean": 24.5079404,
|
|
"min": 11.206266,
|
|
"p50": 14.216713,
|
|
"p95": 83.023614,
|
|
"p99": 91.408766
|
|
},
|
|
"total_tokens": {
|
|
"max": 32,
|
|
"mean": 23.48,
|
|
"min": 4,
|
|
"p50": 25,
|
|
"p95": 32,
|
|
"p99": 32
|
|
}
|
|
}
|
|
],
|
|
"max_num_seqs": 16,
|
|
"phase_counts": {
|
|
"pure_decode": 17417,
|
|
"pure_prefill": 11,
|
|
"true_mixed": 1795
|
|
},
|
|
"prefix": {
|
|
"hit_rate": 0.13543755577854244,
|
|
"hits": 798096,
|
|
"queries": 5892723
|
|
},
|
|
"records": 19223,
|
|
"stream": "/home/admin/cpfs/wjh/fidelity-prefix-pilot-20260714/runs/p1b/cells/tp4_mns16/opprof/opprof-v1-dp0-pid256090-1784007028237607411.jsonl",
|
|
"tensor_parallel_size": 4
|
|
},
|
|
{
|
|
"cell": "tp4_mns64",
|
|
"cudagraph_runtime_mode_counts": {
|
|
"FULL": 21812,
|
|
"NONE": 2809,
|
|
"PIECEWISE": 354
|
|
},
|
|
"dropped_records_before_max": 0,
|
|
"groups": [
|
|
{
|
|
"cudagraph_runtime_mode": "FULL",
|
|
"decode_tokens": {
|
|
"max": 64,
|
|
"mean": 20.639510361269025,
|
|
"min": 1,
|
|
"p50": 15,
|
|
"p95": 62,
|
|
"p99": 64
|
|
},
|
|
"kv_usage": {
|
|
"max": 0.07507819047813735,
|
|
"mean": 0.01804654466537005,
|
|
"min": 4.088391013712389e-05,
|
|
"p50": 0.013655225985813257,
|
|
"p95": 0.05230074204296897,
|
|
"p99": 0.062281526605204474
|
|
},
|
|
"phase": "pure_decode",
|
|
"prefill_tokens": {
|
|
"max": 0,
|
|
"mean": 0.0,
|
|
"min": 0,
|
|
"p50": 0,
|
|
"p95": 0,
|
|
"p99": 0
|
|
},
|
|
"queue_waiting": {
|
|
"max": 0,
|
|
"mean": 0.0,
|
|
"min": 0,
|
|
"p50": 0,
|
|
"p95": 0,
|
|
"p99": 0
|
|
},
|
|
"scheduled_requests": {
|
|
"max": 64,
|
|
"mean": 20.639510361269025,
|
|
"min": 1,
|
|
"p50": 15,
|
|
"p95": 62,
|
|
"p99": 64
|
|
},
|
|
"steps": 21812,
|
|
"submit_to_complete_ms": {
|
|
"max": 198.715632,
|
|
"mean": 21.35240870649184,
|
|
"min": 6.440609,
|
|
"p50": 13.906458,
|
|
"p95": 68.920454,
|
|
"p99": 155.977421
|
|
},
|
|
"total_tokens": {
|
|
"max": 64,
|
|
"mean": 20.639510361269025,
|
|
"min": 1,
|
|
"p50": 15,
|
|
"p95": 62,
|
|
"p99": 64
|
|
}
|
|
},
|
|
{
|
|
"cudagraph_runtime_mode": "NONE",
|
|
"decode_tokens": {
|
|
"max": 0,
|
|
"mean": 0.0,
|
|
"min": 0,
|
|
"p50": 0,
|
|
"p95": 0,
|
|
"p99": 0
|
|
},
|
|
"kv_usage": {
|
|
"max": 0.0025552443835727967,
|
|
"mean": 0.0008350538645516159,
|
|
"min": 0.00032196079233015595,
|
|
"p50": 0.00038839714630312105,
|
|
"p95": 0.0025552443835727967,
|
|
"p99": 0.0025552443835727967
|
|
},
|
|
"phase": "pure_prefill",
|
|
"prefill_tokens": {
|
|
"max": 7950,
|
|
"mean": 2158.0,
|
|
"min": 201,
|
|
"p50": 959,
|
|
"p95": 7950,
|
|
"p99": 7950
|
|
},
|
|
"queue_waiting": {
|
|
"max": 0,
|
|
"mean": 0.0,
|
|
"min": 0,
|
|
"p50": 0,
|
|
"p95": 0,
|
|
"p99": 0
|
|
},
|
|
"scheduled_requests": {
|
|
"max": 1,
|
|
"mean": 1.0,
|
|
"min": 1,
|
|
"p50": 1,
|
|
"p95": 1,
|
|
"p99": 1
|
|
},
|
|
"steps": 10,
|
|
"submit_to_complete_ms": {
|
|
"max": 173.889993,
|
|
"mean": 88.2164954,
|
|
"min": 60.32347,
|
|
"p50": 64.173068,
|
|
"p95": 173.889993,
|
|
"p99": 173.889993
|
|
},
|
|
"total_tokens": {
|
|
"max": 7950,
|
|
"mean": 2158.0,
|
|
"min": 201,
|
|
"p50": 959,
|
|
"p95": 7950,
|
|
"p99": 7950
|
|
}
|
|
},
|
|
{
|
|
"cudagraph_runtime_mode": "PIECEWISE",
|
|
"decode_tokens": {
|
|
"max": 0,
|
|
"mean": 0.0,
|
|
"min": 0,
|
|
"p50": 0,
|
|
"p95": 0,
|
|
"p99": 0
|
|
},
|
|
"kv_usage": {
|
|
"max": 0.00034751323616588614,
|
|
"mean": 0.0003142950591794591,
|
|
"min": 0.00028107688219303206,
|
|
"p50": 0.00028107688219303206,
|
|
"p95": 0.00034751323616588614,
|
|
"p99": 0.00034751323616588614
|
|
},
|
|
"phase": "pure_prefill",
|
|
"prefill_tokens": {
|
|
"max": 23,
|
|
"mean": 13.0,
|
|
"min": 3,
|
|
"p50": 3,
|
|
"p95": 23,
|
|
"p99": 23
|
|
},
|
|
"queue_waiting": {
|
|
"max": 0,
|
|
"mean": 0.0,
|
|
"min": 0,
|
|
"p50": 0,
|
|
"p95": 0,
|
|
"p99": 0
|
|
},
|
|
"scheduled_requests": {
|
|
"max": 1,
|
|
"mean": 1.0,
|
|
"min": 1,
|
|
"p50": 1,
|
|
"p95": 1,
|
|
"p99": 1
|
|
},
|
|
"steps": 2,
|
|
"submit_to_complete_ms": {
|
|
"max": 14.426163,
|
|
"mean": 12.918343,
|
|
"min": 11.410523,
|
|
"p50": 11.410523,
|
|
"p95": 14.426163,
|
|
"p99": 14.426163
|
|
},
|
|
"total_tokens": {
|
|
"max": 23,
|
|
"mean": 13.0,
|
|
"min": 3,
|
|
"p50": 3,
|
|
"p95": 23,
|
|
"p99": 23
|
|
}
|
|
},
|
|
{
|
|
"cudagraph_runtime_mode": "NONE",
|
|
"decode_tokens": {
|
|
"max": 63,
|
|
"mean": 31.312254376563057,
|
|
"min": 1,
|
|
"p50": 27,
|
|
"p95": 62,
|
|
"p99": 63
|
|
},
|
|
"kv_usage": {
|
|
"max": 0.07563012326498908,
|
|
"mean": 0.02885461746554652,
|
|
"min": 0.00024019297205579715,
|
|
"p50": 0.025511559925591287,
|
|
"p95": 0.05791716919806211,
|
|
"p99": 0.06878206831701383
|
|
},
|
|
"phase": "true_mixed",
|
|
"prefill_tokens": {
|
|
"max": 8191,
|
|
"mean": 3841.2243658449447,
|
|
"min": 73,
|
|
"p50": 3432,
|
|
"p95": 8154,
|
|
"p99": 8181
|
|
},
|
|
"queue_waiting": {
|
|
"max": 11,
|
|
"mean": 0.1907824222936763,
|
|
"min": 0,
|
|
"p50": 0,
|
|
"p95": 1,
|
|
"p99": 5
|
|
},
|
|
"scheduled_requests": {
|
|
"max": 64,
|
|
"mean": 32.843515541264736,
|
|
"min": 2,
|
|
"p50": 28,
|
|
"p95": 63,
|
|
"p99": 64
|
|
},
|
|
"steps": 2799,
|
|
"submit_to_complete_ms": {
|
|
"max": 373.577501,
|
|
"mean": 134.91752637549124,
|
|
"min": 55.209363,
|
|
"p50": 114.740321,
|
|
"p95": 314.91308,
|
|
"p99": 343.860853
|
|
},
|
|
"total_tokens": {
|
|
"max": 8192,
|
|
"mean": 3872.5366202215077,
|
|
"min": 129,
|
|
"p50": 3456,
|
|
"p95": 8192,
|
|
"p99": 8192
|
|
}
|
|
},
|
|
{
|
|
"cudagraph_runtime_mode": "PIECEWISE",
|
|
"decode_tokens": {
|
|
"max": 63,
|
|
"mean": 25.65340909090909,
|
|
"min": 1,
|
|
"p50": 20,
|
|
"p95": 60,
|
|
"p99": 63
|
|
},
|
|
"kv_usage": {
|
|
"max": 0.07241562583045447,
|
|
"mean": 0.022061428307276035,
|
|
"min": 0.00047016496657736884,
|
|
"p50": 0.017590302336515484,
|
|
"p95": 0.05158016312680147,
|
|
"p99": 0.05824424047915944
|
|
},
|
|
"phase": "true_mixed",
|
|
"prefill_tokens": {
|
|
"max": 114,
|
|
"mean": 35.32670454545455,
|
|
"min": 2,
|
|
"p50": 25,
|
|
"p95": 87,
|
|
"p99": 106
|
|
},
|
|
"queue_waiting": {
|
|
"max": 0,
|
|
"mean": 0.0,
|
|
"min": 0,
|
|
"p50": 0,
|
|
"p95": 0,
|
|
"p99": 0
|
|
},
|
|
"scheduled_requests": {
|
|
"max": 64,
|
|
"mean": 26.678977272727273,
|
|
"min": 2,
|
|
"p50": 21,
|
|
"p95": 61,
|
|
"p99": 64
|
|
},
|
|
"steps": 352,
|
|
"submit_to_complete_ms": {
|
|
"max": 196.960993,
|
|
"mean": 27.942972144886365,
|
|
"min": 11.037722,
|
|
"p50": 17.402246,
|
|
"p95": 97.041082,
|
|
"p99": 187.504426
|
|
},
|
|
"total_tokens": {
|
|
"max": 128,
|
|
"mean": 60.98011363636363,
|
|
"min": 4,
|
|
"p50": 60,
|
|
"p95": 114,
|
|
"p99": 126
|
|
}
|
|
}
|
|
],
|
|
"max_num_seqs": 64,
|
|
"phase_counts": {
|
|
"pure_decode": 21812,
|
|
"pure_prefill": 12,
|
|
"true_mixed": 3151
|
|
},
|
|
"prefix": {
|
|
"hit_rate": 0.13386788146702433,
|
|
"hits": 1667008,
|
|
"queries": 12452636
|
|
},
|
|
"records": 24975,
|
|
"stream": "/home/admin/cpfs/wjh/fidelity-prefix-pilot-20260714/runs/p1b/cells/tp4_mns64/opprof/opprof-v1-dp0-pid262293-1784007376383569515.jsonl",
|
|
"tensor_parallel_size": 4
|
|
}
|
|
],
|
|
"contains_prompt_text": false,
|
|
"measurement_semantics": "complete_mono_ns - submit_mono_ns from the existing vLLM opprof record; this includes the instrumented submit-to-completion interval and is not claimed to be a single-kernel CUDA-event duration",
|
|
"schema_version": "qwen30_vllm020_opprof_summary.v1"
|
|
}
|