635 lines
18 KiB
JSON
635 lines
18 KiB
JSON
{
|
|
"attention_profile_sha256": "fbcf7e1f95789a6f6d771e24d1fc60958b7daf04eb0db260d27869a19d71d550",
|
|
"cell_version": "v3",
|
|
"cells": [
|
|
{
|
|
"decode_batch": {
|
|
"histogram": {
|
|
"1": 5525
|
|
},
|
|
"max": 1,
|
|
"share_gt_1": 0.0,
|
|
"share_gt_4": 0.0,
|
|
"stages": 5525
|
|
},
|
|
"drain_fraction": 0.00044126232317770095,
|
|
"drain_tail_s": 1.6150201028303854,
|
|
"offered_load": {
|
|
"decode_tokens_per_s": 1.5183060109289617,
|
|
"prefill_tokens_per_s_after_prefix": 189.91639344262296,
|
|
"prefill_tokens_per_s_raw": 201.38743169398907,
|
|
"requests_per_s": 0.008743169398907104
|
|
},
|
|
"prefix_cache_hit_ratio": 0.05698154180238871,
|
|
"requests": 32,
|
|
"rho": 5e-05,
|
|
"subcritical_gate": true,
|
|
"topology": "tp4_mns16",
|
|
"tpot_ms": {
|
|
"count": 32,
|
|
"max": 6.06516500761245,
|
|
"mean": 5.267661756941212,
|
|
"min": 4.642776716764274,
|
|
"p50": 5.27961358181982,
|
|
"p90": 6.065165007589712,
|
|
"p95": 6.06516500761245,
|
|
"p99": 6.06516500761245
|
|
},
|
|
"trace_sha256": "bdc54edafacf3e3f353de7a54d5e43a7e4323eb1e8b3dc55ce333a5e0b347ce5",
|
|
"ttft_ms": {
|
|
"count": 32,
|
|
"max": 9026.404118319988,
|
|
"mean": 1316.2539772416199,
|
|
"min": 76.98844366404956,
|
|
"p50": 401.828087941567,
|
|
"p90": 3363.622890573971,
|
|
"p95": 6721.082190563591,
|
|
"p99": 8741.382807755395
|
|
},
|
|
"waiting_ms": {
|
|
"count": 32,
|
|
"max": 0.0,
|
|
"mean": 0.0,
|
|
"min": 0.0,
|
|
"p50": 0.0,
|
|
"p90": 0.0,
|
|
"p95": 0.0,
|
|
"p99": 0.0
|
|
}
|
|
},
|
|
{
|
|
"decode_batch": {
|
|
"histogram": {
|
|
"1": 33249,
|
|
"2": 294,
|
|
"3": 76
|
|
},
|
|
"max": 3,
|
|
"share_gt_1": 0.011005681311163331,
|
|
"share_gt_4": 0.0,
|
|
"stages": 33619
|
|
},
|
|
"drain_fraction": 0.0006573023445132039,
|
|
"drain_tail_s": 2.4057265809183264,
|
|
"offered_load": {
|
|
"decode_tokens_per_s": 9.343989071038251,
|
|
"prefill_tokens_per_s_after_prefix": 491.8571038251366,
|
|
"prefill_tokens_per_s_raw": 1477.1773224043716,
|
|
"requests_per_s": 0.03224043715846994
|
|
},
|
|
"prefix_cache_hit_ratio": 0.667142227102801,
|
|
"requests": 118,
|
|
"rho": 0.0001,
|
|
"subcritical_gate": true,
|
|
"topology": "tp4_mns16",
|
|
"tpot_ms": {
|
|
"count": 118,
|
|
"max": 109.2514556116484,
|
|
"mean": 6.521761912064516,
|
|
"min": 4.642776716750063,
|
|
"p50": 5.7134254039277,
|
|
"p90": 6.06516500761245,
|
|
"p95": 6.582261050425584,
|
|
"p99": 14.049063410907634
|
|
},
|
|
"trace_sha256": "8d4ac5646f46e84560c17c44013af2bcb7cfe91d4a2be3476d8f3c09565cf55f",
|
|
"ttft_ms": {
|
|
"count": 118,
|
|
"max": 9696.373176562247,
|
|
"mean": 1203.6180593647337,
|
|
"min": 72.1560867923472,
|
|
"p50": 592.0461311603731,
|
|
"p90": 3231.0798294979118,
|
|
"p95": 6885.471905257904,
|
|
"p99": 8918.025104477005
|
|
},
|
|
"waiting_ms": {
|
|
"count": 118,
|
|
"max": 7151.845512636555,
|
|
"mean": 98.59009397263883,
|
|
"min": 0.0,
|
|
"p50": 0.0,
|
|
"p90": 0.0,
|
|
"p95": 0.21863999056675887,
|
|
"p99": 2087.1951396751865
|
|
}
|
|
},
|
|
{
|
|
"decode_batch": {
|
|
"histogram": {
|
|
"1": 71338,
|
|
"2": 4093,
|
|
"3": 1021,
|
|
"4": 33
|
|
},
|
|
"max": 4,
|
|
"share_gt_1": 0.067294240700791,
|
|
"share_gt_4": 0.0,
|
|
"stages": 76485
|
|
},
|
|
"drain_fraction": 0.001483881651363999,
|
|
"drain_tail_s": 5.431006843992236,
|
|
"offered_load": {
|
|
"decode_tokens_per_s": 22.691803278688525,
|
|
"prefill_tokens_per_s_after_prefix": 884.2945355191257,
|
|
"prefill_tokens_per_s_raw": 2459.7437158469947,
|
|
"requests_per_s": 0.0633879781420765
|
|
},
|
|
"prefix_cache_hit_ratio": 0.6406200282639031,
|
|
"requests": 232,
|
|
"rho": 0.0002,
|
|
"subcritical_gate": true,
|
|
"topology": "tp2_mns16",
|
|
"tpot_ms": {
|
|
"count": 231,
|
|
"max": 151.0151050020596,
|
|
"mean": 8.197384891414327,
|
|
"min": 4.790955988028145,
|
|
"p50": 6.377284733844135,
|
|
"p90": 9.066414456096654,
|
|
"p95": 13.235755136549969,
|
|
"p99": 45.72584194314243
|
|
},
|
|
"trace_sha256": "188f9e52674ec268d70d763995f2cdf84ccbcbab86420214b127994569324e9b",
|
|
"ttft_ms": {
|
|
"count": 232,
|
|
"max": 22394.384315265255,
|
|
"mean": 2036.2309970727358,
|
|
"min": 90.95485497891787,
|
|
"p50": 876.6052006358223,
|
|
"p90": 5676.270318256946,
|
|
"p95": 9955.05948414866,
|
|
"p99": 15641.83197487735
|
|
},
|
|
"waiting_ms": {
|
|
"count": 232,
|
|
"max": 13155.607757982125,
|
|
"mean": 338.8558697712575,
|
|
"min": 0.0,
|
|
"p50": 0.0,
|
|
"p90": 5.941389865392926,
|
|
"p95": 1644.4218195787325,
|
|
"p99": 8012.825782587187
|
|
}
|
|
},
|
|
{
|
|
"decode_batch": {
|
|
"histogram": {
|
|
"1": 72379,
|
|
"2": 4138,
|
|
"3": 696
|
|
},
|
|
"max": 3,
|
|
"share_gt_1": 0.06260603784336835,
|
|
"share_gt_4": 0.0,
|
|
"stages": 77213
|
|
},
|
|
"drain_fraction": 0.0010192168060329308,
|
|
"drain_tail_s": 3.730333510080527,
|
|
"offered_load": {
|
|
"decode_tokens_per_s": 22.691803278688525,
|
|
"prefill_tokens_per_s_after_prefix": 880.7972677595628,
|
|
"prefill_tokens_per_s_raw": 2459.7437158469947,
|
|
"requests_per_s": 0.0633879781420765
|
|
},
|
|
"prefix_cache_hit_ratio": 0.6420421114379927,
|
|
"requests": 232,
|
|
"rho": 0.0002,
|
|
"subcritical_gate": true,
|
|
"topology": "tp4_mns16",
|
|
"tpot_ms": {
|
|
"count": 231,
|
|
"max": 109.2514556116484,
|
|
"mean": 6.682483450713786,
|
|
"min": 4.6427767165369005,
|
|
"p50": 5.757631515507455,
|
|
"p90": 6.873691941109428,
|
|
"p95": 8.022302513342353,
|
|
"p99": 28.619774468019706
|
|
},
|
|
"trace_sha256": "188f9e52674ec268d70d763995f2cdf84ccbcbab86420214b127994569324e9b",
|
|
"ttft_ms": {
|
|
"count": 232,
|
|
"max": 12080.695954866542,
|
|
"mean": 1109.6819359234096,
|
|
"min": 72.1560867923472,
|
|
"p50": 527.4056752296019,
|
|
"p90": 2603.3383651315494,
|
|
"p95": 5585.788369509169,
|
|
"p99": 8828.771798959253
|
|
},
|
|
"waiting_ms": {
|
|
"count": 232,
|
|
"max": 7151.845512636555,
|
|
"mean": 100.5913696184332,
|
|
"min": 0.0,
|
|
"p50": 0.0,
|
|
"p90": 2.71668797665825,
|
|
"p95": 337.7136230834021,
|
|
"p99": 2261.809117635907
|
|
}
|
|
},
|
|
{
|
|
"decode_batch": {
|
|
"histogram": {
|
|
"1": 163554,
|
|
"10": 18,
|
|
"11": 5,
|
|
"12": 10,
|
|
"2": 48104,
|
|
"3": 15894,
|
|
"4": 2673,
|
|
"5": 1305,
|
|
"6": 332,
|
|
"7": 181,
|
|
"8": 46,
|
|
"9": 15
|
|
},
|
|
"max": 12,
|
|
"share_gt_1": 0.2954419157652593,
|
|
"share_gt_4": 0.008236515505929689,
|
|
"stages": 232137
|
|
},
|
|
"drain_fraction": 0.001932022033012745,
|
|
"drain_tail_s": 7.071200640826646,
|
|
"offered_load": {
|
|
"decode_tokens_per_s": 90.30054644808743,
|
|
"prefill_tokens_per_s_after_prefix": 1805.1767759562842,
|
|
"prefill_tokens_per_s_raw": 5620.918852459016,
|
|
"requests_per_s": 0.13633879781420766
|
|
},
|
|
"prefix_cache_hit_ratio": 0.6789729811301968,
|
|
"requests": 499,
|
|
"rho": 0.0004,
|
|
"subcritical_gate": true,
|
|
"topology": "tp2_mns16",
|
|
"tpot_ms": {
|
|
"count": 497,
|
|
"max": 556.5990343567938,
|
|
"mean": 18.61788167095804,
|
|
"min": 4.790955988028145,
|
|
"p50": 7.918935117459114,
|
|
"p90": 26.775158132602897,
|
|
"p95": 56.43467403678585,
|
|
"p99": 256.82570583636533
|
|
},
|
|
"trace_sha256": "2751c373bb91eccf42b94e1b28245870d6ffa2f91e1b5dabafe2435bc46da274",
|
|
"ttft_ms": {
|
|
"count": 499,
|
|
"max": 28488.652032648133,
|
|
"mean": 2601.050597929721,
|
|
"min": 90.96449664048123,
|
|
"p50": 942.4448686800133,
|
|
"p90": 7645.075071658986,
|
|
"p95": 12524.265102993284,
|
|
"p99": 19300.755134457228
|
|
},
|
|
"waiting_ms": {
|
|
"count": 499,
|
|
"max": 25277.458416518584,
|
|
"mean": 926.4084886277059,
|
|
"min": 0.0,
|
|
"p50": 2.7291107814733095,
|
|
"p90": 2483.439910098784,
|
|
"p95": 6352.661668917902,
|
|
"p99": 13692.62653031437
|
|
}
|
|
},
|
|
{
|
|
"decode_batch": {
|
|
"histogram": {
|
|
"1": 184431,
|
|
"2": 52296,
|
|
"3": 10583,
|
|
"4": 1254,
|
|
"5": 675
|
|
},
|
|
"max": 5,
|
|
"share_gt_1": 0.2600235115692167,
|
|
"share_gt_4": 0.002708243894414598,
|
|
"stages": 249239
|
|
},
|
|
"drain_fraction": 0.001114813580178276,
|
|
"drain_tail_s": 4.08021770345249,
|
|
"offered_load": {
|
|
"decode_tokens_per_s": 90.30054644808743,
|
|
"prefill_tokens_per_s_after_prefix": 1738.4489071038251,
|
|
"prefill_tokens_per_s_raw": 5620.918852459016,
|
|
"requests_per_s": 0.13633879781420766
|
|
},
|
|
"prefix_cache_hit_ratio": 0.6908465352465023,
|
|
"requests": 499,
|
|
"rho": 0.0004,
|
|
"subcritical_gate": true,
|
|
"topology": "tp4_mns16",
|
|
"tpot_ms": {
|
|
"count": 497,
|
|
"max": 191.36693232402044,
|
|
"mean": 9.29907499058627,
|
|
"min": 4.6427767165369005,
|
|
"p50": 6.554626532726112,
|
|
"p90": 10.321323503714908,
|
|
"p95": 18.835821286381098,
|
|
"p99": 73.17188748931419
|
|
},
|
|
"trace_sha256": "2751c373bb91eccf42b94e1b28245870d6ffa2f91e1b5dabafe2435bc46da274",
|
|
"ttft_ms": {
|
|
"count": 499,
|
|
"max": 14944.866787426236,
|
|
"mean": 1252.450784907019,
|
|
"min": 72.1560867923472,
|
|
"p50": 526.7327146962089,
|
|
"p90": 3295.008250609953,
|
|
"p95": 5487.844406213842,
|
|
"p99": 10840.596450182773
|
|
},
|
|
"waiting_ms": {
|
|
"count": 499,
|
|
"max": 12922.960017376226,
|
|
"mean": 274.43276731097365,
|
|
"min": 0.0,
|
|
"p50": 0.6520015415389935,
|
|
"p90": 164.9360333363801,
|
|
"p95": 2029.720098907419,
|
|
"p99": 6246.634668026156
|
|
}
|
|
},
|
|
{
|
|
"decode_batch": {
|
|
"histogram": {
|
|
"1": 123423,
|
|
"10": 262,
|
|
"11": 84,
|
|
"12": 37,
|
|
"13": 42,
|
|
"14": 108,
|
|
"15": 59,
|
|
"16": 39,
|
|
"2": 64628,
|
|
"3": 34551,
|
|
"4": 17977,
|
|
"5": 5215,
|
|
"6": 2255,
|
|
"7": 1043,
|
|
"8": 349,
|
|
"9": 406
|
|
},
|
|
"max": 16,
|
|
"share_gt_1": 0.5072501377366475,
|
|
"share_gt_4": 0.039520436924600166,
|
|
"stages": 250478
|
|
},
|
|
"drain_fraction": 0.0006878766017147268,
|
|
"drain_tail_s": 2.5176283622759,
|
|
"offered_load": {
|
|
"decode_tokens_per_s": 135.20737704918034,
|
|
"prefill_tokens_per_s_after_prefix": 3107.11174863388,
|
|
"prefill_tokens_per_s_raw": 7722.316120218579,
|
|
"requests_per_s": 0.23743169398907105
|
|
},
|
|
"prefix_cache_hit_ratio": 0.5977800565206239,
|
|
"requests": 869,
|
|
"rho": 0.0008,
|
|
"subcritical_gate": true,
|
|
"topology": "tp2_mns16",
|
|
"tpot_ms": {
|
|
"count": 864,
|
|
"max": 1048.199917263105,
|
|
"mean": 35.531282784807445,
|
|
"min": 4.790955988028145,
|
|
"p50": 10.56810791578755,
|
|
"p90": 58.679343047855866,
|
|
"p95": 113.21830549665293,
|
|
"p99": 497.8300148864786
|
|
},
|
|
"trace_sha256": "2788a21c6bb8be6d03fb5151f2c51efa09c0eb5e1f941af60c00a7cc11d41206",
|
|
"ttft_ms": {
|
|
"count": 869,
|
|
"max": 35023.43713844812,
|
|
"mean": 3519.677566968775,
|
|
"min": 80.48812343014333,
|
|
"p50": 944.5946905507299,
|
|
"p90": 11539.678186253443,
|
|
"p95": 16395.861888973763,
|
|
"p99": 23283.982779742393
|
|
},
|
|
"waiting_ms": {
|
|
"count": 869,
|
|
"max": 33885.372635082604,
|
|
"mean": 1944.5601581730987,
|
|
"min": 0.0,
|
|
"p50": 4.989910657513974,
|
|
"p90": 7660.373377703809,
|
|
"p95": 12268.558603129362,
|
|
"p99": 19104.247337156932
|
|
}
|
|
},
|
|
{
|
|
"decode_batch": {
|
|
"histogram": {
|
|
"1": 185761,
|
|
"2": 81136,
|
|
"3": 28184,
|
|
"4": 10328,
|
|
"5": 2183,
|
|
"6": 759,
|
|
"7": 199,
|
|
"8": 89,
|
|
"9": 18
|
|
},
|
|
"max": 9,
|
|
"share_gt_1": 0.3981636573931581,
|
|
"share_gt_4": 0.010523007739983218,
|
|
"stages": 308657
|
|
},
|
|
"drain_fraction": 0.0005799250803688739,
|
|
"drain_tail_s": 2.1225257941500786,
|
|
"offered_load": {
|
|
"decode_tokens_per_s": 135.20737704918034,
|
|
"prefill_tokens_per_s_after_prefix": 2943.3609289617484,
|
|
"prefill_tokens_per_s_raw": 7722.316120218579,
|
|
"requests_per_s": 0.23743169398907105
|
|
},
|
|
"prefix_cache_hit_ratio": 0.6189897292366545,
|
|
"requests": 869,
|
|
"rho": 0.0008,
|
|
"subcritical_gate": true,
|
|
"topology": "tp4_mns16",
|
|
"tpot_ms": {
|
|
"count": 864,
|
|
"max": 556.1359991102521,
|
|
"mean": 12.147948538437653,
|
|
"min": 4.6427767165369005,
|
|
"p50": 6.843951976861717,
|
|
"p90": 14.306706048821074,
|
|
"p95": 30.43280397112956,
|
|
"p99": 137.6170979713351
|
|
},
|
|
"trace_sha256": "2788a21c6bb8be6d03fb5151f2c51efa09c0eb5e1f941af60c00a7cc11d41206",
|
|
"ttft_ms": {
|
|
"count": 869,
|
|
"max": 17076.806859823137,
|
|
"mean": 1363.8187082248587,
|
|
"min": 67.46058986817616,
|
|
"p50": 457.5338581378219,
|
|
"p90": 4063.5681271006715,
|
|
"p95": 6757.6291299215145,
|
|
"p99": 11100.99635793714
|
|
},
|
|
"waiting_ms": {
|
|
"count": 869,
|
|
"max": 16033.766590147934,
|
|
"mean": 459.6734459851503,
|
|
"min": 0.0,
|
|
"p50": 2.5305451699750847,
|
|
"p90": 796.2202857544993,
|
|
"p95": 3149.129404067843,
|
|
"p99": 9085.03332971697
|
|
}
|
|
},
|
|
{
|
|
"decode_batch": {
|
|
"histogram": {
|
|
"1": 8883,
|
|
"10": 102,
|
|
"11": 206,
|
|
"12": 48,
|
|
"13": 104,
|
|
"14": 98,
|
|
"15": 112,
|
|
"16": 42432,
|
|
"2": 8887,
|
|
"3": 5320,
|
|
"4": 1084,
|
|
"5": 391,
|
|
"6": 1238,
|
|
"7": 983,
|
|
"8": 600,
|
|
"9": 185
|
|
},
|
|
"max": 16,
|
|
"share_gt_1": 0.8743084346214255,
|
|
"share_gt_4": 0.6579457501450342,
|
|
"stages": 70673
|
|
},
|
|
"drain_fraction": 0.26690162220748465,
|
|
"drain_tail_s": 976.8599372793938,
|
|
"offered_load": {
|
|
"decode_tokens_per_s": 222.2658469945355,
|
|
"prefill_tokens_per_s_after_prefix": 6478.3401639344265,
|
|
"prefill_tokens_per_s_raw": 14510.624316939891,
|
|
"requests_per_s": 0.46939890710382515
|
|
},
|
|
"prefix_cache_hit_ratio": 0.5536777047530368,
|
|
"requests": 1718,
|
|
"rho": 0.0016,
|
|
"subcritical_gate": false,
|
|
"topology": "tp2_mns16",
|
|
"tpot_ms": {
|
|
"count": 1710,
|
|
"max": 1075.3968454229532,
|
|
"mean": 82.12803809595945,
|
|
"min": 5.044100516215622,
|
|
"p50": 55.434284109098826,
|
|
"p90": 148.45522338371572,
|
|
"p95": 240.8300400647424,
|
|
"p99": 469.60790088235683
|
|
},
|
|
"trace_sha256": "d785665c86a562c9d9a7492ad710e6393bb883bf8c3ad552a34ff18f22925043",
|
|
"ttft_ms": {
|
|
"count": 1718,
|
|
"max": 899333.5996373689,
|
|
"mean": 363823.5998174686,
|
|
"min": 95.16785391630833,
|
|
"p50": 333469.8464378732,
|
|
"p90": 786932.9549504423,
|
|
"p95": 836208.7308378813,
|
|
"p99": 879481.7276833938
|
|
},
|
|
"waiting_ms": {
|
|
"count": 1718,
|
|
"max": 898008.7571403392,
|
|
"mean": 362060.3850241827,
|
|
"min": 0.0,
|
|
"p50": 332965.98011225695,
|
|
"p90": 785791.1926569697,
|
|
"p95": 832975.7759936375,
|
|
"p99": 872501.6632742453
|
|
}
|
|
},
|
|
{
|
|
"decode_batch": {
|
|
"histogram": {
|
|
"1": 82578,
|
|
"10": 992,
|
|
"11": 686,
|
|
"12": 564,
|
|
"13": 388,
|
|
"14": 207,
|
|
"15": 120,
|
|
"16": 556,
|
|
"2": 76732,
|
|
"3": 49252,
|
|
"4": 30080,
|
|
"5": 19950,
|
|
"6": 9869,
|
|
"7": 6575,
|
|
"8": 3735,
|
|
"9": 1733
|
|
},
|
|
"max": 16,
|
|
"share_gt_1": 0.7092497984275589,
|
|
"share_gt_4": 0.15976156356837795,
|
|
"stages": 284017
|
|
},
|
|
"drain_fraction": 0.0057571275011030374,
|
|
"drain_tail_s": 21.071086654037117,
|
|
"offered_load": {
|
|
"decode_tokens_per_s": 222.2658469945355,
|
|
"prefill_tokens_per_s_after_prefix": 6066.187158469946,
|
|
"prefill_tokens_per_s_raw": 14510.624316939891,
|
|
"requests_per_s": 0.46939890710382515
|
|
},
|
|
"prefix_cache_hit_ratio": 0.5820880455385098,
|
|
"requests": 1718,
|
|
"rho": 0.0016,
|
|
"subcritical_gate": true,
|
|
"topology": "tp4_mns16",
|
|
"tpot_ms": {
|
|
"count": 1710,
|
|
"max": 779.2730316123988,
|
|
"mean": 31.087273162821237,
|
|
"min": 4.6427767165369005,
|
|
"p50": 9.443696854706767,
|
|
"p90": 67.85056779351899,
|
|
"p95": 145.53426807885694,
|
|
"p99": 346.8968267376913
|
|
},
|
|
"trace_sha256": "d785665c86a562c9d9a7492ad710e6393bb883bf8c3ad552a34ff18f22925043",
|
|
"ttft_ms": {
|
|
"count": 1718,
|
|
"max": 24498.655361823694,
|
|
"mean": 2344.154809416248,
|
|
"min": 72.15684780067022,
|
|
"p50": 646.7976478797937,
|
|
"p90": 7657.698621058489,
|
|
"p95": 10053.03089890951,
|
|
"p99": 16706.358496869783
|
|
},
|
|
"waiting_ms": {
|
|
"count": 1718,
|
|
"max": 20607.142196811765,
|
|
"mean": 1395.7184472333554,
|
|
"min": 0.0,
|
|
"p50": 5.521385635063325,
|
|
"p90": 5253.222863654173,
|
|
"p95": 8173.9598212075725,
|
|
"p99": 14808.95660872061
|
|
}
|
|
}
|
|
],
|
|
"schema": "frontier-code-trace-calibration-summary-v1",
|
|
"subcritical_rule": "drain tail <= 10% of the 3660s arrival window",
|
|
"workload_mode": "prefill_decode"
|
|
}
|